Spaces:
Running
Hf testing (#7)
Browse files* Re-order columns for the leaderboard
* Update with correct dataset link
* Hide structure track
* chore: re-order variables
* Order columns on leaderboard correctly
* Fix DownloadButton eagerly fetching leaderboard CSVs on every page load
* Refresh live leaderboards on page load, not just every 30s
* Wire soft-threshold RAE (ST-RAE) credible-interval scoring through evaluation pipeline
* Fix RAE -> ST-RAE fallout in SORT_REGRESSION_LEADERBOARD_BY and
hf_space/head_to_head.py's TRACK_PRIMARY_METRIC/LOWER_IS_BETTER, which would
have broken leaderboard sorting and head-to-head comparisons.
( Fix _resolve_bounds so a missing bound (or both) defaults to the central
estimate rather than crashing or requiring ±inf semantics, so the metric
degrades gracefully to plain RAE when no interval info is available.
- Update conftest.py fixture path for the renamed ground truth test file and
regenerate the stale bootstrap-scores fixture.
- Add unit tests for the soft-threshold RAE math and its wiring through
bootstrap_metrics.
- Update backend/README.md and docs/ to describe ST-RAE and the
credible-interval wiring.
* Add tutorial link
- app.py +1 -1
- config.py +14 -12
- head_to_head.py +5 -4
- leaderboards.py +52 -19
|
@@ -121,7 +121,7 @@ with gr.Blocks(
|
|
| 121 |
# During phase 0 (pre-challenge), there will be placeholder markdown
|
| 122 |
if CURRENT_PHASE in [0, 1, 2]:
|
| 123 |
with gr.TabItem("🏎️ Live Leaderboard"):
|
| 124 |
-
render_live_leaderboards(phase=CURRENT_PHASE)
|
| 125 |
|
| 126 |
# Display the interim leaderboard tab from phase 2 onwards
|
| 127 |
if CURRENT_PHASE in [2, 3, 4]:
|
|
|
|
| 121 |
# During phase 0 (pre-challenge), there will be placeholder markdown
|
| 122 |
if CURRENT_PHASE in [0, 1, 2]:
|
| 123 |
with gr.TabItem("🏎️ Live Leaderboard"):
|
| 124 |
+
render_live_leaderboards(phase=CURRENT_PHASE, demo=demo)
|
| 125 |
|
| 126 |
# Display the interim leaderboard tab from phase 2 onwards
|
| 127 |
if CURRENT_PHASE in [2, 3, 4]:
|
|
@@ -12,6 +12,7 @@ import os
|
|
| 12 |
# 3: Challenge closed, final leaderboard not yet released
|
| 13 |
# 4: Challenge closed, final leaderboard released
|
| 14 |
CURRENT_PHASE = 0
|
|
|
|
| 15 |
|
| 16 |
# Leaderboard and S3 configuration.
|
| 17 |
# Regression, classification, and structure are three fully independent backend
|
|
@@ -39,13 +40,20 @@ AWS_DEFAULT_REGION: str = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
|
|
| 39 |
CHALLENGE_ANNOUCEMENT_LINK = (
|
| 40 |
"https://openadmet.ghost.io/announcing-openadmets-cyp-inhibition-blind-challenge"
|
| 41 |
)
|
| 42 |
-
DATASET_DOWNLOAD_LINK =
|
| 43 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
# -------------------------------- Activity Track Params -------------------------------
|
| 46 |
|
| 47 |
# Dataset sizes and required columns
|
| 48 |
-
ACTIVITY_DATASET_SIZE = 750
|
| 49 |
IDENTIFIER_COLUMNS = ["SMILES", "Molecule_Name"]
|
| 50 |
|
| 51 |
# Drives one leaderboard tab per activity endpoint. Must mirror
|
|
@@ -77,18 +85,12 @@ CLASSIFICATION_ACTIVE = True
|
|
| 77 |
ACTIVITY_ENDPOINTS = REGRESSION_ENDPOINTS + CLASSIFICATION_ENDPOINTS
|
| 78 |
REQUIRED_REGRESSION_COLUMNS = IDENTIFIER_COLUMNS + REGRESSION_ENDPOINTS
|
| 79 |
REQUIRED_CLASSIFICATION_COLUMNS = IDENTIFIER_COLUMNS + CLASSIFICATION_ENDPOINTS
|
| 80 |
-
HOURS_BETWEEN_SUBMISSIONS = 4
|
| 81 |
-
|
| 82 |
-
# Activity bootstrap-results.parquet files hold one row per (Sample, Endpoint) pair,
|
| 83 |
-
# including this synthetic pseudo-endpoint holding the multi-endpoint macro-averaged
|
| 84 |
-
# scores for that bootstrap sample. Must mirror backend.config.MACRO_ENDPOINT_LABEL.
|
| 85 |
-
MACRO_ENDPOINT_LABEL = "MA"
|
| 86 |
|
| 87 |
# Metric names as they appear (unprefixed, once narrowed to a single endpoint) in a
|
| 88 |
# leaderboard's columns, and how each should be labelled in the UI, in display order
|
| 89 |
# (primary/sort metric RAE first). Must mirror backend.config.ACTIVITY_METRICS.
|
| 90 |
ACTIVITY_METRIC_DISPLAY_NAMES = {
|
| 91 |
-
"RAE": "RAE",
|
| 92 |
"MAE": "MAE",
|
| 93 |
"R2": "R²",
|
| 94 |
"Spearman_R": "Spearman ρ",
|
|
@@ -109,7 +111,7 @@ CLASSIFICATION_METRIC_DISPLAY_NAMES = {
|
|
| 109 |
|
| 110 |
# The structure track will be started halfway through the challenge, so we can control
|
| 111 |
# when it is visible on the HF Space.
|
| 112 |
-
STRUCTURE_TRACK_LIVE =
|
| 113 |
STRUCTURE_DATASET_SIZE = 184 # TODO: Update when final dataset is ready
|
| 114 |
|
| 115 |
# Structure track's single leaderboard slug. Must mirror
|
|
@@ -415,4 +417,4 @@ Validation scripts will be released on GitHub. Always run the validation script
|
|
| 415 |
|
| 416 |
**My entry hasn't appeared on the leaderboard. What should I do?**
|
| 417 |
First, check the `#cyp-challenge-submissions` channel on Discord for automated receipts and error logs. If your submission failed validation, re-run the local validation script against your file. If no receipt appears after 2 hours, post a message in `#cyp-challenge` for help.
|
| 418 |
-
"""
|
|
|
|
| 12 |
# 3: Challenge closed, final leaderboard not yet released
|
| 13 |
# 4: Challenge closed, final leaderboard released
|
| 14 |
CURRENT_PHASE = 0
|
| 15 |
+
HOURS_BETWEEN_SUBMISSIONS = 12
|
| 16 |
|
| 17 |
# Leaderboard and S3 configuration.
|
| 18 |
# Regression, classification, and structure are three fully independent backend
|
|
|
|
| 40 |
CHALLENGE_ANNOUCEMENT_LINK = (
|
| 41 |
"https://openadmet.ghost.io/announcing-openadmets-cyp-inhibition-blind-challenge"
|
| 42 |
)
|
| 43 |
+
DATASET_DOWNLOAD_LINK = (
|
| 44 |
+
"https://huggingface.co/datasets/openadmet/cyp-challenge-train-test"
|
| 45 |
+
)
|
| 46 |
+
TUTORIAL_LINK = "https://github.com/OpenADMET/CYP-Challenge-Tutorial"
|
| 47 |
+
|
| 48 |
+
# Activity bootstrap-results.parquet files hold one row per (Sample, Endpoint) pair,
|
| 49 |
+
# including this synthetic pseudo-endpoint holding the multi-endpoint macro-averaged
|
| 50 |
+
# scores for that bootstrap sample. Must mirror backend.config.MACRO_ENDPOINT_LABEL.
|
| 51 |
+
MACRO_ENDPOINT_LABEL = "MA"
|
| 52 |
|
| 53 |
# -------------------------------- Activity Track Params -------------------------------
|
| 54 |
|
| 55 |
# Dataset sizes and required columns
|
| 56 |
+
ACTIVITY_DATASET_SIZE = 750
|
| 57 |
IDENTIFIER_COLUMNS = ["SMILES", "Molecule_Name"]
|
| 58 |
|
| 59 |
# Drives one leaderboard tab per activity endpoint. Must mirror
|
|
|
|
| 85 |
ACTIVITY_ENDPOINTS = REGRESSION_ENDPOINTS + CLASSIFICATION_ENDPOINTS
|
| 86 |
REQUIRED_REGRESSION_COLUMNS = IDENTIFIER_COLUMNS + REGRESSION_ENDPOINTS
|
| 87 |
REQUIRED_CLASSIFICATION_COLUMNS = IDENTIFIER_COLUMNS + CLASSIFICATION_ENDPOINTS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
# Metric names as they appear (unprefixed, once narrowed to a single endpoint) in a
|
| 90 |
# leaderboard's columns, and how each should be labelled in the UI, in display order
|
| 91 |
# (primary/sort metric RAE first). Must mirror backend.config.ACTIVITY_METRICS.
|
| 92 |
ACTIVITY_METRIC_DISPLAY_NAMES = {
|
| 93 |
+
"ST-RAE": "ST-RAE",
|
| 94 |
"MAE": "MAE",
|
| 95 |
"R2": "R²",
|
| 96 |
"Spearman_R": "Spearman ρ",
|
|
|
|
| 111 |
|
| 112 |
# The structure track will be started halfway through the challenge, so we can control
|
| 113 |
# when it is visible on the HF Space.
|
| 114 |
+
STRUCTURE_TRACK_LIVE = False
|
| 115 |
STRUCTURE_DATASET_SIZE = 184 # TODO: Update when final dataset is ready
|
| 116 |
|
| 117 |
# Structure track's single leaderboard slug. Must mirror
|
|
|
|
| 417 |
|
| 418 |
**My entry hasn't appeared on the leaderboard. What should I do?**
|
| 419 |
First, check the `#cyp-challenge-submissions` channel on Discord for automated receipts and error logs. If your submission failed validation, re-run the local validation script against your file. If no receipt appears after 2 hours, post a message in `#cyp-challenge` for help.
|
| 420 |
+
"""
|
|
@@ -15,10 +15,11 @@ IDENTIFIERS_URI = "ground_truth/{track}-identifiers.parquet"
|
|
| 15 |
|
| 16 |
# Track-specific primary metric and sort direction. Activity now scores 8 endpoints
|
| 17 |
# (see ACTIVITY_ENDPOINTS in config.py); this comparison only ever uses the
|
| 18 |
-
# multi-endpoint macro-averaged primary metric (RAE), matching backend's
|
| 19 |
-
# ranking/CLD metric — see MACRO_ENDPOINT_LABEL handling in
|
| 20 |
-
|
| 21 |
-
|
|
|
|
| 22 |
ALPHA_LEVEL = 0.05
|
| 23 |
ACTIVITY_TARGET_PHASE = 2 # Phase-2 compounds used for activity scatter
|
| 24 |
# One scatter tab per activity endpoint.
|
|
|
|
| 15 |
|
| 16 |
# Track-specific primary metric and sort direction. Activity now scores 8 endpoints
|
| 17 |
# (see ACTIVITY_ENDPOINTS in config.py); this comparison only ever uses the
|
| 18 |
+
# multi-endpoint macro-averaged primary metric (ST-RAE), matching backend's
|
| 19 |
+
# leaderboard ranking/CLD metric — see MACRO_ENDPOINT_LABEL handling in
|
| 20 |
+
# _load_bootstrap below. Must mirror backend.config.SORT_REGRESSION_LEADERBOARD_BY.
|
| 21 |
+
TRACK_PRIMARY_METRIC = {"activity": "ST-RAE", "structure": "LDDT-PLI"}
|
| 22 |
+
LOWER_IS_BETTER = ["ST-RAE"]
|
| 23 |
ALPHA_LEVEL = 0.05
|
| 24 |
ACTIVITY_TARGET_PHASE = 2 # Phase-2 compounds used for activity scatter
|
| 25 |
# One scatter tab per activity endpoint.
|
|
@@ -95,7 +95,7 @@ def format_leaderboard_uri(
|
|
| 95 |
def sort_leaderboard_columns(df: pd.DataFrame) -> pd.DataFrame:
|
| 96 |
"""Sort leaderboard columns in a consistent order."""
|
| 97 |
start_cols = ["rank", "username", "Submitted", "model_report_link"]
|
| 98 |
-
end_cols = ["Proprietary Data", "Open Code"
|
| 99 |
start_cols = [c for c in start_cols if c in df.columns]
|
| 100 |
end_cols = [c for c in end_cols if c in df.columns]
|
| 101 |
middle_cols = [c for c in df.columns if c not in start_cols + end_cols]
|
|
@@ -155,7 +155,7 @@ def _prepare_activity_df(
|
|
| 155 |
metric-column schema, since the backend narrows down to one endpoint's columns
|
| 156 |
before saving (see ``aws_leaderboards._narrow_averaged_results_to_endpoint``). So
|
| 157 |
this one function renders all of them for both tracks; ``metric_display_names``
|
| 158 |
-
(regression's RAE/MAE/R2/Spearman/Kendall or classification's
|
| 159 |
MCC/Accuracy/Precision/Recall/F1) and ``metric_prefix`` ("MA-" for a track's
|
| 160 |
master, "" for endpoint tabs whose own tab title already identifies the endpoint)
|
| 161 |
are the only things that differ between calls.
|
|
@@ -188,6 +188,19 @@ def _prepare_activity_df(
|
|
| 188 |
for col in metric_cols:
|
| 189 |
if col in df.columns:
|
| 190 |
df[col] = df[col].round(4)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
df["Submitted"] = pd.to_datetime(df["Submitted"], utc=True).dt.strftime(
|
| 192 |
"%Y-%m-%d %H:%M UTC"
|
| 193 |
)
|
|
@@ -415,7 +428,7 @@ def render_final_leaderboards(phase: int):
|
|
| 415 |
_add_leaderboard_tabs(leaderboard_type="final")
|
| 416 |
|
| 417 |
|
| 418 |
-
def render_live_leaderboards(phase: int):
|
| 419 |
"""Render the live leaderboards for the Gradio app."""
|
| 420 |
if phase == 0:
|
| 421 |
gr.Markdown("# Live Leaderboards (Coming Soon)")
|
|
@@ -425,7 +438,7 @@ def render_live_leaderboards(phase: int):
|
|
| 425 |
)
|
| 426 |
else:
|
| 427 |
gr.Markdown("# Live Leaderboards")
|
| 428 |
-
_add_leaderboard_tabs(leaderboard_type="live")
|
| 429 |
|
| 430 |
|
| 431 |
def _add_activity_leaderboard(
|
|
@@ -442,9 +455,9 @@ def _add_activity_leaderboard(
|
|
| 442 |
leaderboard — only which S3 file is loaded (``endpoint_slug``) and how the metric
|
| 443 |
columns are labelled (``metric_prefix``) differ.
|
| 444 |
"""
|
| 445 |
-
gr.DownloadButton(
|
| 446 |
-
|
| 447 |
-
|
| 448 |
download_activity_leaderboard,
|
| 449 |
leaderboard_type=leaderboard_type,
|
| 450 |
track=track,
|
|
@@ -452,7 +465,7 @@ def _add_activity_leaderboard(
|
|
| 452 |
endpoint_slug=endpoint_slug,
|
| 453 |
metric_prefix=metric_prefix,
|
| 454 |
),
|
| 455 |
-
|
| 456 |
)
|
| 457 |
metric_cols = [f"{metric_prefix}{d}" for d in metric_display_names.values()]
|
| 458 |
return Leaderboard(
|
|
@@ -502,15 +515,15 @@ def _add_structure_leaderboard(
|
|
| 502 |
Same layout/columns regardless of ``endpoint_slug`` — mirrors
|
| 503 |
``_add_activity_leaderboard`` for the structure track's fixed metric set.
|
| 504 |
"""
|
| 505 |
-
gr.DownloadButton(
|
| 506 |
-
|
| 507 |
-
|
| 508 |
download_structure_leaderboard,
|
| 509 |
leaderboard_type=leaderboard_type,
|
| 510 |
track=track,
|
| 511 |
endpoint_slug=endpoint_slug,
|
| 512 |
),
|
| 513 |
-
|
| 514 |
)
|
| 515 |
return Leaderboard(
|
| 516 |
value=load_structure_leaderboard(
|
|
@@ -622,6 +635,7 @@ def _add_structure_track_tabs(
|
|
| 622 |
|
| 623 |
def _wire_activity_timer(
|
| 624 |
timer: gr.Timer,
|
|
|
|
| 625 |
leaderboard_type: str,
|
| 626 |
track: str,
|
| 627 |
lbs: dict[str, Leaderboard],
|
|
@@ -629,10 +643,15 @@ def _wire_activity_timer(
|
|
| 629 |
) -> None:
|
| 630 |
"""Wire a live-refresh timer to every leaderboard component of one
|
| 631 |
regression/classification track.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 632 |
"""
|
| 633 |
for endpoint_slug, lb in lbs.items():
|
| 634 |
metric_prefix = "MA-" if endpoint_slug == MACRO_ENDPOINT_LABEL else ""
|
| 635 |
-
|
|
|
|
| 636 |
fn=partial(
|
| 637 |
load_activity_leaderboard,
|
| 638 |
leaderboard_type=leaderboard_type,
|
|
@@ -646,11 +665,19 @@ def _wire_activity_timer(
|
|
| 646 |
|
| 647 |
|
| 648 |
def _wire_structure_timer(
|
| 649 |
-
timer: gr.Timer,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 650 |
) -> None:
|
| 651 |
-
"""Wire a live-refresh timer to every leaderboard component of the structure track.
|
|
|
|
|
|
|
|
|
|
| 652 |
for endpoint_slug, lb in lbs.items():
|
| 653 |
-
|
|
|
|
| 654 |
fn=partial(
|
| 655 |
load_structure_leaderboard,
|
| 656 |
leaderboard_type=leaderboard_type,
|
|
@@ -661,14 +688,17 @@ def _wire_structure_timer(
|
|
| 661 |
)
|
| 662 |
|
| 663 |
|
| 664 |
-
def _add_leaderboard_tabs(leaderboard_type: str):
|
| 665 |
"""Add the regression, classification, and structure leaderboard tabs to the
|
| 666 |
Gradio app.
|
| 667 |
|
| 668 |
Args:
|
| 669 |
leaderboard_type (str): The leaderboard_type of the leaderboard to display.
|
| 670 |
One of "live", "interim", or "final". Both "interim" and "final" are
|
| 671 |
-
static leaderboards, while "live" is updated every 30 seconds
|
|
|
|
|
|
|
|
|
|
| 672 |
|
| 673 |
"""
|
| 674 |
if leaderboard_type == "live":
|
|
@@ -703,8 +733,10 @@ def _add_leaderboard_tabs(leaderboard_type: str):
|
|
| 703 |
)
|
| 704 |
|
| 705 |
if leaderboard_type == "live":
|
|
|
|
| 706 |
_wire_activity_timer(
|
| 707 |
regression_timer,
|
|
|
|
| 708 |
leaderboard_type,
|
| 709 |
REGRESSION_TRACK,
|
| 710 |
regression_lbs,
|
|
@@ -712,11 +744,12 @@ def _add_leaderboard_tabs(leaderboard_type: str):
|
|
| 712 |
)
|
| 713 |
_wire_activity_timer(
|
| 714 |
classification_timer,
|
|
|
|
| 715 |
leaderboard_type,
|
| 716 |
CLASSIFICATION_TRACK,
|
| 717 |
classification_lbs,
|
| 718 |
CLASSIFICATION_METRIC_DISPLAY_NAMES,
|
| 719 |
)
|
| 720 |
_wire_structure_timer(
|
| 721 |
-
structure_timer, leaderboard_type, STRUCTURE_TRACK, structure_lbs
|
| 722 |
)
|
|
|
|
| 95 |
def sort_leaderboard_columns(df: pd.DataFrame) -> pd.DataFrame:
|
| 96 |
"""Sort leaderboard columns in a consistent order."""
|
| 97 |
start_cols = ["rank", "username", "Submitted", "model_report_link"]
|
| 98 |
+
end_cols = ["Proprietary Data", "Open Code"]
|
| 99 |
start_cols = [c for c in start_cols if c in df.columns]
|
| 100 |
end_cols = [c for c in end_cols if c in df.columns]
|
| 101 |
middle_cols = [c for c in df.columns if c not in start_cols + end_cols]
|
|
|
|
| 155 |
metric-column schema, since the backend narrows down to one endpoint's columns
|
| 156 |
before saving (see ``aws_leaderboards._narrow_averaged_results_to_endpoint``). So
|
| 157 |
this one function renders all of them for both tracks; ``metric_display_names``
|
| 158 |
+
(regression's ST-RAE/MAE/R2/Spearman/Kendall or classification's
|
| 159 |
MCC/Accuracy/Precision/Recall/F1) and ``metric_prefix`` ("MA-" for a track's
|
| 160 |
master, "" for endpoint tabs whose own tab title already identifies the endpoint)
|
| 161 |
are the only things that differ between calls.
|
|
|
|
| 188 |
for col in metric_cols:
|
| 189 |
if col in df.columns:
|
| 190 |
df[col] = df[col].round(4)
|
| 191 |
+
# The source file's physical column order doesn't necessarily match
|
| 192 |
+
# metric_display_names' canonical order (primary/sort metric first) — force it,
|
| 193 |
+
# since sort_leaderboard_columns/gradio_leaderboard's select_columns both preserve
|
| 194 |
+
# whatever order the DataFrame's columns already have rather than reordering them.
|
| 195 |
+
ordered_metric_cols = []
|
| 196 |
+
for col in metric_cols:
|
| 197 |
+
if col in df.columns:
|
| 198 |
+
ordered_metric_cols.append(col)
|
| 199 |
+
std_col = f"{col} std"
|
| 200 |
+
if std_col in df.columns:
|
| 201 |
+
ordered_metric_cols.append(std_col)
|
| 202 |
+
other_cols = [c for c in df.columns if c not in ordered_metric_cols]
|
| 203 |
+
df = df[other_cols + ordered_metric_cols]
|
| 204 |
df["Submitted"] = pd.to_datetime(df["Submitted"], utc=True).dt.strftime(
|
| 205 |
"%Y-%m-%d %H:%M UTC"
|
| 206 |
)
|
|
|
|
| 428 |
_add_leaderboard_tabs(leaderboard_type="final")
|
| 429 |
|
| 430 |
|
| 431 |
+
def render_live_leaderboards(phase: int, demo: gr.Blocks):
|
| 432 |
"""Render the live leaderboards for the Gradio app."""
|
| 433 |
if phase == 0:
|
| 434 |
gr.Markdown("# Live Leaderboards (Coming Soon)")
|
|
|
|
| 438 |
)
|
| 439 |
else:
|
| 440 |
gr.Markdown("# Live Leaderboards")
|
| 441 |
+
_add_leaderboard_tabs(leaderboard_type="live", demo=demo)
|
| 442 |
|
| 443 |
|
| 444 |
def _add_activity_leaderboard(
|
|
|
|
| 455 |
leaderboard — only which S3 file is loaded (``endpoint_slug``) and how the metric
|
| 456 |
columns are labelled (``metric_prefix``) differ.
|
| 457 |
"""
|
| 458 |
+
download_btn = gr.DownloadButton(label="Download CSV", size="sm")
|
| 459 |
+
download_btn.click(
|
| 460 |
+
fn=partial(
|
| 461 |
download_activity_leaderboard,
|
| 462 |
leaderboard_type=leaderboard_type,
|
| 463 |
track=track,
|
|
|
|
| 465 |
endpoint_slug=endpoint_slug,
|
| 466 |
metric_prefix=metric_prefix,
|
| 467 |
),
|
| 468 |
+
outputs=download_btn,
|
| 469 |
)
|
| 470 |
metric_cols = [f"{metric_prefix}{d}" for d in metric_display_names.values()]
|
| 471 |
return Leaderboard(
|
|
|
|
| 515 |
Same layout/columns regardless of ``endpoint_slug`` — mirrors
|
| 516 |
``_add_activity_leaderboard`` for the structure track's fixed metric set.
|
| 517 |
"""
|
| 518 |
+
download_btn = gr.DownloadButton(label="Download CSV", size="sm")
|
| 519 |
+
download_btn.click(
|
| 520 |
+
fn=partial(
|
| 521 |
download_structure_leaderboard,
|
| 522 |
leaderboard_type=leaderboard_type,
|
| 523 |
track=track,
|
| 524 |
endpoint_slug=endpoint_slug,
|
| 525 |
),
|
| 526 |
+
outputs=download_btn,
|
| 527 |
)
|
| 528 |
return Leaderboard(
|
| 529 |
value=load_structure_leaderboard(
|
|
|
|
| 635 |
|
| 636 |
def _wire_activity_timer(
|
| 637 |
timer: gr.Timer,
|
| 638 |
+
demo: gr.Blocks,
|
| 639 |
leaderboard_type: str,
|
| 640 |
track: str,
|
| 641 |
lbs: dict[str, Leaderboard],
|
|
|
|
| 643 |
) -> None:
|
| 644 |
"""Wire a live-refresh timer to every leaderboard component of one
|
| 645 |
regression/classification track.
|
| 646 |
+
|
| 647 |
+
Also fires on ``demo.load`` (i.e. whenever a browser session opens the page), so a
|
| 648 |
+
visitor sees fresh data immediately rather than the snapshot baked in when the
|
| 649 |
+
Blocks graph was built, and doesn't have to wait for the first timer tick.
|
| 650 |
"""
|
| 651 |
for endpoint_slug, lb in lbs.items():
|
| 652 |
metric_prefix = "MA-" if endpoint_slug == MACRO_ENDPOINT_LABEL else ""
|
| 653 |
+
gr.on(
|
| 654 |
+
triggers=[demo.load, timer.tick],
|
| 655 |
fn=partial(
|
| 656 |
load_activity_leaderboard,
|
| 657 |
leaderboard_type=leaderboard_type,
|
|
|
|
| 665 |
|
| 666 |
|
| 667 |
def _wire_structure_timer(
|
| 668 |
+
timer: gr.Timer,
|
| 669 |
+
demo: gr.Blocks,
|
| 670 |
+
leaderboard_type: str,
|
| 671 |
+
track: str,
|
| 672 |
+
lbs: dict[str, Leaderboard],
|
| 673 |
) -> None:
|
| 674 |
+
"""Wire a live-refresh timer to every leaderboard component of the structure track.
|
| 675 |
+
|
| 676 |
+
Also fires on ``demo.load`` — see ``_wire_activity_timer``.
|
| 677 |
+
"""
|
| 678 |
for endpoint_slug, lb in lbs.items():
|
| 679 |
+
gr.on(
|
| 680 |
+
triggers=[demo.load, timer.tick],
|
| 681 |
fn=partial(
|
| 682 |
load_structure_leaderboard,
|
| 683 |
leaderboard_type=leaderboard_type,
|
|
|
|
| 688 |
)
|
| 689 |
|
| 690 |
|
| 691 |
+
def _add_leaderboard_tabs(leaderboard_type: str, demo: gr.Blocks | None = None):
|
| 692 |
"""Add the regression, classification, and structure leaderboard tabs to the
|
| 693 |
Gradio app.
|
| 694 |
|
| 695 |
Args:
|
| 696 |
leaderboard_type (str): The leaderboard_type of the leaderboard to display.
|
| 697 |
One of "live", "interim", or "final". Both "interim" and "final" are
|
| 698 |
+
static leaderboards, while "live" is updated every 30 seconds and on
|
| 699 |
+
every page load.
|
| 700 |
+
demo (gr.Blocks | None): The app's Blocks instance, needed to wire the
|
| 701 |
+
page-load refresh. Required when ``leaderboard_type == "live"``.
|
| 702 |
|
| 703 |
"""
|
| 704 |
if leaderboard_type == "live":
|
|
|
|
| 733 |
)
|
| 734 |
|
| 735 |
if leaderboard_type == "live":
|
| 736 |
+
assert demo is not None, "demo is required to wire live leaderboards"
|
| 737 |
_wire_activity_timer(
|
| 738 |
regression_timer,
|
| 739 |
+
demo,
|
| 740 |
leaderboard_type,
|
| 741 |
REGRESSION_TRACK,
|
| 742 |
regression_lbs,
|
|
|
|
| 744 |
)
|
| 745 |
_wire_activity_timer(
|
| 746 |
classification_timer,
|
| 747 |
+
demo,
|
| 748 |
leaderboard_type,
|
| 749 |
CLASSIFICATION_TRACK,
|
| 750 |
classification_lbs,
|
| 751 |
CLASSIFICATION_METRIC_DISPLAY_NAMES,
|
| 752 |
)
|
| 753 |
_wire_structure_timer(
|
| 754 |
+
structure_timer, demo, leaderboard_type, STRUCTURE_TRACK, structure_lbs
|
| 755 |
)
|