Jon Swain commited on
Commit
df707db
·
1 Parent(s): f281a4d

Hf testing (#7)

Browse files

* Re-order columns for the leaderboard
* Update with correct dataset link
* Hide structure track
* chore: re-order variables
* Order columns on leaderboard correctly
* Fix DownloadButton eagerly fetching leaderboard CSVs on every page load
* Refresh live leaderboards on page load, not just every 30s
* Wire soft-threshold RAE (ST-RAE) credible-interval scoring through evaluation pipeline
* Fix RAE -> ST-RAE fallout in SORT_REGRESSION_LEADERBOARD_BY and
hf_space/head_to_head.py's TRACK_PRIMARY_METRIC/LOWER_IS_BETTER, which would
have broken leaderboard sorting and head-to-head comparisons.
( Fix _resolve_bounds so a missing bound (or both) defaults to the central
estimate rather than crashing or requiring ±inf semantics, so the metric
degrades gracefully to plain RAE when no interval info is available.
- Update conftest.py fixture path for the renamed ground truth test file and
regenerate the stale bootstrap-scores fixture.
- Add unit tests for the soft-threshold RAE math and its wiring through
bootstrap_metrics.
- Update backend/README.md and docs/ to describe ST-RAE and the
credible-interval wiring.
* Add tutorial link

Files changed (4) hide show
  1. app.py +1 -1
  2. config.py +14 -12
  3. head_to_head.py +5 -4
  4. leaderboards.py +52 -19
app.py CHANGED
@@ -121,7 +121,7 @@ with gr.Blocks(
121
  # During phase 0 (pre-challenge), there will be placeholder markdown
122
  if CURRENT_PHASE in [0, 1, 2]:
123
  with gr.TabItem("🏎️ Live Leaderboard"):
124
- render_live_leaderboards(phase=CURRENT_PHASE)
125
 
126
  # Display the interim leaderboard tab from phase 2 onwards
127
  if CURRENT_PHASE in [2, 3, 4]:
 
121
  # During phase 0 (pre-challenge), there will be placeholder markdown
122
  if CURRENT_PHASE in [0, 1, 2]:
123
  with gr.TabItem("🏎️ Live Leaderboard"):
124
+ render_live_leaderboards(phase=CURRENT_PHASE, demo=demo)
125
 
126
  # Display the interim leaderboard tab from phase 2 onwards
127
  if CURRENT_PHASE in [2, 3, 4]:
config.py CHANGED
@@ -12,6 +12,7 @@ import os
12
  # 3: Challenge closed, final leaderboard not yet released
13
  # 4: Challenge closed, final leaderboard released
14
  CURRENT_PHASE = 0
 
15
 
16
  # Leaderboard and S3 configuration.
17
  # Regression, classification, and structure are three fully independent backend
@@ -39,13 +40,20 @@ AWS_DEFAULT_REGION: str = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
39
  CHALLENGE_ANNOUCEMENT_LINK = (
40
  "https://openadmet.ghost.io/announcing-openadmets-cyp-inhibition-blind-challenge"
41
  )
42
- DATASET_DOWNLOAD_LINK = False # TODO: Update with actual link
43
- TUTORIAL_LINK = False # TODO: Update with actual link
 
 
 
 
 
 
 
44
 
45
  # -------------------------------- Activity Track Params -------------------------------
46
 
47
  # Dataset sizes and required columns
48
- ACTIVITY_DATASET_SIZE = 750 # TODO: Update when final dataset is ready
49
  IDENTIFIER_COLUMNS = ["SMILES", "Molecule_Name"]
50
 
51
  # Drives one leaderboard tab per activity endpoint. Must mirror
@@ -77,18 +85,12 @@ CLASSIFICATION_ACTIVE = True
77
  ACTIVITY_ENDPOINTS = REGRESSION_ENDPOINTS + CLASSIFICATION_ENDPOINTS
78
  REQUIRED_REGRESSION_COLUMNS = IDENTIFIER_COLUMNS + REGRESSION_ENDPOINTS
79
  REQUIRED_CLASSIFICATION_COLUMNS = IDENTIFIER_COLUMNS + CLASSIFICATION_ENDPOINTS
80
- HOURS_BETWEEN_SUBMISSIONS = 4
81
-
82
- # Activity bootstrap-results.parquet files hold one row per (Sample, Endpoint) pair,
83
- # including this synthetic pseudo-endpoint holding the multi-endpoint macro-averaged
84
- # scores for that bootstrap sample. Must mirror backend.config.MACRO_ENDPOINT_LABEL.
85
- MACRO_ENDPOINT_LABEL = "MA"
86
 
87
  # Metric names as they appear (unprefixed, once narrowed to a single endpoint) in a
88
  # leaderboard's columns, and how each should be labelled in the UI, in display order
89
  # (primary/sort metric RAE first). Must mirror backend.config.ACTIVITY_METRICS.
90
  ACTIVITY_METRIC_DISPLAY_NAMES = {
91
- "RAE": "RAE",
92
  "MAE": "MAE",
93
  "R2": "R²",
94
  "Spearman_R": "Spearman ρ",
@@ -109,7 +111,7 @@ CLASSIFICATION_METRIC_DISPLAY_NAMES = {
109
 
110
  # The structure track will be started halfway through the challenge, so we can control
111
  # when it is visible on the HF Space.
112
- STRUCTURE_TRACK_LIVE = True
113
  STRUCTURE_DATASET_SIZE = 184 # TODO: Update when final dataset is ready
114
 
115
  # Structure track's single leaderboard slug. Must mirror
@@ -415,4 +417,4 @@ Validation scripts will be released on GitHub. Always run the validation script
415
 
416
  **My entry hasn't appeared on the leaderboard. What should I do?**
417
  First, check the `#cyp-challenge-submissions` channel on Discord for automated receipts and error logs. If your submission failed validation, re-run the local validation script against your file. If no receipt appears after 2 hours, post a message in `#cyp-challenge` for help.
418
- """ # TODO: Proof read
 
12
  # 3: Challenge closed, final leaderboard not yet released
13
  # 4: Challenge closed, final leaderboard released
14
  CURRENT_PHASE = 0
15
+ HOURS_BETWEEN_SUBMISSIONS = 12
16
 
17
  # Leaderboard and S3 configuration.
18
  # Regression, classification, and structure are three fully independent backend
 
40
  CHALLENGE_ANNOUCEMENT_LINK = (
41
  "https://openadmet.ghost.io/announcing-openadmets-cyp-inhibition-blind-challenge"
42
  )
43
+ DATASET_DOWNLOAD_LINK = (
44
+ "https://huggingface.co/datasets/openadmet/cyp-challenge-train-test"
45
+ )
46
+ TUTORIAL_LINK = "https://github.com/OpenADMET/CYP-Challenge-Tutorial"
47
+
48
+ # Activity bootstrap-results.parquet files hold one row per (Sample, Endpoint) pair,
49
+ # including this synthetic pseudo-endpoint holding the multi-endpoint macro-averaged
50
+ # scores for that bootstrap sample. Must mirror backend.config.MACRO_ENDPOINT_LABEL.
51
+ MACRO_ENDPOINT_LABEL = "MA"
52
 
53
  # -------------------------------- Activity Track Params -------------------------------
54
 
55
  # Dataset sizes and required columns
56
+ ACTIVITY_DATASET_SIZE = 750
57
  IDENTIFIER_COLUMNS = ["SMILES", "Molecule_Name"]
58
 
59
  # Drives one leaderboard tab per activity endpoint. Must mirror
 
85
  ACTIVITY_ENDPOINTS = REGRESSION_ENDPOINTS + CLASSIFICATION_ENDPOINTS
86
  REQUIRED_REGRESSION_COLUMNS = IDENTIFIER_COLUMNS + REGRESSION_ENDPOINTS
87
  REQUIRED_CLASSIFICATION_COLUMNS = IDENTIFIER_COLUMNS + CLASSIFICATION_ENDPOINTS
 
 
 
 
 
 
88
 
89
  # Metric names as they appear (unprefixed, once narrowed to a single endpoint) in a
90
  # leaderboard's columns, and how each should be labelled in the UI, in display order
91
  # (primary/sort metric RAE first). Must mirror backend.config.ACTIVITY_METRICS.
92
  ACTIVITY_METRIC_DISPLAY_NAMES = {
93
+ "ST-RAE": "ST-RAE",
94
  "MAE": "MAE",
95
  "R2": "R²",
96
  "Spearman_R": "Spearman ρ",
 
111
 
112
  # The structure track will be started halfway through the challenge, so we can control
113
  # when it is visible on the HF Space.
114
+ STRUCTURE_TRACK_LIVE = False
115
  STRUCTURE_DATASET_SIZE = 184 # TODO: Update when final dataset is ready
116
 
117
  # Structure track's single leaderboard slug. Must mirror
 
417
 
418
  **My entry hasn't appeared on the leaderboard. What should I do?**
419
  First, check the `#cyp-challenge-submissions` channel on Discord for automated receipts and error logs. If your submission failed validation, re-run the local validation script against your file. If no receipt appears after 2 hours, post a message in `#cyp-challenge` for help.
420
+ """
head_to_head.py CHANGED
@@ -15,10 +15,11 @@ IDENTIFIERS_URI = "ground_truth/{track}-identifiers.parquet"
15
 
16
  # Track-specific primary metric and sort direction. Activity now scores 8 endpoints
17
  # (see ACTIVITY_ENDPOINTS in config.py); this comparison only ever uses the
18
- # multi-endpoint macro-averaged primary metric (RAE), matching backend's leaderboard
19
- # ranking/CLD metric — see MACRO_ENDPOINT_LABEL handling in _load_bootstrap below.
20
- TRACK_PRIMARY_METRIC = {"activity": "RAE", "structure": "LDDT-PLI"}
21
- LOWER_IS_BETTER = ["RAE"]
 
22
  ALPHA_LEVEL = 0.05
23
  ACTIVITY_TARGET_PHASE = 2 # Phase-2 compounds used for activity scatter
24
  # One scatter tab per activity endpoint.
 
15
 
16
  # Track-specific primary metric and sort direction. Activity now scores 8 endpoints
17
  # (see ACTIVITY_ENDPOINTS in config.py); this comparison only ever uses the
18
+ # multi-endpoint macro-averaged primary metric (ST-RAE), matching backend's
19
+ # leaderboard ranking/CLD metric — see MACRO_ENDPOINT_LABEL handling in
20
+ # _load_bootstrap below. Must mirror backend.config.SORT_REGRESSION_LEADERBOARD_BY.
21
+ TRACK_PRIMARY_METRIC = {"activity": "ST-RAE", "structure": "LDDT-PLI"}
22
+ LOWER_IS_BETTER = ["ST-RAE"]
23
  ALPHA_LEVEL = 0.05
24
  ACTIVITY_TARGET_PHASE = 2 # Phase-2 compounds used for activity scatter
25
  # One scatter tab per activity endpoint.
leaderboards.py CHANGED
@@ -95,7 +95,7 @@ def format_leaderboard_uri(
95
  def sort_leaderboard_columns(df: pd.DataFrame) -> pd.DataFrame:
96
  """Sort leaderboard columns in a consistent order."""
97
  start_cols = ["rank", "username", "Submitted", "model_report_link"]
98
- end_cols = ["Proprietary Data", "Open Code", "CLD"]
99
  start_cols = [c for c in start_cols if c in df.columns]
100
  end_cols = [c for c in end_cols if c in df.columns]
101
  middle_cols = [c for c in df.columns if c not in start_cols + end_cols]
@@ -155,7 +155,7 @@ def _prepare_activity_df(
155
  metric-column schema, since the backend narrows down to one endpoint's columns
156
  before saving (see ``aws_leaderboards._narrow_averaged_results_to_endpoint``). So
157
  this one function renders all of them for both tracks; ``metric_display_names``
158
- (regression's RAE/MAE/R2/Spearman/Kendall or classification's
159
  MCC/Accuracy/Precision/Recall/F1) and ``metric_prefix`` ("MA-" for a track's
160
  master, "" for endpoint tabs whose own tab title already identifies the endpoint)
161
  are the only things that differ between calls.
@@ -188,6 +188,19 @@ def _prepare_activity_df(
188
  for col in metric_cols:
189
  if col in df.columns:
190
  df[col] = df[col].round(4)
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  df["Submitted"] = pd.to_datetime(df["Submitted"], utc=True).dt.strftime(
192
  "%Y-%m-%d %H:%M UTC"
193
  )
@@ -415,7 +428,7 @@ def render_final_leaderboards(phase: int):
415
  _add_leaderboard_tabs(leaderboard_type="final")
416
 
417
 
418
- def render_live_leaderboards(phase: int):
419
  """Render the live leaderboards for the Gradio app."""
420
  if phase == 0:
421
  gr.Markdown("# Live Leaderboards (Coming Soon)")
@@ -425,7 +438,7 @@ def render_live_leaderboards(phase: int):
425
  )
426
  else:
427
  gr.Markdown("# Live Leaderboards")
428
- _add_leaderboard_tabs(leaderboard_type="live")
429
 
430
 
431
  def _add_activity_leaderboard(
@@ -442,9 +455,9 @@ def _add_activity_leaderboard(
442
  leaderboard — only which S3 file is loaded (``endpoint_slug``) and how the metric
443
  columns are labelled (``metric_prefix``) differ.
444
  """
445
- gr.DownloadButton(
446
- label="Download CSV",
447
- value=partial(
448
  download_activity_leaderboard,
449
  leaderboard_type=leaderboard_type,
450
  track=track,
@@ -452,7 +465,7 @@ def _add_activity_leaderboard(
452
  endpoint_slug=endpoint_slug,
453
  metric_prefix=metric_prefix,
454
  ),
455
- size="sm",
456
  )
457
  metric_cols = [f"{metric_prefix}{d}" for d in metric_display_names.values()]
458
  return Leaderboard(
@@ -502,15 +515,15 @@ def _add_structure_leaderboard(
502
  Same layout/columns regardless of ``endpoint_slug`` — mirrors
503
  ``_add_activity_leaderboard`` for the structure track's fixed metric set.
504
  """
505
- gr.DownloadButton(
506
- label="Download CSV",
507
- value=partial(
508
  download_structure_leaderboard,
509
  leaderboard_type=leaderboard_type,
510
  track=track,
511
  endpoint_slug=endpoint_slug,
512
  ),
513
- size="sm",
514
  )
515
  return Leaderboard(
516
  value=load_structure_leaderboard(
@@ -622,6 +635,7 @@ def _add_structure_track_tabs(
622
 
623
  def _wire_activity_timer(
624
  timer: gr.Timer,
 
625
  leaderboard_type: str,
626
  track: str,
627
  lbs: dict[str, Leaderboard],
@@ -629,10 +643,15 @@ def _wire_activity_timer(
629
  ) -> None:
630
  """Wire a live-refresh timer to every leaderboard component of one
631
  regression/classification track.
 
 
 
 
632
  """
633
  for endpoint_slug, lb in lbs.items():
634
  metric_prefix = "MA-" if endpoint_slug == MACRO_ENDPOINT_LABEL else ""
635
- timer.tick(
 
636
  fn=partial(
637
  load_activity_leaderboard,
638
  leaderboard_type=leaderboard_type,
@@ -646,11 +665,19 @@ def _wire_activity_timer(
646
 
647
 
648
  def _wire_structure_timer(
649
- timer: gr.Timer, leaderboard_type: str, track: str, lbs: dict[str, Leaderboard]
 
 
 
 
650
  ) -> None:
651
- """Wire a live-refresh timer to every leaderboard component of the structure track."""
 
 
 
652
  for endpoint_slug, lb in lbs.items():
653
- timer.tick(
 
654
  fn=partial(
655
  load_structure_leaderboard,
656
  leaderboard_type=leaderboard_type,
@@ -661,14 +688,17 @@ def _wire_structure_timer(
661
  )
662
 
663
 
664
- def _add_leaderboard_tabs(leaderboard_type: str):
665
  """Add the regression, classification, and structure leaderboard tabs to the
666
  Gradio app.
667
 
668
  Args:
669
  leaderboard_type (str): The leaderboard_type of the leaderboard to display.
670
  One of "live", "interim", or "final". Both "interim" and "final" are
671
- static leaderboards, while "live" is updated every 30 seconds.
 
 
 
672
 
673
  """
674
  if leaderboard_type == "live":
@@ -703,8 +733,10 @@ def _add_leaderboard_tabs(leaderboard_type: str):
703
  )
704
 
705
  if leaderboard_type == "live":
 
706
  _wire_activity_timer(
707
  regression_timer,
 
708
  leaderboard_type,
709
  REGRESSION_TRACK,
710
  regression_lbs,
@@ -712,11 +744,12 @@ def _add_leaderboard_tabs(leaderboard_type: str):
712
  )
713
  _wire_activity_timer(
714
  classification_timer,
 
715
  leaderboard_type,
716
  CLASSIFICATION_TRACK,
717
  classification_lbs,
718
  CLASSIFICATION_METRIC_DISPLAY_NAMES,
719
  )
720
  _wire_structure_timer(
721
- structure_timer, leaderboard_type, STRUCTURE_TRACK, structure_lbs
722
  )
 
95
  def sort_leaderboard_columns(df: pd.DataFrame) -> pd.DataFrame:
96
  """Sort leaderboard columns in a consistent order."""
97
  start_cols = ["rank", "username", "Submitted", "model_report_link"]
98
+ end_cols = ["Proprietary Data", "Open Code"]
99
  start_cols = [c for c in start_cols if c in df.columns]
100
  end_cols = [c for c in end_cols if c in df.columns]
101
  middle_cols = [c for c in df.columns if c not in start_cols + end_cols]
 
155
  metric-column schema, since the backend narrows down to one endpoint's columns
156
  before saving (see ``aws_leaderboards._narrow_averaged_results_to_endpoint``). So
157
  this one function renders all of them for both tracks; ``metric_display_names``
158
+ (regression's ST-RAE/MAE/R2/Spearman/Kendall or classification's
159
  MCC/Accuracy/Precision/Recall/F1) and ``metric_prefix`` ("MA-" for a track's
160
  master, "" for endpoint tabs whose own tab title already identifies the endpoint)
161
  are the only things that differ between calls.
 
188
  for col in metric_cols:
189
  if col in df.columns:
190
  df[col] = df[col].round(4)
191
+ # The source file's physical column order doesn't necessarily match
192
+ # metric_display_names' canonical order (primary/sort metric first) — force it,
193
+ # since sort_leaderboard_columns/gradio_leaderboard's select_columns both preserve
194
+ # whatever order the DataFrame's columns already have rather than reordering them.
195
+ ordered_metric_cols = []
196
+ for col in metric_cols:
197
+ if col in df.columns:
198
+ ordered_metric_cols.append(col)
199
+ std_col = f"{col} std"
200
+ if std_col in df.columns:
201
+ ordered_metric_cols.append(std_col)
202
+ other_cols = [c for c in df.columns if c not in ordered_metric_cols]
203
+ df = df[other_cols + ordered_metric_cols]
204
  df["Submitted"] = pd.to_datetime(df["Submitted"], utc=True).dt.strftime(
205
  "%Y-%m-%d %H:%M UTC"
206
  )
 
428
  _add_leaderboard_tabs(leaderboard_type="final")
429
 
430
 
431
+ def render_live_leaderboards(phase: int, demo: gr.Blocks):
432
  """Render the live leaderboards for the Gradio app."""
433
  if phase == 0:
434
  gr.Markdown("# Live Leaderboards (Coming Soon)")
 
438
  )
439
  else:
440
  gr.Markdown("# Live Leaderboards")
441
+ _add_leaderboard_tabs(leaderboard_type="live", demo=demo)
442
 
443
 
444
  def _add_activity_leaderboard(
 
455
  leaderboard — only which S3 file is loaded (``endpoint_slug``) and how the metric
456
  columns are labelled (``metric_prefix``) differ.
457
  """
458
+ download_btn = gr.DownloadButton(label="Download CSV", size="sm")
459
+ download_btn.click(
460
+ fn=partial(
461
  download_activity_leaderboard,
462
  leaderboard_type=leaderboard_type,
463
  track=track,
 
465
  endpoint_slug=endpoint_slug,
466
  metric_prefix=metric_prefix,
467
  ),
468
+ outputs=download_btn,
469
  )
470
  metric_cols = [f"{metric_prefix}{d}" for d in metric_display_names.values()]
471
  return Leaderboard(
 
515
  Same layout/columns regardless of ``endpoint_slug`` — mirrors
516
  ``_add_activity_leaderboard`` for the structure track's fixed metric set.
517
  """
518
+ download_btn = gr.DownloadButton(label="Download CSV", size="sm")
519
+ download_btn.click(
520
+ fn=partial(
521
  download_structure_leaderboard,
522
  leaderboard_type=leaderboard_type,
523
  track=track,
524
  endpoint_slug=endpoint_slug,
525
  ),
526
+ outputs=download_btn,
527
  )
528
  return Leaderboard(
529
  value=load_structure_leaderboard(
 
635
 
636
  def _wire_activity_timer(
637
  timer: gr.Timer,
638
+ demo: gr.Blocks,
639
  leaderboard_type: str,
640
  track: str,
641
  lbs: dict[str, Leaderboard],
 
643
  ) -> None:
644
  """Wire a live-refresh timer to every leaderboard component of one
645
  regression/classification track.
646
+
647
+ Also fires on ``demo.load`` (i.e. whenever a browser session opens the page), so a
648
+ visitor sees fresh data immediately rather than the snapshot baked in when the
649
+ Blocks graph was built, and doesn't have to wait for the first timer tick.
650
  """
651
  for endpoint_slug, lb in lbs.items():
652
  metric_prefix = "MA-" if endpoint_slug == MACRO_ENDPOINT_LABEL else ""
653
+ gr.on(
654
+ triggers=[demo.load, timer.tick],
655
  fn=partial(
656
  load_activity_leaderboard,
657
  leaderboard_type=leaderboard_type,
 
665
 
666
 
667
  def _wire_structure_timer(
668
+ timer: gr.Timer,
669
+ demo: gr.Blocks,
670
+ leaderboard_type: str,
671
+ track: str,
672
+ lbs: dict[str, Leaderboard],
673
  ) -> None:
674
+ """Wire a live-refresh timer to every leaderboard component of the structure track.
675
+
676
+ Also fires on ``demo.load`` — see ``_wire_activity_timer``.
677
+ """
678
  for endpoint_slug, lb in lbs.items():
679
+ gr.on(
680
+ triggers=[demo.load, timer.tick],
681
  fn=partial(
682
  load_structure_leaderboard,
683
  leaderboard_type=leaderboard_type,
 
688
  )
689
 
690
 
691
+ def _add_leaderboard_tabs(leaderboard_type: str, demo: gr.Blocks | None = None):
692
  """Add the regression, classification, and structure leaderboard tabs to the
693
  Gradio app.
694
 
695
  Args:
696
  leaderboard_type (str): The leaderboard_type of the leaderboard to display.
697
  One of "live", "interim", or "final". Both "interim" and "final" are
698
+ static leaderboards, while "live" is updated every 30 seconds and on
699
+ every page load.
700
+ demo (gr.Blocks | None): The app's Blocks instance, needed to wire the
701
+ page-load refresh. Required when ``leaderboard_type == "live"``.
702
 
703
  """
704
  if leaderboard_type == "live":
 
733
  )
734
 
735
  if leaderboard_type == "live":
736
+ assert demo is not None, "demo is required to wire live leaderboards"
737
  _wire_activity_timer(
738
  regression_timer,
739
+ demo,
740
  leaderboard_type,
741
  REGRESSION_TRACK,
742
  regression_lbs,
 
744
  )
745
  _wire_activity_timer(
746
  classification_timer,
747
+ demo,
748
  leaderboard_type,
749
  CLASSIFICATION_TRACK,
750
  classification_lbs,
751
  CLASSIFICATION_METRIC_DISPLAY_NAMES,
752
  )
753
  _wire_structure_timer(
754
+ structure_timer, demo, leaderboard_type, STRUCTURE_TRACK, structure_lbs
755
  )