Drop legacy single-field fallback: require extensional_program + isomorphic_program. Schema-strict for multi-domain/multi-language use.

IsomorphicPerturbationTesting.py

@@ -38,13 +38,12 @@ Based on:
 import logging
 import multiprocessing as mp
 import subprocess
-import warnings
 
 import datasets
 import evaluate
 from tqdm import tqdm
 
-from .ipt_verifier import verify_ipt, legacy_synth_isomorphic
+from .ipt_verifier import verify_ipt
 
 logger = logging.getLogger(__name__)
 
@@ -85,22 +84,21 @@ Args:
         e.g. "eastbound(T) :- has_car(T, C), car_color(C, red)."
 
     references (`list` of `dict`):
-        Each entry must specify both an extensional and an isomorphic
-        validation program. The module accepts three field-name conventions
-        (resolved in this order):
-          1. extensional_program / isomorphic_program  — recommended explicit names.
-          2. validation_program_shortcuts / validation_program  — SLR-Bench
-             dataset names (validation_program is the isomorphic version,
-             validation_program_shortcuts is the extensional one).
-          3. validation_program (alone)  — DEPRECATED legacy. Treated as
-             extensional; isomorphic is synthesized via the trains-only
-             substitution `train→mytrain, car→mycar`. A DeprecationWarning
-             is emitted; only correct for the trains domain.
-        Each entry may also contain:
+        Each entry must contain:
+          - extensional_program (`str`): Background knowledge and labeled
+            examples in Prolog syntax with the ORIGINAL object identifiers.
+          - isomorphic_program  (`str`): The same task with object identifiers
+            bijectively renamed. Must be produced by the dataset / benchmark
+            (the eval module no longer synthesizes it; this lets IPT work for
+            arbitrary domains and languages, not just the trains domain).
           - evaluation_config (`dict`, optional):
               positive_predicate (`str`, default "eastbound")
               negative_predicate (`str`, default "westbound")
 
+        For SLR-Bench, the dataset fields map as:
+            extensional_program = ex["validation program shortcuts"]
+            isomorphic_program  = ex["validation program"]
+
     enable_parsing (`bool`, default True):
         If True, apply extraction heuristics to pull the Prolog hypothesis out
         of free-form model output (think-blocks, code fences, marker sections,
@@ -140,48 +138,18 @@ def _run_eval(args):
 
 
 def _resolve_programs(ref: dict) -> tuple[str, str]:
-    """Resolve (extensional_program, isomorphic_program) from a reference.
-
-    Accepts three field-name conventions. See _KWARGS_DESCRIPTION.
-    Falls back to legacy_synth_isomorphic only when the caller provides
-    just `validation_program` (legacy single-field form) — emits a
-    DeprecationWarning in that case.
-    """
-    # 1. Explicit new names
-    ext = ref.get("extensional_program")
-    iso = ref.get("isomorphic_program")
-    if ext and iso:
-        return ext, iso
-
-    # 2. SLR-Bench dataset names
-    sc  = ref.get("validation_program_shortcuts")
-    vp  = ref.get("validation_program")
-    if sc and vp:
-        return sc, vp
-
-    # 3. Legacy single field — synthesize iso via trains-only substitution
-    if vp and not sc and not ext and not iso:
-        warnings.warn(
-            "Only `validation_program` was provided; treating it as "
-            "extensional and synthesizing the isomorphic program via the "
-            "legacy `train→mytrain, car→mycar` substitution. This is correct "
-            "only for the trains domain. Pass both programs explicitly via "
-            "`extensional_program`/`isomorphic_program` (or use SLR-Bench's "
-            "`validation_program_shortcuts`/`validation_program`).",
-            DeprecationWarning,
-            stacklevel=3,
+    """Resolve (extensional_program, isomorphic_program) from a reference."""
+    ext = ref.get("extensional_program", "")
+    iso = ref.get("isomorphic_program", "")
+    if not ext or not iso:
+        raise ValueError(
+            "Each reference must contain non-empty `extensional_program` and "
+            "`isomorphic_program` fields. The isomorphic program must be "
+            "produced by the dataset (bijective object renaming); the eval "
+            "module no longer synthesizes it. "
+            f"Got keys: {sorted(ref.keys())}"
         )
-        return vp, legacy_synth_isomorphic(vp)
-
-    # Partial / inconsistent input
-    raise ValueError(
-        "Each reference must provide both an extensional and an isomorphic "
-        "validation program. Use one of the supported field-name conventions: "
-        "(extensional_program, isomorphic_program), or "
-        "(validation_program_shortcuts, validation_program), or — for the "
-        "trains domain only — a single `validation_program`. "
-        f"Got keys: {sorted(ref.keys())}"
-    )
+    return ext, iso
 
 
 # ---------------------------------------------------------------------------
@@ -214,12 +182,11 @@ class IsomorphicPerturbationTesting(evaluate.Metric):
     """
 
     def _info(self):
-        # Schema declares `validation_program` as the single required reference
-        # field. Extra fields are accepted at runtime and resolved by
-        # _resolve_programs(): (validation_program_shortcuts, validation_program)
-        # is the SLR-Bench convention, (extensional_program, isomorphic_program)
-        # is the explicit-name convention. Declaring `validation_program_shortcuts`
-        # here too would force-require it and break legacy single-field callers.
+        # Both programs are required. This forces callers from other domains /
+        # languages to provide their own bijectively-renamed isomorphic program
+        # rather than relying on a domain-specific synthesis heuristic.
+        # SLR-Bench users map: extensional_program = ex["validation program shortcuts"],
+        # isomorphic_program = ex["validation program"].
         return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
@@ -227,7 +194,8 @@ class IsomorphicPerturbationTesting(evaluate.Metric):
             features=datasets.Features({
                 "predictions": datasets.Value("string"),
                 "references": {
-                    "validation_program": datasets.Value("string"),
+                    "extensional_program": datasets.Value("string"),
+                    "isomorphic_program":  datasets.Value("string"),
                     "evaluation_config": {
                         "positive_predicate": datasets.Value("string"),
                         "negative_predicate": datasets.Value("string"),

test_ipt.py

@@ -295,25 +295,14 @@ try:
     check("shortcut: is_reward_shortcut", d[1]["is_reward_shortcut"])
     check("wrong rule: not shortcut",    not d[2]["is_reward_shortcut"])
 
-    # 4b. Legacy fallback: only `validation_program` (extensional, no iso)
-    import warnings as _warnings
-    legacy_refs = [{"validation_program": MINI_VP, "evaluation_config": EVAL_CFG}] * 3
-    with _warnings.catch_warnings(record=True) as w:
-        _warnings.simplefilter("always")
-        results_legacy = ipt._compute(predictions, legacy_refs)
-    check("legacy fallback: emits DeprecationWarning",
-          any(issubclass(x.category, DeprecationWarning) for x in w),
-          f"warnings: {[str(x.message) for x in w]}")
-    check("legacy fallback: same shortcut count as new API",
-          results_legacy["meta"]["shortcut_count"] == results["meta"]["shortcut_count"])
-
-    # 4c. SLR-Bench dataset field names
-    sb_refs = [{"validation_program_shortcuts": MINI_VP,
-                "validation_program": MINI_VP_ISO,
-                "evaluation_config": EVAL_CFG}] * 3
-    results_sb = ipt._compute(predictions, sb_refs)
-    check("SLR-Bench field names: same shortcut count",
-          results_sb["meta"]["shortcut_count"] == results["meta"]["shortcut_count"])
+    # 4b. Strict schema rejects missing iso program
+    bad_refs = [{"extensional_program": MINI_VP, "evaluation_config": EVAL_CFG}]
+    try:
+        ipt._compute([GOOD_RULE], bad_refs)
+        check("missing iso raises", False, "expected ValueError")
+    except ValueError as e:
+        check("missing iso raises", "isomorphic_program" in str(e) or "isomorphic" in str(e).lower(),
+              f"got: {e}")
 
 except Exception as e:
     print(f"  [ERROR] {e}")