regular input

cider.py

@@ -63,12 +63,20 @@ class CIDEr(evaluate.Metric):
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
             # This defines the format of each prediction and reference
-            features=datasets.Features(
-                {
-                    "predictions": datasets.List((datasets.Value("string"))),
-                    "references": datasets.List(datasets.Value("string")),
-                }
-            ),
+            features=[
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string"),
+                        "references": datasets.Value("string"),
+                    }
+                ),
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string"),
+                        "references": datasets.Sequence(datasets.Value("string")),
+                    }
+                ),
+            ],
             # Homepage of the module for documentation
             homepage="https://huggingface.co/spaces/sunhill/cider",
             # Additional links to the codebase or references
@@ -84,10 +92,6 @@ class CIDEr(evaluate.Metric):
             ],
         )
 
-    def _download_and_prepare(self, dl_manager):
-        """Optional: download external resources useful to compute the scores"""
-        pass
-
     def _compute(self, predictions, references):
         """Returns the scores"""
         assert len(predictions) == len(references), (
@@ -96,6 +100,15 @@ class CIDEr(evaluate.Metric):
         )
         cider_scorer = CiderScorer(n=4, sigma=6.0)
         for pred, ref in zip(predictions, references):
-            cider_scorer += (pred[0], ref)
+            assert isinstance(pred, str), (
+                f"Each prediction should be a string. Got {type(pred)}."
+            )
+            if isinstance(ref, str):
+                ref = [ref]
+            assert isinstance(ref, list) and all(isinstance(r, str) for r in ref), (
+                "Each reference should be a list of strings. "
+                f"Got {type(ref)} with elements of type {[type(r) for r in ref]}."
+            )
+            cider_scorer += (pred, ref)
         score, _ = cider_scorer.compute_score()
         return {"cider_score": score.item()}

tests.py

@@ -3,7 +3,7 @@ import evaluate
 
 test_cases = [
     {
-        "predictions": [["train traveling down a track in front of a road"]],
+        "predictions": ["train traveling down a track in front of a road"],
         "references": [
             [
                 "a train traveling down tracks next to lights",
@@ -12,12 +12,18 @@ test_cases = [
                 "a passenger train pulls into a train station",
                 "a train coming down the tracks arriving at a station",
             ]
-        ]
+        ],
+    },
+    {
+        "predictions": ["birthday cake sitting on top of a white plate"],
+        "references": [
+            "a blue plate filled with marshmallows chocolate chips and banana"
+        ],
     },
     {
         "predictions": [
-            ["plane is flying through the sky"],
-            ["birthday cake sitting on top of a white plate"],
+            "plane is flying through the sky",
+            "birthday cake sitting on top of a white plate",
         ],
         "references": [
             [
@@ -28,16 +34,16 @@ test_cases = [
                 "the plane is flying over top of the cars",
             ],
             ["a blue plate filled with marshmallows chocolate chips and banana"],
-        ]
+        ],
     },
 ]
 
-metric = evaluate.load("sunhill/cider")
+metric = evaluate.load("./cider.py")
 for i, test_case in enumerate(test_cases):
     results = metric.compute(
         predictions=test_case["predictions"], references=test_case["references"]
     )
-    print(f"Test case {i+1}:")
+    print(f"Test case {i + 1}:")
     print("Predictions:", test_case["predictions"])
     print("References:", test_case["references"])
     print(results)