Expand escaping of \n \r \t

test_of_time_accuracy.py

@@ -89,43 +89,65 @@ class TestOfTimeAccuracy(evaluate.Metric):
     @staticmethod
     def _extract_first_json_object(text: str) -> dict | None:
         """
-        Extracts the first valid JSON object from a string.
+        Extract the first valid JSON object from text.
 
-        Scans through the text and returns the first valid JSON dictionary found.
-        This is useful for parsing LLM outputs that may contain JSON mixed with
-        other text or markdown formatting.
+        Handles common LLM output issues like unescaped newlines in string
+        values (LLMs produce human-readable output, not strict JSON).
 
         Args:
             text: String that may contain JSON objects
 
         Returns:
-            The first JSON dictionary found, or None if no valid JSON dict exists
+            The first JSON dictionary found, or None if no valid JSON exists
         """
+        # Fix unescaped control chars in strings (common LLM issue)
+        text = TestOfTimeAccuracy._escape_control_chars_in_strings(text)
+
         decoder = json.JSONDecoder()
-        idx, end = 0, len(text)
-
-        while idx < end:
-            try:
-                obj, next_idx = decoder.raw_decode(text, idx)
-                if isinstance(obj, dict):
-                    return obj
-                idx = next_idx
-            except json.JSONDecodeError:
-                # Try escaping newlines and parsing again from this position
+        idx = 0
+        while idx < len(text):
+            if text[idx] == '{':
                 try:
-                    # Find the potential JSON substring and escape newlines
-                    remaining = text[idx:]
-                    fixed = remaining.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
-                    obj, _ = decoder.raw_decode(fixed, 0)
+                    obj, _ = decoder.raw_decode(text, idx)
                     if isinstance(obj, dict):
                         return obj
-                except (json.JSONDecodeError, ValueError):
+                except json.JSONDecodeError:
                     pass
-                idx += 1
-            except ValueError:
-                idx += 1
+            idx += 1
         return None
 
+    @staticmethod
+    def _escape_control_chars_in_strings(text: str) -> str:
+        """
+        Escape literal control characters inside JSON string values.
+
+        LLMs produce newlines/tabs for readability, but JSON requires them
+        to be escaped within strings.
+        """
+        result = []
+        in_string = False
+        i = 0
+        while i < len(text):
+            char = text[i]
+            if char == '\\' and in_string and i + 1 < len(text):
+                # Preserve existing escape sequences
+                result.append(char)
+                result.append(text[i + 1])
+                i += 2
+                continue
+            if char == '"':
+                in_string = not in_string
+            if in_string and char == '\n':
+                result.append('\\n')
+            elif in_string and char == '\r':
+                result.append('\\r')
+            elif in_string and char == '\t':
+                result.append('\\t')
+            else:
+                result.append(char)
+            i += 1
+        return ''.join(result)
+
     @staticmethod
     def _parse_reference_label(label_str: str) -> dict | None:
         """

tests/test_arithmetic_scoring.py

@@ -12,8 +12,8 @@ arithmetic_test_cases = {
         ' "To find the date of the second most important game, we need to subtract 7 days from the date of the most important game. We can do this by counting back 7 days from April 14, 2005. April 14 - 7 days = April 7, 2005", "answer": "2005-04-07"}',
         '\n```json\n{\n  "explanation": "Step 1: Determine the time it takes the robot to carry a single box. The robot takes 4 hours, 34 minutes, and 30 seconds to carry 2 boxes. We divide this time by 2 to find the time per box.\\n- Hours: 4 / 2 = 2 hours\\n- Minutes: 34 / 2 = 17 minutes\\n- Seconds: 30 / 2 = 15 seconds\\nSo, it takes the robot 2 hours, 17 minutes, and 15 seconds to carry one box.\\n\\nStep 2: Calculate the total time to carry 25 boxes. We multiply the time per box by the total number of boxes (25).\\n- Total Hours: 2 hours/box * 25 boxes = 50 hours\\n- Total Minutes: 17 minutes/box * 25 boxes = 425 minutes\\n- Total Seconds: 15 seconds/box * 25 boxes = 375 seconds\\n\\nStep 3: Convert the calculated time into the standard H:M:S format by carrying over excess seconds and minutes.\\n- Convert seconds to minutes: 375 seconds is equal to 6 minutes and 15 seconds (since 375 / 60 = 6 with a remainder of 15). We add the 6 minutes to our minutes total.\\n- New total: 50 hours, (425 + 6) minutes, 15 seconds -> 50 hours, 431 minutes, 15 seconds.\\n- Convert minutes to hours: 431 minutes is equal to 7 hours and 11 minutes (since 431 / 60 = 7 with a remainder of 11). We add the 7 hours to our hours total.\\n- New total: (50 + 7) hours, 11 minutes, 15 seconds -> 57 hours, 11 minutes, 15 seconds.\\n\\nThe final time is 57 hours, 11 minutes, and 15 seconds.",\n  "H": 57,\n  "M": 11,\n  "S": 15\n}\n```',
         '{"explanation": "some explanation", "age": "3319"}',
-        'JSON = {"explanation": "Let\'s analyze the available time slots for a 30-minute meeting, considering the constraints and schedules of Clara, William, and Alice.\n\n**1. Define Working Hours:** 9:00 AM to 5:00 PM\n\n**2. Identify Booked Time Slots:**\n\n*   Clara: 9:30 AM - 12:30 PM\n*   William: 1:30 PM - 3:30 PM\n*   Alice: 9:00 AM - 11:00 AM (flexible, but this constraint impacts the meeting time)\n\n**3. Determine Available Time Slots for Clara and William:**\n\n*   Clara is available: 9:00 AM - 9:30 AM and 12:30 PM - 5:00 PM\n*   William is available: 9:00 AM - 1:30 PM and 3:30 PM - 5:00 PM\n\n**4. Find the Intersection of Clara and William\'s Availability:**\n\n*   From 9:00 AM to 9:30 AM: Both are available.\n*   From 9:30 AM to 12:30 PM: Clara is booked.\n*   From 12:30 PM to 1:30 PM: Both are available.\n*   From 1:30 PM to 3:30 PM: William is booked.\n*   From 3:30 PM to 5:00 PM: Both are available.\n\nSo, the available time slots for both Clara and William are:\n* 9:00 AM - 9:30 AM\n* 12:30 PM - 1:30 PM\n* 3:30 PM - 5:00 PM\n\n**5. Consider Alice\'s Flexibility:** Since Alice is flexible and can shift her meetings, we don\'t need to consider her booked time slot (9:00 AM - 11:00 AM) when determining the possibilities.\n\n**6. Calculate Possible Meeting Start Times:**\n\n*   **9:00 AM - 9:30 AM:** Possible start time: 9:00 AM. (1 possibility)\n*   **12:30 PM - 1:30 PM:** Possible start times: 12:30 PM, 1:00 PM. (2 possibilities)\n*   **3:30 PM - 5:00 PM:** Possible start times: 3:30 PM, 4:00 PM, 4:30 PM. (3 possibilities)\n\n**7. Total Possible Meeting Times:** 1 + 2 + 3 = 6\n\n", "answer": 6}'
-
+        'JSON = {"explanation": "Let\'s analyze the available time slots for a 30-minute meeting, considering the constraints and schedules of Clara, William, and Alice.\n\n**1. Define Working Hours:** 9:00 AM to 5:00 PM\n\n**2. Identify Booked Time Slots:**\n\n*   Clara: 9:30 AM - 12:30 PM\n*   William: 1:30 PM - 3:30 PM\n*   Alice: 9:00 AM - 11:00 AM (flexible, but this constraint impacts the meeting time)\n\n**3. Determine Available Time Slots for Clara and William:**\n\n*   Clara is available: 9:00 AM - 9:30 AM and 12:30 PM - 5:00 PM\n*   William is available: 9:00 AM - 1:30 PM and 3:30 PM - 5:00 PM\n\n**4. Find the Intersection of Clara and William\'s Availability:**\n\n*   From 9:00 AM to 9:30 AM: Both are available.\n*   From 9:30 AM to 12:30 PM: Clara is booked.\n*   From 12:30 PM to 1:30 PM: Both are available.\n*   From 1:30 PM to 3:30 PM: William is booked.\n*   From 3:30 PM to 5:00 PM: Both are available.\n\nSo, the available time slots for both Clara and William are:\n* 9:00 AM - 9:30 AM\n* 12:30 PM - 1:30 PM\n* 3:30 PM - 5:00 PM\n\n**5. Consider Alice\'s Flexibility:** Since Alice is flexible and can shift her meetings, we don\'t need to consider her booked time slot (9:00 AM - 11:00 AM) when determining the possibilities.\n\n**6. Calculate Possible Meeting Start Times:**\n\n*   **9:00 AM - 9:30 AM:** Possible start time: 9:00 AM. (1 possibility)\n*   **12:30 PM - 1:30 PM:** Possible start times: 12:30 PM, 1:00 PM. (2 possibilities)\n*   **3:30 PM - 5:00 PM:** Possible start times: 3:30 PM, 4:00 PM, 4:30 PM. (3 possibilities)\n\n**7. Total Possible Meeting Times:** 1 + 2 + 3 = 6\n\n", "answer": 6}',
+        '{\n  "explanation": "First, we calculate the difference in hours, minutes, and seconds between the two times. The start time is 08:30:33 and the end time is 14:33:31. Since the end time is later than the start time, we do not need to consider the next day scenario. \n  Hours difference: 14 - 8 = 6\n  Minutes difference: 33 - 30 = 3\n  Seconds difference: 31 - 33 = -2, but since we cannot have negative seconds, we borrow 1 minute from the minutes difference, making it 2 and adding 60 to the seconds difference, resulting in 59 - 2 = 59",\n  "hours": 6,\n  "minutes": 2,\n  "seconds": 58\n}',
     ],
     "references": [
         '{"answer": "352 BC"}',
@@ -25,9 +25,21 @@ arithmetic_test_cases = {
         '{"H": 57.0, "M": 11.0, "S": 15.0}',
         '{"answer": 3319}',
         '{"answer": 6}',
+        "{'hours': 6, 'minutes': 2, 'seconds': 58}",
+    ],
+    "result": {"accuracy": 7 / 10},
+    "per_item_accuracy": [
+        True,
+        True,
+        True,
+        False,
+        False,
+        False,
+        True,
+        True,
+        True,
+        True,
     ],
-    "result": {"accuracy": 6 / 9},
-    "per_item_accuracy": [True, True, True, False,False, False, True, True, True],
 }