Upload add_sub_baseline_10K

add_sub_baseline_10K/config.json

@@ -4,9 +4,7 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": null,
   "dtype": "float32",
-  "eos_token_id": null,
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 510,
@@ -22,15 +20,12 @@
   "num_attention_heads": 3,
   "num_hidden_layers": 2,
   "num_key_value_heads": 3,
-  "pad_token_id": null,
   "rms_norm_eps": 1e-06,
-  "rope_parameters": {
-    "rope_theta": 10000.0,
-    "rope_type": "default"
-  },
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
   "sliding_window": null,
   "tie_word_embeddings": false,
-  "transformers_version": "5.5.0",
+  "transformers_version": "4.57.6",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151645

add_sub_baseline_10K/generation_config.json

@@ -1,7 +1,4 @@
 {
   "_from_model_config": true,
-  "output_attentions": false,
-  "output_hidden_states": false,
-  "transformers_version": "5.5.0",
-  "use_cache": true
+  "transformers_version": "4.57.6"
 }

add_sub_baseline_10K/metrics.json

@@ -3,327 +3,34 @@
     "step": [
       50,
       100,
-      150,
-      200,
-      250,
-      300,
-      350,
-      400,
-      450,
-      500,
-      550,
-      600,
-      650,
-      700,
-      750,
-      800,
-      850,
-      900,
-      950,
-      1000,
-      1050,
-      1100,
-      1150,
-      1200,
-      1250,
-      1300,
-      1350,
-      1400,
-      1450,
-      1500,
-      1550,
-      1600,
-      1650,
-      1700,
-      1750,
-      1800,
-      1850,
-      1900,
-      1950,
-      2000,
-      2050,
-      2100,
-      2150,
-      2200,
-      2250,
-      2300,
-      2350,
-      2400,
-      2450,
-      2500,
-      2550,
-      2600,
-      2650,
-      2700,
-      2750,
-      2800,
-      2850,
-      2900,
-      2950,
-      3000,
-      3050,
-      3100
+      150
     ],
     "loss": [
-      7.406970977783203,
-      4.287904262542725,
-      2.028409719467163,
-      1.9122778177261353,
-      1.7641465663909912,
-      1.6944198608398438,
-      1.5549181699752808,
-      1.4273613691329956,
-      1.1411476135253906,
-      0.7707713842391968,
-      0.652624249458313,
-      0.5090309977531433,
-      0.4863796830177307,
-      0.4217025935649872,
-      0.3141794204711914,
-      0.26072242856025696,
-      0.20925267040729523,
-      0.1722976416349411,
-      0.16701845824718475,
-      0.14609448611736298,
-      0.15161116421222687,
-      0.11793585866689682,
-      0.12784767150878906,
-      0.17903172969818115,
-      0.07992793619632721,
-      0.09357694536447525,
-      0.07849029451608658,
-      0.08744285255670547,
-      0.047625500708818436,
-      0.06857192516326904,
-      0.058801162987947464,
-      0.08054264634847641,
-      0.0829831138253212,
-      0.08300244808197021,
-      0.04127458110451698,
-      0.045694876462221146,
-      0.0545954629778862,
-      0.04024777188897133,
-      0.07766951620578766,
-      0.024591432884335518,
-      0.06888258457183838,
-      0.03272942453622818,
-      0.05081992596387863,
-      0.07547096163034439,
-      0.04898810759186745,
-      0.04332876205444336,
-      0.018458085134625435,
-      0.03549834340810776,
-      0.020787600427865982,
-      0.01855045184493065,
-      0.018782520666718483,
-      0.013612723909318447,
-      0.0217971820384264,
-      0.01920527033507824,
-      0.01275918073952198,
-      0.023383263498544693,
-      0.017817409709095955,
-      0.01498924009501934,
-      0.021258754655718803,
-      0.022163305431604385,
-      0.012838104739785194,
-      0.015554169192910194
+      7.406985759735107,
+      4.287901878356934,
+      2.494678020477295
     ],
     "base_loss": [
-      7.406970977783203,
-      4.287904262542725,
-      2.028409719467163,
-      1.9122778177261353,
-      1.7641465663909912,
-      1.6944198608398438,
-      1.5549181699752808,
-      1.4273613691329956,
-      1.1411476135253906,
-      0.7707713842391968,
-      0.652624249458313,
-      0.5090309977531433,
-      0.4863796830177307,
-      0.4217025935649872,
-      0.3141794204711914,
-      0.26072242856025696,
-      0.20925267040729523,
-      0.1722976416349411,
-      0.16701845824718475,
-      0.14609448611736298,
-      0.15161116421222687,
-      0.11793585866689682,
-      0.12784767150878906,
-      0.17903172969818115,
-      0.07992793619632721,
-      0.09357694536447525,
-      0.07849029451608658,
-      0.08744285255670547,
-      0.047625500708818436,
-      0.06857192516326904,
-      0.058801162987947464,
-      0.08054264634847641,
-      0.0829831138253212,
-      0.08300244808197021,
-      0.04127458110451698,
-      0.045694876462221146,
-      0.0545954629778862,
-      0.04024777188897133,
-      0.07766951620578766,
-      0.024591432884335518,
-      0.06888258457183838,
-      0.03272942453622818,
-      0.05081992596387863,
-      0.07547096163034439,
-      0.04898810759186745,
-      0.04332876205444336,
-      0.018458085134625435,
-      0.03549834340810776,
-      0.020787600427865982,
-      0.01855045184493065,
-      0.018782520666718483,
-      0.013612723909318447,
-      0.0217971820384264,
-      0.01920527033507824,
-      0.01275918073952198,
-      0.023383263498544693,
-      0.017817409709095955,
-      0.01498924009501934,
-      0.021258754655718803,
-      0.022163305431604385,
-      0.012838104739785194,
-      0.015554169192910194
+      7.406985759735107,
+      4.287901878356934,
+      2.494678020477295
     ],
     "lr": [
       3.9200000000000004e-05,
       7.92e-05,
-      7.994872780244471e-05,
-      7.979084217550451e-05,
-      7.952674320281786e-05,
-      7.915713584145437e-05,
-      7.868300668109943e-05,
-      7.810562131055899e-05,
-      7.742652093953451e-05,
-      7.664751828468545e-05,
-      7.57706927309605e-05,
-      7.47983847811137e-05,
-      7.373318980822093e-05,
-      7.25779511278734e-05,
-      7.133575240854014e-05,
-      7.000990944035905e-05,
-      6.860396128432721e-05,
-      6.712166082551651e-05,
-      6.556696475553043e-05,
-      6.394402301094192e-05,
-      6.225716769590408e-05,
-      6.0510901518502626e-05,
-      5.8709885771716645e-05,
-      5.685892789107026e-05,
-      5.4962968622187084e-05,
-      5.3027068832501364e-05,
-      5.1056396002328924e-05,
-      4.9056210431357356e-05,
-      4.703185119737419e-05,
-      4.498872190471344e-05,
-      4.293227626046202e-05,
-      4.086800351692724e-05,
-      3.880141381922381e-05,
-      3.673802349709187e-05,
-      3.468334034020644e-05,
-      3.2642848896282794e-05,
-      3.062199583122133e-05,
-      2.862617539037015e-05,
-      2.6660714999713274e-05,
-      2.4730861045419232e-05,
-      2.284176486970851e-05,
-      2.099846902042102e-05,
-      1.9205893790987304e-05,
-      1.7468824086732586e-05,
-      1.579189665257094e-05,
-      1.4179587696182778e-05,
-      1.2636200939713001e-05,
-      1.1165856131883247e-05,
-      9.772478051182794e-06,
-      8.459786029491775e-06,
-      7.231284024101261e-06,
-      6.090251264630804e-06,
-      5.039733499809587e-06,
-      4.082534867486105e-06,
-      3.221210409567612e-06,
-      2.458059251869167e-06,
-      1.7951184670772902e-06,
-      1.2341576372097185e-06,
-      7.766741300856728e-07,
-      4.23889102415056e-07,
-      1.7674424017557922e-07,
-      3.58992449786566e-08
+      3.825713572096903e-06
     ],
     "eval_step": [
-      156,
-      312,
-      468,
-      624,
-      780,
-      936,
-      1092,
-      1248,
-      1404,
-      1560,
-      1716,
-      1872,
-      2028,
-      2184,
-      2340,
-      2496,
-      2652,
-      2808,
-      2964,
-      3120
+      156
     ],
     "eval_epoch": [
-      1,
-      2,
-      3,
-      4,
-      5,
-      6,
-      7,
-      8,
-      9,
-      10,
-      11,
-      12,
-      13,
-      14,
-      15,
-      16,
-      17,
-      18,
-      19,
-      20
+      1
     ],
     "eval_accuracy": [
-      0.005263157894736842,
-      0.005263157894736842,
-      0.023157894736842106,
-      0.10421052631578948,
-      0.4073684210526316,
-      0.6189473684210526,
-      0.6663157894736842,
-      0.7210526315789474,
-      0.7315789473684211,
-      0.76,
-      0.7884210526315789,
-      0.783157894736842,
-      0.7631578947368421,
-      0.8126315789473684,
-      0.8147368421052632,
-      0.8126315789473684,
-      0.8105263157894737,
-      0.8189473684210526,
-      0.82,
-      0.8094736842105263
+      0.004210526315789474
     ]
   },
-  "final_accuracy": 0.76,
+  "final_accuracy": 0.005,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
@@ -334,330 +41,330 @@
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.97,
-        "digit_accuracy": 0.9957142857142857,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.31142857142857144,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9950413223140496,
+            "accuracy": 0.2049586776859504,
             "count": 605
           },
           "SS": {
-            "accuracy": 1.0,
+            "accuracy": 0.9894736842105263,
             "count": 95
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 1.0,
-        "digit_accuracy": 1.0,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.19428571428571428,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.22549019607843138,
             "count": 204
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.09467455621301775,
             "count": 169
           },
           "SS": {
-            "accuracy": 1.0,
+            "accuracy": 0.6774193548387096,
             "count": 31
           },
           "UC": {
-            "accuracy": 1.0,
+            "accuracy": 0.17905405405405406,
             "count": 296
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.97,
-        "digit_accuracy": 0.9957142857142857,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.31285714285714283,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.294478527607362,
             "count": 163
           },
           "SC": {
-            "accuracy": 0.9846153846153847,
+            "accuracy": 0.12307692307692308,
             "count": 130
           },
           "SS": {
-            "accuracy": 0.9885057471264368,
+            "accuracy": 0.5402298850574713,
             "count": 87
           },
           "UC": {
-            "accuracy": 1.0,
+            "accuracy": 0.24630541871921183,
             "count": 203
           },
           "US": {
-            "accuracy": 1.0,
+            "accuracy": 0.49572649572649574,
             "count": 117
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.67,
-        "digit_accuracy": 0.9528571428571428,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.36857142857142855,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.2727272727272727,
             "count": 121
           },
           "SC": {
-            "accuracy": 0.9834710743801653,
+            "accuracy": 0.08264462809917356,
             "count": 121
           },
           "SS": {
-            "accuracy": 1.0,
+            "accuracy": 0.40816326530612246,
             "count": 49
           },
           "UC": {
-            "accuracy": 0.8333333333333334,
+            "accuracy": 0.3118279569892473,
             "count": 186
           },
           "US": {
-            "accuracy": 1.0,
+            "accuracy": 0.6143497757847534,
             "count": 223
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.6,
-        "digit_accuracy": 0.9171428571428571,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.37142857142857144,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.38461538461538464,
             "count": 104
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.09433962264150944,
             "count": 106
           },
           "SS": {
-            "accuracy": 1.0,
+            "accuracy": 0.6086956521739131,
             "count": 23
           },
           "UC": {
-            "accuracy": 0.8,
+            "accuracy": 0.325,
             "count": 160
           },
           "US": {
-            "accuracy": 0.9153094462540716,
+            "accuracy": 0.46905537459283386,
             "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.5,
-        "digit_accuracy": 0.8371428571428572,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.22,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.4,
             "count": 100
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.05,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.65,
+            "accuracy": 0.21,
             "count": 100
           },
           "US": {
-            "accuracy": 0.8025,
+            "accuracy": 0.22,
             "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.5,
-        "digit_accuracy": 0.8142857142857143,
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.5328571428571428,
         "n_examples": 100,
         "per_subtask": {
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.13,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.76,
+            "accuracy": 0.6,
             "count": 100
           },
           "US": {
-            "accuracy": 0.788,
+            "accuracy": 0.6,
             "count": 500
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.97,
-        "digit_accuracy": 0.9957142857142857,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.19714285714285715,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9955257270693513,
+            "accuracy": 0.21029082774049218,
             "count": 447
           },
           "SC": {
-            "accuracy": 0.99375,
+            "accuracy": 0.103125,
             "count": 320
           },
           "SS": {
-            "accuracy": 1.0,
+            "accuracy": 0.5535714285714286,
             "count": 56
           },
           "UC": {
-            "accuracy": 0.996219281663516,
+            "accuracy": 0.18714555765595464,
             "count": 529
           },
           "US": {
-            "accuracy": 1.0,
+            "accuracy": 0.3958333333333333,
             "count": 48
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.99,
-        "digit_accuracy": 0.9985714285714286,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.12142857142857143,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.998,
+            "accuracy": 0.164,
             "count": 500
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 100
           },
           "UC": {
-            "accuracy": 1.0,
+            "accuracy": 0.03,
             "count": 100
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.93,
-        "digit_accuracy": 0.99,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.1357142857142857,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.995,
+            "accuracy": 0.2075,
             "count": 400
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.01,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.9743589743589743,
+            "accuracy": 0.0641025641025641,
             "count": 156
           },
           "US": {
-            "accuracy": 0.9772727272727273,
+            "accuracy": 0.022727272727272728,
             "count": 44
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.85,
-        "digit_accuracy": 0.9785714285714285,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.15571428571428572,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.25,
             "count": 300
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.07,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.9246231155778895,
+            "accuracy": 0.08040201005025126,
             "count": 199
           },
           "US": {
-            "accuracy": 1.0,
+            "accuracy": 0.10891089108910891,
             "count": 101
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.78,
-        "digit_accuracy": 0.9657142857142857,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.17285714285714285,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.365,
             "count": 200
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.04,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.9242424242424242,
+            "accuracy": 0.07954545454545454,
             "count": 264
           },
           "US": {
-            "accuracy": 0.9705882352941176,
+            "accuracy": 0.16911764705882354,
             "count": 136
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.79,
-        "digit_accuracy": 0.9614285714285714,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.19857142857142857,
         "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
+            "accuracy": 0.51,
             "count": 100
           },
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.07,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.9354838709677419,
+            "accuracy": 0.09032258064516129,
             "count": 310
           },
           "US": {
-            "accuracy": 0.9631578947368421,
+            "accuracy": 0.2789473684210526,
             "count": 190
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.74,
-        "digit_accuracy": 0.9542857142857143,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.36142857142857143,
         "n_examples": 100,
         "per_subtask": {
           "SC": {
-            "accuracy": 1.0,
+            "accuracy": 0.15,
             "count": 100
           },
           "UC": {
-            "accuracy": 0.9459459459459459,
+            "accuracy": 0.20270270270270271,
             "count": 370
           },
           "US": {
-            "accuracy": 0.9478260869565217,
+            "accuracy": 0.7086956521739131,
             "count": 230
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.97,
-        "digit_accuracy": 0.9957142857142857,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.29285714285714287,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9951219512195122,
+            "accuracy": 0.1951219512195122,
             "count": 615
           },
           "ME": {
@@ -667,16 +374,16 @@
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.97,
-        "digit_accuracy": 0.9957142857142857,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.22428571428571428,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9931506849315068,
+            "accuracy": 0.3698630136986301,
             "count": 292
           },
           "MB": {
-            "accuracy": 0.9930555555555556,
+            "accuracy": 0.0,
             "count": 144
           },
           "ME": {
@@ -684,49 +391,49 @@
             "count": 25
           },
           "UB": {
-            "accuracy": 1.0,
+            "accuracy": 0.100418410041841,
             "count": 239
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.94,
-        "digit_accuracy": 0.9914285714285714,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.35428571428571426,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
+            "accuracy": 0.6208530805687204,
             "count": 211
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 115
           },
           "ME": {
-            "accuracy": 0.9882352941176471,
+            "accuracy": 1.0,
             "count": 85
           },
           "UB": {
-            "accuracy": 0.9723756906077348,
+            "accuracy": 0.17679558011049723,
             "count": 181
           },
           "UD": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 108
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.38,
-        "digit_accuracy": 0.9028571428571428,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.3,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
+            "accuracy": 0.7597765363128491,
             "count": 179
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 103
           },
           "ME": {
@@ -734,41 +441,41 @@
             "count": 56
           },
           "UB": {
-            "accuracy": 0.610738255033557,
+            "accuracy": 0.12080536912751678,
             "count": 149
           },
           "UD": {
-            "accuracy": 0.9530516431924883,
+            "accuracy": 0.0,
             "count": 213
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.18,
-        "digit_accuracy": 0.7942857142857143,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.18571428571428572,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
+            "accuracy": 0.5,
             "count": 200
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 100
           },
           "UB": {
-            "accuracy": 0.47,
+            "accuracy": 0.3,
             "count": 100
           },
           "UD": {
-            "accuracy": 0.6966666666666667,
+            "accuracy": 0.0,
             "count": 300
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.13,
-        "digit_accuracy": 0.6685714285714286,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.18714285714285714,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
@@ -776,30 +483,30 @@
             "count": 100
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 100
           },
           "UB": {
-            "accuracy": 0.46,
+            "accuracy": 0.31,
             "count": 100
           },
           "UD": {
-            "accuracy": 0.555,
+            "accuracy": 0.0,
             "count": 400
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.95,
-        "digit_accuracy": 0.9928571428571429,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.2307142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9883333333333333,
+            "accuracy": 0.3616666666666667,
             "count": 600
           },
           "MB": {
-            "accuracy": 0.9962546816479401,
+            "accuracy": 0.0,
             "count": 267
           },
           "ME": {
@@ -807,64 +514,64 @@
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9954441913439636,
+            "accuracy": 0.12072892938496584,
             "count": 439
           },
           "UD": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 41
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.79,
-        "digit_accuracy": 0.9685714285714285,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.19285714285714287,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9966666666666667,
+            "accuracy": 0.3333333333333333,
             "count": 300
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 100
           },
           "UB": {
-            "accuracy": 0.8984771573604061,
+            "accuracy": 0.17766497461928935,
             "count": 197
           },
           "UD": {
-            "accuracy": 0.9902912621359223,
+            "accuracy": 0.0,
             "count": 103
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.61,
-        "digit_accuracy": 0.9342857142857143,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.19428571428571428,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.98,
+            "accuracy": 0.5,
             "count": 200
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 100
           },
           "UB": {
-            "accuracy": 0.8744939271255061,
+            "accuracy": 0.145748987854251,
             "count": 247
           },
           "UD": {
-            "accuracy": 0.9281045751633987,
+            "accuracy": 0.0,
             "count": 153
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.66,
-        "digit_accuracy": 0.9342857142857143,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.19,
         "n_examples": 100,
         "per_subtask": {
           "MD": {
@@ -872,23 +579,23 @@
             "count": 100
           },
           "MB": {
-            "accuracy": 1.0,
+            "accuracy": 0.0,
             "count": 100
           },
           "UB": {
-            "accuracy": 0.8993288590604027,
+            "accuracy": 0.11073825503355705,
             "count": 298
           },
           "UD": {
-            "accuracy": 0.9207920792079208,
+            "accuracy": 0.0,
             "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.76,
-      "digit_accuracy": 0.9431318681318681,
+      "overall_accuracy": 0.005,
+      "digit_accuracy": 0.24736263736263736,
       "total_examples": 2600,
       "n_splits": 24
     }

add_sub_baseline_10K/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9024dda0f3d40530abfd72b50df46a8e6a8a4dde5fed1a429b758171421411ab
+oid sha256:93cfd7c10000750b8a27620e26b0445a5a43d40542a11ffc61a0136c8f555e0b
 size 650266922

add_sub_baseline_10K/train_config.json

@@ -30,13 +30,13 @@
   "vq_abs_pretrain_target_vectors": 20000,
   "batch_size": 64,
   "gradient_accumulation_steps": 1,
-  "num_epochs": 20,
+  "num_epochs": 1,
   "emb_warmup_steps": 0,
   "log_every": 50,
   "eval_every": 156,
   "save_every": 999999,
   "eval_samples": 100,
-  "output_dir": "ckpt/sweep/as_baseline_10K_2L3H510d",
+  "output_dir": "ckpt/smoke_test_baseline",
   "eval_K": 4,
   "alpha_traj": 0.0,
   "corrupt_method": "shuffle",
@@ -66,23 +66,24 @@
   "mode": "baseline",
   "device": "cuda",
   "push_to_hub": true,
-  "no_wandb": false,
+  "no_wandb": true,
   "n_params": 162490082,
   "run_name": "add_sub_baseline_10K",
-  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
-  "timestamp": "2026-04-16T01:10:30.945547+00:00",
+  "git_commit": "2cf43f15c420a037503a723221cfde984733f98c",
+  "timestamp": "2026-04-19T15:25:53.286620+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
   "train_dataset": "fixed_train/train_10K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "dvsplayi",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/dvsplayi",
+  "wandb_run_id": null,
+  "wandb_url": null,
   "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
   "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
   "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
-  "final_accuracy": 0.76,
-  "sft_accuracy": 0.76,
+  "config_hash": "bd3188fbcb22",
+  "final_accuracy": 0.005,
+  "sft_accuracy": 0.005,
   "eval_method": "ArithmeticEvaluator"
 }