Upload add_sub_sorl_v1_abs1_K1_10K

add_sub_sorl_v1_abs1_K1_10K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151645
+}

add_sub_sorl_v1_abs1_K1_10K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs1_K1_10K/metrics.json

@@ -0,0 +1,1687 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      207,
+      257,
+      307,
+      364,
+      414,
+      464,
+      521,
+      571,
+      621,
+      678,
+      728,
+      778,
+      835,
+      885,
+      935,
+      992,
+      1042,
+      1092,
+      1149,
+      1199,
+      1249,
+      1306,
+      1356,
+      1406,
+      1463,
+      1513,
+      1563,
+      1620,
+      1670,
+      1720,
+      1777,
+      1827,
+      1877,
+      1934,
+      1984,
+      2034,
+      2091,
+      2141,
+      2191,
+      2248,
+      2298,
+      2348,
+      2405,
+      2455,
+      2505,
+      2562,
+      2612,
+      2662,
+      2719,
+      2769,
+      2819,
+      2876,
+      2926,
+      2976,
+      3033,
+      3083,
+      3133
+    ],
+    "loss": [
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN
+    ],
+    "base_loss": [
+      7.768276691436768,
+      4.110814094543457,
+      2.00011944770813,
+      1.8781509399414062,
+      1.8382772207260132,
+      1.812304139137268,
+      1.7306851148605347,
+      1.727476716041565,
+      1.7197872400283813,
+      1.6057711839675903,
+      1.2709606885910034,
+      1.1420778036117554,
+      0.902216911315918,
+      0.8765246272087097,
+      0.771567165851593,
+      0.7372206449508667,
+      0.6973976492881775,
+      0.702940046787262,
+      0.6661996841430664,
+      0.594870388507843,
+      0.5054638981819153,
+      0.5699015259742737,
+      0.5439233183860779,
+      0.5391989946365356,
+      0.4917656481266022,
+      0.5458092093467712,
+      0.4743604362010956,
+      0.4410257935523987,
+      0.44940850138664246,
+      0.4665364623069763,
+      0.39998432993888855,
+      0.3829936385154724,
+      0.37692418694496155,
+      0.43536049127578735,
+      0.41529184579849243,
+      0.4333970248699188,
+      0.41430020332336426,
+      0.3589570224285126,
+      0.32946068048477173,
+      0.2883765399456024,
+      0.32201504707336426,
+      0.31255021691322327,
+      0.2970984876155853,
+      0.3347127139568329,
+      0.310171902179718,
+      0.3680216372013092,
+      0.2886590361595154,
+      0.29018694162368774,
+      0.26444265246391296,
+      0.273633748292923,
+      0.2644065022468567,
+      0.26346877217292786,
+      0.20881788432598114,
+      0.25836777687072754,
+      0.21619343757629395,
+      0.2508084177970886,
+      0.2066682130098343,
+      0.18159745633602142,
+      0.21431173384189606,
+      0.19220055639743805
+    ],
+    "info_loss": [
+      -0.29820680618286133,
+      -0.020511627197265625,
+      -0.03582203388214111,
+      -0.046578407287597656,
+      -0.10162782669067383,
+      -0.11258316040039062,
+      -0.45531249046325684,
+      -0.804049015045166,
+      -0.9047041535377502,
+      -0.904850423336029,
+      -0.6043012142181396,
+      -0.5437737107276917,
+      -0.38619208335876465,
+      -0.3382325768470764,
+      -0.3524826765060425,
+      -0.36228618025779724,
+      -0.3772541284561157,
+      -0.3371272087097168,
+      -0.3112085163593292,
+      -0.2882551848888397,
+      -0.2722354233264923,
+      -0.3066837787628174,
+      -0.29322195053100586,
+      -0.34488701820373535,
+      -0.2819674611091614,
+      -0.3375912308692932,
+      -0.3093259334564209,
+      -0.2779213786125183,
+      -0.2858109176158905,
+      -0.31344014406204224,
+      -0.29580312967300415,
+      -0.24960872530937195,
+      -0.24778319895267487,
+      -0.2526150643825531,
+      -0.258733868598938,
+      -0.2963833212852478,
+      -0.28865474462509155,
+      -0.24197861552238464,
+      -0.24057714641094208,
+      -0.2397194355726242,
+      -0.23951296508312225,
+      -0.24607932567596436,
+      -0.2575826346874237,
+      -0.28247666358947754,
+      -0.2385159730911255,
+      -0.2948923110961914,
+      -0.22245699167251587,
+      -0.24565109610557556,
+      -0.2091638743877411,
+      -0.21068799495697021,
+      -0.22219160199165344,
+      -0.2074809968471527,
+      -0.17261289060115814,
+      -0.225511372089386,
+      -0.18519316613674164,
+      -0.2172529101371765,
+      -0.18351851403713226,
+      -0.16235585510730743,
+      -0.19041140377521515,
+      -0.17723910510540009
+    ],
+    "abs_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "zipf_loss": [
+      11.741195678710938,
+      0.9040621519088745,
+      0.049326345324516296,
+      0.014370954595506191,
+      0.008595990017056465,
+      0.00705260643735528,
+      0.014052295126020908,
+      0.03310779482126236,
+      0.027424722909927368,
+      0.02580416575074196,
+      0.02558719925582409,
+      0.02553989179432392,
+      0.02478562295436859,
+      0.01847648434340954,
+      0.015505120158195496,
+      0.017019525170326233,
+      0.012050194665789604,
+      0.013144981116056442,
+      0.009881021454930305,
+      0.010805479250848293,
+      0.009166087955236435,
+      0.009159133769571781,
+      0.0075935060158371925,
+      0.007109250873327255,
+      0.006107652559876442,
+      0.005024964455515146,
+      0.004673627205193043,
+      0.006059933453798294,
+      0.0038989605382084846,
+      0.004231364466249943,
+      0.005021039396524429,
+      0.002918974496424198,
+      0.002982738194987178,
+      0.003413921222090721,
+      0.003438219428062439,
+      0.0028688814491033554,
+      0.0031602769158780575,
+      0.003254554932937026,
+      0.003583212150260806,
+      0.0021136049181222916,
+      0.002153565175831318,
+      0.0020841355435550213,
+      0.0022789854556322098,
+      0.0021481704898178577,
+      0.0018883869051933289,
+      0.0018621227936819196,
+      0.0018467966001480818,
+      0.001444419613108039,
+      0.0018124142661690712,
+      0.00142430339474231,
+      0.0017039082013070583,
+      0.0014954449143260717,
+      0.0015363717684522271,
+      0.0015524711925536394,
+      0.0016609512967988849,
+      0.001424457412213087,
+      0.0015847194008529186,
+      0.0014708241214975715,
+      0.0015044735046103597,
+      0.0013292990624904633
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN
+    ],
+    "lr": [
+      3.9200000000000004e-05,
+      7.92e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      7.946710526315791e-05,
+      7.650657894736843e-05,
+      7.354605263157895e-05,
+      7.017105263157896e-05,
+      6.721052631578948e-05,
+      6.425e-05,
+      6.0875e-05,
+      5.791447368421054e-05,
+      5.495394736842105e-05,
+      5.157894736842105e-05,
+      4.861842105263157e-05,
+      4.565789473684212e-05,
+      4.2282894736842104e-05,
+      3.9322368421052625e-05,
+      3.636184210526315e-05,
+      3.2986842105263165e-05,
+      3.0026315789473686e-05,
+      2.7065789473684206e-05,
+      2.3690789473684223e-05,
+      2.0730263157894743e-05,
+      1.7769736842105264e-05,
+      1.4394736842105275e-05,
+      1.1434210526315796e-05,
+      8.473684210526318e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      150,
+      307,
+      464,
+      621,
+      778,
+      935,
+      1092,
+      1199,
+      1356,
+      1513,
+      1670,
+      1827,
+      1984,
+      2141,
+      2298,
+      2455,
+      2612,
+      2769,
+      2926,
+      3083
+    ],
+    "eval_accuracy": [
+      0.02,
+      0.01,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.845,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.36,
+        "digit_accuracy": 0.8785714285714286,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8661157024793389,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9578947368421052,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.9085714285714286,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8921568627450981,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9585798816568047,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.9032258064516129,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.8918918918918919,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.37,
+        "digit_accuracy": 0.8814285714285715,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9386503067484663,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9076923076923077,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9080459770114943,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.7684729064039408,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9487179487179487,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.7985714285714286,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9008264462809917,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9421487603305785,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.8775510204081632,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7043010752688172,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.726457399103139,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.21,
+        "digit_accuracy": 0.7085714285714285,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9326923076923077,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9339622641509434,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.5407166123778502,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.19,
+        "digit_accuracy": 0.5757142857142857,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.46,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.4,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.58,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.492,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.495,
+        "digit_accuracy": 0.91,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8926174496644296,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.95625,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.8960302457466919,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.875,
+            "count": 48
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.39,
+        "digit_accuracy": 0.8728571428571429,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.866,
+            "count": 500
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.79,
+            "count": 100
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.49,
+        "digit_accuracy": 0.89,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.905,
+            "count": 400
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8205128205128205,
+            "count": 156
+          },
+          "US": {
+            "accuracy": 0.7727272727272727,
+            "count": 44
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.25,
+        "digit_accuracy": 0.8357142857142857,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9333333333333333,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6532663316582915,
+            "count": 199
+          },
+          "US": {
+            "accuracy": 0.7524752475247525,
+            "count": 101
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.35,
+        "digit_accuracy": 0.8342857142857143,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.945,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.75,
+            "count": 264
+          },
+          "US": {
+            "accuracy": 0.7352941176470589,
+            "count": 136
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.19,
+        "digit_accuracy": 0.7742857142857142,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 310
+          },
+          "US": {
+            "accuracy": 0.6736842105263158,
+            "count": 190
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.22,
+        "digit_accuracy": 0.8014285714285714,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7594594594594595,
+            "count": 370
+          },
+          "US": {
+            "accuracy": 0.782608695652174,
+            "count": 230
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.56,
+        "digit_accuracy": 0.9214285714285714,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9170731707317074,
+            "count": 615
+          },
+          "ME": {
+            "accuracy": 0.9529411764705882,
+            "count": 85
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.67,
+        "digit_accuracy": 0.9471428571428572,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9657534246575342,
+            "count": 292
+          },
+          "MB": {
+            "accuracy": 0.9583333333333334,
+            "count": 144
+          },
+          "ME": {
+            "accuracy": 0.88,
+            "count": 25
+          },
+          "UB": {
+            "accuracy": 0.9246861924686193,
+            "count": 239
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.35,
+        "digit_accuracy": 0.8714285714285714,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9620853080568721,
+            "count": 211
+          },
+          "MB": {
+            "accuracy": 0.9304347826086956,
+            "count": 115
+          },
+          "ME": {
+            "accuracy": 0.9764705882352941,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.6795580110497238,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.8703703703703703,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.15,
+        "digit_accuracy": 0.8014285714285714,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9888268156424581,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.941747572815534,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.6308724832214765,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.647887323943662,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.27,
+        "digit_accuracy": 0.7414285714285714,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.71,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.5066666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.5971428571428572,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.62,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.39,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.57,
+        "digit_accuracy": 0.92,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.965,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9438202247191011,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9622641509433962,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.8428246013667426,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.8780487804878049,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.8657142857142858,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9633333333333334,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7157360406091371,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.7669902912621359,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.27,
+        "digit_accuracy": 0.8185714285714286,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.98,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7125506072874493,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.673202614379085,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.18,
+        "digit_accuracy": 0.7742857142857142,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7046979865771812,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.6584158415841584,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.3503846153846154,
+      "digit_accuracy": 0.8206043956043956,
+      "total_examples": 2600,
+      "n_splits": 24
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.99,
+        "digit_accuracy": 0.9985714285714286,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9983471074380166,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.99,
+        "digit_accuracy": 0.9985714285714286,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9940828402366864,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9914285714285714,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9923076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9885057471264368,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9852216748768473,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9914529914529915,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.91,
+        "digit_accuracy": 0.9871428571428571,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.9516129032258065,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.81,
+        "digit_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.88125,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.993485342019544,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.54,
+        "digit_accuracy": 0.8842857142857142,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.63,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.89,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.62,
+        "digit_accuracy": 0.8571428571428571,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.72,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.856,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.975,
+        "digit_accuracy": 0.9964285714285714,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9905482041587902,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 500
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.98,
+            "count": 100
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 400
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9807692307692307,
+            "count": 156
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 44
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.91,
+        "digit_accuracy": 0.9842857142857143,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9547738693467337,
+            "count": 199
+          },
+          "US": {
+            "accuracy": 0.9801980198019802,
+            "count": 101
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9810606060606061,
+            "count": 264
+          },
+          "US": {
+            "accuracy": 0.9852941176470589,
+            "count": 136
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9806451612903225,
+            "count": 310
+          },
+          "US": {
+            "accuracy": 0.9947368421052631,
+            "count": 190
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.86,
+        "digit_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9621621621621622,
+            "count": 370
+          },
+          "US": {
+            "accuracy": 0.9695652173913043,
+            "count": 230
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9967479674796748,
+            "count": 615
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.97,
+        "digit_accuracy": 0.9957142857142857,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9965753424657534,
+            "count": 292
+          },
+          "MB": {
+            "accuracy": 0.9861111111111112,
+            "count": 144
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 25
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 239
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 211
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 115
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.994475138121547,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.9907407407407407,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.7,
+        "digit_accuracy": 0.95,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.8187919463087249,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.9624413145539906,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.38,
+        "digit_accuracy": 0.8828571428571429,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.8933333333333333,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.18,
+        "digit_accuracy": 0.7628571428571429,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.745,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.95,
+        "digit_accuracy": 0.9928571428571429,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9933333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9887640449438202,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9931662870159453,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.95,
+        "digit_accuracy": 0.9928571428571429,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9796954314720813,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.9902912621359223,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.78,
+        "digit_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9311740890688259,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.9477124183006536,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.81,
+        "digit_accuracy": 0.9642857142857143,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9362416107382551,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.9702970297029703,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.845,
+      "digit_accuracy": 0.9651098901098901,
+      "total_examples": 2600,
+      "n_splits": 24
+    }
+  },
+  "sorl_overall_accuracy": 0.845,
+  "sft_overall_accuracy": 0.3503846153846154
+}

add_sub_sorl_v1_abs1_K1_10K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b657f0d2d656ff6596e9436a64636c48f9bd79a42360a8f96f74689d1ad83f91
+size 650266922

add_sub_sorl_v1_abs1_K1_10K/train_config.json

@@ -0,0 +1,89 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 8e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs1_K1_10K_2L3H510d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 1,
+  "dataset_size": 10000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162490082,
+  "run_name": "add_sub_sorl_v1_abs1_K1_10K",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-16T04:12:31.461525+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_10K_seed42.pt",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "oqawpqvh",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/oqawpqvh",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "config_hash": "0a607564f712",
+  "final_accuracy": 0.845,
+  "sft_accuracy": 0.3503846153846154,
+  "eval_method": "ArithmeticEvaluator"
+}