Upload folder using huggingface_hub

README.md

@@ -0,0 +1,56 @@
+---
+library_name: transformers
+license: mit
+base_model: sbintuitions/tiny-lm
+tags:
+- generated_from_trainer
+datasets:
+- HuggingFaceFW/fineweb
+model-index:
+- name: output-tiny-lm-fineweb
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# output-tiny-lm-fineweb
+
+This model is a fine-tuned version of [sbintuitions/tiny-lm](https://huggingface.co/sbintuitions/tiny-lm) on the HuggingFaceFW/fineweb dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 128
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.01
+- training_steps: 20000
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.57.3
+- Pytorch 2.9.1+cu130
+- Datasets 4.4.1
+- Tokenizers 0.22.1

all_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 1.0,
+    "num_input_tokens_seen": 655360000,
+    "total_flos": 1.23866185728e+16,
+    "train_loss": 1.4302955017089845,
+    "train_runtime": 4951.1263,
+    "train_samples": 0,
+    "train_samples_per_second": 517.054,
+    "train_steps_per_second": 4.039
+}

bit_projection_weights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b0ae34ff7325d9251433afe0add80c29dfb19c95ce713dd15d98617dd2ac420
+size 9938

chat_template.jinja

@@ -0,0 +1 @@
+{# -------------------------------Optional: Define available tools------------------------------- #}{%- if tools %}{%- for tool in tools %}{{- "\x11" -}} {# Start Of Tool Definition #}{{- "\x0E" -}} {# Start Of Attention Block #}{{- tool['function']['name'] + '\n' -}}{# Loop through all argument names and their descriptions #}{%- for argument in tool['function']['parameters']['properties'] %}{{- argument + ': ' + tool['function']['parameters']['properties'][argument]['description'] + '\n' -}}{%- endfor %}{{- '\x0F' -}} {# End Of Attention Block #}{%- endfor %}{%- endif %}{# -------------------------------Main conversation message loop------------------------------- #}{% for message in messages %}{{- "\x01" -}} {# Start Of Text Block #}{# Print the role tag, e.g. user or assistant #}{{- message.role + "\n" -}}{# If not the "assistant", we wrap with an attention block #}{% if message.role != "assistant" %}{{- "\x0E" -}}{% endif %}{# If the message contains normal content, print it #}{% if message.content %}{{- message.content -}}{% endif %}{# If the assistant called any tools, print those tool calls #}{% if message.tool_calls %}{% for call in message.tool_calls %}{{- "\x1A" -}} {# Start Of Tool Call #}{{- call.type + " " + call.function.name+ "(" + call.function.arguments | tojson + ")" -}}{{- "\x1B" -}} {# End Of Tool Call #}{% endfor %}{% endif %}{# If not the "assistant", we wrap with an attention block #}{% if message.role != "assistant" %}{{- "\x0F" -}}{% endif %}{{- "\x17" -}} {# End Of Text Block #}{{- "\n" -}} {# Newline after each message #}{% endfor %}{# -------------------------------Add a final assistant prompt marker to continue generation------------------------------- #}{%- if add_generation_prompt %}{{- "\x01assistant\n" -}}{%- endif %}

config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "eos_token_id": 3,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 640,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 256
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    3,
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af0c67f5b41359f3c33dcc00a8756fa0dbb2254abb6fb8f7e324b2cddb671f7e
+size 12596248

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "\u0002",
+  "eos_token": "\u0003",
+  "pad_token": "\u0000"
+}

tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "\u0000",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "\u0002",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "\u0003",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "\u0002",
+  "bos_token_id": 2,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "\u0003",
+  "eos_token_id": 3,
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "\u0000",
+  "pad_token_id": 0,
+  "tokenizer_class": "UTF8Tokenizer"
+}

train_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 1.0,
+    "num_input_tokens_seen": 655360000,
+    "total_flos": 1.23866185728e+16,
+    "train_loss": 1.4302955017089845,
+    "train_runtime": 4951.1263,
+    "train_samples": 0,
+    "train_samples_per_second": 517.054,
+    "train_steps_per_second": 4.039
+}

trainer_state.json

@@ -0,0 +1,2044 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 20000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005,
+      "grad_norm": 0.5784105658531189,
+      "learning_rate": 0.00014849999999999998,
+      "loss": 4.9555,
+      "num_input_tokens_seen": 3276800,
+      "step": 100,
+      "train_runtime": 30.2422,
+      "train_tokens_per_second": 108352.003
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.457616925239563,
+      "learning_rate": 0.0002985,
+      "loss": 3.0038,
+      "num_input_tokens_seen": 6553600,
+      "step": 200,
+      "train_runtime": 55.436,
+      "train_tokens_per_second": 118219.297
+    },
+    {
+      "epoch": 0.015,
+      "grad_norm": 2.997375249862671,
+      "learning_rate": 0.0002999814948722491,
+      "loss": 2.4591,
+      "num_input_tokens_seen": 9830400,
+      "step": 300,
+      "train_runtime": 86.3647,
+      "train_tokens_per_second": 113824.237
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 2.3580925464630127,
+      "learning_rate": 0.0002999252345933521,
+      "loss": 2.147,
+      "num_input_tokens_seen": 13107200,
+      "step": 400,
+      "train_runtime": 109.562,
+      "train_tokens_per_second": 119632.766
+    },
+    {
+      "epoch": 0.025,
+      "grad_norm": 2.9443857669830322,
+      "learning_rate": 0.000299831231438409,
+      "loss": 1.9844,
+      "num_input_tokens_seen": 16384000,
+      "step": 500,
+      "train_runtime": 133.562,
+      "train_tokens_per_second": 122669.618
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 2.476471185684204,
+      "learning_rate": 0.0002996995090722112,
+      "loss": 1.8654,
+      "num_input_tokens_seen": 19660800,
+      "step": 600,
+      "train_runtime": 157.7502,
+      "train_tokens_per_second": 124632.515
+    },
+    {
+      "epoch": 0.035,
+      "grad_norm": 3.6577234268188477,
+      "learning_rate": 0.00029953010065516004,
+      "loss": 1.8002,
+      "num_input_tokens_seen": 22937600,
+      "step": 700,
+      "train_runtime": 182.0366,
+      "train_tokens_per_second": 126005.43
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 2.630089044570923,
+      "learning_rate": 0.0002993230488349186,
+      "loss": 1.7327,
+      "num_input_tokens_seen": 26214400,
+      "step": 800,
+      "train_runtime": 205.9679,
+      "train_tokens_per_second": 127274.199
+    },
+    {
+      "epoch": 0.045,
+      "grad_norm": 2.3071346282958984,
+      "learning_rate": 0.00029907840573567524,
+      "loss": 1.7033,
+      "num_input_tokens_seen": 29491200,
+      "step": 900,
+      "train_runtime": 229.9223,
+      "train_tokens_per_second": 128265.936
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 2.2288811206817627,
+      "learning_rate": 0.00029879623294502204,
+      "loss": 1.6789,
+      "num_input_tokens_seen": 32768000,
+      "step": 1000,
+      "train_runtime": 258.3246,
+      "train_tokens_per_second": 126848.171
+    },
+    {
+      "epoch": 0.055,
+      "grad_norm": 2.1029598712921143,
+      "learning_rate": 0.00029847660149844995,
+      "loss": 1.6491,
+      "num_input_tokens_seen": 36044800,
+      "step": 1100,
+      "train_runtime": 282.4776,
+      "train_tokens_per_second": 127602.309
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 1.4104539155960083,
+      "learning_rate": 0.0002981195918614664,
+      "loss": 1.6318,
+      "num_input_tokens_seen": 39321600,
+      "step": 1200,
+      "train_runtime": 306.4453,
+      "train_tokens_per_second": 128315.216
+    },
+    {
+      "epoch": 0.065,
+      "grad_norm": 1.5436947345733643,
+      "learning_rate": 0.0002977252939093383,
+      "loss": 1.6277,
+      "num_input_tokens_seen": 42598400,
+      "step": 1300,
+      "train_runtime": 330.6399,
+      "train_tokens_per_second": 128836.255
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 1.8562994003295898,
+      "learning_rate": 0.00029729380690446654,
+      "loss": 1.6016,
+      "num_input_tokens_seen": 45875200,
+      "step": 1400,
+      "train_runtime": 354.9473,
+      "train_tokens_per_second": 129245.099
+    },
+    {
+      "epoch": 0.075,
+      "grad_norm": 1.7223314046859741,
+      "learning_rate": 0.000296825239471397,
+      "loss": 1.5832,
+      "num_input_tokens_seen": 49152000,
+      "step": 1500,
+      "train_runtime": 380.0343,
+      "train_tokens_per_second": 129335.708
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 1.294960618019104,
+      "learning_rate": 0.00029631970956947514,
+      "loss": 1.5697,
+      "num_input_tokens_seen": 52428800,
+      "step": 1600,
+      "train_runtime": 403.7379,
+      "train_tokens_per_second": 129858.509
+    },
+    {
+      "epoch": 0.085,
+      "grad_norm": 1.6293072700500488,
+      "learning_rate": 0.0002957773444631505,
+      "loss": 1.5668,
+      "num_input_tokens_seen": 55705600,
+      "step": 1700,
+      "train_runtime": 427.714,
+      "train_tokens_per_second": 130240.289
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 1.2281590700149536,
+      "learning_rate": 0.000295198280689938,
+      "loss": 1.5866,
+      "num_input_tokens_seen": 58982400,
+      "step": 1800,
+      "train_runtime": 452.5238,
+      "train_tokens_per_second": 130340.981
+    },
+    {
+      "epoch": 0.095,
+      "grad_norm": 1.2015036344528198,
+      "learning_rate": 0.000294582664026046,
+      "loss": 1.5497,
+      "num_input_tokens_seen": 62259200,
+      "step": 1900,
+      "train_runtime": 476.482,
+      "train_tokens_per_second": 130664.34
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.569770336151123,
+      "learning_rate": 0.00029393064944967733,
+      "loss": 1.5564,
+      "num_input_tokens_seen": 65536000,
+      "step": 2000,
+      "train_runtime": 505.6146,
+      "train_tokens_per_second": 129616.522
+    },
+    {
+      "epoch": 0.105,
+      "grad_norm": 1.0509306192398071,
+      "learning_rate": 0.0002932424011020149,
+      "loss": 1.5755,
+      "num_input_tokens_seen": 68812800,
+      "step": 2100,
+      "train_runtime": 529.3779,
+      "train_tokens_per_second": 129988.045
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.2682929039001465,
+      "learning_rate": 0.0002925180922458996,
+      "loss": 1.532,
+      "num_input_tokens_seen": 72089600,
+      "step": 2200,
+      "train_runtime": 553.4037,
+      "train_tokens_per_second": 130265.845
+    },
+    {
+      "epoch": 0.115,
+      "grad_norm": 0.8587242960929871,
+      "learning_rate": 0.00029175790522221253,
+      "loss": 1.5221,
+      "num_input_tokens_seen": 75366400,
+      "step": 2300,
+      "train_runtime": 577.2313,
+      "train_tokens_per_second": 130565.344
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.9015905261039734,
+      "learning_rate": 0.00029096203140397157,
+      "loss": 1.5253,
+      "num_input_tokens_seen": 78643200,
+      "step": 2400,
+      "train_runtime": 601.9877,
+      "train_tokens_per_second": 130639.212
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 0.9860896468162537,
+      "learning_rate": 0.0002901306711481544,
+      "loss": 1.518,
+      "num_input_tokens_seen": 81920000,
+      "step": 2500,
+      "train_runtime": 625.982,
+      "train_tokens_per_second": 130866.382
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 0.9716039896011353,
+      "learning_rate": 0.00028926403374525953,
+      "loss": 1.5216,
+      "num_input_tokens_seen": 85196800,
+      "step": 2600,
+      "train_runtime": 649.7987,
+      "train_tokens_per_second": 131112.604
+    },
+    {
+      "epoch": 0.135,
+      "grad_norm": 1.115633487701416,
+      "learning_rate": 0.00028836233736661843,
+      "loss": 1.504,
+      "num_input_tokens_seen": 88473600,
+      "step": 2700,
+      "train_runtime": 678.8811,
+      "train_tokens_per_second": 130322.683
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.8320448398590088,
+      "learning_rate": 0.0002874258090094726,
+      "loss": 1.5041,
+      "num_input_tokens_seen": 91750400,
+      "step": 2800,
+      "train_runtime": 703.1036,
+      "train_tokens_per_second": 130493.433
+    },
+    {
+      "epoch": 0.145,
+      "grad_norm": 1.2689387798309326,
+      "learning_rate": 0.00028645468443982747,
+      "loss": 1.4988,
+      "num_input_tokens_seen": 95027200,
+      "step": 2900,
+      "train_runtime": 727.3731,
+      "train_tokens_per_second": 130644.376
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.9444186091423035,
+      "learning_rate": 0.0002854492081331002,
+      "loss": 1.4737,
+      "num_input_tokens_seen": 98304000,
+      "step": 3000,
+      "train_runtime": 750.7174,
+      "train_tokens_per_second": 130946.747
+    },
+    {
+      "epoch": 0.155,
+      "grad_norm": 0.9993096590042114,
+      "learning_rate": 0.00028440963321257385,
+      "loss": 1.4902,
+      "num_input_tokens_seen": 101580800,
+      "step": 3100,
+      "train_runtime": 774.175,
+      "train_tokens_per_second": 131211.683
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.8590114712715149,
+      "learning_rate": 0.00028333622138567544,
+      "loss": 1.4691,
+      "num_input_tokens_seen": 104857600,
+      "step": 3200,
+      "train_runtime": 804.4318,
+      "train_tokens_per_second": 130349.895
+    },
+    {
+      "epoch": 0.165,
+      "grad_norm": 0.8465983867645264,
+      "learning_rate": 0.000282229242878092,
+      "loss": 1.4961,
+      "num_input_tokens_seen": 108134400,
+      "step": 3300,
+      "train_runtime": 822.6712,
+      "train_tokens_per_second": 131443.042
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 0.8805415630340576,
+      "learning_rate": 0.00028108897636574284,
+      "loss": 1.5042,
+      "num_input_tokens_seen": 111411200,
+      "step": 3400,
+      "train_runtime": 847.8836,
+      "train_tokens_per_second": 131399.175
+    },
+    {
+      "epoch": 0.175,
+      "grad_norm": 0.8398001194000244,
+      "learning_rate": 0.0002799157089046248,
+      "loss": 1.4675,
+      "num_input_tokens_seen": 114688000,
+      "step": 3500,
+      "train_runtime": 871.6987,
+      "train_tokens_per_second": 131568.398
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.7862799763679504,
+      "learning_rate": 0.00027870973585854665,
+      "loss": 1.4679,
+      "num_input_tokens_seen": 117964800,
+      "step": 3600,
+      "train_runtime": 896.6814,
+      "train_tokens_per_second": 131557.096
+    },
+    {
+      "epoch": 0.185,
+      "grad_norm": 0.7606053352355957,
+      "learning_rate": 0.00027747136082477365,
+      "loss": 1.4592,
+      "num_input_tokens_seen": 121241600,
+      "step": 3700,
+      "train_runtime": 925.0842,
+      "train_tokens_per_second": 131060.069
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.7865683436393738,
+      "learning_rate": 0.00027620089555759807,
+      "loss": 1.463,
+      "num_input_tokens_seen": 124518400,
+      "step": 3800,
+      "train_runtime": 948.479,
+      "train_tokens_per_second": 131282.194
+    },
+    {
+      "epoch": 0.195,
+      "grad_norm": 0.8784666657447815,
+      "learning_rate": 0.0002748986598898566,
+      "loss": 1.4662,
+      "num_input_tokens_seen": 127795200,
+      "step": 3900,
+      "train_runtime": 972.3642,
+      "train_tokens_per_second": 131427.304
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.7535898685455322,
+      "learning_rate": 0.00027356498165241475,
+      "loss": 1.4661,
+      "num_input_tokens_seen": 131072000,
+      "step": 4000,
+      "train_runtime": 995.9673,
+      "train_tokens_per_second": 131602.716
+    },
+    {
+      "epoch": 0.205,
+      "grad_norm": 0.7886632084846497,
+      "learning_rate": 0.00027220019659163653,
+      "loss": 1.4523,
+      "num_input_tokens_seen": 134348800,
+      "step": 4100,
+      "train_runtime": 1025.0973,
+      "train_tokens_per_second": 131059.561
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.7285377383232117,
+      "learning_rate": 0.00027080464828486246,
+      "loss": 1.4382,
+      "num_input_tokens_seen": 137625600,
+      "step": 4200,
+      "train_runtime": 1049.2312,
+      "train_tokens_per_second": 131168.036
+    },
+    {
+      "epoch": 0.215,
+      "grad_norm": 0.6270678639411926,
+      "learning_rate": 0.0002693786880539158,
+      "loss": 1.4381,
+      "num_input_tokens_seen": 140902400,
+      "step": 4300,
+      "train_runtime": 1072.6988,
+      "train_tokens_per_second": 131353.186
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.9621595144271851,
+      "learning_rate": 0.0002679226748766589,
+      "loss": 1.456,
+      "num_input_tokens_seen": 144179200,
+      "step": 4400,
+      "train_runtime": 1097.9055,
+      "train_tokens_per_second": 131322.044
+    },
+    {
+      "epoch": 0.225,
+      "grad_norm": 0.6201126575469971,
+      "learning_rate": 0.0002664369752966228,
+      "loss": 1.4744,
+      "num_input_tokens_seen": 147456000,
+      "step": 4500,
+      "train_runtime": 1122.0946,
+      "train_tokens_per_second": 131411.382
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.8038685917854309,
+      "learning_rate": 0.00026492196333073164,
+      "loss": 1.4537,
+      "num_input_tokens_seen": 150732800,
+      "step": 4600,
+      "train_runtime": 1146.3348,
+      "train_tokens_per_second": 131491.08
+    },
+    {
+      "epoch": 0.235,
+      "grad_norm": 0.8983737826347351,
+      "learning_rate": 0.0002633780203751459,
+      "loss": 1.4298,
+      "num_input_tokens_seen": 154009600,
+      "step": 4700,
+      "train_runtime": 1170.6825,
+      "train_tokens_per_second": 131555.396
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.9596767425537109,
+      "learning_rate": 0.0002618055351092481,
+      "loss": 1.4341,
+      "num_input_tokens_seen": 157286400,
+      "step": 4800,
+      "train_runtime": 1194.1389,
+      "train_tokens_per_second": 131715.33
+    },
+    {
+      "epoch": 0.245,
+      "grad_norm": 0.734665036201477,
+      "learning_rate": 0.0002602049033977945,
+      "loss": 1.4458,
+      "num_input_tokens_seen": 160563200,
+      "step": 4900,
+      "train_runtime": 1217.5015,
+      "train_tokens_per_second": 131879.268
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.5745177268981934,
+      "learning_rate": 0.0002585765281912588,
+      "loss": 1.444,
+      "num_input_tokens_seen": 163840000,
+      "step": 5000,
+      "train_runtime": 1247.5321,
+      "train_tokens_per_second": 131331.29
+    },
+    {
+      "epoch": 0.255,
+      "grad_norm": 0.7568549513816833,
+      "learning_rate": 0.00025692081942439113,
+      "loss": 1.4307,
+      "num_input_tokens_seen": 167116800,
+      "step": 5100,
+      "train_runtime": 1270.9694,
+      "train_tokens_per_second": 131487.668
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.7054229378700256,
+      "learning_rate": 0.0002552381939130192,
+      "loss": 1.4325,
+      "num_input_tokens_seen": 170393600,
+      "step": 5200,
+      "train_runtime": 1294.8044,
+      "train_tokens_per_second": 131597.945
+    },
+    {
+      "epoch": 0.265,
+      "grad_norm": 0.5529087781906128,
+      "learning_rate": 0.00025352907524911716,
+      "loss": 1.4271,
+      "num_input_tokens_seen": 173670400,
+      "step": 5300,
+      "train_runtime": 1318.1084,
+      "train_tokens_per_second": 131757.299
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.669465184211731,
+      "learning_rate": 0.00025179389369416885,
+      "loss": 1.4056,
+      "num_input_tokens_seen": 176947200,
+      "step": 5400,
+      "train_runtime": 1341.5822,
+      "train_tokens_per_second": 131894.417
+    },
+    {
+      "epoch": 0.275,
+      "grad_norm": 0.662200927734375,
+      "learning_rate": 0.0002500330860708513,
+      "loss": 1.4634,
+      "num_input_tokens_seen": 180224000,
+      "step": 5500,
+      "train_runtime": 1366.4603,
+      "train_tokens_per_second": 131891.134
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.8842505216598511,
+      "learning_rate": 0.00024824709565306733,
+      "loss": 1.422,
+      "num_input_tokens_seen": 183500800,
+      "step": 5600,
+      "train_runtime": 1390.3365,
+      "train_tokens_per_second": 131983.017
+    },
+    {
+      "epoch": 0.285,
+      "grad_norm": 0.5874491930007935,
+      "learning_rate": 0.00024643637205435363,
+      "loss": 1.4204,
+      "num_input_tokens_seen": 186777600,
+      "step": 5700,
+      "train_runtime": 1420.1755,
+      "train_tokens_per_second": 131517.271
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 0.6088038682937622,
+      "learning_rate": 0.00024460137111469296,
+      "loss": 1.4286,
+      "num_input_tokens_seen": 190054400,
+      "step": 5800,
+      "train_runtime": 1443.4282,
+      "train_tokens_per_second": 131668.763
+    },
+    {
+      "epoch": 0.295,
+      "grad_norm": 0.5753700733184814,
+      "learning_rate": 0.00024274255478575854,
+      "loss": 1.3925,
+      "num_input_tokens_seen": 193331200,
+      "step": 5900,
+      "train_runtime": 1467.8392,
+      "train_tokens_per_second": 131711.428
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.0680171251296997,
+      "learning_rate": 0.00024086039101462085,
+      "loss": 1.4237,
+      "num_input_tokens_seen": 196608000,
+      "step": 6000,
+      "train_runtime": 1492.1963,
+      "train_tokens_per_second": 131757.468
+    },
+    {
+      "epoch": 0.305,
+      "grad_norm": 0.6099634170532227,
+      "learning_rate": 0.00023895535362594388,
+      "loss": 1.4171,
+      "num_input_tokens_seen": 199884800,
+      "step": 6100,
+      "train_runtime": 1515.7396,
+      "train_tokens_per_second": 131872.784
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 0.6650487184524536,
+      "learning_rate": 0.0002370279222027026,
+      "loss": 1.4142,
+      "num_input_tokens_seen": 203161600,
+      "step": 6200,
+      "train_runtime": 1539.3684,
+      "train_tokens_per_second": 131977.242
+    },
+    {
+      "epoch": 0.315,
+      "grad_norm": 0.6537796258926392,
+      "learning_rate": 0.00023507858196545036,
+      "loss": 1.4055,
+      "num_input_tokens_seen": 206438400,
+      "step": 6300,
+      "train_runtime": 1569.0555,
+      "train_tokens_per_second": 131568.574
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.9217991828918457,
+      "learning_rate": 0.00023310782365016728,
+      "loss": 1.4001,
+      "num_input_tokens_seen": 209715200,
+      "step": 6400,
+      "train_runtime": 1593.9139,
+      "train_tokens_per_second": 131572.477
+    },
+    {
+      "epoch": 0.325,
+      "grad_norm": 0.5520761609077454,
+      "learning_rate": 0.00023111614338472018,
+      "loss": 1.3995,
+      "num_input_tokens_seen": 212992000,
+      "step": 6500,
+      "train_runtime": 1618.0503,
+      "train_tokens_per_second": 131634.969
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 0.6622788906097412,
+      "learning_rate": 0.00022910404256396485,
+      "loss": 1.4155,
+      "num_input_tokens_seen": 216268800,
+      "step": 6600,
+      "train_runtime": 1641.7748,
+      "train_tokens_per_second": 131728.667
+    },
+    {
+      "epoch": 0.335,
+      "grad_norm": 0.5474684834480286,
+      "learning_rate": 0.00022707202772352265,
+      "loss": 1.41,
+      "num_input_tokens_seen": 219545600,
+      "step": 6700,
+      "train_runtime": 1665.6292,
+      "train_tokens_per_second": 131809.411
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.631249189376831,
+      "learning_rate": 0.00022502061041226298,
+      "loss": 1.3895,
+      "num_input_tokens_seen": 222822400,
+      "step": 6800,
+      "train_runtime": 1689.8435,
+      "train_tokens_per_second": 131859.781
+    },
+    {
+      "epoch": 0.345,
+      "grad_norm": 0.6114600300788879,
+      "learning_rate": 0.00022295030706352356,
+      "loss": 1.4042,
+      "num_input_tokens_seen": 226099200,
+      "step": 6900,
+      "train_runtime": 1713.7744,
+      "train_tokens_per_second": 131930.552
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.5166604518890381,
+      "learning_rate": 0.00022086163886510086,
+      "loss": 1.3822,
+      "num_input_tokens_seen": 229376000,
+      "step": 7000,
+      "train_runtime": 1737.7511,
+      "train_tokens_per_second": 131995.885
+    },
+    {
+      "epoch": 0.355,
+      "grad_norm": 0.7250663638114929,
+      "learning_rate": 0.00021875513162804427,
+      "loss": 1.4204,
+      "num_input_tokens_seen": 232652800,
+      "step": 7100,
+      "train_runtime": 1762.0205,
+      "train_tokens_per_second": 132037.511
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.7085418701171875,
+      "learning_rate": 0.00021663131565428554,
+      "loss": 1.3858,
+      "num_input_tokens_seen": 235929600,
+      "step": 7200,
+      "train_runtime": 1785.6176,
+      "train_tokens_per_second": 132127.731
+    },
+    {
+      "epoch": 0.365,
+      "grad_norm": 0.7583802938461304,
+      "learning_rate": 0.00021449072560313843,
+      "loss": 1.3885,
+      "num_input_tokens_seen": 239206400,
+      "step": 7300,
+      "train_runtime": 1814.7871,
+      "train_tokens_per_second": 131809.624
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 0.5545434951782227,
+      "learning_rate": 0.00021233390035670105,
+      "loss": 1.3765,
+      "num_input_tokens_seen": 242483200,
+      "step": 7400,
+      "train_runtime": 1838.4345,
+      "train_tokens_per_second": 131896.568
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 0.7899689078330994,
+      "learning_rate": 0.00021016138288419497,
+      "loss": 1.394,
+      "num_input_tokens_seen": 245760000,
+      "step": 7500,
+      "train_runtime": 1862.6752,
+      "train_tokens_per_second": 131939.27
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.668602466583252,
+      "learning_rate": 0.0002079737201052759,
+      "loss": 1.3727,
+      "num_input_tokens_seen": 249036800,
+      "step": 7600,
+      "train_runtime": 1887.7021,
+      "train_tokens_per_second": 131925.902
+    },
+    {
+      "epoch": 0.385,
+      "grad_norm": 0.5739292502403259,
+      "learning_rate": 0.0002057714627523492,
+      "loss": 1.3855,
+      "num_input_tokens_seen": 252313600,
+      "step": 7700,
+      "train_runtime": 1911.7352,
+      "train_tokens_per_second": 131981.456
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 0.5914629697799683,
+      "learning_rate": 0.00020355516523192651,
+      "loss": 1.4026,
+      "num_input_tokens_seen": 255590400,
+      "step": 7800,
+      "train_runtime": 1935.658,
+      "train_tokens_per_second": 132043.16
+    },
+    {
+      "epoch": 0.395,
+      "grad_norm": 0.6528250575065613,
+      "learning_rate": 0.00020132538548505634,
+      "loss": 1.3895,
+      "num_input_tokens_seen": 258867200,
+      "step": 7900,
+      "train_runtime": 1959.0455,
+      "train_tokens_per_second": 132139.453
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.6278071403503418,
+      "learning_rate": 0.00019908268484686558,
+      "loss": 1.3902,
+      "num_input_tokens_seen": 262144000,
+      "step": 8000,
+      "train_runtime": 1982.9613,
+      "train_tokens_per_second": 132198.243
+    },
+    {
+      "epoch": 0.405,
+      "grad_norm": 1.0000662803649902,
+      "learning_rate": 0.00019682762790524657,
+      "loss": 1.3806,
+      "num_input_tokens_seen": 265420800,
+      "step": 8100,
+      "train_runtime": 2007.4327,
+      "train_tokens_per_second": 132219.03
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.5706632137298584,
+      "learning_rate": 0.00019456078235872488,
+      "loss": 1.4018,
+      "num_input_tokens_seen": 268697600,
+      "step": 8200,
+      "train_runtime": 2037.3211,
+      "train_tokens_per_second": 131887.703
+    },
+    {
+      "epoch": 0.415,
+      "grad_norm": 0.52730792760849,
+      "learning_rate": 0.0001922827188735443,
+      "loss": 1.3884,
+      "num_input_tokens_seen": 271974400,
+      "step": 8300,
+      "train_runtime": 2061.1352,
+      "train_tokens_per_second": 131953.693
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.7067908644676208,
+      "learning_rate": 0.00018999401094000473,
+      "loss": 1.3724,
+      "num_input_tokens_seen": 275251200,
+      "step": 8400,
+      "train_runtime": 2085.2036,
+      "train_tokens_per_second": 132002.073
+    },
+    {
+      "epoch": 0.425,
+      "grad_norm": 0.5584085583686829,
+      "learning_rate": 0.00018769523472808897,
+      "loss": 1.3828,
+      "num_input_tokens_seen": 278528000,
+      "step": 8500,
+      "train_runtime": 2108.8848,
+      "train_tokens_per_second": 132073.599
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.7685338854789734,
+      "learning_rate": 0.0001853869689424151,
+      "loss": 1.384,
+      "num_input_tokens_seen": 281804800,
+      "step": 8600,
+      "train_runtime": 2133.0174,
+      "train_tokens_per_second": 132115.568
+    },
+    {
+      "epoch": 0.435,
+      "grad_norm": 0.71221923828125,
+      "learning_rate": 0.00018306979467655062,
+      "loss": 1.3768,
+      "num_input_tokens_seen": 285081600,
+      "step": 8700,
+      "train_runtime": 2157.3919,
+      "train_tokens_per_second": 132141.775
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.587790310382843,
+      "learning_rate": 0.0001807442952667254,
+      "loss": 1.3646,
+      "num_input_tokens_seen": 288358400,
+      "step": 8800,
+      "train_runtime": 2185.9477,
+      "train_tokens_per_second": 131914.592
+    },
+    {
+      "epoch": 0.445,
+      "grad_norm": 0.6176910400390625,
+      "learning_rate": 0.00017841105614497952,
+      "loss": 1.3793,
+      "num_input_tokens_seen": 291635200,
+      "step": 8900,
+      "train_runtime": 2210.0556,
+      "train_tokens_per_second": 131958.309
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.5924071073532104,
+      "learning_rate": 0.00017607066469178443,
+      "loss": 1.3727,
+      "num_input_tokens_seen": 294912000,
+      "step": 9000,
+      "train_runtime": 2233.2228,
+      "train_tokens_per_second": 132056.685
+    },
+    {
+      "epoch": 0.455,
+      "grad_norm": 0.6892787218093872,
+      "learning_rate": 0.00017372371008817256,
+      "loss": 1.3598,
+      "num_input_tokens_seen": 298188800,
+      "step": 9100,
+      "train_runtime": 2258.9526,
+      "train_tokens_per_second": 132003.123
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.5958895087242126,
+      "learning_rate": 0.00017137078316741442,
+      "loss": 1.3913,
+      "num_input_tokens_seen": 301465600,
+      "step": 9200,
+      "train_runtime": 2283.0976,
+      "train_tokens_per_second": 132042.364
+    },
+    {
+      "epoch": 0.465,
+      "grad_norm": 0.5727524757385254,
+      "learning_rate": 0.00016901247626627952,
+      "loss": 1.3556,
+      "num_input_tokens_seen": 304742400,
+      "step": 9300,
+      "train_runtime": 2307.8709,
+      "train_tokens_per_second": 132044.818
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.7339876890182495,
+      "learning_rate": 0.00016664938307591853,
+      "loss": 1.352,
+      "num_input_tokens_seen": 308019200,
+      "step": 9400,
+      "train_runtime": 2330.8551,
+      "train_tokens_per_second": 132148.583
+    },
+    {
+      "epoch": 0.475,
+      "grad_norm": 0.6451588869094849,
+      "learning_rate": 0.00016428209849240496,
+      "loss": 1.3603,
+      "num_input_tokens_seen": 311296000,
+      "step": 9500,
+      "train_runtime": 2354.8224,
+      "train_tokens_per_second": 132195.107
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.5640471577644348,
+      "learning_rate": 0.00016191121846697302,
+      "loss": 1.353,
+      "num_input_tokens_seen": 314572800,
+      "step": 9600,
+      "train_runtime": 2384.4134,
+      "train_tokens_per_second": 131928.799
+    },
+    {
+      "epoch": 0.485,
+      "grad_norm": 0.7020093202590942,
+      "learning_rate": 0.00015953733985599023,
+      "loss": 1.3514,
+      "num_input_tokens_seen": 317849600,
+      "step": 9700,
+      "train_runtime": 2408.1089,
+      "train_tokens_per_second": 131991.375
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.5688868761062622,
+      "learning_rate": 0.000157161060270702,
+      "loss": 1.3626,
+      "num_input_tokens_seen": 321126400,
+      "step": 9800,
+      "train_runtime": 2431.721,
+      "train_tokens_per_second": 132057.253
+    },
+    {
+      "epoch": 0.495,
+      "grad_norm": 0.702392041683197,
+      "learning_rate": 0.00015478297792678616,
+      "loss": 1.3396,
+      "num_input_tokens_seen": 324403200,
+      "step": 9900,
+      "train_runtime": 2455.4838,
+      "train_tokens_per_second": 132113.76
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.6182683110237122,
+      "learning_rate": 0.00015240369149375544,
+      "loss": 1.3433,
+      "num_input_tokens_seen": 327680000,
+      "step": 10000,
+      "train_runtime": 2479.369,
+      "train_tokens_per_second": 132162.661
+    },
+    {
+      "epoch": 0.505,
+      "grad_norm": 0.7529293298721313,
+      "learning_rate": 0.00015002379994424547,
+      "loss": 1.3443,
+      "num_input_tokens_seen": 330956800,
+      "step": 10100,
+      "train_runtime": 2504.1688,
+      "train_tokens_per_second": 132162.336
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 0.5510791540145874,
+      "learning_rate": 0.00014764390240322691,
+      "loss": 1.3602,
+      "num_input_tokens_seen": 334233600,
+      "step": 10200,
+      "train_runtime": 2529.0077,
+      "train_tokens_per_second": 132159.975
+    },
+    {
+      "epoch": 0.515,
+      "grad_norm": 0.6009634137153625,
+      "learning_rate": 0.00014526459799717842,
+      "loss": 1.3558,
+      "num_input_tokens_seen": 337510400,
+      "step": 10300,
+      "train_runtime": 2553.0238,
+      "train_tokens_per_second": 132200.255
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.6439092755317688,
+      "learning_rate": 0.0001428864857032605,
+      "loss": 1.3526,
+      "num_input_tokens_seen": 340787200,
+      "step": 10400,
+      "train_runtime": 2578.3654,
+      "train_tokens_per_second": 132171.799
+    },
+    {
+      "epoch": 0.525,
+      "grad_norm": 0.6969897747039795,
+      "learning_rate": 0.00014051016419852538,
+      "loss": 1.3567,
+      "num_input_tokens_seen": 344064000,
+      "step": 10500,
+      "train_runtime": 2607.8404,
+      "train_tokens_per_second": 131934.456
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 0.5618866682052612,
+      "learning_rate": 0.00013813623170920404,
+      "loss": 1.3636,
+      "num_input_tokens_seen": 347340800,
+      "step": 10600,
+      "train_runtime": 2631.6887,
+      "train_tokens_per_second": 131984.0
+    },
+    {
+      "epoch": 0.535,
+      "grad_norm": 0.7540677785873413,
+      "learning_rate": 0.00013576528586010548,
+      "loss": 1.3507,
+      "num_input_tokens_seen": 350617600,
+      "step": 10700,
+      "train_runtime": 2655.342,
+      "train_tokens_per_second": 132042.35
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.5177021622657776,
+      "learning_rate": 0.0001333979235241679,
+      "loss": 1.3391,
+      "num_input_tokens_seen": 353894400,
+      "step": 10800,
+      "train_runtime": 2678.6561,
+      "train_tokens_per_second": 132116.4
+    },
+    {
+      "epoch": 0.545,
+      "grad_norm": 0.5168038606643677,
+      "learning_rate": 0.0001310347406721994,
+      "loss": 1.3596,
+      "num_input_tokens_seen": 357171200,
+      "step": 10900,
+      "train_runtime": 2701.9577,
+      "train_tokens_per_second": 132189.781
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.5508900284767151,
+      "learning_rate": 0.00012867633222284514,
+      "loss": 1.3495,
+      "num_input_tokens_seen": 360448000,
+      "step": 11000,
+      "train_runtime": 2730.6775,
+      "train_tokens_per_second": 131999.474
+    },
+    {
+      "epoch": 0.555,
+      "grad_norm": 0.4799763560295105,
+      "learning_rate": 0.0001263232918928202,
+      "loss": 1.3444,
+      "num_input_tokens_seen": 363724800,
+      "step": 11100,
+      "train_runtime": 2755.3984,
+      "train_tokens_per_second": 132004.431
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.5217195153236389,
+      "learning_rate": 0.00012397621204744406,
+      "loss": 1.3813,
+      "num_input_tokens_seen": 367001600,
+      "step": 11200,
+      "train_runtime": 2780.3769,
+      "train_tokens_per_second": 131997.07
+    },
+    {
+      "epoch": 0.565,
+      "grad_norm": 0.5274467468261719,
+      "learning_rate": 0.00012163568355151628,
+      "loss": 1.3503,
+      "num_input_tokens_seen": 370278400,
+      "step": 11300,
+      "train_runtime": 2804.6323,
+      "train_tokens_per_second": 132023.868
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 0.5383314490318298,
+      "learning_rate": 0.00011930229562056919,
+      "loss": 1.3684,
+      "num_input_tokens_seen": 373555200,
+      "step": 11400,
+      "train_runtime": 2828.7646,
+      "train_tokens_per_second": 132055.952
+    },
+    {
+      "epoch": 0.575,
+      "grad_norm": 0.5812146663665771,
+      "learning_rate": 0.00011697663567253592,
+      "loss": 1.3536,
+      "num_input_tokens_seen": 376832000,
+      "step": 11500,
+      "train_runtime": 2852.3871,
+      "train_tokens_per_second": 132111.102
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.6360482573509216,
+      "learning_rate": 0.00011465928917987139,
+      "loss": 1.3566,
+      "num_input_tokens_seen": 380108800,
+      "step": 11600,
+      "train_runtime": 2875.7456,
+      "train_tokens_per_second": 132177.479
+    },
+    {
+      "epoch": 0.585,
+      "grad_norm": 0.7270791530609131,
+      "learning_rate": 0.00011235083952216253,
+      "loss": 1.3634,
+      "num_input_tokens_seen": 383385600,
+      "step": 11700,
+      "train_runtime": 2899.9756,
+      "train_tokens_per_second": 132203.042
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 0.6202597618103027,
+      "learning_rate": 0.00011005186783926572,
+      "loss": 1.3424,
+      "num_input_tokens_seen": 386662400,
+      "step": 11800,
+      "train_runtime": 2923.5674,
+      "train_tokens_per_second": 132257.049
+    },
+    {
+      "epoch": 0.595,
+      "grad_norm": 0.6157307624816895,
+      "learning_rate": 0.00010776295288500768,
+      "loss": 1.3455,
+      "num_input_tokens_seen": 389939200,
+      "step": 11900,
+      "train_runtime": 2952.7095,
+      "train_tokens_per_second": 132061.485
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.5824424028396606,
+      "learning_rate": 0.00010548467088148766,
+      "loss": 1.3406,
+      "num_input_tokens_seen": 393216000,
+      "step": 12000,
+      "train_runtime": 2975.865,
+      "train_tokens_per_second": 132135.028
+    },
+    {
+      "epoch": 0.605,
+      "grad_norm": 0.5702685713768005,
+      "learning_rate": 0.00010321759537401644,
+      "loss": 1.3371,
+      "num_input_tokens_seen": 396492800,
+      "step": 12100,
+      "train_runtime": 3000.5381,
+      "train_tokens_per_second": 132140.564
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 0.5175092816352844,
+      "learning_rate": 0.0001009622970867292,
+      "loss": 1.3338,
+      "num_input_tokens_seen": 399769600,
+      "step": 12200,
+      "train_runtime": 3024.9092,
+      "train_tokens_per_second": 132159.207
+    },
+    {
+      "epoch": 0.615,
+      "grad_norm": 0.6169431805610657,
+      "learning_rate": 9.871934377890893e-05,
+      "loss": 1.3471,
+      "num_input_tokens_seen": 403046400,
+      "step": 12300,
+      "train_runtime": 3048.4879,
+      "train_tokens_per_second": 132211.909
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.512392520904541,
+      "learning_rate": 9.648930010205619e-05,
+      "loss": 1.3327,
+      "num_input_tokens_seen": 406323200,
+      "step": 12400,
+      "train_runtime": 3072.2957,
+      "train_tokens_per_second": 132253.935
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 0.7224143147468567,
+      "learning_rate": 9.4272727457741e-05,
+      "loss": 1.3438,
+      "num_input_tokens_seen": 409600000,
+      "step": 12500,
+      "train_runtime": 3096.0293,
+      "train_tokens_per_second": 132298.492
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.6021597981452942,
+      "learning_rate": 9.20701838562727e-05,
+      "loss": 1.3284,
+      "num_input_tokens_seen": 412876800,
+      "step": 12600,
+      "train_runtime": 3125.5109,
+      "train_tokens_per_second": 132098.979
+    },
+    {
+      "epoch": 0.635,
+      "grad_norm": 0.6892530918121338,
+      "learning_rate": 8.988222377622442e-05,
+      "loss": 1.3507,
+      "num_input_tokens_seen": 416153600,
+      "step": 12700,
+      "train_runtime": 3149.5923,
+      "train_tokens_per_second": 132129.355
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.1807767152786255,
+      "learning_rate": 8.770939802484568e-05,
+      "loss": 1.3229,
+      "num_input_tokens_seen": 419430400,
+      "step": 12800,
+      "train_runtime": 3174.0646,
+      "train_tokens_per_second": 132142.993
+    },
+    {
+      "epoch": 0.645,
+      "grad_norm": 1.419129490852356,
+      "learning_rate": 8.555225359939956e-05,
+      "loss": 1.3552,
+      "num_input_tokens_seen": 422707200,
+      "step": 12900,
+      "train_runtime": 3197.4391,
+      "train_tokens_per_second": 132201.8
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.8219085931777954,
+      "learning_rate": 8.341133354945939e-05,
+      "loss": 1.3221,
+      "num_input_tokens_seen": 425984000,
+      "step": 13000,
+      "train_runtime": 3220.7263,
+      "train_tokens_per_second": 132263.333
+    },
+    {
+      "epoch": 0.655,
+      "grad_norm": 0.4796520173549652,
+      "learning_rate": 8.12871768401986e-05,
+      "loss": 1.3212,
+      "num_input_tokens_seen": 429260800,
+      "step": 13100,
+      "train_runtime": 3250.0068,
+      "train_tokens_per_second": 132079.971
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.6295695900917053,
+      "learning_rate": 7.918031821670926e-05,
+      "loss": 1.3547,
+      "num_input_tokens_seen": 432537600,
+      "step": 13200,
+      "train_runtime": 3273.9064,
+      "train_tokens_per_second": 132116.666
+    },
+    {
+      "epoch": 0.665,
+      "grad_norm": 0.7862270474433899,
+      "learning_rate": 7.709128806938292e-05,
+      "loss": 1.3495,
+      "num_input_tokens_seen": 435814400,
+      "step": 13300,
+      "train_runtime": 3298.5265,
+      "train_tokens_per_second": 132123.965
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.548321008682251,
+      "learning_rate": 7.502061230038749e-05,
+      "loss": 1.3587,
+      "num_input_tokens_seen": 439091200,
+      "step": 13400,
+      "train_runtime": 3322.8737,
+      "train_tokens_per_second": 132142.007
+    },
+    {
+      "epoch": 0.675,
+      "grad_norm": 1.1466425657272339,
+      "learning_rate": 7.296881219127452e-05,
+      "loss": 1.3253,
+      "num_input_tokens_seen": 442368000,
+      "step": 13500,
+      "train_runtime": 3346.3922,
+      "train_tokens_per_second": 132192.514
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.6317051649093628,
+      "learning_rate": 7.093640427174874e-05,
+      "loss": 1.3333,
+      "num_input_tokens_seen": 445644800,
+      "step": 13600,
+      "train_runtime": 3370.0791,
+      "train_tokens_per_second": 132235.711
+    },
+    {
+      "epoch": 0.685,
+      "grad_norm": 0.7266567349433899,
+      "learning_rate": 6.892390018963525e-05,
+      "loss": 1.3336,
+      "num_input_tokens_seen": 448921600,
+      "step": 13700,
+      "train_runtime": 3394.8242,
+      "train_tokens_per_second": 132237.069
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.54411381483078,
+      "learning_rate": 6.693180658207431e-05,
+      "loss": 1.3142,
+      "num_input_tokens_seen": 452198400,
+      "step": 13800,
+      "train_runtime": 3418.9202,
+      "train_tokens_per_second": 132263.513
+    },
+    {
+      "epoch": 0.695,
+      "grad_norm": 0.6269710063934326,
+      "learning_rate": 6.496062494797838e-05,
+      "loss": 1.3443,
+      "num_input_tokens_seen": 455475200,
+      "step": 13900,
+      "train_runtime": 3442.671,
+      "train_tokens_per_second": 132302.853
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.6389264464378357,
+      "learning_rate": 6.301085152178248e-05,
+      "loss": 1.3257,
+      "num_input_tokens_seen": 458752000,
+      "step": 14000,
+      "train_runtime": 3466.1379,
+      "train_tokens_per_second": 132352.497
+    },
+    {
+      "epoch": 0.705,
+      "grad_norm": 0.5984766483306885,
+      "learning_rate": 6.108297714851969e-05,
+      "loss": 1.3206,
+      "num_input_tokens_seen": 462028800,
+      "step": 14100,
+      "train_runtime": 3496.6779,
+      "train_tokens_per_second": 132133.647
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.7461857199668884,
+      "learning_rate": 5.9177487160253855e-05,
+      "loss": 1.3363,
+      "num_input_tokens_seen": 465305600,
+      "step": 14200,
+      "train_runtime": 3521.4286,
+      "train_tokens_per_second": 132135.464
+    },
+    {
+      "epoch": 0.715,
+      "grad_norm": 0.6152383685112,
+      "learning_rate": 5.729486125389922e-05,
+      "loss": 1.3509,
+      "num_input_tokens_seen": 468582400,
+      "step": 14300,
+      "train_runtime": 3539.6668,
+      "train_tokens_per_second": 132380.37
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 1.0594799518585205,
+      "learning_rate": 5.5435573370460045e-05,
+      "loss": 1.3429,
+      "num_input_tokens_seen": 471859200,
+      "step": 14400,
+      "train_runtime": 3570.9274,
+      "train_tokens_per_second": 132139.118
+    },
+    {
+      "epoch": 0.725,
+      "grad_norm": 0.6609026193618774,
+      "learning_rate": 5.3600091575717944e-05,
+      "loss": 1.3192,
+      "num_input_tokens_seen": 475136000,
+      "step": 14500,
+      "train_runtime": 3595.0028,
+      "train_tokens_per_second": 132165.684
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 0.6246200203895569,
+      "learning_rate": 5.178887794239904e-05,
+      "loss": 1.3237,
+      "num_input_tokens_seen": 478412800,
+      "step": 14600,
+      "train_runtime": 3618.4736,
+      "train_tokens_per_second": 132213.982
+    },
+    {
+      "epoch": 0.735,
+      "grad_norm": 0.6195717453956604,
+      "learning_rate": 5.00023884338496e-05,
+      "loss": 1.3175,
+      "num_input_tokens_seen": 481689600,
+      "step": 14700,
+      "train_runtime": 3642.0847,
+      "train_tokens_per_second": 132256.56
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.7129009366035461,
+      "learning_rate": 4.82410727892497e-05,
+      "loss": 1.3324,
+      "num_input_tokens_seen": 484966400,
+      "step": 14800,
+      "train_runtime": 3665.9901,
+      "train_tokens_per_second": 132287.972
+    },
+    {
+      "epoch": 0.745,
+      "grad_norm": 1.336125373840332,
+      "learning_rate": 4.650537441039379e-05,
+      "loss": 1.3359,
+      "num_input_tokens_seen": 488243200,
+      "step": 14900,
+      "train_runtime": 3689.171,
+      "train_tokens_per_second": 132344.964
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.6141300797462463,
+      "learning_rate": 4.479573025006664e-05,
+      "loss": 1.3461,
+      "num_input_tokens_seen": 491520000,
+      "step": 15000,
+      "train_runtime": 3712.6329,
+      "train_tokens_per_second": 132391.222
+    },
+    {
+      "epoch": 0.755,
+      "grad_norm": 0.5975731015205383,
+      "learning_rate": 4.311257070204293e-05,
+      "loss": 1.3182,
+      "num_input_tokens_seen": 494796800,
+      "step": 15100,
+      "train_runtime": 3742.4523,
+      "train_tokens_per_second": 132211.918
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.5820591449737549,
+      "learning_rate": 4.145631949273772e-05,
+      "loss": 1.3308,
+      "num_input_tokens_seen": 498073600,
+      "step": 15200,
+      "train_runtime": 3767.5955,
+      "train_tokens_per_second": 132199.328
+    },
+    {
+      "epoch": 0.765,
+      "grad_norm": 0.5704116225242615,
+      "learning_rate": 3.982739357453573e-05,
+      "loss": 1.3205,
+      "num_input_tokens_seen": 501350400,
+      "step": 15300,
+      "train_runtime": 3791.6072,
+      "train_tokens_per_second": 132226.355
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 0.6396017670631409,
+      "learning_rate": 3.8226203020825794e-05,
+      "loss": 1.3236,
+      "num_input_tokens_seen": 504627200,
+      "step": 15400,
+      "train_runtime": 3815.7237,
+      "train_tokens_per_second": 132249.409
+    },
+    {
+      "epoch": 0.775,
+      "grad_norm": 0.5288236737251282,
+      "learning_rate": 3.665315092276703e-05,
+      "loss": 1.3172,
+      "num_input_tokens_seen": 507904000,
+      "step": 15500,
+      "train_runtime": 3839.9672,
+      "train_tokens_per_second": 132267.796
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.5525203943252563,
+      "learning_rate": 3.510863328781284e-05,
+      "loss": 1.3332,
+      "num_input_tokens_seen": 511180800,
+      "step": 15600,
+      "train_runtime": 3863.4067,
+      "train_tokens_per_second": 132313.486
+    },
+    {
+      "epoch": 0.785,
+      "grad_norm": 0.5880547165870667,
+      "learning_rate": 3.359303894001809e-05,
+      "loss": 1.3175,
+      "num_input_tokens_seen": 514457600,
+      "step": 15700,
+      "train_runtime": 3887.3027,
+      "train_tokens_per_second": 132343.077
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 0.6689568161964417,
+      "learning_rate": 3.210674942215488e-05,
+      "loss": 1.3132,
+      "num_input_tokens_seen": 517734400,
+      "step": 15800,
+      "train_runtime": 3911.3563,
+      "train_tokens_per_second": 132366.975
+    },
+    {
+      "epoch": 0.795,
+      "grad_norm": 0.7104946970939636,
+      "learning_rate": 3.065013889966106e-05,
+      "loss": 1.3231,
+      "num_input_tokens_seen": 521011200,
+      "step": 15900,
+      "train_runtime": 3940.5019,
+      "train_tokens_per_second": 132219.501
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.5700929760932922,
+      "learning_rate": 2.922357406644594e-05,
+      "loss": 1.3239,
+      "num_input_tokens_seen": 524288000,
+      "step": 16000,
+      "train_runtime": 3964.1157,
+      "train_tokens_per_second": 132258.5
+    },
+    {
+      "epoch": 0.805,
+      "grad_norm": 0.5516105890274048,
+      "learning_rate": 2.7827414052577485e-05,
+      "loss": 1.3176,
+      "num_input_tokens_seen": 527564800,
+      "step": 16100,
+      "train_runtime": 3988.691,
+      "train_tokens_per_second": 132265.148
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 0.66648268699646,
+      "learning_rate": 2.6462010333872835e-05,
+      "loss": 1.3176,
+      "num_input_tokens_seen": 530841600,
+      "step": 16200,
+      "train_runtime": 4013.8341,
+      "train_tokens_per_second": 132253.001
+    },
+    {
+      "epoch": 0.815,
+      "grad_norm": 0.6033647060394287,
+      "learning_rate": 2.512770664341634e-05,
+      "loss": 1.3266,
+      "num_input_tokens_seen": 534118400,
+      "step": 16300,
+      "train_runtime": 4038.866,
+      "train_tokens_per_second": 132244.645
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.6393758654594421,
+      "learning_rate": 2.3824838885026542e-05,
+      "loss": 1.3145,
+      "num_input_tokens_seen": 537395200,
+      "step": 16400,
+      "train_runtime": 4063.16,
+      "train_tokens_per_second": 132260.407
+    },
+    {
+      "epoch": 0.825,
+      "grad_norm": 0.6355799436569214,
+      "learning_rate": 2.2553735048694176e-05,
+      "loss": 1.3327,
+      "num_input_tokens_seen": 540672000,
+      "step": 16500,
+      "train_runtime": 4087.2347,
+      "train_tokens_per_second": 132283.082
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 0.5949875116348267,
+      "learning_rate": 2.13147151280126e-05,
+      "loss": 1.3195,
+      "num_input_tokens_seen": 543948800,
+      "step": 16600,
+      "train_runtime": 4110.6472,
+      "train_tokens_per_second": 132326.802
+    },
+    {
+      "epoch": 0.835,
+      "grad_norm": 0.5466151237487793,
+      "learning_rate": 2.0108091039620746e-05,
+      "loss": 1.2963,
+      "num_input_tokens_seen": 547225600,
+      "step": 16700,
+      "train_runtime": 4134.8391,
+      "train_tokens_per_second": 132345.078
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.6407220959663391,
+      "learning_rate": 1.893416654468022e-05,
+      "loss": 1.3068,
+      "num_input_tokens_seen": 550502400,
+      "step": 16800,
+      "train_runtime": 4158.5574,
+      "train_tokens_per_second": 132378.213
+    },
+    {
+      "epoch": 0.845,
+      "grad_norm": 0.8987123966217041,
+      "learning_rate": 1.7793237172404756e-05,
+      "loss": 1.3141,
+      "num_input_tokens_seen": 553779200,
+      "step": 16900,
+      "train_runtime": 4182.0603,
+      "train_tokens_per_second": 132417.795
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.5130665898323059,
+      "learning_rate": 1.6685590145662397e-05,
+      "loss": 1.3224,
+      "num_input_tokens_seen": 557056000,
+      "step": 17000,
+      "train_runtime": 4211.0014,
+      "train_tokens_per_second": 132285.874
+    },
+    {
+      "epoch": 0.855,
+      "grad_norm": 0.6242457032203674,
+      "learning_rate": 1.5611504308668742e-05,
+      "loss": 1.3044,
+      "num_input_tokens_seen": 560332800,
+      "step": 17100,
+      "train_runtime": 4235.4217,
+      "train_tokens_per_second": 132296.813
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 3.367483377456665,
+      "learning_rate": 1.4571250056789274e-05,
+      "loss": 1.3335,
+      "num_input_tokens_seen": 563609600,
+      "step": 17200,
+      "train_runtime": 4260.3571,
+      "train_tokens_per_second": 132291.634
+    },
+    {
+      "epoch": 0.865,
+      "grad_norm": 0.758712887763977,
+      "learning_rate": 1.356508926846892e-05,
+      "loss": 1.343,
+      "num_input_tokens_seen": 566886400,
+      "step": 17300,
+      "train_runtime": 4284.7372,
+      "train_tokens_per_second": 132303.658
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.700606644153595,
+      "learning_rate": 1.2593275239305378e-05,
+      "loss": 1.3026,
+      "num_input_tokens_seen": 570163200,
+      "step": 17400,
+      "train_runtime": 4309.3996,
+      "train_tokens_per_second": 132306.877
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 0.6233837008476257,
+      "learning_rate": 1.1656052618283552e-05,
+      "loss": 1.3068,
+      "num_input_tokens_seen": 573440000,
+      "step": 17500,
+      "train_runtime": 4333.6553,
+      "train_tokens_per_second": 132322.477
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.7638545632362366,
+      "learning_rate": 1.0753657346186217e-05,
+      "loss": 1.3114,
+      "num_input_tokens_seen": 576716800,
+      "step": 17600,
+      "train_runtime": 4357.7488,
+      "train_tokens_per_second": 132342.827
+    },
+    {
+      "epoch": 0.885,
+      "grad_norm": 0.6683318614959717,
+      "learning_rate": 9.886316596197341e-06,
+      "loss": 1.3141,
+      "num_input_tokens_seen": 579993600,
+      "step": 17700,
+      "train_runtime": 4381.4392,
+      "train_tokens_per_second": 132375.134
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.6613823175430298,
+      "learning_rate": 9.054248716712404e-06,
+      "loss": 1.3556,
+      "num_input_tokens_seen": 583270400,
+      "step": 17800,
+      "train_runtime": 4406.3414,
+      "train_tokens_per_second": 132370.679
+    },
+    {
+      "epoch": 0.895,
+      "grad_norm": 0.6166129112243652,
+      "learning_rate": 8.257663176370389e-06,
+      "loss": 1.2998,
+      "num_input_tokens_seen": 586547200,
+      "step": 17900,
+      "train_runtime": 4429.85,
+      "train_tokens_per_second": 132407.914
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.5475023984909058,
+      "learning_rate": 7.496760511321115e-06,
+      "loss": 1.3117,
+      "num_input_tokens_seen": 589824000,
+      "step": 18000,
+      "train_runtime": 4458.7377,
+      "train_tokens_per_second": 132284.974
+    },
+    {
+      "epoch": 0.905,
+      "grad_norm": 0.611223042011261,
+      "learning_rate": 6.7717322747414036e-06,
+      "loss": 1.3084,
+      "num_input_tokens_seen": 593100800,
+      "step": 18100,
+      "train_runtime": 4482.5986,
+      "train_tokens_per_second": 132311.824
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.8295536041259766,
+      "learning_rate": 6.0827609886125505e-06,
+      "loss": 1.3056,
+      "num_input_tokens_seen": 596377600,
+      "step": 18200,
+      "train_runtime": 4506.3151,
+      "train_tokens_per_second": 132342.632
+    },
+    {
+      "epoch": 0.915,
+      "grad_norm": 0.9818552136421204,
+      "learning_rate": 5.430020097771398e-06,
+      "loss": 1.3115,
+      "num_input_tokens_seen": 599654400,
+      "step": 18300,
+      "train_runtime": 4531.2552,
+      "train_tokens_per_second": 132337.372
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 1.4284358024597168,
+      "learning_rate": 4.8136739262464974e-06,
+      "loss": 1.3283,
+      "num_input_tokens_seen": 602931200,
+      "step": 18400,
+      "train_runtime": 4555.0502,
+      "train_tokens_per_second": 132365.435
+    },
+    {
+      "epoch": 0.925,
+      "grad_norm": 0.6662764549255371,
+      "learning_rate": 4.233877635890481e-06,
+      "loss": 1.3041,
+      "num_input_tokens_seen": 606208000,
+      "step": 18500,
+      "train_runtime": 4579.4146,
+      "train_tokens_per_second": 132376.747
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.5408564805984497,
+      "learning_rate": 3.690777187318844e-06,
+      "loss": 1.2923,
+      "num_input_tokens_seen": 609484800,
+      "step": 18600,
+      "train_runtime": 4604.2652,
+      "train_tokens_per_second": 132373.955
+    },
+    {
+      "epoch": 0.935,
+      "grad_norm": 0.7473659515380859,
+      "learning_rate": 3.1845093031651335e-06,
+      "loss": 1.2959,
+      "num_input_tokens_seen": 612761600,
+      "step": 18700,
+      "train_runtime": 4627.7441,
+      "train_tokens_per_second": 132410.433
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.6809543371200562,
+      "learning_rate": 2.715201433661801e-06,
+      "loss": 1.3088,
+      "num_input_tokens_seen": 616038400,
+      "step": 18800,
+      "train_runtime": 4652.153,
+      "train_tokens_per_second": 132420.065
+    },
+    {
+      "epoch": 0.945,
+      "grad_norm": 0.5511806011199951,
+      "learning_rate": 2.282971724555249e-06,
+      "loss": 1.2991,
+      "num_input_tokens_seen": 619315200,
+      "step": 18900,
+      "train_runtime": 4676.0174,
+      "train_tokens_per_second": 132445.017
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.5333660244941711,
+      "learning_rate": 1.8879289873632907e-06,
+      "loss": 1.3112,
+      "num_input_tokens_seen": 622592000,
+      "step": 19000,
+      "train_runtime": 4705.3038,
+      "train_tokens_per_second": 132317.068
+    },
+    {
+      "epoch": 0.955,
+      "grad_norm": 0.5114225745201111,
+      "learning_rate": 1.530172671982427e-06,
+      "loss": 1.304,
+      "num_input_tokens_seen": 625868800,
+      "step": 19100,
+      "train_runtime": 4728.0888,
+      "train_tokens_per_second": 132372.471
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.7311033010482788,
+      "learning_rate": 1.2097928416518577e-06,
+      "loss": 1.3016,
+      "num_input_tokens_seen": 629145600,
+      "step": 19200,
+      "train_runtime": 4751.8878,
+      "train_tokens_per_second": 132399.086
+    },
+    {
+      "epoch": 0.965,
+      "grad_norm": 0.5196536183357239,
+      "learning_rate": 9.268701502805509e-07,
+      "loss": 1.3029,
+      "num_input_tokens_seen": 632422400,
+      "step": 19300,
+      "train_runtime": 4776.4621,
+      "train_tokens_per_second": 132403.941
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 0.6195040345191956,
+      "learning_rate": 6.814758221430683e-07,
+      "loss": 1.3103,
+      "num_input_tokens_seen": 635699200,
+      "step": 19400,
+      "train_runtime": 4800.6355,
+      "train_tokens_per_second": 132419.8
+    },
+    {
+      "epoch": 0.975,
+      "grad_norm": 0.6139786243438721,
+      "learning_rate": 4.7367163394922416e-07,
+      "loss": 1.2979,
+      "num_input_tokens_seen": 638976000,
+      "step": 19500,
+      "train_runtime": 4825.134,
+      "train_tokens_per_second": 132426.581
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.49304258823394775,
+      "learning_rate": 3.035098992920926e-07,
+      "loss": 1.3025,
+      "num_input_tokens_seen": 642252800,
+      "step": 19600,
+      "train_runtime": 4849.4238,
+      "train_tokens_per_second": 132438.992
+    },
+    {
+      "epoch": 0.985,
+      "grad_norm": 0.5793161988258362,
+      "learning_rate": 1.710334554783921e-07,
+      "loss": 1.3106,
+      "num_input_tokens_seen": 645529600,
+      "step": 19700,
+      "train_runtime": 4879.0091,
+      "train_tokens_per_second": 132307.52
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 0.6929790377616882,
+      "learning_rate": 7.627565274441194e-08,
+      "loss": 1.3241,
+      "num_input_tokens_seen": 648806400,
+      "step": 19800,
+      "train_runtime": 4902.4093,
+      "train_tokens_per_second": 132344.397
+    },
+    {
+      "epoch": 0.995,
+      "grad_norm": 0.6057826280593872,
+      "learning_rate": 1.9260345860244718e-08,
+      "loss": 1.3207,
+      "num_input_tokens_seen": 652083200,
+      "step": 19900,
+      "train_runtime": 4926.578,
+      "train_tokens_per_second": 132360.27
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.596373438835144,
+      "learning_rate": 1.8881245067436934e-12,
+      "loss": 1.3027,
+      "num_input_tokens_seen": 655360000,
+      "step": 20000,
+      "train_runtime": 4950.9316,
+      "train_tokens_per_second": 132371.046
+    },
+    {
+      "epoch": 1.0,
+      "num_input_tokens_seen": 655360000,
+      "step": 20000,
+      "total_flos": 1.23866185728e+16,
+      "train_loss": 1.4302955017089845,
+      "train_runtime": 4951.1263,
+      "train_samples_per_second": 517.054,
+      "train_steps_per_second": 4.039
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 20000,
+  "num_input_tokens_seen": 655360000,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.23866185728e+16,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5641ba09bd0c0ab3f39b0e2bc742cb033a6a6494c1a96dd5f0c0ab5c802ebc52
+size 5841