Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: microsoft/Phi-3-mini-4k-instruct
+base_model: meta-llama/Llama-3.2-3B-Instruct
 library_name: transformers
 model_name: model_tuned
 tags:
@@ -11,7 +11,7 @@ licence: license
 
 # Model Card for model_tuned
 
-This model is a fine-tuned version of [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct).
+This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/vedaantj/ft-llm/runs/083810)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/vedaantj/ft-llm/runs/098452)
 
 This model was trained with SFT.
 
@@ -35,7 +35,7 @@ This model was trained with SFT.
 
 - TRL: 0.12.2
 - Transformers: 4.46.3
-- Pytorch: 2.1.2+cu118
+- Pytorch: 2.4.1
 - Datasets: 3.1.0
 - Tokenizers: 0.20.3
 

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 2.985645933014354,
-    "total_flos": 9149727375360.0,
-    "train_loss": 0.5307981822735224,
-    "train_runtime": 3368.0081,
-    "train_samples": 5060,
-    "train_samples_per_second": 1.115,
-    "train_steps_per_second": 0.093
+    "epoch": 14.917333333333334,
+    "total_flos": 6.545011747156132e+17,
+    "train_loss": 4.23463837784251,
+    "train_runtime": 3346.972,
+    "train_samples": 4498,
+    "train_samples_per_second": 20.159,
+    "train_steps_per_second": 0.417
 }

config.json

@@ -1,36 +1,40 @@
 {
-  "_name_or_path": "microsoft/Phi-3-mini-4k-instruct",
+  "_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
   "architectures": [
-    "Phi3ForCausalLM"
+    "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "microsoft/Phi-3-mini-4k-instruct--configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "microsoft/Phi-3-mini-4k-instruct--modeling_phi3.Phi3ForCausalLM"
-  },
-  "bos_token_id": 1,
-  "embd_pdrop": 0.0,
-  "eos_token_id": 32000,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 3072,
   "initializer_range": 0.02,
   "intermediate_size": 8192,
-  "max_position_embeddings": 4096,
-  "model_type": "phi3",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "resid_pdrop": 0.0,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "sliding_window": 2047,
-  "tie_word_embeddings": false,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.46.3",
   "use_cache": false,
-  "vocab_size": 32064
+  "vocab_size": 128256
 }

generation_config.json

@@ -1,11 +1,12 @@
 {
-  "_from_model_config": true,
-  "bos_token_id": 1,
+  "bos_token_id": 128000,
+  "do_sample": true,
   "eos_token_id": [
-    32000,
-    32001,
-    32007
+    128001,
+    128008,
+    128009
   ],
-  "pad_token_id": 32000,
+  "temperature": 0.6,
+  "top_p": 0.9,
   "transformers_version": "4.46.3"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b9c0ec47315587eb34dba5cd70d3b39b30b09d78dfaa552bc1f5bb2c0e3107d
-size 4972489328
+oid sha256:3bab372755153620bdc2c4abe368c749f1781f0625d652b5f9736343debd0b24
+size 4965799096

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f479150a15a9c19f7567422a44c4e4f3d6ff3167f5199c0d2f6adb837c6b2e68
-size 2669692552
+oid sha256:2cb8070968d9750f6bc09aa79466db170fbeb51afede8cab1aab61eff48fc7b4
+size 2247734992

model.safetensors.index.json

@@ -1,202 +1,262 @@
 {
   "metadata": {
-    "total_size": 7642159104
+    "total_size": 7213504512
   },
   "weight_map": {
     "lm_head.weight": "model-00002-of-00002.safetensors",
     "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.norm.weight": "model-00002-of-00002.safetensors"
   }
 }

special_tokens_map.json

@@ -1,30 +1,17 @@
 {
   "bos_token": {
-    "content": "<s>",
+    "content": "<|begin_of_text|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|eot_id|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<|eot_id|>"
 }

tokenizer_config.json

@@ -1,131 +1,2063 @@
 {
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "add_prefix_space": null,
   "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
+    "128000": {
+      "content": "<|begin_of_text|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "1": {
-      "content": "<s>",
+    "128001": {
+      "content": "<|end_of_text|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "2": {
-      "content": "</s>",
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "32000": {
-      "content": "<|endoftext|>",
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32001": {
-      "content": "<|assistant|>",
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32002": {
-      "content": "<|placeholder1|>",
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32003": {
-      "content": "<|placeholder2|>",
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32004": {
-      "content": "<|placeholder3|>",
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32005": {
-      "content": "<|placeholder4|>",
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32006": {
-      "content": "<|system|>",
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32007": {
-      "content": "<|end|>",
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32008": {
-      "content": "<|placeholder5|>",
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32009": {
-      "content": "<|placeholder6|>",
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32010": {
-      "content": "<|user|>",
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "legacy": false,
-  "model_max_length": 4096,
-  "pad_token": "<|endoftext|>",
-  "padding_side": "left",
-  "sp_model_kwargs": {},
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 2048,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 2.985645933014354,
-    "total_flos": 9149727375360.0,
-    "train_loss": 0.5307981822735224,
-    "train_runtime": 3368.0081,
-    "train_samples": 5060,
-    "train_samples_per_second": 1.115,
-    "train_steps_per_second": 0.093
+    "epoch": 14.917333333333334,
+    "total_flos": 6.545011747156132e+17,
+    "train_loss": 4.23463837784251,
+    "train_runtime": 3346.972,
+    "train_samples": 4498,
+    "train_samples_per_second": 20.159,
+    "train_steps_per_second": 0.417
 }

trainer_state.json

@@ -1,492 +1,2107 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.985645933014354,
+  "epoch": 14.917333333333334,
   "eval_steps": 500,
-  "global_step": 312,
+  "global_step": 1395,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.009569377990430622,
-      "grad_norm": 2.0939094431170635,
-      "learning_rate": 6.25e-07,
-      "loss": 1.0265,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 2.017279624938965,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 1.1741,
       "step": 1
     },
     {
-      "epoch": 0.04784688995215311,
-      "grad_norm": 1.2308584432431018,
-      "learning_rate": 3.125e-06,
-      "loss": 1.0462,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 1.1414889097213745,
+      "learning_rate": 7.142857142857142e-05,
+      "loss": 1.0112,
       "step": 5
     },
     {
-      "epoch": 0.09569377990430622,
-      "grad_norm": 0.8192528176324341,
-      "learning_rate": 6.25e-06,
-      "loss": 0.9416,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 0.5752452611923218,
+      "learning_rate": 0.00014285714285714284,
+      "loss": 0.9233,
       "step": 10
     },
     {
-      "epoch": 0.14354066985645933,
-      "grad_norm": 0.5587998098246274,
-      "learning_rate": 9.375000000000001e-06,
-      "loss": 0.8065,
+      "epoch": 0.16,
+      "grad_norm": 0.4000088572502136,
+      "learning_rate": 0.00021428571428571427,
+      "loss": 0.8458,
       "step": 15
     },
     {
-      "epoch": 0.19138755980861244,
-      "grad_norm": 0.3008165110686449,
-      "learning_rate": 1.25e-05,
-      "loss": 0.6851,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 0.6516547799110413,
+      "learning_rate": 0.0002857142857142857,
+      "loss": 0.9087,
       "step": 20
     },
     {
-      "epoch": 0.23923444976076555,
-      "grad_norm": 0.20347128493031447,
-      "learning_rate": 1.5625e-05,
-      "loss": 0.6537,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 0.8725354075431824,
+      "learning_rate": 0.00035714285714285714,
+      "loss": 0.9859,
       "step": 25
     },
     {
-      "epoch": 0.28708133971291866,
-      "grad_norm": 0.17128021850811456,
-      "learning_rate": 1.8750000000000002e-05,
-      "loss": 0.6333,
+      "epoch": 0.32,
+      "grad_norm": 0.7446133494377136,
+      "learning_rate": 0.00042857142857142855,
+      "loss": 0.9609,
       "step": 30
     },
     {
-      "epoch": 0.3349282296650718,
-      "grad_norm": 0.16652191969621177,
-      "learning_rate": 1.9994335583335336e-05,
-      "loss": 0.6149,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 0.4507102370262146,
+      "learning_rate": 0.0005,
+      "loss": 1.2413,
       "step": 35
     },
     {
-      "epoch": 0.3827751196172249,
-      "grad_norm": 0.14198363233390676,
-      "learning_rate": 1.9959742939952393e-05,
-      "loss": 0.6082,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.1437228918075562,
+      "learning_rate": 0.0005714285714285714,
+      "loss": 1.0784,
       "step": 40
     },
     {
-      "epoch": 0.430622009569378,
-      "grad_norm": 0.21713283105061185,
-      "learning_rate": 1.9893813260530368e-05,
-      "loss": 0.5895,
+      "epoch": 0.48,
+      "grad_norm": 1.6952320337295532,
+      "learning_rate": 0.0006428571428571429,
+      "loss": 1.2622,
       "step": 45
     },
     {
-      "epoch": 0.4784688995215311,
-      "grad_norm": 0.13924953869885492,
-      "learning_rate": 1.9796753984232357e-05,
-      "loss": 0.5561,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 12.92784309387207,
+      "learning_rate": 0.0007142857142857143,
+      "loss": 2.0382,
       "step": 50
     },
     {
-      "epoch": 0.5263157894736842,
-      "grad_norm": 0.13684768549424628,
-      "learning_rate": 1.9668870495450064e-05,
-      "loss": 0.569,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 4.855193138122559,
+      "learning_rate": 0.0007857142857142857,
+      "loss": 5.5645,
       "step": 55
     },
     {
-      "epoch": 0.5741626794258373,
-      "grad_norm": 0.13894406081826732,
-      "learning_rate": 1.9510565162951538e-05,
-      "loss": 0.5712,
+      "epoch": 0.64,
+      "grad_norm": 7.7960357666015625,
+      "learning_rate": 0.0008571428571428571,
+      "loss": 12.1792,
       "step": 60
     },
     {
-      "epoch": 0.6220095693779905,
-      "grad_norm": 0.14374947839898614,
-      "learning_rate": 1.9322336073880143e-05,
-      "loss": 0.5702,
+      "epoch": 0.6933333333333334,
+      "grad_norm": 9.617668151855469,
+      "learning_rate": 0.0009285714285714287,
+      "loss": 12.4584,
       "step": 65
     },
     {
-      "epoch": 0.6698564593301436,
-      "grad_norm": 0.14479528353076987,
-      "learning_rate": 1.9104775466588162e-05,
-      "loss": 0.5664,
+      "epoch": 0.7466666666666667,
+      "grad_norm": 1.5412602424621582,
+      "learning_rate": 0.001,
+      "loss": 7.3645,
       "step": 70
     },
     {
-      "epoch": 0.7177033492822966,
-      "grad_norm": 0.14766483692101193,
-      "learning_rate": 1.88585678672358e-05,
-      "loss": 0.5477,
+      "epoch": 0.8,
+      "grad_norm": 1.459766149520874,
+      "learning_rate": 0.0010714285714285715,
+      "loss": 6.9284,
       "step": 75
     },
     {
-      "epoch": 0.7655502392344498,
-      "grad_norm": 0.15525222782036766,
-      "learning_rate": 1.8584487936018663e-05,
-      "loss": 0.5383,
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.6621876955032349,
+      "learning_rate": 0.0011428571428571427,
+      "loss": 6.7362,
       "step": 80
     },
     {
-      "epoch": 0.8133971291866029,
-      "grad_norm": 0.14420000030755972,
-      "learning_rate": 1.8283398029800167e-05,
-      "loss": 0.5509,
+      "epoch": 0.9066666666666666,
+      "grad_norm": 1.8117409944534302,
+      "learning_rate": 0.0012142857142857142,
+      "loss": 6.7313,
       "step": 85
     },
     {
-      "epoch": 0.861244019138756,
-      "grad_norm": 0.14021638409143689,
-      "learning_rate": 1.795624548881781e-05,
-      "loss": 0.5492,
+      "epoch": 0.96,
+      "grad_norm": 1.2151100635528564,
+      "learning_rate": 0.0012857142857142859,
+      "loss": 6.6705,
       "step": 90
     },
     {
-      "epoch": 0.9090909090909091,
-      "grad_norm": 0.13897720896535773,
-      "learning_rate": 1.7604059656000313e-05,
-      "loss": 0.535,
-      "step": 95
+      "epoch": 0.992,
+      "eval_loss": 6.7252326011657715,
+      "eval_runtime": 12.4963,
+      "eval_samples_per_second": 90.027,
+      "eval_steps_per_second": 15.045,
+      "step": 93
     },
     {
-      "epoch": 0.9569377990430622,
-      "grad_norm": 0.144999235947906,
-      "learning_rate": 1.7227948638273918e-05,
-      "loss": 0.558,
-      "step": 100
+      "epoch": 1.016,
+      "grad_norm": 0.7800785303115845,
+      "learning_rate": 0.0013571428571428573,
+      "loss": 7.3155,
+      "step": 95
     },
     {
-      "epoch": 0.9952153110047847,
-      "eval_loss": 0.5750399827957153,
-      "eval_runtime": 7.9492,
-      "eval_samples_per_second": 17.109,
-      "eval_steps_per_second": 5.787,
-      "step": 104
+      "epoch": 1.0693333333333332,
+      "grad_norm": 0.6527447700500488,
+      "learning_rate": 0.0014285714285714286,
+      "loss": 6.5969,
+      "step": 100
     },
     {
-      "epoch": 1.0047846889952152,
-      "grad_norm": 0.2609579886738764,
-      "learning_rate": 1.682909582004807e-05,
-      "loss": 0.5801,
+      "epoch": 1.1226666666666667,
+      "grad_norm": 0.8018523454666138,
+      "learning_rate": 0.0015,
+      "loss": 6.4691,
       "step": 105
     },
     {
-      "epoch": 1.0526315789473684,
-      "grad_norm": 0.14024934432114472,
-      "learning_rate": 1.6408756139850243e-05,
-      "loss": 0.53,
+      "epoch": 1.176,
+      "grad_norm": 0.5544389486312866,
+      "learning_rate": 0.0015714285714285715,
+      "loss": 6.6514,
       "step": 110
     },
     {
-      "epoch": 1.1004784688995215,
-      "grad_norm": 0.14458482486273788,
-      "learning_rate": 1.5968252141825038e-05,
-      "loss": 0.5322,
+      "epoch": 1.2293333333333334,
+      "grad_norm": 0.6455843448638916,
+      "learning_rate": 0.0016428571428571427,
+      "loss": 6.4806,
       "step": 115
     },
     {
-      "epoch": 1.1483253588516746,
-      "grad_norm": 0.14629086236317015,
-      "learning_rate": 1.5508969814521026e-05,
-      "loss": 0.5226,
+      "epoch": 1.2826666666666666,
+      "grad_norm": 0.6757238507270813,
+      "learning_rate": 0.0017142857142857142,
+      "loss": 6.6304,
       "step": 120
     },
     {
-      "epoch": 1.1961722488038278,
-      "grad_norm": 0.15275249319420742,
-      "learning_rate": 1.5032354230058004e-05,
-      "loss": 0.5237,
+      "epoch": 1.336,
+      "grad_norm": 0.5105107426643372,
+      "learning_rate": 0.0017857142857142859,
+      "loss": 6.4649,
       "step": 125
     },
     {
-      "epoch": 1.244019138755981,
-      "grad_norm": 0.15270769602744638,
-      "learning_rate": 1.4539904997395468e-05,
-      "loss": 0.5101,
+      "epoch": 1.3893333333333333,
+      "grad_norm": 0.7170645594596863,
+      "learning_rate": 0.0018571428571428573,
+      "loss": 6.6033,
       "step": 130
     },
     {
-      "epoch": 1.291866028708134,
-      "grad_norm": 0.14854577877982414,
-      "learning_rate": 1.4033171544008053e-05,
-      "loss": 0.5084,
+      "epoch": 1.4426666666666668,
+      "grad_norm": 0.7564003467559814,
+      "learning_rate": 0.0019285714285714286,
+      "loss": 6.6749,
       "step": 135
     },
     {
-      "epoch": 1.339712918660287,
-      "grad_norm": 0.14092662389201455,
-      "learning_rate": 1.3513748240813429e-05,
-      "loss": 0.518,
+      "epoch": 1.496,
+      "grad_norm": 0.4599800407886505,
+      "learning_rate": 0.002,
+      "loss": 6.3382,
       "step": 140
     },
     {
-      "epoch": 1.38755980861244,
-      "grad_norm": 0.13607051872752182,
-      "learning_rate": 1.2983269385691562e-05,
-      "loss": 0.5184,
+      "epoch": 1.5493333333333332,
+      "grad_norm": 0.837786853313446,
+      "learning_rate": 0.0019999216720722225,
+      "loss": 6.8014,
       "step": 145
     },
     {
-      "epoch": 1.4354066985645932,
-      "grad_norm": 0.14966143078522062,
-      "learning_rate": 1.2443404061378941e-05,
-      "loss": 0.5062,
+      "epoch": 1.6026666666666667,
+      "grad_norm": 1.4660875797271729,
+      "learning_rate": 0.001999686700559419,
+      "loss": 6.5512,
       "step": 150
     },
     {
-      "epoch": 1.4832535885167464,
-      "grad_norm": 0.15381766051199638,
-      "learning_rate": 1.1895850883916786e-05,
-      "loss": 0.5049,
+      "epoch": 1.6560000000000001,
+      "grad_norm": 0.9593129754066467,
+      "learning_rate": 0.001999295122271253,
+      "loss": 6.5249,
       "step": 155
     },
     {
-      "epoch": 1.5311004784688995,
-      "grad_norm": 0.1654965470842917,
-      "learning_rate": 1.1342332658176556e-05,
-      "loss": 0.5027,
+      "epoch": 1.7093333333333334,
+      "grad_norm": 0.712748110294342,
+      "learning_rate": 0.0019987469985507556,
+      "loss": 6.5814,
       "step": 160
     },
     {
-      "epoch": 1.5789473684210527,
-      "grad_norm": 0.15124728351536673,
-      "learning_rate": 1.0784590957278452e-05,
-      "loss": 0.4928,
+      "epoch": 1.7626666666666666,
+      "grad_norm": 0.5598364472389221,
+      "learning_rate": 0.0019980424152647174,
+      "loss": 6.5057,
       "step": 165
     },
     {
-      "epoch": 1.6267942583732058,
-      "grad_norm": 0.1445164484047255,
-      "learning_rate": 1.0224380642958052e-05,
-      "loss": 0.4926,
+      "epoch": 1.8159999999999998,
+      "grad_norm": 1.0086586475372314,
+      "learning_rate": 0.001997181482790236,
+      "loss": 6.3741,
       "step": 170
     },
     {
-      "epoch": 1.674641148325359,
-      "grad_norm": 0.14421144356382473,
-      "learning_rate": 9.663464344122064e-06,
-      "loss": 0.4953,
+      "epoch": 1.8693333333333333,
+      "grad_norm": 1.0380948781967163,
+      "learning_rate": 0.0019961643359974247,
+      "loss": 6.5918,
       "step": 175
     },
     {
-      "epoch": 1.722488038277512,
-      "grad_norm": 0.14068706673794015,
-      "learning_rate": 9.103606910965666e-06,
-      "loss": 0.5027,
+      "epoch": 1.9226666666666667,
+      "grad_norm": 0.7874330282211304,
+      "learning_rate": 0.0019949911342282845,
+      "loss": 6.3813,
       "step": 180
     },
     {
-      "epoch": 1.7703349282296652,
-      "grad_norm": 0.14082548483803073,
-      "learning_rate": 8.546569862100876e-06,
-      "loss": 0.4853,
+      "epoch": 1.976,
+      "grad_norm": 0.5932402610778809,
+      "learning_rate": 0.0019936620612717427,
+      "loss": 6.35,
       "step": 185
     },
     {
-      "epoch": 1.8181818181818183,
-      "grad_norm": 0.14468108210583894,
-      "learning_rate": 7.994105842167274e-06,
-      "loss": 0.498,
+      "epoch": 1.9973333333333332,
+      "eval_loss": 6.313483238220215,
+      "eval_runtime": 12.8744,
+      "eval_samples_per_second": 87.383,
+      "eval_steps_per_second": 14.603,
+      "step": 187
+    },
+    {
+      "epoch": 2.032,
+      "grad_norm": 0.777173638343811,
+      "learning_rate": 0.00199217732533486,
+      "loss": 6.6836,
       "step": 190
     },
     {
-      "epoch": 1.8660287081339713,
-      "grad_norm": 0.14087947795424693,
-      "learning_rate": 7.447953107363574e-06,
-      "loss": 0.4763,
+      "epoch": 2.0853333333333333,
+      "grad_norm": 0.9935985207557678,
+      "learning_rate": 0.0019905371590102153,
+      "loss": 6.1532,
       "step": 195
     },
     {
-      "epoch": 1.9138755980861244,
-      "grad_norm": 0.13646275065164523,
-      "learning_rate": 6.909830056250527e-06,
-      "loss": 0.4549,
+      "epoch": 2.1386666666666665,
+      "grad_norm": 0.5834164023399353,
+      "learning_rate": 0.0019887418192394667,
+      "loss": 6.0439,
       "step": 200
     },
     {
-      "epoch": 1.9617224880382775,
-      "grad_norm": 0.13978124622450475,
-      "learning_rate": 6.381429823033281e-06,
-      "loss": 0.4876,
+      "epoch": 2.192,
+      "grad_norm": 0.7577049136161804,
+      "learning_rate": 0.001986791587273103,
+      "loss": 6.0436,
       "step": 205
     },
     {
-      "epoch": 2.0,
-      "eval_loss": 0.5740731954574585,
-      "eval_runtime": 11.5294,
-      "eval_samples_per_second": 11.796,
-      "eval_steps_per_second": 3.99,
-      "step": 209
-    },
-    {
-      "epoch": 2.0095693779904304,
-      "grad_norm": 0.1395628254838664,
-      "learning_rate": 5.864414950334796e-06,
-      "loss": 0.5258,
+      "epoch": 2.2453333333333334,
+      "grad_norm": 0.9114102125167847,
+      "learning_rate": 0.0019846867686263803,
+      "loss": 6.0193,
       "step": 210
     },
     {
-      "epoch": 2.0574162679425836,
-      "grad_norm": 0.1646035873905585,
-      "learning_rate": 5.360412158221661e-06,
-      "loss": 0.4763,
+      "epoch": 2.2986666666666666,
+      "grad_norm": 0.9455011487007141,
+      "learning_rate": 0.001982427693031465,
+      "loss": 5.9565,
       "step": 215
     },
     {
-      "epoch": 2.1052631578947367,
-      "grad_norm": 0.1430640678690393,
-      "learning_rate": 4.871007225940939e-06,
-      "loss": 0.4613,
+      "epoch": 2.352,
+      "grad_norm": 1.504512071609497,
+      "learning_rate": 0.001980014714385777,
+      "loss": 6.2788,
       "step": 220
     },
     {
-      "epoch": 2.15311004784689,
-      "grad_norm": 0.1461980161567961,
-      "learning_rate": 4.397740002471973e-06,
-      "loss": 0.4446,
+      "epoch": 2.405333333333333,
+      "grad_norm": 1.2779139280319214,
+      "learning_rate": 0.001977448210696551,
+      "loss": 6.0278,
       "step": 225
     },
     {
-      "epoch": 2.200956937799043,
-      "grad_norm": 0.14316992446211674,
-      "learning_rate": 3.942099561591802e-06,
-      "loss": 0.4654,
+      "epoch": 2.458666666666667,
+      "grad_norm": 0.6228100061416626,
+      "learning_rate": 0.001974728584021618,
+      "loss": 5.846,
       "step": 230
     },
     {
-      "epoch": 2.248803827751196,
-      "grad_norm": 0.16075469053450084,
-      "learning_rate": 3.505519516698165e-06,
-      "loss": 0.4513,
+      "epoch": 2.512,
+      "grad_norm": 0.5647552609443665,
+      "learning_rate": 0.0019718562604064214,
+      "loss": 5.7982,
       "step": 235
     },
     {
-      "epoch": 2.2966507177033493,
-      "grad_norm": 0.1555203656237632,
-      "learning_rate": 3.089373510131354e-06,
-      "loss": 0.4645,
+      "epoch": 2.5653333333333332,
+      "grad_norm": 0.6691617965698242,
+      "learning_rate": 0.0019688316898172744,
+      "loss": 5.6841,
       "step": 240
     },
     {
-      "epoch": 2.3444976076555024,
-      "grad_norm": 0.14495139725052797,
-      "learning_rate": 2.694970891187225e-06,
-      "loss": 0.4397,
+      "epoch": 2.618666666666667,
+      "grad_norm": 0.8721797466278076,
+      "learning_rate": 0.0019656553460708705,
+      "loss": 5.6921,
       "step": 245
     },
     {
-      "epoch": 2.3923444976076556,
-      "grad_norm": 0.14820564306015635,
-      "learning_rate": 2.323552596419889e-06,
-      "loss": 0.4573,
+      "epoch": 2.672,
+      "grad_norm": 0.49442487955093384,
+      "learning_rate": 0.0019623277267600573,
+      "loss": 5.6087,
       "step": 250
     },
     {
-      "epoch": 2.4401913875598087,
-      "grad_norm": 0.14625430854110832,
-      "learning_rate": 1.9762872451962214e-06,
-      "loss": 0.4453,
+      "epoch": 2.7253333333333334,
+      "grad_norm": 0.5820254683494568,
+      "learning_rate": 0.001958849353175884,
+      "loss": 5.4145,
       "step": 255
     },
     {
-      "epoch": 2.488038277511962,
-      "grad_norm": 0.1417828182794398,
-      "learning_rate": 1.6542674627869738e-06,
-      "loss": 0.454,
+      "epoch": 2.7786666666666666,
+      "grad_norm": 0.9541387557983398,
+      "learning_rate": 0.001955220770225941,
+      "loss": 5.4061,
       "step": 260
     },
     {
-      "epoch": 2.535885167464115,
-      "grad_norm": 0.2347199249792442,
-      "learning_rate": 1.3585064425634542e-06,
-      "loss": 0.4469,
+      "epoch": 2.832,
+      "grad_norm": 0.8580039739608765,
+      "learning_rate": 0.0019514425463489948,
+      "loss": 5.3056,
       "step": 265
     },
     {
-      "epoch": 2.583732057416268,
-      "grad_norm": 0.14511063554793874,
-      "learning_rate": 1.0899347581163222e-06,
-      "loss": 0.4483,
+      "epoch": 2.8853333333333335,
+      "grad_norm": 0.9031959772109985,
+      "learning_rate": 0.001947515273425939,
+      "loss": 5.3682,
       "step": 270
     },
     {
-      "epoch": 2.6315789473684212,
-      "grad_norm": 0.14598006035506486,
-      "learning_rate": 8.493974353268019e-07,
-      "loss": 0.459,
+      "epoch": 2.9386666666666668,
+      "grad_norm": 0.739450216293335,
+      "learning_rate": 0.0019434395666870735,
+      "loss": 5.2505,
       "step": 275
     },
     {
-      "epoch": 2.679425837320574,
-      "grad_norm": 0.15533898050231443,
-      "learning_rate": 6.37651293602628e-07,
-      "loss": 0.4493,
+      "epoch": 2.992,
+      "grad_norm": 0.8577784299850464,
+      "learning_rate": 0.0019392160646157241,
+      "loss": 5.0786,
       "step": 280
     },
     {
-      "epoch": 2.7272727272727275,
-      "grad_norm": 0.14147230186128865,
-      "learning_rate": 4.553625646441928e-07,
-      "loss": 0.4344,
+      "epoch": 2.992,
+      "eval_loss": 5.260681629180908,
+      "eval_runtime": 12.495,
+      "eval_samples_per_second": 90.036,
+      "eval_steps_per_second": 15.046,
+      "step": 280
+    },
+    {
+      "epoch": 3.048,
+      "grad_norm": 0.7697703242301941,
+      "learning_rate": 0.001934845428848222,
+      "loss": 5.5708,
       "step": 285
     },
     {
-      "epoch": 2.77511961722488,
-      "grad_norm": 0.1738953363945195,
-      "learning_rate": 3.0310479623313125e-07,
-      "loss": 0.4513,
+      "epoch": 3.1013333333333333,
+      "grad_norm": 0.8419788479804993,
+      "learning_rate": 0.0019303283440702521,
+      "loss": 4.9402,
       "step": 290
     },
     {
-      "epoch": 2.8229665071770333,
-      "grad_norm": 0.15001004355811276,
-      "learning_rate": 1.81357047638816e-07,
-      "loss": 0.4474,
+      "epoch": 3.1546666666666665,
+      "grad_norm": 1.6958568096160889,
+      "learning_rate": 0.0019256655179095952,
+      "loss": 4.9352,
       "step": 295
     },
     {
-      "epoch": 2.8708133971291865,
-      "grad_norm": 0.15435665748487531,
-      "learning_rate": 9.0502382320653e-08,
-      "loss": 0.4543,
+      "epoch": 3.208,
+      "grad_norm": 0.706997275352478,
+      "learning_rate": 0.0019208576808252725,
+      "loss": 4.9083,
       "step": 300
     },
     {
-      "epoch": 2.9186602870813396,
-      "grad_norm": 0.14366671363851222,
-      "learning_rate": 3.082666266872036e-08,
-      "loss": 0.4558,
+      "epoch": 3.2613333333333334,
+      "grad_norm": 0.7924236059188843,
+      "learning_rate": 0.0019159055859931163,
+      "loss": 4.7812,
       "step": 305
     },
     {
-      "epoch": 2.9665071770334928,
-      "grad_norm": 0.1455740191931236,
-      "learning_rate": 2.5176505749346937e-09,
-      "loss": 0.4556,
+      "epoch": 3.3146666666666667,
+      "grad_norm": 0.6681360006332397,
+      "learning_rate": 0.0019108100091877788,
+      "loss": 4.7953,
       "step": 310
     },
     {
-      "epoch": 2.985645933014354,
-      "eval_loss": 0.5831425786018372,
-      "eval_runtime": 13.8937,
-      "eval_samples_per_second": 9.789,
-      "eval_steps_per_second": 3.311,
-      "step": 312
+      "epoch": 3.368,
+      "grad_norm": 0.9580642580986023,
+      "learning_rate": 0.0019055717486612038,
+      "loss": 4.8082,
+      "step": 315
+    },
+    {
+      "epoch": 3.421333333333333,
+      "grad_norm": 1.4874638319015503,
+      "learning_rate": 0.0019001916250175764,
+      "loss": 4.7807,
+      "step": 320
+    },
+    {
+      "epoch": 3.474666666666667,
+      "grad_norm": 2.9284310340881348,
+      "learning_rate": 0.0018946704810847688,
+      "loss": 4.7192,
+      "step": 325
+    },
+    {
+      "epoch": 3.528,
+      "grad_norm": 0.8562856912612915,
+      "learning_rate": 0.0018890091817823072,
+      "loss": 4.7759,
+      "step": 330
+    },
+    {
+      "epoch": 3.5813333333333333,
+      "grad_norm": 0.8265694975852966,
+      "learning_rate": 0.0018832086139858775,
+      "loss": 4.7143,
+      "step": 335
+    },
+    {
+      "epoch": 3.634666666666667,
+      "grad_norm": 1.2272521257400513,
+      "learning_rate": 0.0018772696863883906,
+      "loss": 4.775,
+      "step": 340
+    },
+    {
+      "epoch": 3.6879999999999997,
+      "grad_norm": 1.2595627307891846,
+      "learning_rate": 0.0018711933293576303,
+      "loss": 4.5926,
+      "step": 345
+    },
+    {
+      "epoch": 3.7413333333333334,
+      "grad_norm": 1.0950653553009033,
+      "learning_rate": 0.0018649804947905057,
+      "loss": 4.813,
+      "step": 350
+    },
+    {
+      "epoch": 3.7946666666666666,
+      "grad_norm": 1.4552189111709595,
+      "learning_rate": 0.0018586321559639317,
+      "loss": 4.717,
+      "step": 355
+    },
+    {
+      "epoch": 3.848,
+      "grad_norm": 1.0459569692611694,
+      "learning_rate": 0.001852149307382358,
+      "loss": 4.6884,
+      "step": 360
+    },
+    {
+      "epoch": 3.9013333333333335,
+      "grad_norm": 1.0256690979003906,
+      "learning_rate": 0.0018455329646219765,
+      "loss": 4.5672,
+      "step": 365
+    },
+    {
+      "epoch": 3.9546666666666668,
+      "grad_norm": 1.2847387790679932,
+      "learning_rate": 0.0018387841641716223,
+      "loss": 4.4933,
+      "step": 370
+    },
+    {
+      "epoch": 3.997333333333333,
+      "eval_loss": 4.830221176147461,
+      "eval_runtime": 12.4939,
+      "eval_samples_per_second": 90.044,
+      "eval_steps_per_second": 15.047,
+      "step": 374
+    },
+    {
+      "epoch": 4.010666666666666,
+      "grad_norm": 1.6286957263946533,
+      "learning_rate": 0.001831903963270404,
+      "loss": 5.0358,
+      "step": 375
+    },
+    {
+      "epoch": 4.064,
+      "grad_norm": 1.2841631174087524,
+      "learning_rate": 0.00182489343974208,
+      "loss": 4.6172,
+      "step": 380
+    },
+    {
+      "epoch": 4.117333333333334,
+      "grad_norm": 1.1748610734939575,
+      "learning_rate": 0.001817753691826212,
+      "loss": 4.6433,
+      "step": 385
+    },
+    {
+      "epoch": 4.1706666666666665,
+      "grad_norm": 1.6473082304000854,
+      "learning_rate": 0.0018104858380061178,
+      "loss": 4.5933,
+      "step": 390
+    },
+    {
+      "epoch": 4.224,
+      "grad_norm": 1.9459190368652344,
+      "learning_rate": 0.0018030910168336557,
+      "loss": 4.631,
+      "step": 395
+    },
+    {
+      "epoch": 4.277333333333333,
+      "grad_norm": 1.5279489755630493,
+      "learning_rate": 0.0017955703867508633,
+      "loss": 4.5583,
+      "step": 400
+    },
+    {
+      "epoch": 4.330666666666667,
+      "grad_norm": 1.3347927331924438,
+      "learning_rate": 0.0017879251259084804,
+      "loss": 4.7295,
+      "step": 405
+    },
+    {
+      "epoch": 4.384,
+      "grad_norm": 2.52054500579834,
+      "learning_rate": 0.0017801564319813853,
+      "loss": 4.4718,
+      "step": 410
+    },
+    {
+      "epoch": 4.437333333333333,
+      "grad_norm": 1.1874818801879883,
+      "learning_rate": 0.0017722655219809715,
+      "loss": 4.5063,
+      "step": 415
+    },
+    {
+      "epoch": 4.490666666666667,
+      "grad_norm": 1.491890788078308,
+      "learning_rate": 0.0017642536320644964,
+      "loss": 4.3769,
+      "step": 420
+    },
+    {
+      "epoch": 4.5440000000000005,
+      "grad_norm": 1.1082266569137573,
+      "learning_rate": 0.0017561220173414297,
+      "loss": 4.3971,
+      "step": 425
+    },
+    {
+      "epoch": 4.597333333333333,
+      "grad_norm": 0.9449096918106079,
+      "learning_rate": 0.0017478719516768322,
+      "loss": 4.443,
+      "step": 430
+    },
+    {
+      "epoch": 4.650666666666667,
+      "grad_norm": 1.4695461988449097,
+      "learning_rate": 0.0017395047274917994,
+      "loss": 4.3679,
+      "step": 435
+    },
+    {
+      "epoch": 4.704,
+      "grad_norm": 2.962554693222046,
+      "learning_rate": 0.001731021655560995,
+      "loss": 4.2775,
+      "step": 440
+    },
+    {
+      "epoch": 4.757333333333333,
+      "grad_norm": 0.9070279598236084,
+      "learning_rate": 0.0017224240648073094,
+      "loss": 4.5664,
+      "step": 445
+    },
+    {
+      "epoch": 4.810666666666666,
+      "grad_norm": 1.7772026062011719,
+      "learning_rate": 0.0017137133020936782,
+      "loss": 4.4455,
+      "step": 450
+    },
+    {
+      "epoch": 4.864,
+      "grad_norm": 2.937232255935669,
+      "learning_rate": 0.0017048907320120865,
+      "loss": 4.4177,
+      "step": 455
+    },
+    {
+      "epoch": 4.917333333333334,
+      "grad_norm": 6.296092510223389,
+      "learning_rate": 0.0016959577366697988,
+      "loss": 4.5929,
+      "step": 460
+    },
+    {
+      "epoch": 4.970666666666666,
+      "grad_norm": 22.26595687866211,
+      "learning_rate": 0.0016869157154728437,
+      "loss": 4.5319,
+      "step": 465
+    },
+    {
+      "epoch": 4.992,
+      "eval_loss": 4.809427738189697,
+      "eval_runtime": 12.4879,
+      "eval_samples_per_second": 90.087,
+      "eval_steps_per_second": 15.055,
+      "step": 467
+    },
+    {
+      "epoch": 5.026666666666666,
+      "grad_norm": 6.585814476013184,
+      "learning_rate": 0.001677766084906787,
+      "loss": 5.1022,
+      "step": 470
+    },
+    {
+      "epoch": 5.08,
+      "grad_norm": 5.868590831756592,
+      "learning_rate": 0.001668510278314833,
+      "loss": 4.5102,
+      "step": 475
+    },
+    {
+      "epoch": 5.133333333333334,
+      "grad_norm": 4.387262344360352,
+      "learning_rate": 0.0016591497456732824,
+      "loss": 4.6548,
+      "step": 480
+    },
+    {
+      "epoch": 5.1866666666666665,
+      "grad_norm": 9.387950897216797,
+      "learning_rate": 0.0016496859533643852,
+      "loss": 4.6778,
+      "step": 485
+    },
+    {
+      "epoch": 5.24,
+      "grad_norm": 51.944549560546875,
+      "learning_rate": 0.0016401203839466211,
+      "loss": 5.041,
+      "step": 490
+    },
+    {
+      "epoch": 5.293333333333333,
+      "grad_norm": 14.20453929901123,
+      "learning_rate": 0.0016304545359224522,
+      "loss": 5.2576,
+      "step": 495
+    },
+    {
+      "epoch": 5.346666666666667,
+      "grad_norm": 3.1657214164733887,
+      "learning_rate": 0.00162068992350357,
+      "loss": 5.1287,
+      "step": 500
+    },
+    {
+      "epoch": 5.4,
+      "grad_norm": 4.8425726890563965,
+      "learning_rate": 0.001610828076373687,
+      "loss": 4.9648,
+      "step": 505
+    },
+    {
+      "epoch": 5.453333333333333,
+      "grad_norm": 1.9821243286132812,
+      "learning_rate": 0.0016008705394489032,
+      "loss": 4.9847,
+      "step": 510
+    },
+    {
+      "epoch": 5.506666666666667,
+      "grad_norm": 6.5806379318237305,
+      "learning_rate": 0.0015908188726356843,
+      "loss": 4.9543,
+      "step": 515
+    },
+    {
+      "epoch": 5.5600000000000005,
+      "grad_norm": 18.127779006958008,
+      "learning_rate": 0.0015806746505864946,
+      "loss": 4.9112,
+      "step": 520
+    },
+    {
+      "epoch": 5.613333333333333,
+      "grad_norm": 12.010746955871582,
+      "learning_rate": 0.0015704394624531184,
+      "loss": 4.9836,
+      "step": 525
+    },
+    {
+      "epoch": 5.666666666666667,
+      "grad_norm": 46.7609977722168,
+      "learning_rate": 0.0015601149116377094,
+      "loss": 4.8966,
+      "step": 530
+    },
+    {
+      "epoch": 5.72,
+      "grad_norm": 117.95108795166016,
+      "learning_rate": 0.0015497026155416088,
+      "loss": 4.8016,
+      "step": 535
+    },
+    {
+      "epoch": 5.773333333333333,
+      "grad_norm": 41.65292739868164,
+      "learning_rate": 0.0015392042053119699,
+      "loss": 5.1113,
+      "step": 540
+    },
+    {
+      "epoch": 5.826666666666666,
+      "grad_norm": 26.858564376831055,
+      "learning_rate": 0.0015286213255862294,
+      "loss": 4.9389,
+      "step": 545
+    },
+    {
+      "epoch": 5.88,
+      "grad_norm": 8.647442817687988,
+      "learning_rate": 0.0015179556342344643,
+      "loss": 4.9543,
+      "step": 550
+    },
+    {
+      "epoch": 5.933333333333334,
+      "grad_norm": 3.5178844928741455,
+      "learning_rate": 0.0015072088020996791,
+      "loss": 5.1129,
+      "step": 555
+    },
+    {
+      "epoch": 5.986666666666666,
+      "grad_norm": 2.328364849090576,
+      "learning_rate": 0.001496382512736056,
+      "loss": 4.8257,
+      "step": 560
+    },
+    {
+      "epoch": 5.997333333333334,
+      "eval_loss": 5.105650901794434,
+      "eval_runtime": 12.5011,
+      "eval_samples_per_second": 89.992,
+      "eval_steps_per_second": 15.039,
+      "step": 561
+    },
+    {
+      "epoch": 6.042666666666666,
+      "grad_norm": 2.6246631145477295,
+      "learning_rate": 0.0014854784621452176,
+      "loss": 5.4436,
+      "step": 565
+    },
+    {
+      "epoch": 6.096,
+      "grad_norm": 1.641006588935852,
+      "learning_rate": 0.0014744983585105386,
+      "loss": 4.721,
+      "step": 570
+    },
+    {
+      "epoch": 6.149333333333334,
+      "grad_norm": 2.2615890502929688,
+      "learning_rate": 0.001463443921929548,
+      "loss": 4.6645,
+      "step": 575
+    },
+    {
+      "epoch": 6.2026666666666666,
+      "grad_norm": 2.4459476470947266,
+      "learning_rate": 0.0014523168841444657,
+      "loss": 4.6295,
+      "step": 580
+    },
+    {
+      "epoch": 6.256,
+      "grad_norm": 27.94793128967285,
+      "learning_rate": 0.001441118988270916,
+      "loss": 4.6528,
+      "step": 585
+    },
+    {
+      "epoch": 6.309333333333333,
+      "grad_norm": 5.710525035858154,
+      "learning_rate": 0.0014298519885248572,
+      "loss": 4.549,
+      "step": 590
+    },
+    {
+      "epoch": 6.362666666666667,
+      "grad_norm": 4.791785717010498,
+      "learning_rate": 0.0014185176499477742,
+      "loss": 4.5602,
+      "step": 595
+    },
+    {
+      "epoch": 6.416,
+      "grad_norm": 2.0223538875579834,
+      "learning_rate": 0.0014071177481301737,
+      "loss": 4.5802,
+      "step": 600
+    },
+    {
+      "epoch": 6.469333333333333,
+      "grad_norm": 1.6422585248947144,
+      "learning_rate": 0.0013956540689334285,
+      "loss": 4.4584,
+      "step": 605
+    },
+    {
+      "epoch": 6.522666666666667,
+      "grad_norm": 8.107535362243652,
+      "learning_rate": 0.001384128408210011,
+      "loss": 4.5343,
+      "step": 610
+    },
+    {
+      "epoch": 6.576,
+      "grad_norm": 2.697594404220581,
+      "learning_rate": 0.0013725425715221625,
+      "loss": 4.6172,
+      "step": 615
+    },
+    {
+      "epoch": 6.629333333333333,
+      "grad_norm": 1.561109185218811,
+      "learning_rate": 0.0013608983738590413,
+      "loss": 4.4385,
+      "step": 620
+    },
+    {
+      "epoch": 6.682666666666667,
+      "grad_norm": 2.0785770416259766,
+      "learning_rate": 0.0013491976393523951,
+      "loss": 4.4098,
+      "step": 625
+    },
+    {
+      "epoch": 6.736,
+      "grad_norm": 5.022526264190674,
+      "learning_rate": 0.0013374422009907983,
+      "loss": 4.3741,
+      "step": 630
+    },
+    {
+      "epoch": 6.789333333333333,
+      "grad_norm": 1.8558008670806885,
+      "learning_rate": 0.0013256339003325053,
+      "loss": 4.325,
+      "step": 635
+    },
+    {
+      "epoch": 6.842666666666666,
+      "grad_norm": 1.2337331771850586,
+      "learning_rate": 0.0013137745872169578,
+      "loss": 4.3192,
+      "step": 640
+    },
+    {
+      "epoch": 6.896,
+      "grad_norm": 1.0328541994094849,
+      "learning_rate": 0.0013018661194749985,
+      "loss": 4.3117,
+      "step": 645
+    },
+    {
+      "epoch": 6.949333333333334,
+      "grad_norm": 6.057741641998291,
+      "learning_rate": 0.00128991036263783,
+      "loss": 4.2177,
+      "step": 650
+    },
+    {
+      "epoch": 6.992,
+      "eval_loss": 4.5589494705200195,
+      "eval_runtime": 12.5045,
+      "eval_samples_per_second": 89.968,
+      "eval_steps_per_second": 15.035,
+      "step": 654
+    },
+    {
+      "epoch": 7.005333333333334,
+      "grad_norm": 10.887760162353516,
+      "learning_rate": 0.001277909189644768,
+      "loss": 4.7952,
+      "step": 655
+    },
+    {
+      "epoch": 7.058666666666666,
+      "grad_norm": 3.197256565093994,
+      "learning_rate": 0.001265864480549836,
+      "loss": 4.166,
+      "step": 660
+    },
+    {
+      "epoch": 7.112,
+      "grad_norm": 3.441707134246826,
+      "learning_rate": 0.001253778122227242,
+      "loss": 4.2237,
+      "step": 665
+    },
+    {
+      "epoch": 7.165333333333333,
+      "grad_norm": 1.8096449375152588,
+      "learning_rate": 0.0012416520080757892,
+      "loss": 4.0627,
+      "step": 670
+    },
+    {
+      "epoch": 7.218666666666667,
+      "grad_norm": 1.5362603664398193,
+      "learning_rate": 0.0012294880377222647,
+      "loss": 4.1205,
+      "step": 675
+    },
+    {
+      "epoch": 7.272,
+      "grad_norm": 2.7223100662231445,
+      "learning_rate": 0.0012172881167238515,
+      "loss": 4.1721,
+      "step": 680
+    },
+    {
+      "epoch": 7.325333333333333,
+      "grad_norm": 1.2539554834365845,
+      "learning_rate": 0.0012050541562696109,
+      "loss": 4.159,
+      "step": 685
+    },
+    {
+      "epoch": 7.378666666666667,
+      "grad_norm": 3.9169719219207764,
+      "learning_rate": 0.001192788072881085,
+      "loss": 4.207,
+      "step": 690
+    },
+    {
+      "epoch": 7.432,
+      "grad_norm": 2.505176544189453,
+      "learning_rate": 0.0011804917881120607,
+      "loss": 3.9257,
+      "step": 695
+    },
+    {
+      "epoch": 7.485333333333333,
+      "grad_norm": 1.2009811401367188,
+      "learning_rate": 0.0011681672282475495,
+      "loss": 4.0998,
+      "step": 700
+    },
+    {
+      "epoch": 7.538666666666667,
+      "grad_norm": 1.2483587265014648,
+      "learning_rate": 0.0011558163240020208,
+      "loss": 4.0612,
+      "step": 705
+    },
+    {
+      "epoch": 7.592,
+      "grad_norm": 1.8018878698349,
+      "learning_rate": 0.001143441010216946,
+      "loss": 4.1208,
+      "step": 710
+    },
+    {
+      "epoch": 7.645333333333333,
+      "grad_norm": 1.5484060049057007,
+      "learning_rate": 0.0011310432255576942,
+      "loss": 4.0344,
+      "step": 715
+    },
+    {
+      "epoch": 7.698666666666667,
+      "grad_norm": 2.459238290786743,
+      "learning_rate": 0.0011186249122098282,
+      "loss": 4.0085,
+      "step": 720
+    },
+    {
+      "epoch": 7.752,
+      "grad_norm": 2.3227972984313965,
+      "learning_rate": 0.0011061880155748497,
+      "loss": 4.0162,
+      "step": 725
+    },
+    {
+      "epoch": 7.8053333333333335,
+      "grad_norm": 4.121925354003906,
+      "learning_rate": 0.0010937344839654416,
+      "loss": 4.1103,
+      "step": 730
+    },
+    {
+      "epoch": 7.858666666666666,
+      "grad_norm": 2.338094711303711,
+      "learning_rate": 0.0010812662683002527,
+      "loss": 4.1566,
+      "step": 735
+    },
+    {
+      "epoch": 7.912,
+      "grad_norm": 1.8548033237457275,
+      "learning_rate": 0.001068785321798276,
+      "loss": 4.0535,
+      "step": 740
+    },
+    {
+      "epoch": 7.965333333333334,
+      "grad_norm": 1.4415037631988525,
+      "learning_rate": 0.0010562935996728628,
+      "loss": 4.0313,
+      "step": 745
+    },
+    {
+      "epoch": 7.997333333333334,
+      "eval_loss": 4.3429412841796875,
+      "eval_runtime": 12.4844,
+      "eval_samples_per_second": 90.112,
+      "eval_steps_per_second": 15.059,
+      "step": 748
+    },
+    {
+      "epoch": 8.021333333333333,
+      "grad_norm": 1.2015221118927002,
+      "learning_rate": 0.001043793058825431,
+      "loss": 4.3842,
+      "step": 750
+    },
+    {
+      "epoch": 8.074666666666667,
+      "grad_norm": 2.2034997940063477,
+      "learning_rate": 0.0010312856575389016,
+      "loss": 3.8739,
+      "step": 755
+    },
+    {
+      "epoch": 8.128,
+      "grad_norm": 1.0531103610992432,
+      "learning_rate": 0.0010187733551709234,
+      "loss": 3.9207,
+      "step": 760
+    },
+    {
+      "epoch": 8.181333333333333,
+      "grad_norm": 1.6261956691741943,
+      "learning_rate": 0.0010062581118469298,
+      "loss": 3.9147,
+      "step": 765
+    },
+    {
+      "epoch": 8.234666666666667,
+      "grad_norm": 4.931313991546631,
+      "learning_rate": 0.0009937418881530702,
+      "loss": 3.9194,
+      "step": 770
+    },
+    {
+      "epoch": 8.288,
+      "grad_norm": 1.1570011377334595,
+      "learning_rate": 0.0009812266448290766,
+      "loss": 3.8607,
+      "step": 775
+    },
+    {
+      "epoch": 8.341333333333333,
+      "grad_norm": 2.3009421825408936,
+      "learning_rate": 0.0009687143424610986,
+      "loss": 3.8516,
+      "step": 780
+    },
+    {
+      "epoch": 8.394666666666666,
+      "grad_norm": 1.5181440114974976,
+      "learning_rate": 0.0009562069411745691,
+      "loss": 3.8939,
+      "step": 785
+    },
+    {
+      "epoch": 8.448,
+      "grad_norm": 1.7607407569885254,
+      "learning_rate": 0.0009437064003271373,
+      "loss": 3.9431,
+      "step": 790
+    },
+    {
+      "epoch": 8.501333333333333,
+      "grad_norm": 1.155339241027832,
+      "learning_rate": 0.0009312146782017244,
+      "loss": 3.8956,
+      "step": 795
+    },
+    {
+      "epoch": 8.554666666666666,
+      "grad_norm": 0.9878433346748352,
+      "learning_rate": 0.0009187337316997475,
+      "loss": 3.8884,
+      "step": 800
+    },
+    {
+      "epoch": 8.608,
+      "grad_norm": 0.9723697304725647,
+      "learning_rate": 0.0009062655160345587,
+      "loss": 3.714,
+      "step": 805
+    },
+    {
+      "epoch": 8.661333333333333,
+      "grad_norm": 6.659087181091309,
+      "learning_rate": 0.0008938119844251507,
+      "loss": 3.8302,
+      "step": 810
+    },
+    {
+      "epoch": 8.714666666666666,
+      "grad_norm": 1.3487255573272705,
+      "learning_rate": 0.0008813750877901723,
+      "loss": 3.9658,
+      "step": 815
+    },
+    {
+      "epoch": 8.768,
+      "grad_norm": 1.3927958011627197,
+      "learning_rate": 0.0008689567744423059,
+      "loss": 3.8516,
+      "step": 820
+    },
+    {
+      "epoch": 8.821333333333333,
+      "grad_norm": 0.8693830370903015,
+      "learning_rate": 0.0008565589897830542,
+      "loss": 3.8388,
+      "step": 825
+    },
+    {
+      "epoch": 8.874666666666666,
+      "grad_norm": 1.2397363185882568,
+      "learning_rate": 0.0008441836759979795,
+      "loss": 3.7521,
+      "step": 830
+    },
+    {
+      "epoch": 8.928,
+      "grad_norm": 1.1434884071350098,
+      "learning_rate": 0.0008318327717524509,
+      "loss": 3.7411,
+      "step": 835
+    },
+    {
+      "epoch": 8.981333333333334,
+      "grad_norm": 0.8602774143218994,
+      "learning_rate": 0.0008195082118879396,
+      "loss": 3.7408,
+      "step": 840
+    },
+    {
+      "epoch": 8.992,
+      "eval_loss": 4.147874355316162,
+      "eval_runtime": 12.5024,
+      "eval_samples_per_second": 89.983,
+      "eval_steps_per_second": 15.037,
+      "step": 841
+    },
+    {
+      "epoch": 9.037333333333333,
+      "grad_norm": 1.2394230365753174,
+      "learning_rate": 0.0008072119271189156,
+      "loss": 4.0861,
+      "step": 845
+    },
+    {
+      "epoch": 9.090666666666667,
+      "grad_norm": 0.9018468260765076,
+      "learning_rate": 0.0007949458437303892,
+      "loss": 3.6608,
+      "step": 850
+    },
+    {
+      "epoch": 9.144,
+      "grad_norm": 2.0389747619628906,
+      "learning_rate": 0.0007827118832761486,
+      "loss": 3.6369,
+      "step": 855
+    },
+    {
+      "epoch": 9.197333333333333,
+      "grad_norm": 1.0341744422912598,
+      "learning_rate": 0.0007705119622777351,
+      "loss": 3.6354,
+      "step": 860
+    },
+    {
+      "epoch": 9.250666666666667,
+      "grad_norm": 0.8617631196975708,
+      "learning_rate": 0.0007583479919242108,
+      "loss": 3.6106,
+      "step": 865
+    },
+    {
+      "epoch": 9.304,
+      "grad_norm": 0.719153642654419,
+      "learning_rate": 0.000746221877772758,
+      "loss": 3.7636,
+      "step": 870
+    },
+    {
+      "epoch": 9.357333333333333,
+      "grad_norm": 3.26902437210083,
+      "learning_rate": 0.0007341355194501638,
+      "loss": 3.6817,
+      "step": 875
+    },
+    {
+      "epoch": 9.410666666666666,
+      "grad_norm": 0.9911563396453857,
+      "learning_rate": 0.0007220908103552318,
+      "loss": 3.7094,
+      "step": 880
+    },
+    {
+      "epoch": 9.464,
+      "grad_norm": 0.8281224370002747,
+      "learning_rate": 0.0007100896373621699,
+      "loss": 3.6431,
+      "step": 885
+    },
+    {
+      "epoch": 9.517333333333333,
+      "grad_norm": 0.8220219612121582,
+      "learning_rate": 0.0006981338805250015,
+      "loss": 3.6953,
+      "step": 890
+    },
+    {
+      "epoch": 9.570666666666666,
+      "grad_norm": 0.8096312284469604,
+      "learning_rate": 0.0006862254127830425,
+      "loss": 3.5947,
+      "step": 895
+    },
+    {
+      "epoch": 9.624,
+      "grad_norm": 0.998589813709259,
+      "learning_rate": 0.000674366099667495,
+      "loss": 3.6492,
+      "step": 900
+    },
+    {
+      "epoch": 9.677333333333333,
+      "grad_norm": 1.0131206512451172,
+      "learning_rate": 0.0006625577990092018,
+      "loss": 3.638,
+      "step": 905
+    },
+    {
+      "epoch": 9.730666666666666,
+      "grad_norm": 0.6822465062141418,
+      "learning_rate": 0.0006508023606476051,
+      "loss": 3.626,
+      "step": 910
+    },
+    {
+      "epoch": 9.784,
+      "grad_norm": 0.7446674704551697,
+      "learning_rate": 0.0006391016261409591,
+      "loss": 3.6361,
+      "step": 915
+    },
+    {
+      "epoch": 9.837333333333333,
+      "grad_norm": 1.5426651239395142,
+      "learning_rate": 0.0006274574284778378,
+      "loss": 3.6579,
+      "step": 920
+    },
+    {
+      "epoch": 9.890666666666666,
+      "grad_norm": 1.0557122230529785,
+      "learning_rate": 0.0006158715917899892,
+      "loss": 3.5978,
+      "step": 925
+    },
+    {
+      "epoch": 9.943999999999999,
+      "grad_norm": 1.271697759628296,
+      "learning_rate": 0.0006043459310665716,
+      "loss": 3.5695,
+      "step": 930
+    },
+    {
+      "epoch": 9.997333333333334,
+      "grad_norm": 1.3581335544586182,
+      "learning_rate": 0.0005928822518698263,
+      "loss": 4.0306,
+      "step": 935
+    },
+    {
+      "epoch": 9.997333333333334,
+      "eval_loss": 4.033357620239258,
+      "eval_runtime": 12.5191,
+      "eval_samples_per_second": 89.863,
+      "eval_steps_per_second": 15.017,
+      "step": 935
+    },
+    {
+      "epoch": 10.053333333333333,
+      "grad_norm": 1.4836759567260742,
+      "learning_rate": 0.000581482350052226,
+      "loss": 3.5213,
+      "step": 940
+    },
+    {
+      "epoch": 10.106666666666667,
+      "grad_norm": 0.8610721230506897,
+      "learning_rate": 0.0005701480114751431,
+      "loss": 3.4489,
+      "step": 945
+    },
+    {
+      "epoch": 10.16,
+      "grad_norm": 1.1154193878173828,
+      "learning_rate": 0.0005588810117290842,
+      "loss": 3.4965,
+      "step": 950
+    },
+    {
+      "epoch": 10.213333333333333,
+      "grad_norm": 0.9979135990142822,
+      "learning_rate": 0.0005476831158555345,
+      "loss": 3.5145,
+      "step": 955
+    },
+    {
+      "epoch": 10.266666666666667,
+      "grad_norm": 18.88866424560547,
+      "learning_rate": 0.0005365560780704523,
+      "loss": 3.4903,
+      "step": 960
+    },
+    {
+      "epoch": 10.32,
+      "grad_norm": 1.4323471784591675,
+      "learning_rate": 0.0005255016414894615,
+      "loss": 3.5408,
+      "step": 965
+    },
+    {
+      "epoch": 10.373333333333333,
+      "grad_norm": 3.502009868621826,
+      "learning_rate": 0.0005145215378547825,
+      "loss": 3.5039,
+      "step": 970
+    },
+    {
+      "epoch": 10.426666666666666,
+      "grad_norm": 4.632591247558594,
+      "learning_rate": 0.0005036174872639443,
+      "loss": 3.5314,
+      "step": 975
+    },
+    {
+      "epoch": 10.48,
+      "grad_norm": 1.0967168807983398,
+      "learning_rate": 0.0004927911979003214,
+      "loss": 3.5151,
+      "step": 980
+    },
+    {
+      "epoch": 10.533333333333333,
+      "grad_norm": 0.8381322622299194,
+      "learning_rate": 0.000482044365765536,
+      "loss": 3.4655,
+      "step": 985
+    },
+    {
+      "epoch": 10.586666666666666,
+      "grad_norm": 0.9333806037902832,
+      "learning_rate": 0.0004713786744137709,
+      "loss": 3.4428,
+      "step": 990
+    },
+    {
+      "epoch": 10.64,
+      "grad_norm": 1.3139746189117432,
+      "learning_rate": 0.00046079579468803045,
+      "loss": 3.4655,
+      "step": 995
+    },
+    {
+      "epoch": 10.693333333333333,
+      "grad_norm": 13.540165901184082,
+      "learning_rate": 0.0004502973844583914,
+      "loss": 3.4847,
+      "step": 1000
+    },
+    {
+      "epoch": 10.746666666666666,
+      "grad_norm": 0.730549156665802,
+      "learning_rate": 0.00043988508836229046,
+      "loss": 3.5064,
+      "step": 1005
+    },
+    {
+      "epoch": 10.8,
+      "grad_norm": 1.1248104572296143,
+      "learning_rate": 0.00042956053754688174,
+      "loss": 3.4264,
+      "step": 1010
+    },
+    {
+      "epoch": 10.853333333333333,
+      "grad_norm": 2.549948215484619,
+      "learning_rate": 0.0004193253494135054,
+      "loss": 3.4826,
+      "step": 1015
+    },
+    {
+      "epoch": 10.906666666666666,
+      "grad_norm": 2.570146083831787,
+      "learning_rate": 0.00040918112736431565,
+      "loss": 3.5088,
+      "step": 1020
+    },
+    {
+      "epoch": 10.96,
+      "grad_norm": 0.9071534276008606,
+      "learning_rate": 0.0003991294605510969,
+      "loss": 3.4673,
+      "step": 1025
+    },
+    {
+      "epoch": 10.992,
+      "eval_loss": 3.9098801612854004,
+      "eval_runtime": 12.4879,
+      "eval_samples_per_second": 90.087,
+      "eval_steps_per_second": 15.055,
+      "step": 1028
+    },
+    {
+      "epoch": 11.016,
+      "grad_norm": 0.7501585483551025,
+      "learning_rate": 0.0003891719236263128,
+      "loss": 3.7099,
+      "step": 1030
+    },
+    {
+      "epoch": 11.069333333333333,
+      "grad_norm": 0.7575392723083496,
+      "learning_rate": 0.0003793100764964299,
+      "loss": 3.3254,
+      "step": 1035
+    },
+    {
+      "epoch": 11.122666666666667,
+      "grad_norm": 1.5098010301589966,
+      "learning_rate": 0.00036954546407754797,
+      "loss": 3.3361,
+      "step": 1040
+    },
+    {
+      "epoch": 11.176,
+      "grad_norm": 0.652353048324585,
+      "learning_rate": 0.00035987961605337895,
+      "loss": 3.3419,
+      "step": 1045
+    },
+    {
+      "epoch": 11.229333333333333,
+      "grad_norm": 1.4576791524887085,
+      "learning_rate": 0.0003503140466356151,
+      "loss": 3.4373,
+      "step": 1050
+    },
+    {
+      "epoch": 11.282666666666668,
+      "grad_norm": 0.8805112242698669,
+      "learning_rate": 0.0003408502543267175,
+      "loss": 3.361,
+      "step": 1055
+    },
+    {
+      "epoch": 11.336,
+      "grad_norm": 2.8293251991271973,
+      "learning_rate": 0.0003314897216851673,
+      "loss": 3.3537,
+      "step": 1060
+    },
+    {
+      "epoch": 11.389333333333333,
+      "grad_norm": 1.0685662031173706,
+      "learning_rate": 0.00032223391509321334,
+      "loss": 3.295,
+      "step": 1065
+    },
+    {
+      "epoch": 11.442666666666666,
+      "grad_norm": 0.5625657439231873,
+      "learning_rate": 0.00031308428452715643,
+      "loss": 3.3364,
+      "step": 1070
+    },
+    {
+      "epoch": 11.496,
+      "grad_norm": 0.8257743120193481,
+      "learning_rate": 0.00030404226333020115,
+      "loss": 3.2609,
+      "step": 1075
+    },
+    {
+      "epoch": 11.549333333333333,
+      "grad_norm": 1.0789713859558105,
+      "learning_rate": 0.0002951092679879136,
+      "loss": 3.3134,
+      "step": 1080
+    },
+    {
+      "epoch": 11.602666666666666,
+      "grad_norm": 1.148529052734375,
+      "learning_rate": 0.0002862866979063219,
+      "loss": 3.2776,
+      "step": 1085
+    },
+    {
+      "epoch": 11.656,
+      "grad_norm": 3.4152815341949463,
+      "learning_rate": 0.0002775759351926909,
+      "loss": 3.2817,
+      "step": 1090
+    },
+    {
+      "epoch": 11.709333333333333,
+      "grad_norm": 5.510611057281494,
+      "learning_rate": 0.00026897834443900527,
+      "loss": 3.4051,
+      "step": 1095
+    },
+    {
+      "epoch": 11.762666666666666,
+      "grad_norm": 3.2792694568634033,
+      "learning_rate": 0.0002604952725082005,
+      "loss": 3.3752,
+      "step": 1100
+    },
+    {
+      "epoch": 11.816,
+      "grad_norm": 0.8379424214363098,
+      "learning_rate": 0.00025212804832316785,
+      "loss": 3.3568,
+      "step": 1105
+    },
+    {
+      "epoch": 11.869333333333334,
+      "grad_norm": 1.0004689693450928,
+      "learning_rate": 0.00024387798265857076,
+      "loss": 3.2548,
+      "step": 1110
+    },
+    {
+      "epoch": 11.922666666666666,
+      "grad_norm": 1.1289111375808716,
+      "learning_rate": 0.00023574636793550374,
+      "loss": 3.2763,
+      "step": 1115
+    },
+    {
+      "epoch": 11.975999999999999,
+      "grad_norm": 0.7427679300308228,
+      "learning_rate": 0.00022773447801902857,
+      "loss": 3.3279,
+      "step": 1120
+    },
+    {
+      "epoch": 11.997333333333334,
+      "eval_loss": 3.8246541023254395,
+      "eval_runtime": 12.4686,
+      "eval_samples_per_second": 90.226,
+      "eval_steps_per_second": 15.078,
+      "step": 1122
+    },
+    {
+      "epoch": 12.032,
+      "grad_norm": 0.7016487717628479,
+      "learning_rate": 0.00021984356801861506,
+      "loss": 3.545,
+      "step": 1125
+    },
+    {
+      "epoch": 12.085333333333333,
+      "grad_norm": 1.0700368881225586,
+      "learning_rate": 0.00021207487409151982,
+      "loss": 3.213,
+      "step": 1130
+    },
+    {
+      "epoch": 12.138666666666667,
+      "grad_norm": 0.548744261264801,
+      "learning_rate": 0.0002044296132491369,
+      "loss": 3.1303,
+      "step": 1135
+    },
+    {
+      "epoch": 12.192,
+      "grad_norm": 0.5944454669952393,
+      "learning_rate": 0.00019690898316634432,
+      "loss": 3.2735,
+      "step": 1140
+    },
+    {
+      "epoch": 12.245333333333333,
+      "grad_norm": 0.559898853302002,
+      "learning_rate": 0.0001895141619938825,
+      "loss": 3.221,
+      "step": 1145
+    },
+    {
+      "epoch": 12.298666666666668,
+      "grad_norm": 0.5505184531211853,
+      "learning_rate": 0.00018224630817378828,
+      "loss": 3.247,
+      "step": 1150
+    },
+    {
+      "epoch": 12.352,
+      "grad_norm": 4.3635735511779785,
+      "learning_rate": 0.00017510656025792004,
+      "loss": 3.091,
+      "step": 1155
+    },
+    {
+      "epoch": 12.405333333333333,
+      "grad_norm": 0.553667426109314,
+      "learning_rate": 0.00016809603672959616,
+      "loss": 3.2349,
+      "step": 1160
+    },
+    {
+      "epoch": 12.458666666666666,
+      "grad_norm": 1.5174871683120728,
+      "learning_rate": 0.00016121583582837774,
+      "loss": 3.2065,
+      "step": 1165
+    },
+    {
+      "epoch": 12.512,
+      "grad_norm": 0.7590833306312561,
+      "learning_rate": 0.0001544670353780234,
+      "loss": 3.1883,
+      "step": 1170
+    },
+    {
+      "epoch": 12.565333333333333,
+      "grad_norm": 1.4404692649841309,
+      "learning_rate": 0.00014785069261764183,
+      "loss": 3.2056,
+      "step": 1175
+    },
+    {
+      "epoch": 12.618666666666666,
+      "grad_norm": 0.6769624948501587,
+      "learning_rate": 0.0001413678440360684,
+      "loss": 3.2165,
+      "step": 1180
+    },
+    {
+      "epoch": 12.672,
+      "grad_norm": 0.5693238377571106,
+      "learning_rate": 0.00013501950520949436,
+      "loss": 3.1955,
+      "step": 1185
+    },
+    {
+      "epoch": 12.725333333333333,
+      "grad_norm": 0.9455315470695496,
+      "learning_rate": 0.00012880667064237006,
+      "loss": 3.2017,
+      "step": 1190
+    },
+    {
+      "epoch": 12.778666666666666,
+      "grad_norm": 0.8543607592582703,
+      "learning_rate": 0.00012273031361160957,
+      "loss": 3.2268,
+      "step": 1195
+    },
+    {
+      "epoch": 12.832,
+      "grad_norm": 1.906198501586914,
+      "learning_rate": 0.00011679138601412254,
+      "loss": 3.2457,
+      "step": 1200
+    },
+    {
+      "epoch": 12.885333333333334,
+      "grad_norm": 0.5271804928779602,
+      "learning_rate": 0.00011099081821769297,
+      "loss": 3.2146,
+      "step": 1205
+    },
+    {
+      "epoch": 12.938666666666666,
+      "grad_norm": 1.0702733993530273,
+      "learning_rate": 0.00010532951891523124,
+      "loss": 3.2083,
+      "step": 1210
+    },
+    {
+      "epoch": 12.992,
+      "grad_norm": 2.5151164531707764,
+      "learning_rate": 9.980837498242356e-05,
+      "loss": 3.2354,
+      "step": 1215
+    },
+    {
+      "epoch": 12.992,
+      "eval_loss": 3.785278558731079,
+      "eval_runtime": 12.6277,
+      "eval_samples_per_second": 89.09,
+      "eval_steps_per_second": 14.888,
+      "step": 1215
+    },
+    {
+      "epoch": 13.048,
+      "grad_norm": 1.2420941591262817,
+      "learning_rate": 9.442825133879607e-05,
+      "loss": 3.4289,
+      "step": 1220
+    },
+    {
+      "epoch": 13.101333333333333,
+      "grad_norm": 0.4141329824924469,
+      "learning_rate": 8.918999081222156e-05,
+      "loss": 3.0674,
+      "step": 1225
+    },
+    {
+      "epoch": 13.154666666666667,
+      "grad_norm": 0.6331799030303955,
+      "learning_rate": 8.4094414006884e-05,
+      "loss": 3.1639,
+      "step": 1230
+    },
+    {
+      "epoch": 13.208,
+      "grad_norm": 0.558097779750824,
+      "learning_rate": 7.914231917472747e-05,
+      "loss": 3.1318,
+      "step": 1235
+    },
+    {
+      "epoch": 13.261333333333333,
+      "grad_norm": 0.5015540719032288,
+      "learning_rate": 7.433448209040495e-05,
+      "loss": 3.1447,
+      "step": 1240
+    },
+    {
+      "epoch": 13.314666666666668,
+      "grad_norm": 3.2629053592681885,
+      "learning_rate": 6.967165592974789e-05,
+      "loss": 3.1929,
+      "step": 1245
+    },
+    {
+      "epoch": 13.368,
+      "grad_norm": 0.35517868399620056,
+      "learning_rate": 6.515457115177803e-05,
+      "loss": 3.1434,
+      "step": 1250
+    },
+    {
+      "epoch": 13.421333333333333,
+      "grad_norm": 2.8700928688049316,
+      "learning_rate": 6.0783935384275736e-05,
+      "loss": 3.1381,
+      "step": 1255
+    },
+    {
+      "epoch": 13.474666666666666,
+      "grad_norm": 1.7031766176223755,
+      "learning_rate": 5.656043331292682e-05,
+      "loss": 3.1343,
+      "step": 1260
+    },
+    {
+      "epoch": 13.528,
+      "grad_norm": 0.6974568963050842,
+      "learning_rate": 5.2484726574061225e-05,
+      "loss": 3.154,
+      "step": 1265
+    },
+    {
+      "epoch": 13.581333333333333,
+      "grad_norm": 0.656442403793335,
+      "learning_rate": 4.855745365100539e-05,
+      "loss": 3.0817,
+      "step": 1270
+    },
+    {
+      "epoch": 13.634666666666666,
+      "grad_norm": 0.7731483578681946,
+      "learning_rate": 4.477922977405913e-05,
+      "loss": 3.1515,
+      "step": 1275
+    },
+    {
+      "epoch": 13.688,
+      "grad_norm": 0.6341891884803772,
+      "learning_rate": 4.115064682411607e-05,
+      "loss": 3.1462,
+      "step": 1280
+    },
+    {
+      "epoch": 13.741333333333333,
+      "grad_norm": 0.5872439742088318,
+      "learning_rate": 3.7672273239942934e-05,
+      "loss": 3.1376,
+      "step": 1285
+    },
+    {
+      "epoch": 13.794666666666666,
+      "grad_norm": 0.6876071691513062,
+      "learning_rate": 3.434465392912956e-05,
+      "loss": 3.1342,
+      "step": 1290
+    },
+    {
+      "epoch": 13.848,
+      "grad_norm": 0.5463366508483887,
+      "learning_rate": 3.1168310182725815e-05,
+      "loss": 3.1465,
+      "step": 1295
+    },
+    {
+      "epoch": 13.901333333333334,
+      "grad_norm": 0.49031180143356323,
+      "learning_rate": 2.8143739593578856e-05,
+      "loss": 3.0675,
+      "step": 1300
+    },
+    {
+      "epoch": 13.954666666666666,
+      "grad_norm": 0.5363944172859192,
+      "learning_rate": 2.5271415978382117e-05,
+      "loss": 3.0656,
+      "step": 1305
+    },
+    {
+      "epoch": 13.997333333333334,
+      "eval_loss": 3.7691831588745117,
+      "eval_runtime": 12.4823,
+      "eval_samples_per_second": 90.128,
+      "eval_steps_per_second": 15.061,
+      "step": 1309
+    },
+    {
+      "epoch": 14.010666666666667,
+      "grad_norm": 0.5036697387695312,
+      "learning_rate": 2.255178930344903e-05,
+      "loss": 3.4255,
+      "step": 1310
+    },
+    {
+      "epoch": 14.064,
+      "grad_norm": 0.6493708491325378,
+      "learning_rate": 1.998528561422297e-05,
+      "loss": 3.0921,
+      "step": 1315
+    },
+    {
+      "epoch": 14.117333333333333,
+      "grad_norm": 0.5409650206565857,
+      "learning_rate": 1.757230696853518e-05,
+      "loss": 3.074,
+      "step": 1320
+    },
+    {
+      "epoch": 14.170666666666667,
+      "grad_norm": 0.7467535734176636,
+      "learning_rate": 1.531323137361995e-05,
+      "loss": 3.0659,
+      "step": 1325
+    },
+    {
+      "epoch": 14.224,
+      "grad_norm": 1.0386552810668945,
+      "learning_rate": 1.3208412726897322e-05,
+      "loss": 3.1497,
+      "step": 1330
+    },
+    {
+      "epoch": 14.277333333333333,
+      "grad_norm": 0.49301984906196594,
+      "learning_rate": 1.1258180760533088e-05,
+      "loss": 3.0785,
+      "step": 1335
+    },
+    {
+      "epoch": 14.330666666666666,
+      "grad_norm": 1.8429055213928223,
+      "learning_rate": 9.462840989784672e-06,
+      "loss": 3.0583,
+      "step": 1340
+    },
+    {
+      "epoch": 14.384,
+      "grad_norm": 1.5353028774261475,
+      "learning_rate": 7.822674665139752e-06,
+      "loss": 3.0546,
+      "step": 1345
+    },
+    {
+      "epoch": 14.437333333333333,
+      "grad_norm": 0.3447898328304291,
+      "learning_rate": 6.337938728257053e-06,
+      "loss": 3.0634,
+      "step": 1350
+    },
+    {
+      "epoch": 14.490666666666666,
+      "grad_norm": 0.923072874546051,
+      "learning_rate": 5.008865771715221e-06,
+      "loss": 3.1333,
+      "step": 1355
+    },
+    {
+      "epoch": 14.544,
+      "grad_norm": 0.5001369714736938,
+      "learning_rate": 3.8356640025752896e-06,
+      "loss": 3.0982,
+      "step": 1360
+    },
+    {
+      "epoch": 14.597333333333333,
+      "grad_norm": 0.559633195400238,
+      "learning_rate": 2.8185172097641155e-06,
+      "loss": 3.0525,
+      "step": 1365
+    },
+    {
+      "epoch": 14.650666666666666,
+      "grad_norm": 0.49595069885253906,
+      "learning_rate": 1.957584735282847e-06,
+      "loss": 3.1459,
+      "step": 1370
+    },
+    {
+      "epoch": 14.704,
+      "grad_norm": 0.6536312103271484,
+      "learning_rate": 1.2530014492446729e-06,
+      "loss": 3.1156,
+      "step": 1375
+    },
+    {
+      "epoch": 14.757333333333333,
+      "grad_norm": 0.38376209139823914,
+      "learning_rate": 7.048777287472774e-07,
+      "loss": 3.0831,
+      "step": 1380
+    },
+    {
+      "epoch": 14.810666666666666,
+      "grad_norm": 0.5727828741073608,
+      "learning_rate": 3.132994405808942e-07,
+      "loss": 3.0888,
+      "step": 1385
+    },
+    {
+      "epoch": 14.864,
+      "grad_norm": 0.45092472434043884,
+      "learning_rate": 7.832792777739961e-08,
+      "loss": 3.1226,
+      "step": 1390
+    },
+    {
+      "epoch": 14.917333333333334,
+      "grad_norm": 0.4628413915634155,
+      "learning_rate": 0.0,
+      "loss": 3.0981,
+      "step": 1395
+    },
+    {
+      "epoch": 14.917333333333334,
+      "eval_loss": 3.767644166946411,
+      "eval_runtime": 12.5131,
+      "eval_samples_per_second": 89.906,
+      "eval_steps_per_second": 15.024,
+      "step": 1395
     },
     {
-      "epoch": 2.985645933014354,
-      "step": 312,
-      "total_flos": 9149727375360.0,
-      "train_loss": 0.5307981822735224,
-      "train_runtime": 3368.0081,
-      "train_samples_per_second": 1.115,
-      "train_steps_per_second": 0.093
+      "epoch": 14.917333333333334,
+      "step": 1395,
+      "total_flos": 6.545011747156132e+17,
+      "train_loss": 4.23463837784251,
+      "train_runtime": 3346.972,
+      "train_samples_per_second": 20.159,
+      "train_steps_per_second": 0.417
     }
   ],
   "logging_steps": 5,
-  "max_steps": 312,
+  "max_steps": 1395,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 15,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -500,8 +2115,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 9149727375360.0,
-  "train_batch_size": 1,
+  "total_flos": 6.545011747156132e+17,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc535384880fd37a44458e5a0ed40df06e0cd47adbe05540cc8dc5625583ac8c
+oid sha256:83721718b2300c4998041c8f16898e602ab8bde0007f0d047de03da688b8be14
 size 7224