Model save

README.md

@@ -4,6 +4,7 @@ base_model: EleutherAI/pythia-2.8b-deduped
 tags:
 - trl
 - sft
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: pythia-2_8b_sft-gpt-turbo
@@ -35,9 +36,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 32
+- train_batch_size: 16
 - eval_batch_size: 4
 - seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
-    "total_flos": 7.533261039139123e+17,
-    "train_loss": 1.0007318160669976,
-    "train_runtime": 6142.5782,
-    "train_samples": 51577,
-    "train_samples_per_second": 8.397,
-    "train_steps_per_second": 0.262
+    "total_flos": 4.7710598211200614e+17,
+    "train_loss": 1.2026635146722562,
+    "train_runtime": 2875.3707,
+    "train_samples": 41954,
+    "train_samples_per_second": 14.591,
+    "train_steps_per_second": 0.456
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e51ee42459117c226841d3f881d312d056ddd2d28c9ffacf26e8fe4049c85b27
+oid sha256:18f01819f93756c7e3055d02f82a12514166e9002f375a6492205f19ba78ce59
 size 4978209248

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72dcf9a085323aee478eec6df877a9b49e719c2d24e57b5f087626f6e7495610
+oid sha256:208094d7e45292a7eed2812c6da4f68a27eb6cca0041fac99d416ae32fac0449
 size 572254856

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
-    "total_flos": 7.533261039139123e+17,
-    "train_loss": 1.0007318160669976,
-    "train_runtime": 6142.5782,
-    "train_samples": 51577,
-    "train_samples_per_second": 8.397,
-    "train_steps_per_second": 0.262
+    "total_flos": 4.7710598211200614e+17,
+    "train_loss": 1.2026635146722562,
+    "train_runtime": 2875.3707,
+    "train_samples": 41954,
+    "train_samples_per_second": 14.591,
+    "train_steps_per_second": 0.456
 }

trainer_state.json

@@ -3,1150 +3,940 @@
   "best_model_checkpoint": null,
   "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1612,
+  "global_step": 1312,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.00620347394540943,
-      "grad_norm": 4.46875,
-      "learning_rate": 1.234567901234568e-06,
-      "loss": 1.4227,
+      "epoch": 0.007621951219512195,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.5151515151515152e-06,
+      "loss": 1.6918,
       "step": 10
     },
     {
-      "epoch": 0.01240694789081886,
-      "grad_norm": 5.84375,
-      "learning_rate": 2.469135802469136e-06,
-      "loss": 1.4156,
+      "epoch": 0.01524390243902439,
+      "grad_norm": 5.0625,
+      "learning_rate": 3.0303030303030305e-06,
+      "loss": 1.6227,
       "step": 20
     },
     {
-      "epoch": 0.018610421836228287,
-      "grad_norm": 3.890625,
-      "learning_rate": 3.7037037037037037e-06,
-      "loss": 1.3094,
+      "epoch": 0.022865853658536585,
+      "grad_norm": 4.34375,
+      "learning_rate": 4.5454545454545455e-06,
+      "loss": 1.6367,
       "step": 30
     },
     {
-      "epoch": 0.02481389578163772,
-      "grad_norm": 4.03125,
-      "learning_rate": 4.938271604938272e-06,
-      "loss": 1.2648,
+      "epoch": 0.03048780487804878,
+      "grad_norm": 4.21875,
+      "learning_rate": 6.060606060606061e-06,
+      "loss": 1.493,
       "step": 40
     },
     {
-      "epoch": 0.031017369727047148,
+      "epoch": 0.038109756097560975,
       "grad_norm": 2.921875,
-      "learning_rate": 6.17283950617284e-06,
-      "loss": 1.1773,
+      "learning_rate": 7.5757575757575764e-06,
+      "loss": 1.441,
       "step": 50
     },
     {
-      "epoch": 0.03722084367245657,
-      "grad_norm": 2.5625,
-      "learning_rate": 7.4074074074074075e-06,
-      "loss": 1.1281,
+      "epoch": 0.04573170731707317,
+      "grad_norm": 3.078125,
+      "learning_rate": 9.090909090909091e-06,
+      "loss": 1.3488,
       "step": 60
     },
     {
-      "epoch": 0.043424317617866005,
-      "grad_norm": 2.390625,
-      "learning_rate": 8.641975308641975e-06,
-      "loss": 1.1273,
+      "epoch": 0.053353658536585365,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.0606060606060606e-05,
+      "loss": 1.3289,
       "step": 70
     },
     {
-      "epoch": 0.04962779156327544,
-      "grad_norm": 2.234375,
-      "learning_rate": 9.876543209876543e-06,
-      "loss": 1.0973,
+      "epoch": 0.06097560975609756,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.2121212121212122e-05,
+      "loss": 1.2742,
       "step": 80
     },
     {
-      "epoch": 0.05583126550868486,
-      "grad_norm": 2.359375,
-      "learning_rate": 1.1111111111111113e-05,
-      "loss": 1.084,
+      "epoch": 0.06859756097560976,
+      "grad_norm": 2.890625,
+      "learning_rate": 1.3636363636363637e-05,
+      "loss": 1.2758,
       "step": 90
     },
     {
-      "epoch": 0.062034739454094295,
-      "grad_norm": 2.796875,
-      "learning_rate": 1.234567901234568e-05,
-      "loss": 1.0523,
+      "epoch": 0.07621951219512195,
+      "grad_norm": 2.859375,
+      "learning_rate": 1.5151515151515153e-05,
+      "loss": 1.2707,
       "step": 100
     },
     {
-      "epoch": 0.06823821339950373,
-      "grad_norm": 2.375,
-      "learning_rate": 1.3580246913580248e-05,
-      "loss": 1.118,
+      "epoch": 0.08384146341463415,
+      "grad_norm": 2.890625,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 1.273,
       "step": 110
     },
     {
-      "epoch": 0.07444168734491315,
-      "grad_norm": 2.21875,
-      "learning_rate": 1.4814814814814815e-05,
-      "loss": 1.0496,
+      "epoch": 0.09146341463414634,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.8181818181818182e-05,
+      "loss": 1.2414,
       "step": 120
     },
     {
-      "epoch": 0.08064516129032258,
-      "grad_norm": 2.28125,
-      "learning_rate": 1.6049382716049385e-05,
-      "loss": 1.0219,
+      "epoch": 0.09908536585365854,
+      "grad_norm": 2.75,
+      "learning_rate": 1.96969696969697e-05,
+      "loss": 1.275,
       "step": 130
     },
     {
-      "epoch": 0.08684863523573201,
-      "grad_norm": 2.171875,
-      "learning_rate": 1.728395061728395e-05,
-      "loss": 1.0566,
+      "epoch": 0.10670731707317073,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.986440677966102e-05,
+      "loss": 1.2547,
       "step": 140
     },
     {
-      "epoch": 0.09305210918114144,
-      "grad_norm": 2.1875,
-      "learning_rate": 1.851851851851852e-05,
-      "loss": 1.0754,
+      "epoch": 0.11432926829268293,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.969491525423729e-05,
+      "loss": 1.2391,
       "step": 150
     },
     {
-      "epoch": 0.09925558312655088,
-      "grad_norm": 2.09375,
-      "learning_rate": 1.9753086419753087e-05,
-      "loss": 1.0496,
+      "epoch": 0.12195121951219512,
+      "grad_norm": 2.75,
+      "learning_rate": 1.9525423728813562e-05,
+      "loss": 1.2379,
       "step": 160
     },
     {
-      "epoch": 0.1054590570719603,
-      "grad_norm": 2.390625,
-      "learning_rate": 1.9889655172413794e-05,
-      "loss": 1.0562,
+      "epoch": 0.12957317073170732,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.9355932203389832e-05,
+      "loss": 1.2105,
       "step": 170
     },
     {
-      "epoch": 0.11166253101736973,
-      "grad_norm": 2.078125,
-      "learning_rate": 1.9751724137931037e-05,
-      "loss": 1.0219,
+      "epoch": 0.13719512195121952,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.9186440677966102e-05,
+      "loss": 1.2359,
       "step": 180
     },
     {
-      "epoch": 0.11786600496277916,
-      "grad_norm": 2.40625,
-      "learning_rate": 1.961379310344828e-05,
-      "loss": 1.0008,
+      "epoch": 0.1448170731707317,
+      "grad_norm": 2.75,
+      "learning_rate": 1.9016949152542375e-05,
+      "loss": 1.2867,
       "step": 190
     },
     {
-      "epoch": 0.12406947890818859,
-      "grad_norm": 2.25,
-      "learning_rate": 1.9475862068965518e-05,
-      "loss": 1.0355,
+      "epoch": 0.1524390243902439,
+      "grad_norm": 2.8125,
+      "learning_rate": 1.8847457627118645e-05,
+      "loss": 1.1762,
       "step": 200
     },
     {
-      "epoch": 0.13027295285359802,
-      "grad_norm": 2.234375,
-      "learning_rate": 1.933793103448276e-05,
-      "loss": 1.0281,
+      "epoch": 0.1600609756097561,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.8677966101694918e-05,
+      "loss": 1.1992,
       "step": 210
     },
     {
-      "epoch": 0.13647642679900746,
-      "grad_norm": 2.453125,
-      "learning_rate": 1.9200000000000003e-05,
-      "loss": 1.0484,
+      "epoch": 0.1676829268292683,
+      "grad_norm": 2.75,
+      "learning_rate": 1.8508474576271188e-05,
+      "loss": 1.232,
       "step": 220
     },
     {
-      "epoch": 0.14267990074441686,
-      "grad_norm": 2.390625,
-      "learning_rate": 1.906206896551724e-05,
-      "loss": 1.0082,
+      "epoch": 0.17530487804878048,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.8338983050847458e-05,
+      "loss": 1.2172,
       "step": 230
     },
     {
-      "epoch": 0.1488833746898263,
-      "grad_norm": 2.171875,
-      "learning_rate": 1.8924137931034484e-05,
-      "loss": 1.0145,
+      "epoch": 0.18292682926829268,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.816949152542373e-05,
+      "loss": 1.182,
       "step": 240
     },
     {
-      "epoch": 0.15508684863523572,
-      "grad_norm": 2.265625,
-      "learning_rate": 1.8786206896551726e-05,
-      "loss": 1.0531,
+      "epoch": 0.19054878048780488,
+      "grad_norm": 2.875,
+      "learning_rate": 1.8e-05,
+      "loss": 1.2492,
       "step": 250
     },
     {
-      "epoch": 0.16129032258064516,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.864827586206897e-05,
-      "loss": 0.993,
+      "epoch": 0.19817073170731708,
+      "grad_norm": 2.71875,
+      "learning_rate": 1.7830508474576274e-05,
+      "loss": 1.2312,
       "step": 260
     },
     {
-      "epoch": 0.1674937965260546,
-      "grad_norm": 2.265625,
-      "learning_rate": 1.8510344827586207e-05,
-      "loss": 1.0422,
+      "epoch": 0.20579268292682926,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.7661016949152543e-05,
+      "loss": 1.241,
       "step": 270
     },
     {
-      "epoch": 0.17369727047146402,
-      "grad_norm": 2.1875,
-      "learning_rate": 1.837241379310345e-05,
-      "loss": 1.0027,
+      "epoch": 0.21341463414634146,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.7491525423728813e-05,
+      "loss": 1.1879,
       "step": 280
     },
     {
-      "epoch": 0.17990074441687345,
-      "grad_norm": 2.0,
-      "learning_rate": 1.8234482758620692e-05,
-      "loss": 0.9922,
+      "epoch": 0.22103658536585366,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.7322033898305086e-05,
+      "loss": 1.2211,
       "step": 290
     },
     {
-      "epoch": 0.18610421836228289,
-      "grad_norm": 2.28125,
-      "learning_rate": 1.809655172413793e-05,
-      "loss": 1.0164,
+      "epoch": 0.22865853658536586,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.715254237288136e-05,
+      "loss": 1.2152,
       "step": 300
     },
     {
-      "epoch": 0.19230769230769232,
-      "grad_norm": 2.15625,
-      "learning_rate": 1.7958620689655173e-05,
-      "loss": 1.0242,
+      "epoch": 0.23628048780487804,
+      "grad_norm": 2.8125,
+      "learning_rate": 1.698305084745763e-05,
+      "loss": 1.1938,
       "step": 310
     },
     {
-      "epoch": 0.19851116625310175,
-      "grad_norm": 2.1875,
-      "learning_rate": 1.7820689655172416e-05,
-      "loss": 1.0602,
+      "epoch": 0.24390243902439024,
+      "grad_norm": 2.71875,
+      "learning_rate": 1.68135593220339e-05,
+      "loss": 1.2266,
       "step": 320
     },
     {
-      "epoch": 0.20471464019851116,
-      "grad_norm": 2.203125,
-      "learning_rate": 1.7682758620689658e-05,
-      "loss": 1.0082,
+      "epoch": 0.25152439024390244,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.6644067796610172e-05,
+      "loss": 1.1672,
       "step": 330
     },
     {
-      "epoch": 0.2109181141439206,
-      "grad_norm": 2.171875,
-      "learning_rate": 1.7544827586206897e-05,
-      "loss": 1.0031,
+      "epoch": 0.25914634146341464,
+      "grad_norm": 2.71875,
+      "learning_rate": 1.6474576271186442e-05,
+      "loss": 1.2104,
       "step": 340
     },
     {
-      "epoch": 0.21712158808933002,
-      "grad_norm": 2.4375,
-      "learning_rate": 1.740689655172414e-05,
-      "loss": 1.0031,
+      "epoch": 0.26676829268292684,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.6305084745762715e-05,
+      "loss": 1.2098,
       "step": 350
     },
     {
-      "epoch": 0.22332506203473945,
-      "grad_norm": 2.09375,
-      "learning_rate": 1.7268965517241382e-05,
-      "loss": 0.9684,
+      "epoch": 0.27439024390243905,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.6135593220338985e-05,
+      "loss": 1.1684,
       "step": 360
     },
     {
-      "epoch": 0.22952853598014888,
-      "grad_norm": 2.046875,
-      "learning_rate": 1.713103448275862e-05,
-      "loss": 0.984,
+      "epoch": 0.2820121951219512,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.5966101694915255e-05,
+      "loss": 1.1734,
       "step": 370
     },
     {
-      "epoch": 0.23573200992555832,
-      "grad_norm": 2.28125,
-      "learning_rate": 1.6993103448275863e-05,
-      "loss": 0.9828,
+      "epoch": 0.2896341463414634,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.5796610169491528e-05,
+      "loss": 1.2008,
       "step": 380
     },
     {
-      "epoch": 0.24193548387096775,
-      "grad_norm": 2.5,
-      "learning_rate": 1.6855172413793106e-05,
-      "loss": 1.0297,
+      "epoch": 0.2972560975609756,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.5627118644067798e-05,
+      "loss": 1.1848,
       "step": 390
     },
     {
-      "epoch": 0.24813895781637718,
-      "grad_norm": 1.8828125,
-      "learning_rate": 1.6717241379310348e-05,
-      "loss": 0.9895,
+      "epoch": 0.3048780487804878,
+      "grad_norm": 2.625,
+      "learning_rate": 1.545762711864407e-05,
+      "loss": 1.2033,
       "step": 400
     },
     {
-      "epoch": 0.2543424317617866,
-      "grad_norm": 2.140625,
-      "learning_rate": 1.6579310344827587e-05,
-      "loss": 1.0195,
+      "epoch": 0.3125,
+      "grad_norm": 2.625,
+      "learning_rate": 1.528813559322034e-05,
+      "loss": 1.1781,
       "step": 410
     },
     {
-      "epoch": 0.26054590570719605,
-      "grad_norm": 2.0625,
-      "learning_rate": 1.644137931034483e-05,
-      "loss": 0.9945,
+      "epoch": 0.3201219512195122,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.511864406779661e-05,
+      "loss": 1.2043,
       "step": 420
     },
     {
-      "epoch": 0.2667493796526055,
-      "grad_norm": 2.078125,
-      "learning_rate": 1.630344827586207e-05,
-      "loss": 1.0043,
+      "epoch": 0.3277439024390244,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.4949152542372882e-05,
+      "loss": 1.1699,
       "step": 430
     },
     {
-      "epoch": 0.2729528535980149,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.616551724137931e-05,
-      "loss": 1.0293,
+      "epoch": 0.3353658536585366,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.4779661016949153e-05,
+      "loss": 1.1871,
       "step": 440
     },
     {
-      "epoch": 0.27915632754342434,
-      "grad_norm": 2.265625,
-      "learning_rate": 1.6027586206896553e-05,
-      "loss": 1.0043,
+      "epoch": 0.3429878048780488,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.4610169491525426e-05,
+      "loss": 1.1418,
       "step": 450
     },
     {
-      "epoch": 0.2853598014888337,
-      "grad_norm": 2.203125,
-      "learning_rate": 1.5889655172413795e-05,
-      "loss": 1.0004,
+      "epoch": 0.35060975609756095,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.4440677966101698e-05,
+      "loss": 1.1824,
       "step": 460
     },
     {
-      "epoch": 0.29156327543424315,
-      "grad_norm": 2.40625,
-      "learning_rate": 1.5751724137931034e-05,
-      "loss": 1.0094,
+      "epoch": 0.35823170731707316,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.4271186440677966e-05,
+      "loss": 1.176,
       "step": 470
     },
     {
-      "epoch": 0.2977667493796526,
-      "grad_norm": 2.203125,
-      "learning_rate": 1.5613793103448276e-05,
-      "loss": 0.9801,
+      "epoch": 0.36585365853658536,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.4101694915254239e-05,
+      "loss": 1.1785,
       "step": 480
     },
     {
-      "epoch": 0.303970223325062,
-      "grad_norm": 2.359375,
-      "learning_rate": 1.547586206896552e-05,
-      "loss": 1.009,
+      "epoch": 0.37347560975609756,
+      "grad_norm": 2.796875,
+      "learning_rate": 1.393220338983051e-05,
+      "loss": 1.1758,
       "step": 490
     },
     {
-      "epoch": 0.31017369727047145,
-      "grad_norm": 2.375,
-      "learning_rate": 1.533793103448276e-05,
-      "loss": 1.0008,
+      "epoch": 0.38109756097560976,
+      "grad_norm": 2.75,
+      "learning_rate": 1.3762711864406782e-05,
+      "loss": 1.1988,
       "step": 500
     },
     {
-      "epoch": 0.3163771712158809,
-      "grad_norm": 2.078125,
-      "learning_rate": 1.5200000000000002e-05,
-      "loss": 0.9852,
+      "epoch": 0.38871951219512196,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.3593220338983053e-05,
+      "loss": 1.1822,
       "step": 510
     },
     {
-      "epoch": 0.3225806451612903,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.5062068965517243e-05,
-      "loss": 0.9887,
+      "epoch": 0.39634146341463417,
+      "grad_norm": 2.953125,
+      "learning_rate": 1.3423728813559323e-05,
+      "loss": 1.1801,
       "step": 520
     },
     {
-      "epoch": 0.32878411910669975,
-      "grad_norm": 2.3125,
-      "learning_rate": 1.4924137931034483e-05,
-      "loss": 1.0188,
+      "epoch": 0.40396341463414637,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.3254237288135595e-05,
+      "loss": 1.2043,
       "step": 530
     },
     {
-      "epoch": 0.3349875930521092,
-      "grad_norm": 2.4375,
-      "learning_rate": 1.4786206896551726e-05,
-      "loss": 0.9668,
+      "epoch": 0.4115853658536585,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.3084745762711866e-05,
+      "loss": 1.2098,
       "step": 540
     },
     {
-      "epoch": 0.3411910669975186,
-      "grad_norm": 2.125,
-      "learning_rate": 1.4648275862068966e-05,
-      "loss": 1.0008,
+      "epoch": 0.4192073170731707,
+      "grad_norm": 2.625,
+      "learning_rate": 1.2915254237288137e-05,
+      "loss": 1.1973,
       "step": 550
     },
     {
-      "epoch": 0.34739454094292804,
-      "grad_norm": 2.125,
-      "learning_rate": 1.4510344827586209e-05,
-      "loss": 0.9785,
+      "epoch": 0.4268292682926829,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.2745762711864407e-05,
+      "loss": 1.2066,
       "step": 560
     },
     {
-      "epoch": 0.3535980148883375,
-      "grad_norm": 2.296875,
-      "learning_rate": 1.4372413793103451e-05,
-      "loss": 1.0262,
+      "epoch": 0.4344512195121951,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.2576271186440679e-05,
+      "loss": 1.1543,
       "step": 570
     },
     {
-      "epoch": 0.3598014888337469,
-      "grad_norm": 2.125,
-      "learning_rate": 1.4234482758620692e-05,
-      "loss": 0.9633,
+      "epoch": 0.4420731707317073,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.240677966101695e-05,
+      "loss": 1.1703,
       "step": 580
     },
     {
-      "epoch": 0.36600496277915634,
-      "grad_norm": 2.046875,
-      "learning_rate": 1.4096551724137932e-05,
-      "loss": 1.0066,
+      "epoch": 0.4496951219512195,
+      "grad_norm": 3.09375,
+      "learning_rate": 1.2237288135593222e-05,
+      "loss": 1.1566,
       "step": 590
     },
     {
-      "epoch": 0.37220843672456577,
-      "grad_norm": 2.390625,
-      "learning_rate": 1.3958620689655173e-05,
-      "loss": 0.9812,
+      "epoch": 0.4573170731707317,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.2067796610169493e-05,
+      "loss": 1.1623,
       "step": 600
     },
     {
-      "epoch": 0.3784119106699752,
-      "grad_norm": 1.9921875,
-      "learning_rate": 1.3820689655172415e-05,
-      "loss": 0.9266,
+      "epoch": 0.4649390243902439,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.1898305084745763e-05,
+      "loss": 1.198,
       "step": 610
     },
     {
-      "epoch": 0.38461538461538464,
-      "grad_norm": 2.28125,
-      "learning_rate": 1.3682758620689656e-05,
-      "loss": 1.0223,
+      "epoch": 0.4725609756097561,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.1728813559322034e-05,
+      "loss": 1.1971,
       "step": 620
     },
     {
-      "epoch": 0.39081885856079407,
-      "grad_norm": 2.1875,
-      "learning_rate": 1.3544827586206896e-05,
-      "loss": 0.9871,
+      "epoch": 0.4801829268292683,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.1559322033898306e-05,
+      "loss": 1.159,
       "step": 630
     },
     {
-      "epoch": 0.3970223325062035,
-      "grad_norm": 2.4375,
-      "learning_rate": 1.340689655172414e-05,
-      "loss": 1.0594,
+      "epoch": 0.4878048780487805,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.1389830508474577e-05,
+      "loss": 1.184,
       "step": 640
     },
     {
-      "epoch": 0.4032258064516129,
-      "grad_norm": 2.34375,
-      "learning_rate": 1.3268965517241381e-05,
-      "loss": 0.9625,
+      "epoch": 0.4954268292682927,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.1220338983050849e-05,
+      "loss": 1.1859,
       "step": 650
     },
     {
-      "epoch": 0.4094292803970223,
-      "grad_norm": 2.359375,
-      "learning_rate": 1.3131034482758622e-05,
-      "loss": 1.0172,
+      "epoch": 0.5030487804878049,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.1050847457627118e-05,
+      "loss": 1.1434,
       "step": 660
     },
     {
-      "epoch": 0.41563275434243174,
-      "grad_norm": 2.5,
-      "learning_rate": 1.2993103448275863e-05,
-      "loss": 1.0176,
+      "epoch": 0.510670731707317,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.088135593220339e-05,
+      "loss": 1.1775,
       "step": 670
     },
     {
-      "epoch": 0.4218362282878412,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.2855172413793105e-05,
-      "loss": 0.9344,
+      "epoch": 0.5182926829268293,
+      "grad_norm": 2.515625,
+      "learning_rate": 1.0711864406779661e-05,
+      "loss": 1.15,
       "step": 680
     },
     {
-      "epoch": 0.4280397022332506,
-      "grad_norm": 2.15625,
-      "learning_rate": 1.2717241379310346e-05,
-      "loss": 0.9543,
+      "epoch": 0.5259146341463414,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.0542372881355933e-05,
+      "loss": 1.1613,
       "step": 690
     },
     {
-      "epoch": 0.43424317617866004,
-      "grad_norm": 2.46875,
-      "learning_rate": 1.2579310344827586e-05,
-      "loss": 0.9949,
+      "epoch": 0.5335365853658537,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.0372881355932204e-05,
+      "loss": 1.1602,
       "step": 700
     },
     {
-      "epoch": 0.44044665012406947,
-      "grad_norm": 1.96875,
-      "learning_rate": 1.2441379310344829e-05,
-      "loss": 0.9859,
+      "epoch": 0.5411585365853658,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.0203389830508474e-05,
+      "loss": 1.1879,
       "step": 710
     },
     {
-      "epoch": 0.4466501240694789,
-      "grad_norm": 2.46875,
-      "learning_rate": 1.2303448275862071e-05,
-      "loss": 1.0063,
+      "epoch": 0.5487804878048781,
+      "grad_norm": 2.625,
+      "learning_rate": 1.0033898305084746e-05,
+      "loss": 1.1941,
       "step": 720
     },
     {
-      "epoch": 0.45285359801488834,
-      "grad_norm": 2.078125,
-      "learning_rate": 1.2165517241379312e-05,
-      "loss": 0.959,
+      "epoch": 0.5564024390243902,
+      "grad_norm": 2.8125,
+      "learning_rate": 9.864406779661017e-06,
+      "loss": 1.1574,
       "step": 730
     },
     {
-      "epoch": 0.45905707196029777,
-      "grad_norm": 2.140625,
-      "learning_rate": 1.2027586206896554e-05,
-      "loss": 0.9574,
+      "epoch": 0.5640243902439024,
+      "grad_norm": 2.75,
+      "learning_rate": 9.69491525423729e-06,
+      "loss": 1.1793,
       "step": 740
     },
     {
-      "epoch": 0.4652605459057072,
-      "grad_norm": 2.171875,
-      "learning_rate": 1.1889655172413795e-05,
-      "loss": 0.9863,
+      "epoch": 0.5716463414634146,
+      "grad_norm": 2.703125,
+      "learning_rate": 9.52542372881356e-06,
+      "loss": 1.1711,
       "step": 750
     },
     {
-      "epoch": 0.47146401985111663,
-      "grad_norm": 2.09375,
-      "learning_rate": 1.1751724137931035e-05,
-      "loss": 0.9859,
+      "epoch": 0.5792682926829268,
+      "grad_norm": 2.703125,
+      "learning_rate": 9.355932203389831e-06,
+      "loss": 1.1875,
       "step": 760
     },
     {
-      "epoch": 0.47766749379652607,
-      "grad_norm": 2.390625,
-      "learning_rate": 1.1613793103448276e-05,
-      "loss": 0.9984,
+      "epoch": 0.586890243902439,
+      "grad_norm": 2.8125,
+      "learning_rate": 9.186440677966101e-06,
+      "loss": 1.19,
       "step": 770
     },
     {
-      "epoch": 0.4838709677419355,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.1475862068965518e-05,
-      "loss": 0.9984,
+      "epoch": 0.5945121951219512,
+      "grad_norm": 2.9375,
+      "learning_rate": 9.016949152542374e-06,
+      "loss": 1.141,
       "step": 780
     },
     {
-      "epoch": 0.49007444168734493,
-      "grad_norm": 2.078125,
-      "learning_rate": 1.1337931034482759e-05,
-      "loss": 0.9402,
+      "epoch": 0.6021341463414634,
+      "grad_norm": 2.75,
+      "learning_rate": 8.847457627118646e-06,
+      "loss": 1.1699,
       "step": 790
     },
     {
-      "epoch": 0.49627791563275436,
-      "grad_norm": 2.078125,
-      "learning_rate": 1.1200000000000001e-05,
-      "loss": 0.9902,
+      "epoch": 0.6097560975609756,
+      "grad_norm": 2.65625,
+      "learning_rate": 8.677966101694915e-06,
+      "loss": 1.1809,
       "step": 800
     },
     {
-      "epoch": 0.5024813895781638,
-      "grad_norm": 2.46875,
-      "learning_rate": 1.1062068965517244e-05,
-      "loss": 1.0008,
+      "epoch": 0.6173780487804879,
+      "grad_norm": 2.703125,
+      "learning_rate": 8.508474576271187e-06,
+      "loss": 1.1824,
       "step": 810
     },
     {
-      "epoch": 0.5086848635235732,
-      "grad_norm": 2.09375,
-      "learning_rate": 1.0924137931034484e-05,
-      "loss": 0.9727,
+      "epoch": 0.625,
+      "grad_norm": 2.796875,
+      "learning_rate": 8.338983050847458e-06,
+      "loss": 1.1729,
       "step": 820
     },
     {
-      "epoch": 0.5148883374689827,
-      "grad_norm": 2.53125,
-      "learning_rate": 1.0786206896551725e-05,
-      "loss": 0.9672,
+      "epoch": 0.6326219512195121,
+      "grad_norm": 2.765625,
+      "learning_rate": 8.16949152542373e-06,
+      "loss": 1.1113,
       "step": 830
     },
     {
-      "epoch": 0.5210918114143921,
-      "grad_norm": 2.203125,
-      "learning_rate": 1.0648275862068967e-05,
-      "loss": 0.9371,
+      "epoch": 0.6402439024390244,
+      "grad_norm": 2.5,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 1.1434,
       "step": 840
     },
     {
-      "epoch": 0.5272952853598015,
-      "grad_norm": 2.015625,
-      "learning_rate": 1.0510344827586208e-05,
-      "loss": 1.0145,
+      "epoch": 0.6478658536585366,
+      "grad_norm": 2.515625,
+      "learning_rate": 7.830508474576271e-06,
+      "loss": 1.1643,
       "step": 850
     },
     {
-      "epoch": 0.533498759305211,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.0372413793103449e-05,
-      "loss": 0.9672,
+      "epoch": 0.6554878048780488,
+      "grad_norm": 2.65625,
+      "learning_rate": 7.661016949152543e-06,
+      "loss": 1.1738,
       "step": 860
     },
     {
-      "epoch": 0.5397022332506204,
-      "grad_norm": 2.40625,
-      "learning_rate": 1.023448275862069e-05,
-      "loss": 0.9602,
+      "epoch": 0.663109756097561,
+      "grad_norm": 2.609375,
+      "learning_rate": 7.491525423728814e-06,
+      "loss": 1.125,
       "step": 870
     },
     {
-      "epoch": 0.5459057071960298,
-      "grad_norm": 2.203125,
-      "learning_rate": 1.0096551724137933e-05,
-      "loss": 0.9871,
+      "epoch": 0.6707317073170732,
+      "grad_norm": 2.5625,
+      "learning_rate": 7.3220338983050855e-06,
+      "loss": 1.1723,
       "step": 880
     },
     {
-      "epoch": 0.5521091811414393,
-      "grad_norm": 2.359375,
-      "learning_rate": 9.958620689655172e-06,
-      "loss": 0.9797,
+      "epoch": 0.6783536585365854,
+      "grad_norm": 2.546875,
+      "learning_rate": 7.152542372881357e-06,
+      "loss": 1.1617,
       "step": 890
     },
     {
-      "epoch": 0.5583126550868487,
-      "grad_norm": 2.390625,
-      "learning_rate": 9.820689655172415e-06,
-      "loss": 0.9848,
+      "epoch": 0.6859756097560976,
+      "grad_norm": 2.6875,
+      "learning_rate": 6.9830508474576275e-06,
+      "loss": 1.1445,
       "step": 900
     },
     {
-      "epoch": 0.5645161290322581,
-      "grad_norm": 2.390625,
-      "learning_rate": 9.682758620689657e-06,
-      "loss": 0.9699,
+      "epoch": 0.6935975609756098,
+      "grad_norm": 2.546875,
+      "learning_rate": 6.813559322033899e-06,
+      "loss": 1.1807,
       "step": 910
     },
     {
-      "epoch": 0.5707196029776674,
-      "grad_norm": 2.234375,
-      "learning_rate": 9.544827586206898e-06,
-      "loss": 0.9641,
+      "epoch": 0.7012195121951219,
+      "grad_norm": 2.65625,
+      "learning_rate": 6.64406779661017e-06,
+      "loss": 1.1865,
       "step": 920
     },
     {
-      "epoch": 0.5769230769230769,
-      "grad_norm": 2.53125,
-      "learning_rate": 9.406896551724138e-06,
-      "loss": 1.0109,
+      "epoch": 0.7088414634146342,
+      "grad_norm": 2.734375,
+      "learning_rate": 6.474576271186441e-06,
+      "loss": 1.198,
       "step": 930
     },
     {
-      "epoch": 0.5831265508684863,
-      "grad_norm": 1.9765625,
-      "learning_rate": 9.26896551724138e-06,
-      "loss": 0.9906,
+      "epoch": 0.7164634146341463,
+      "grad_norm": 2.765625,
+      "learning_rate": 6.3050847457627125e-06,
+      "loss": 1.1734,
       "step": 940
     },
     {
-      "epoch": 0.5893300248138957,
-      "grad_norm": 2.34375,
-      "learning_rate": 9.131034482758621e-06,
-      "loss": 0.9938,
+      "epoch": 0.7240853658536586,
+      "grad_norm": 2.609375,
+      "learning_rate": 6.135593220338983e-06,
+      "loss": 1.1633,
       "step": 950
     },
     {
-      "epoch": 0.5955334987593052,
-      "grad_norm": 2.046875,
-      "learning_rate": 8.993103448275864e-06,
-      "loss": 0.923,
+      "epoch": 0.7317073170731707,
+      "grad_norm": 2.578125,
+      "learning_rate": 5.9661016949152555e-06,
+      "loss": 1.1953,
       "step": 960
     },
     {
-      "epoch": 0.6017369727047146,
-      "grad_norm": 2.0625,
-      "learning_rate": 8.855172413793104e-06,
-      "loss": 0.9766,
+      "epoch": 0.739329268292683,
+      "grad_norm": 2.5,
+      "learning_rate": 5.796610169491525e-06,
+      "loss": 1.1879,
       "step": 970
     },
     {
-      "epoch": 0.607940446650124,
-      "grad_norm": 2.25,
-      "learning_rate": 8.717241379310347e-06,
-      "loss": 0.9832,
+      "epoch": 0.7469512195121951,
+      "grad_norm": 2.609375,
+      "learning_rate": 5.6271186440677975e-06,
+      "loss": 1.1664,
       "step": 980
     },
     {
-      "epoch": 0.6141439205955335,
-      "grad_norm": 2.15625,
-      "learning_rate": 8.579310344827587e-06,
-      "loss": 0.9184,
+      "epoch": 0.7545731707317073,
+      "grad_norm": 2.765625,
+      "learning_rate": 5.457627118644067e-06,
+      "loss": 1.1789,
       "step": 990
     },
     {
-      "epoch": 0.6203473945409429,
-      "grad_norm": 2.359375,
-      "learning_rate": 8.441379310344828e-06,
-      "loss": 0.9367,
+      "epoch": 0.7621951219512195,
+      "grad_norm": 2.890625,
+      "learning_rate": 5.28813559322034e-06,
+      "loss": 1.1738,
       "step": 1000
     },
     {
-      "epoch": 0.6265508684863523,
-      "grad_norm": 2.40625,
-      "learning_rate": 8.30344827586207e-06,
-      "loss": 0.9969,
+      "epoch": 0.7698170731707317,
+      "grad_norm": 2.6875,
+      "learning_rate": 5.118644067796611e-06,
+      "loss": 1.1582,
       "step": 1010
     },
     {
-      "epoch": 0.6327543424317618,
-      "grad_norm": 2.4375,
-      "learning_rate": 8.165517241379311e-06,
-      "loss": 0.9941,
+      "epoch": 0.7774390243902439,
+      "grad_norm": 2.9375,
+      "learning_rate": 4.949152542372882e-06,
+      "loss": 1.1391,
       "step": 1020
     },
     {
-      "epoch": 0.6389578163771712,
-      "grad_norm": 2.0625,
-      "learning_rate": 8.027586206896553e-06,
-      "loss": 0.9363,
+      "epoch": 0.7850609756097561,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.779661016949153e-06,
+      "loss": 1.1652,
       "step": 1030
     },
     {
-      "epoch": 0.6451612903225806,
-      "grad_norm": 2.3125,
-      "learning_rate": 7.889655172413794e-06,
-      "loss": 0.9961,
+      "epoch": 0.7926829268292683,
+      "grad_norm": 2.703125,
+      "learning_rate": 4.610169491525424e-06,
+      "loss": 1.1426,
       "step": 1040
     },
     {
-      "epoch": 0.6513647642679901,
-      "grad_norm": 1.9140625,
-      "learning_rate": 7.751724137931035e-06,
-      "loss": 0.9738,
+      "epoch": 0.8003048780487805,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.440677966101695e-06,
+      "loss": 1.1713,
       "step": 1050
     },
     {
-      "epoch": 0.6575682382133995,
-      "grad_norm": 2.296875,
-      "learning_rate": 7.613793103448276e-06,
-      "loss": 0.9641,
+      "epoch": 0.8079268292682927,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.271186440677967e-06,
+      "loss": 1.1727,
       "step": 1060
     },
     {
-      "epoch": 0.6637717121588089,
-      "grad_norm": 2.1875,
-      "learning_rate": 7.475862068965518e-06,
-      "loss": 0.9797,
+      "epoch": 0.8155487804878049,
+      "grad_norm": 2.59375,
+      "learning_rate": 4.101694915254237e-06,
+      "loss": 1.1695,
       "step": 1070
     },
     {
-      "epoch": 0.6699751861042184,
-      "grad_norm": 2.296875,
-      "learning_rate": 7.337931034482759e-06,
-      "loss": 0.9379,
+      "epoch": 0.823170731707317,
+      "grad_norm": 2.671875,
+      "learning_rate": 3.932203389830509e-06,
+      "loss": 1.1672,
       "step": 1080
     },
     {
-      "epoch": 0.6761786600496278,
-      "grad_norm": 2.015625,
-      "learning_rate": 7.2000000000000005e-06,
-      "loss": 0.952,
+      "epoch": 0.8307926829268293,
+      "grad_norm": 2.734375,
+      "learning_rate": 3.76271186440678e-06,
+      "loss": 1.2016,
       "step": 1090
     },
     {
-      "epoch": 0.6823821339950372,
-      "grad_norm": 2.046875,
-      "learning_rate": 7.062068965517241e-06,
-      "loss": 0.9668,
+      "epoch": 0.8384146341463414,
+      "grad_norm": 2.90625,
+      "learning_rate": 3.5932203389830512e-06,
+      "loss": 1.1727,
       "step": 1100
     },
     {
-      "epoch": 0.6885856079404467,
-      "grad_norm": 2.328125,
-      "learning_rate": 6.9241379310344835e-06,
-      "loss": 0.9824,
+      "epoch": 0.8460365853658537,
+      "grad_norm": 2.84375,
+      "learning_rate": 3.4237288135593223e-06,
+      "loss": 1.1799,
       "step": 1110
     },
     {
-      "epoch": 0.6947890818858561,
-      "grad_norm": 2.296875,
-      "learning_rate": 6.786206896551725e-06,
-      "loss": 0.9926,
+      "epoch": 0.8536585365853658,
+      "grad_norm": 2.8125,
+      "learning_rate": 3.2542372881355933e-06,
+      "loss": 1.1555,
       "step": 1120
     },
     {
-      "epoch": 0.7009925558312655,
-      "grad_norm": 1.78125,
-      "learning_rate": 6.648275862068966e-06,
-      "loss": 0.934,
+      "epoch": 0.8612804878048781,
+      "grad_norm": 2.6875,
+      "learning_rate": 3.0847457627118648e-06,
+      "loss": 1.1678,
       "step": 1130
     },
     {
-      "epoch": 0.707196029776675,
-      "grad_norm": 2.265625,
-      "learning_rate": 6.510344827586207e-06,
-      "loss": 0.9621,
+      "epoch": 0.8689024390243902,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.915254237288136e-06,
+      "loss": 1.1578,
       "step": 1140
     },
     {
-      "epoch": 0.7133995037220844,
-      "grad_norm": 1.9765625,
-      "learning_rate": 6.372413793103449e-06,
-      "loss": 0.982,
+      "epoch": 0.8765243902439024,
+      "grad_norm": 2.375,
+      "learning_rate": 2.745762711864407e-06,
+      "loss": 1.1535,
       "step": 1150
     },
     {
-      "epoch": 0.7196029776674938,
-      "grad_norm": 2.09375,
-      "learning_rate": 6.23448275862069e-06,
-      "loss": 0.9883,
+      "epoch": 0.8841463414634146,
+      "grad_norm": 2.703125,
+      "learning_rate": 2.576271186440678e-06,
+      "loss": 1.1602,
       "step": 1160
     },
     {
-      "epoch": 0.7258064516129032,
-      "grad_norm": 2.40625,
-      "learning_rate": 6.096551724137932e-06,
-      "loss": 0.9762,
+      "epoch": 0.8917682926829268,
+      "grad_norm": 2.78125,
+      "learning_rate": 2.4067796610169493e-06,
+      "loss": 1.1852,
       "step": 1170
     },
     {
-      "epoch": 0.7320099255583127,
-      "grad_norm": 2.15625,
-      "learning_rate": 5.958620689655172e-06,
-      "loss": 0.977,
+      "epoch": 0.899390243902439,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.2372881355932204e-06,
+      "loss": 1.1605,
       "step": 1180
     },
     {
-      "epoch": 0.7382133995037221,
-      "grad_norm": 2.171875,
-      "learning_rate": 5.820689655172415e-06,
-      "loss": 1.0129,
+      "epoch": 0.9070121951219512,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.0677966101694914e-06,
+      "loss": 1.1451,
       "step": 1190
     },
     {
-      "epoch": 0.7444168734491315,
-      "grad_norm": 2.25,
-      "learning_rate": 5.682758620689655e-06,
-      "loss": 0.9887,
+      "epoch": 0.9146341463414634,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.8983050847457629e-06,
+      "loss": 1.1496,
       "step": 1200
     },
     {
-      "epoch": 0.750620347394541,
-      "grad_norm": 2.140625,
-      "learning_rate": 5.544827586206897e-06,
-      "loss": 0.9516,
+      "epoch": 0.9222560975609756,
+      "grad_norm": 2.859375,
+      "learning_rate": 1.728813559322034e-06,
+      "loss": 1.1582,
       "step": 1210
     },
     {
-      "epoch": 0.7568238213399504,
-      "grad_norm": 2.171875,
-      "learning_rate": 5.4068965517241375e-06,
-      "loss": 0.952,
+      "epoch": 0.9298780487804879,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.5593220338983054e-06,
+      "loss": 1.227,
       "step": 1220
     },
     {
-      "epoch": 0.7630272952853598,
-      "grad_norm": 2.28125,
-      "learning_rate": 5.26896551724138e-06,
-      "loss": 0.9922,
+      "epoch": 0.9375,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.3898305084745764e-06,
+      "loss": 1.1977,
       "step": 1230
     },
     {
-      "epoch": 0.7692307692307693,
-      "grad_norm": 1.9375,
-      "learning_rate": 5.131034482758621e-06,
-      "loss": 0.9914,
+      "epoch": 0.9451219512195121,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.2203389830508477e-06,
+      "loss": 1.1885,
       "step": 1240
     },
     {
-      "epoch": 0.7754342431761787,
-      "grad_norm": 2.28125,
-      "learning_rate": 4.993103448275862e-06,
-      "loss": 0.9449,
+      "epoch": 0.9527439024390244,
+      "grad_norm": 2.6875,
+      "learning_rate": 1.0508474576271187e-06,
+      "loss": 1.1549,
       "step": 1250
     },
     {
-      "epoch": 0.7816377171215881,
-      "grad_norm": 2.359375,
-      "learning_rate": 4.8551724137931035e-06,
-      "loss": 1.0051,
+      "epoch": 0.9603658536585366,
+      "grad_norm": 2.53125,
+      "learning_rate": 8.813559322033899e-07,
+      "loss": 1.1781,
       "step": 1260
     },
     {
-      "epoch": 0.7878411910669976,
-      "grad_norm": 2.15625,
-      "learning_rate": 4.717241379310345e-06,
-      "loss": 0.9926,
+      "epoch": 0.9679878048780488,
+      "grad_norm": 2.5625,
+      "learning_rate": 7.118644067796611e-07,
+      "loss": 1.1957,
       "step": 1270
     },
     {
-      "epoch": 0.794044665012407,
-      "grad_norm": 2.109375,
-      "learning_rate": 4.5793103448275865e-06,
-      "loss": 0.9477,
+      "epoch": 0.975609756097561,
+      "grad_norm": 2.640625,
+      "learning_rate": 5.423728813559322e-07,
+      "loss": 1.1898,
       "step": 1280
     },
     {
-      "epoch": 0.8002481389578163,
-      "grad_norm": 2.21875,
-      "learning_rate": 4.441379310344828e-06,
-      "loss": 0.9477,
+      "epoch": 0.9832317073170732,
+      "grad_norm": 2.625,
+      "learning_rate": 3.7288135593220347e-07,
+      "loss": 1.1531,
       "step": 1290
     },
     {
-      "epoch": 0.8064516129032258,
-      "grad_norm": 2.03125,
-      "learning_rate": 4.3034482758620695e-06,
-      "loss": 0.9543,
+      "epoch": 0.9908536585365854,
+      "grad_norm": 2.6875,
+      "learning_rate": 2.0338983050847458e-07,
+      "loss": 1.1773,
       "step": 1300
     },
     {
-      "epoch": 0.8126550868486352,
-      "grad_norm": 2.3125,
-      "learning_rate": 4.16551724137931e-06,
-      "loss": 0.9492,
+      "epoch": 0.9984756097560976,
+      "grad_norm": 2.640625,
+      "learning_rate": 3.3898305084745764e-08,
+      "loss": 1.0754,
       "step": 1310
     },
-    {
-      "epoch": 0.8188585607940446,
-      "grad_norm": 2.421875,
-      "learning_rate": 4.027586206896552e-06,
-      "loss": 0.9809,
-      "step": 1320
-    },
-    {
-      "epoch": 0.825062034739454,
-      "grad_norm": 2.390625,
-      "learning_rate": 3.889655172413793e-06,
-      "loss": 0.9812,
-      "step": 1330
-    },
-    {
-      "epoch": 0.8312655086848635,
-      "grad_norm": 2.109375,
-      "learning_rate": 3.7517241379310347e-06,
-      "loss": 0.9781,
-      "step": 1340
-    },
-    {
-      "epoch": 0.8374689826302729,
-      "grad_norm": 2.234375,
-      "learning_rate": 3.6137931034482758e-06,
-      "loss": 0.9754,
-      "step": 1350
-    },
-    {
-      "epoch": 0.8436724565756824,
-      "grad_norm": 2.109375,
-      "learning_rate": 3.4758620689655177e-06,
-      "loss": 0.9641,
-      "step": 1360
-    },
-    {
-      "epoch": 0.8498759305210918,
-      "grad_norm": 2.671875,
-      "learning_rate": 3.3379310344827588e-06,
-      "loss": 0.9723,
-      "step": 1370
-    },
-    {
-      "epoch": 0.8560794044665012,
-      "grad_norm": 2.375,
-      "learning_rate": 3.2000000000000003e-06,
-      "loss": 0.9938,
-      "step": 1380
-    },
-    {
-      "epoch": 0.8622828784119106,
-      "grad_norm": 2.359375,
-      "learning_rate": 3.0620689655172414e-06,
-      "loss": 0.9523,
-      "step": 1390
-    },
-    {
-      "epoch": 0.8684863523573201,
-      "grad_norm": 1.9609375,
-      "learning_rate": 2.924137931034483e-06,
-      "loss": 0.9508,
-      "step": 1400
-    },
-    {
-      "epoch": 0.8746898263027295,
-      "grad_norm": 2.28125,
-      "learning_rate": 2.7862068965517244e-06,
-      "loss": 0.9512,
-      "step": 1410
-    },
-    {
-      "epoch": 0.8808933002481389,
-      "grad_norm": 2.109375,
-      "learning_rate": 2.648275862068966e-06,
-      "loss": 0.9547,
-      "step": 1420
-    },
-    {
-      "epoch": 0.8870967741935484,
-      "grad_norm": 2.28125,
-      "learning_rate": 2.510344827586207e-06,
-      "loss": 0.9539,
-      "step": 1430
-    },
-    {
-      "epoch": 0.8933002481389578,
-      "grad_norm": 2.265625,
-      "learning_rate": 2.3724137931034484e-06,
-      "loss": 0.9879,
-      "step": 1440
-    },
-    {
-      "epoch": 0.8995037220843672,
-      "grad_norm": 2.40625,
-      "learning_rate": 2.23448275862069e-06,
-      "loss": 0.9684,
-      "step": 1450
-    },
-    {
-      "epoch": 0.9057071960297767,
-      "grad_norm": 2.09375,
-      "learning_rate": 2.0965517241379315e-06,
-      "loss": 0.9348,
-      "step": 1460
-    },
-    {
-      "epoch": 0.9119106699751861,
-      "grad_norm": 2.25,
-      "learning_rate": 1.9586206896551725e-06,
-      "loss": 0.9711,
-      "step": 1470
-    },
-    {
-      "epoch": 0.9181141439205955,
-      "grad_norm": 2.328125,
-      "learning_rate": 1.820689655172414e-06,
-      "loss": 0.9602,
-      "step": 1480
-    },
-    {
-      "epoch": 0.924317617866005,
-      "grad_norm": 2.234375,
-      "learning_rate": 1.6827586206896553e-06,
-      "loss": 1.0289,
-      "step": 1490
-    },
-    {
-      "epoch": 0.9305210918114144,
-      "grad_norm": 2.359375,
-      "learning_rate": 1.5448275862068968e-06,
-      "loss": 0.9785,
-      "step": 1500
-    },
-    {
-      "epoch": 0.9367245657568238,
-      "grad_norm": 2.4375,
-      "learning_rate": 1.4068965517241381e-06,
-      "loss": 1.0277,
-      "step": 1510
-    },
-    {
-      "epoch": 0.9429280397022333,
-      "grad_norm": 2.109375,
-      "learning_rate": 1.2689655172413794e-06,
-      "loss": 0.982,
-      "step": 1520
-    },
-    {
-      "epoch": 0.9491315136476427,
-      "grad_norm": 2.265625,
-      "learning_rate": 1.1310344827586207e-06,
-      "loss": 0.9723,
-      "step": 1530
-    },
-    {
-      "epoch": 0.9553349875930521,
-      "grad_norm": 2.125,
-      "learning_rate": 9.931034482758622e-07,
-      "loss": 0.9535,
-      "step": 1540
-    },
-    {
-      "epoch": 0.9615384615384616,
-      "grad_norm": 2.21875,
-      "learning_rate": 8.551724137931035e-07,
-      "loss": 0.9652,
-      "step": 1550
-    },
-    {
-      "epoch": 0.967741935483871,
-      "grad_norm": 2.515625,
-      "learning_rate": 7.172413793103449e-07,
-      "loss": 0.9527,
-      "step": 1560
-    },
-    {
-      "epoch": 0.9739454094292804,
-      "grad_norm": 2.25,
-      "learning_rate": 5.793103448275863e-07,
-      "loss": 0.9809,
-      "step": 1570
-    },
-    {
-      "epoch": 0.9801488833746899,
-      "grad_norm": 2.28125,
-      "learning_rate": 4.413793103448276e-07,
-      "loss": 0.9852,
-      "step": 1580
-    },
-    {
-      "epoch": 0.9863523573200993,
-      "grad_norm": 2.109375,
-      "learning_rate": 3.0344827586206897e-07,
-      "loss": 0.9164,
-      "step": 1590
-    },
-    {
-      "epoch": 0.9925558312655087,
-      "grad_norm": 2.40625,
-      "learning_rate": 1.6551724137931037e-07,
-      "loss": 0.9355,
-      "step": 1600
-    },
-    {
-      "epoch": 0.9987593052109182,
-      "grad_norm": 2.015625,
-      "learning_rate": 2.7586206896551723e-08,
-      "loss": 0.9949,
-      "step": 1610
-    },
     {
       "epoch": 1.0,
-      "step": 1612,
-      "total_flos": 7.533261039139123e+17,
-      "train_loss": 1.0007318160669976,
-      "train_runtime": 6142.5782,
-      "train_samples_per_second": 8.397,
-      "train_steps_per_second": 0.262
+      "step": 1312,
+      "total_flos": 4.7710598211200614e+17,
+      "train_loss": 1.2026635146722562,
+      "train_runtime": 2875.3707,
+      "train_samples_per_second": 14.591,
+      "train_steps_per_second": 0.456
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1612,
+  "max_steps": 1312,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -1162,8 +952,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.533261039139123e+17,
-  "train_batch_size": 32,
+  "total_flos": 4.7710598211200614e+17,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b890d1a6a193b79f8bfcf860eb52ccf74c423753ac6370315ce2b62acfd64a1
+oid sha256:21fa790fc457d5b6b0d17aa60aa932cb9b96c1c3eeb7b854b96ec867dcb5facf
 size 5176