Duplicate from HuggingFaceTB/smollm2-360M-8k-lc100k-dpo-ultaf-ep2

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,87 @@
+---
+base_model: loubnabnl/smollm2-360M-8k-lc100k-mix1-ep2
+tags:
+- alignment-handbook
+- trl
+- dpo
+- generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+model-index:
+- name: smollm2-360M-8k-lc100k-dpo-ultaf-ep2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/loubnabnl/huggingface/runs/3nedrmyg)
+# smollm2-360M-8k-lc100k-dpo-ultaf-ep2
+
+This model is a fine-tuned version of [loubnabnl/smollm2-360M-8k-lc100k-mix1-ep2](https://huggingface.co/loubnabnl/smollm2-360M-8k-lc100k-mix1-ep2) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6348
+- Rewards/chosen: -0.0342
+- Rewards/rejected: -0.3910
+- Rewards/accuracies: 0.6190
+- Rewards/margins: 0.3568
+- Logps/rejected: -323.7198
+- Logps/chosen: -375.6464
+- Logits/rejected: -1.6969
+- Logits/chosen: -1.6408
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 2
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.7098        | 0.2094 | 100  | 0.7162          | -0.0109        | -0.0675          | 0.5278             | 0.0566          | -323.0727      | -375.5997    | -1.6983         | -1.6387       |
+| 0.6825        | 0.4187 | 200  | 0.6842          | -0.0010        | -0.1880          | 0.5794             | 0.1870          | -323.3139      | -375.5800    | -1.6938         | -1.6358       |
+| 0.663         | 0.6281 | 300  | 0.6617          | 0.0225         | -0.2389          | 0.6032             | 0.2614          | -323.4156      | -375.5330    | -1.6893         | -1.6317       |
+| 0.6547        | 0.8375 | 400  | 0.6591          | 0.0001         | -0.3516          | 0.6389             | 0.3517          | -323.6410      | -375.5778    | -1.6980         | -1.6414       |
+| 0.6456        | 1.0468 | 500  | 0.6430          | 0.0133         | -0.3566          | 0.6667             | 0.3699          | -323.6510      | -375.5514    | -1.6931         | -1.6365       |
+| 0.6054        | 1.2562 | 600  | 0.6423          | -0.0329        | -0.3895          | 0.6349             | 0.3566          | -323.7167      | -375.6438    | -1.6991         | -1.6431       |
+| 0.6129        | 1.4656 | 700  | 0.6431          | -0.0449        | -0.4183          | 0.6349             | 0.3735          | -323.7745      | -375.6677    | -1.6979         | -1.6414       |
+| 0.5972        | 1.6750 | 800  | 0.6384          | -0.0695        | -0.4139          | 0.6429             | 0.3444          | -323.7656      | -375.7169    | -1.6965         | -1.6399       |
+| 0.6207        | 1.8843 | 900  | 0.6362          | -0.0627        | -0.4222          | 0.6786             | 0.3595          | -323.7822      | -375.7033    | -1.6976         | -1.6407       |
+
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.1.2
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 1.9973828840617638,
+    "eval_logits/chosen": -1.6407532691955566,
+    "eval_logits/rejected": -1.6968854665756226,
+    "eval_logps/chosen": -375.6463623046875,
+    "eval_logps/rejected": -323.7197570800781,
+    "eval_loss": 0.6348475217819214,
+    "eval_rewards/accuracies": 0.6190476417541504,
+    "eval_rewards/chosen": -0.034213583916425705,
+    "eval_rewards/margins": 0.3567626178264618,
+    "eval_rewards/rejected": -0.3909761905670166,
+    "eval_runtime": 22.3598,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 89.446,
+    "eval_steps_per_second": 2.818,
+    "total_flos": 0.0,
+    "train_loss": 0.6516540072998911,
+    "train_runtime": 5944.7081,
+    "train_samples": 61134,
+    "train_samples_per_second": 20.568,
+    "train_steps_per_second": 0.16
+}

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "loubnabnl/smollm2-360M-8k-lc100k-mix1-ep2",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 960,
+  "initializer_range": 0.02,
+  "intermediate_size": 2560,
+  "is_llama_config": true,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 15,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 5,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_interleaved": false,
+  "rope_scaling": null,
+  "rope_theta": 100000,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 49152
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.9973828840617638,
+    "eval_logits/chosen": -1.6407532691955566,
+    "eval_logits/rejected": -1.6968854665756226,
+    "eval_logps/chosen": -375.6463623046875,
+    "eval_logps/rejected": -323.7197570800781,
+    "eval_loss": 0.6348475217819214,
+    "eval_rewards/accuracies": 0.6190476417541504,
+    "eval_rewards/chosen": -0.034213583916425705,
+    "eval_rewards/margins": 0.3567626178264618,
+    "eval_rewards/rejected": -0.3909761905670166,
+    "eval_runtime": 22.3598,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 89.446,
+    "eval_steps_per_second": 2.818
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.42.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6bffe7435d7ddc10fd3b9a9efd429dafbacb1cb17015fb5562664e7532bf86e
+size 723674912

runs/Oct31_09-01-58_ip-26-0-172-142/events.out.tfevents.1730365788.ip-26-0-172-142.451351.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a59625dee7fb61068e6a6ea53682ee0d91da3f92189019d5ceed597d21ebf27
+size 78139

runs/Oct31_09-01-58_ip-26-0-172-142/events.out.tfevents.1730371773.ip-26-0-172-142.451351.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac5691a0cdfdd383d29b69e85c4aaf632c695ce8fd31683bb9512f310fbf1250
+size 828

runs/Oct31_09-19-57_ip-26-0-174-36/events.out.tfevents.1730366818.ip-26-0-174-36.3233632.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44f4f653d8d22e5db793f6743dc486f1d3f0919a66d5bbd69ebfd22b9e1f598b
+size 38996

runs/Oct31_09-20-43_ip-26-0-161-142/events.out.tfevents.1730366856.ip-26-0-161-142.1301887.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:684365d127b952fdfd215e337f4f8dc5bc4d37ef88fb608632d244ec6f86a449
+size 37621

special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,154 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.9973828840617638,
+    "total_flos": 0.0,
+    "train_loss": 0.6516540072998911,
+    "train_runtime": 5944.7081,
+    "train_samples": 61134,
+    "train_samples_per_second": 20.568,
+    "train_steps_per_second": 0.16
+}

trainer_state.json

@@ -0,0 +1,1626 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9973828840617638,
+  "eval_steps": 100,
+  "global_step": 954,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002093692750588851,
+      "grad_norm": 37.29612828590162,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -1.5181711912155151,
+      "logits/rejected": -1.4237494468688965,
+      "logps/chosen": -331.7369384765625,
+      "logps/rejected": -353.880859375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 34.09183299219303,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -1.4457752704620361,
+      "logits/rejected": -1.4434000253677368,
+      "logps/chosen": -409.06182861328125,
+      "logps/rejected": -344.4476318359375,
+      "loss": 0.7192,
+      "rewards/accuracies": 0.3611111044883728,
+      "rewards/chosen": -0.012955614365637302,
+      "rewards/margins": 0.009454314596951008,
+      "rewards/rejected": -0.02240992709994316,
+      "step": 10
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 33.674655892230355,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -1.2904545068740845,
+      "logits/rejected": -1.4261707067489624,
+      "logps/chosen": -361.19525146484375,
+      "logps/rejected": -322.1033630371094,
+      "loss": 0.7204,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.05613602325320244,
+      "rewards/margins": 0.04490422457456589,
+      "rewards/rejected": 0.011231804266571999,
+      "step": 20
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 34.76216235748818,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -1.326378583908081,
+      "logits/rejected": -1.334900140762329,
+      "logps/chosen": -390.536376953125,
+      "logps/rejected": -312.28826904296875,
+      "loss": 0.7412,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.034933071583509445,
+      "rewards/margins": -0.05697251483798027,
+      "rewards/rejected": 0.022039445117115974,
+      "step": 30
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 34.40909279988205,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -1.3933440446853638,
+      "logits/rejected": -1.4404706954956055,
+      "logps/chosen": -329.0964050292969,
+      "logps/rejected": -291.3802490234375,
+      "loss": 0.725,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.02061540260910988,
+      "rewards/margins": 0.004305871669203043,
+      "rewards/rejected": 0.016309529542922974,
+      "step": 40
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 33.558772768765095,
+      "learning_rate": 5.208333333333334e-07,
+      "logits/chosen": -1.3897579908370972,
+      "logits/rejected": -1.4129403829574585,
+      "logps/chosen": -353.4942321777344,
+      "logps/rejected": -314.30755615234375,
+      "loss": 0.7195,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.021332601085305214,
+      "rewards/margins": 0.062744140625,
+      "rewards/rejected": -0.04141153767704964,
+      "step": 50
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 37.25091651222252,
+      "learning_rate": 6.249999999999999e-07,
+      "logits/chosen": -1.4710047245025635,
+      "logits/rejected": -1.4629329442977905,
+      "logps/chosen": -328.04437255859375,
+      "logps/rejected": -303.51922607421875,
+      "loss": 0.7164,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.010426114313304424,
+      "rewards/margins": 0.02291293814778328,
+      "rewards/rejected": -0.01248682476580143,
+      "step": 60
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 36.76489316948809,
+      "learning_rate": 7.291666666666666e-07,
+      "logits/chosen": -1.3989927768707275,
+      "logits/rejected": -1.3778417110443115,
+      "logps/chosen": -402.1053466796875,
+      "logps/rejected": -349.107421875,
+      "loss": 0.7271,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.033832818269729614,
+      "rewards/margins": 0.09618775546550751,
+      "rewards/rejected": -0.062354933470487595,
+      "step": 70
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 36.668613144227514,
+      "learning_rate": 8.333333333333333e-07,
+      "logits/chosen": -1.14809250831604,
+      "logits/rejected": -1.3004900217056274,
+      "logps/chosen": -393.3706970214844,
+      "logps/rejected": -302.55267333984375,
+      "loss": 0.7166,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.022442013025283813,
+      "rewards/margins": 0.0886504203081131,
+      "rewards/rejected": -0.06620840728282928,
+      "step": 80
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 37.50656065265379,
+      "learning_rate": 9.374999999999999e-07,
+      "logits/chosen": -1.4117727279663086,
+      "logits/rejected": -1.4133552312850952,
+      "logps/chosen": -344.2930603027344,
+      "logps/rejected": -297.4874572753906,
+      "loss": 0.7401,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.002166506601497531,
+      "rewards/margins": -0.026871949434280396,
+      "rewards/rejected": 0.024705441668629646,
+      "step": 90
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 40.39537923810557,
+      "learning_rate": 9.999463737538052e-07,
+      "logits/chosen": -1.4038206338882446,
+      "logits/rejected": -1.4591708183288574,
+      "logps/chosen": -382.0655822753906,
+      "logps/rejected": -325.4857482910156,
+      "loss": 0.7098,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.009398235008120537,
+      "rewards/margins": 0.012028699740767479,
+      "rewards/rejected": -0.002630466129630804,
+      "step": 100
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "eval_logits/chosen": -1.638710618019104,
+      "eval_logits/rejected": -1.6983067989349365,
+      "eval_logps/chosen": -375.59967041015625,
+      "eval_logps/rejected": -323.07269287109375,
+      "eval_loss": 0.7161566615104675,
+      "eval_rewards/accuracies": 0.5277777910232544,
+      "eval_rewards/chosen": -0.010860972106456757,
+      "eval_rewards/margins": 0.05658983066678047,
+      "eval_rewards/rejected": -0.06745080649852753,
+      "eval_runtime": 23.8466,
+      "eval_samples_per_second": 83.869,
+      "eval_steps_per_second": 2.642,
+      "step": 100
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 31.68914195854589,
+      "learning_rate": 9.993432105822034e-07,
+      "logits/chosen": -1.3994014263153076,
+      "logits/rejected": -1.5090693235397339,
+      "logps/chosen": -380.85809326171875,
+      "logps/rejected": -315.415771484375,
+      "loss": 0.7244,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.0006190292770043015,
+      "rewards/margins": 0.017153877764940262,
+      "rewards/rejected": -0.016534846276044846,
+      "step": 110
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 32.5897566012542,
+      "learning_rate": 9.980706626858607e-07,
+      "logits/chosen": -1.3891736268997192,
+      "logits/rejected": -1.4548366069793701,
+      "logps/chosen": -330.4214782714844,
+      "logps/rejected": -290.9881286621094,
+      "loss": 0.7015,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.002393510192632675,
+      "rewards/margins": 0.015673214569687843,
+      "rewards/rejected": -0.013279703445732594,
+      "step": 120
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 37.434859226068475,
+      "learning_rate": 9.961304359538434e-07,
+      "logits/chosen": -1.3691463470458984,
+      "logits/rejected": -1.4777592420578003,
+      "logps/chosen": -375.37957763671875,
+      "logps/rejected": -303.9295959472656,
+      "loss": 0.6959,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.023477336391806602,
+      "rewards/margins": 0.07401027530431747,
+      "rewards/rejected": -0.05053293704986572,
+      "step": 130
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 32.710975845053845,
+      "learning_rate": 9.935251313189563e-07,
+      "logits/chosen": -1.2984201908111572,
+      "logits/rejected": -1.3567609786987305,
+      "logps/chosen": -403.44390869140625,
+      "logps/rejected": -315.7667541503906,
+      "loss": 0.694,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.1049453616142273,
+      "rewards/margins": 0.1717820167541504,
+      "rewards/rejected": -0.0668366402387619,
+      "step": 140
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 33.59099498042739,
+      "learning_rate": 9.902582412711118e-07,
+      "logits/chosen": -1.3101998567581177,
+      "logits/rejected": -1.4272937774658203,
+      "logps/chosen": -362.3497619628906,
+      "logps/rejected": -304.9016418457031,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.030804574489593506,
+      "rewards/margins": 0.16174563765525818,
+      "rewards/rejected": -0.13094103336334229,
+      "step": 150
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 33.64536927205325,
+      "learning_rate": 9.86334145175542e-07,
+      "logits/chosen": -1.372686743736267,
+      "logits/rejected": -1.3871064186096191,
+      "logps/chosen": -333.40252685546875,
+      "logps/rejected": -309.45538330078125,
+      "loss": 0.698,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02940761111676693,
+      "rewards/margins": 0.14784011244773865,
+      "rewards/rejected": -0.11843249946832657,
+      "step": 160
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 32.519654688788634,
+      "learning_rate": 9.817581034021272e-07,
+      "logits/chosen": -1.469236135482788,
+      "logits/rejected": -1.5464330911636353,
+      "logps/chosen": -325.42401123046875,
+      "logps/rejected": -279.6885986328125,
+      "loss": 0.6943,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.008727139793336391,
+      "rewards/margins": 0.06822283565998077,
+      "rewards/rejected": -0.05949569493532181,
+      "step": 170
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 31.217980241389647,
+      "learning_rate": 9.765362502737097e-07,
+      "logits/chosen": -1.392663598060608,
+      "logits/rejected": -1.3267390727996826,
+      "logps/chosen": -328.3432312011719,
+      "logps/rejected": -328.40679931640625,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0019484326476231217,
+      "rewards/margins": 0.09815844148397446,
+      "rewards/rejected": -0.10010688006877899,
+      "step": 180
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 35.19062151474422,
+      "learning_rate": 9.706755858428485e-07,
+      "logits/chosen": -1.317312479019165,
+      "logits/rejected": -1.3148291110992432,
+      "logps/chosen": -339.4427490234375,
+      "logps/rejected": -339.53936767578125,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.017630616202950478,
+      "rewards/margins": 0.09675482660531998,
+      "rewards/rejected": -0.1143854409456253,
+      "step": 190
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 38.736703713611334,
+      "learning_rate": 9.641839665080363e-07,
+      "logits/chosen": -1.3673009872436523,
+      "logits/rejected": -1.3681590557098389,
+      "logps/chosen": -341.8690490722656,
+      "logps/rejected": -310.6811828613281,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.02894829586148262,
+      "rewards/margins": 0.07689115405082703,
+      "rewards/rejected": -0.10583944618701935,
+      "step": 200
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "eval_logits/chosen": -1.635803461074829,
+      "eval_logits/rejected": -1.69376540184021,
+      "eval_logps/chosen": -375.5799865722656,
+      "eval_logps/rejected": -323.3138732910156,
+      "eval_loss": 0.6841831207275391,
+      "eval_rewards/accuracies": 0.579365074634552,
+      "eval_rewards/chosen": -0.001013976288959384,
+      "eval_rewards/margins": 0.18699844181537628,
+      "eval_rewards/rejected": -0.18801240622997284,
+      "eval_runtime": 22.6938,
+      "eval_samples_per_second": 88.13,
+      "eval_steps_per_second": 2.776,
+      "step": 200
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 29.329351201852436,
+      "learning_rate": 9.570700944819582e-07,
+      "logits/chosen": -1.1939313411712646,
+      "logits/rejected": -1.3856732845306396,
+      "logps/chosen": -384.10784912109375,
+      "logps/rejected": -317.24755859375,
+      "loss": 0.684,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0037221908569335938,
+      "rewards/margins": 0.16761450469493866,
+      "rewards/rejected": -0.16389232873916626,
+      "step": 210
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 29.84515748563858,
+      "learning_rate": 9.493435061259129e-07,
+      "logits/chosen": -1.3803870677947998,
+      "logits/rejected": -1.3274301290512085,
+      "logps/chosen": -313.02777099609375,
+      "logps/rejected": -297.11004638671875,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.023346154019236565,
+      "rewards/margins": 0.06815574318170547,
+      "rewards/rejected": -0.09150189161300659,
+      "step": 220
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 31.662572343473325,
+      "learning_rate": 9.4101455916603e-07,
+      "logits/chosen": -1.3570753335952759,
+      "logits/rejected": -1.3131479024887085,
+      "logps/chosen": -331.6525573730469,
+      "logps/rejected": -329.0929260253906,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.025835394859313965,
+      "rewards/margins": 0.16715176403522491,
+      "rewards/rejected": -0.1929871290922165,
+      "step": 230
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 34.26947764899909,
+      "learning_rate": 9.320944188084241e-07,
+      "logits/chosen": -1.3401970863342285,
+      "logits/rejected": -1.3203916549682617,
+      "logps/chosen": -341.75750732421875,
+      "logps/rejected": -315.9219970703125,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.01570424810051918,
+      "rewards/margins": 0.13407179713249207,
+      "rewards/rejected": -0.11836756765842438,
+      "step": 240
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 32.570026830574825,
+      "learning_rate": 9.225950427718974e-07,
+      "logits/chosen": -1.3138176202774048,
+      "logits/rejected": -1.4214763641357422,
+      "logps/chosen": -388.63800048828125,
+      "logps/rejected": -327.308349609375,
+      "loss": 0.662,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.034160248935222626,
+      "rewards/margins": 0.23999419808387756,
+      "rewards/rejected": -0.20583395659923553,
+      "step": 250
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 31.875700958447943,
+      "learning_rate": 9.125291652582547e-07,
+      "logits/chosen": -1.171736717224121,
+      "logits/rejected": -1.2927379608154297,
+      "logps/chosen": -367.10638427734375,
+      "logps/rejected": -301.8586730957031,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.03132578730583191,
+      "rewards/margins": 0.2260134220123291,
+      "rewards/rejected": -0.2573392391204834,
+      "step": 260
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 29.95901746509434,
+      "learning_rate": 9.019102798817195e-07,
+      "logits/chosen": -1.4621995687484741,
+      "logits/rejected": -1.563369631767273,
+      "logps/chosen": -379.08746337890625,
+      "logps/rejected": -326.1041259765625,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010546261444687843,
+      "rewards/margins": 0.17825904488563538,
+      "rewards/rejected": -0.16771277785301208,
+      "step": 270
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 35.30660280902673,
+      "learning_rate": 8.90752621580335e-07,
+      "logits/chosen": -1.4563312530517578,
+      "logits/rejected": -1.5124019384384155,
+      "logps/chosen": -360.3949279785156,
+      "logps/rejected": -296.45318603515625,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.05117585510015488,
+      "rewards/margins": 0.1499311625957489,
+      "rewards/rejected": -0.20110702514648438,
+      "step": 280
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 30.02627471974457,
+      "learning_rate": 8.79071147533597e-07,
+      "logits/chosen": -1.3743504285812378,
+      "logits/rejected": -1.3512884378433228,
+      "logps/chosen": -341.1376647949219,
+      "logps/rejected": -331.7826843261719,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.004191230051219463,
+      "rewards/margins": 0.26077696681022644,
+      "rewards/rejected": -0.26496821641921997,
+      "step": 290
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 32.675301753581984,
+      "learning_rate": 8.668815171119019e-07,
+      "logits/chosen": -1.2676564455032349,
+      "logits/rejected": -1.319390058517456,
+      "logps/chosen": -378.872314453125,
+      "logps/rejected": -322.4703369140625,
+      "loss": 0.663,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.022002484649419785,
+      "rewards/margins": 0.1741904467344284,
+      "rewards/rejected": -0.1961929351091385,
+      "step": 300
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "eval_logits/chosen": -1.6317138671875,
+      "eval_logits/rejected": -1.6892949342727661,
+      "eval_logps/chosen": -375.532958984375,
+      "eval_logps/rejected": -323.4156494140625,
+      "eval_loss": 0.6616633534431458,
+      "eval_rewards/accuracies": 0.60317462682724,
+      "eval_rewards/chosen": 0.022496730089187622,
+      "eval_rewards/margins": 0.2614184617996216,
+      "eval_rewards/rejected": -0.23892174661159515,
+      "eval_runtime": 22.3915,
+      "eval_samples_per_second": 89.32,
+      "eval_steps_per_second": 2.814,
+      "step": 300
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 32.37940998012059,
+      "learning_rate": 8.54200070884685e-07,
+      "logits/chosen": -1.3149330615997314,
+      "logits/rejected": -1.4230272769927979,
+      "logps/chosen": -389.00384521484375,
+      "logps/rejected": -293.93487548828125,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.028895879164338112,
+      "rewards/margins": 0.29164841771125793,
+      "rewards/rejected": -0.3205442726612091,
+      "step": 310
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 34.74994897373754,
+      "learning_rate": 8.410438087153911e-07,
+      "logits/chosen": -1.2203506231307983,
+      "logits/rejected": -1.28737211227417,
+      "logps/chosen": -353.5164489746094,
+      "logps/rejected": -297.2350769042969,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.03598624840378761,
+      "rewards/margins": 0.1709606647491455,
+      "rewards/rejected": -0.20694692432880402,
+      "step": 320
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 27.89021442930517,
+      "learning_rate": 8.274303669726426e-07,
+      "logits/chosen": -1.2289667129516602,
+      "logits/rejected": -1.3027209043502808,
+      "logps/chosen": -351.02435302734375,
+      "logps/rejected": -311.8251953125,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.005795622244477272,
+      "rewards/margins": 0.3018319308757782,
+      "rewards/rejected": -0.30762752890586853,
+      "step": 330
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 32.72596936653152,
+      "learning_rate": 8.133779948881513e-07,
+      "logits/chosen": -1.3401914834976196,
+      "logits/rejected": -1.387083888053894,
+      "logps/chosen": -355.665771484375,
+      "logps/rejected": -318.4053955078125,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04015441983938217,
+      "rewards/margins": 0.22958505153656006,
+      "rewards/rejected": -0.2697394788265228,
+      "step": 340
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 35.42610267799739,
+      "learning_rate": 7.989055300930704e-07,
+      "logits/chosen": -1.3034248352050781,
+      "logits/rejected": -1.3343088626861572,
+      "logps/chosen": -340.1492614746094,
+      "logps/rejected": -289.6612854003906,
+      "loss": 0.6554,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.1225026398897171,
+      "rewards/margins": 0.12164878845214844,
+      "rewards/rejected": -0.24415142834186554,
+      "step": 350
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 29.02918447808193,
+      "learning_rate": 7.840323733655778e-07,
+      "logits/chosen": -1.2503941059112549,
+      "logits/rejected": -1.4153048992156982,
+      "logps/chosen": -406.9992980957031,
+      "logps/rejected": -322.87994384765625,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.024089232087135315,
+      "rewards/margins": 0.31749090552330017,
+      "rewards/rejected": -0.29340168833732605,
+      "step": 360
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 33.710516075595095,
+      "learning_rate": 7.687784626235447e-07,
+      "logits/chosen": -1.312897801399231,
+      "logits/rejected": -1.3840960264205933,
+      "logps/chosen": -369.8164978027344,
+      "logps/rejected": -291.6910705566406,
+      "loss": 0.643,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.02891460619866848,
+      "rewards/margins": 0.3528975546360016,
+      "rewards/rejected": -0.3239828944206238,
+      "step": 370
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 30.30908317538956,
+      "learning_rate": 7.531642461971514e-07,
+      "logits/chosen": -1.4397472143173218,
+      "logits/rejected": -1.5266616344451904,
+      "logps/chosen": -368.24224853515625,
+      "logps/rejected": -316.00567626953125,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07685438543558121,
+      "rewards/margins": 0.29695773124694824,
+      "rewards/rejected": -0.37381213903427124,
+      "step": 380
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 34.60440393056815,
+      "learning_rate": 7.372106554172801e-07,
+      "logits/chosen": -1.3900893926620483,
+      "logits/rejected": -1.348008394241333,
+      "logps/chosen": -369.2901306152344,
+      "logps/rejected": -338.5638427734375,
+      "loss": 0.661,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0064844414591789246,
+      "rewards/margins": 0.22314131259918213,
+      "rewards/rejected": -0.22962574660778046,
+      "step": 390
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 31.77115164859215,
+      "learning_rate": 7.209390765564318e-07,
+      "logits/chosen": -1.4996167421340942,
+      "logits/rejected": -1.4596645832061768,
+      "logps/chosen": -308.35113525390625,
+      "logps/rejected": -281.60693359375,
+      "loss": 0.6547,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04348212480545044,
+      "rewards/margins": 0.27009111642837524,
+      "rewards/rejected": -0.3135732412338257,
+      "step": 400
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "eval_logits/chosen": -1.6414421796798706,
+      "eval_logits/rejected": -1.6979553699493408,
+      "eval_logps/chosen": -375.57781982421875,
+      "eval_logps/rejected": -323.6410217285156,
+      "eval_loss": 0.6590762734413147,
+      "eval_rewards/accuracies": 0.6388888955116272,
+      "eval_rewards/chosen": 7.240355625981465e-05,
+      "eval_rewards/margins": 0.3516833484172821,
+      "eval_rewards/rejected": -0.3516109585762024,
+      "eval_runtime": 22.0937,
+      "eval_samples_per_second": 90.524,
+      "eval_steps_per_second": 2.851,
+      "step": 400
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 31.789442088224007,
+      "learning_rate": 7.043713221597773e-07,
+      "logits/chosen": -1.2454522848129272,
+      "logits/rejected": -1.4237592220306396,
+      "logps/chosen": -399.0449523925781,
+      "logps/rejected": -326.0782775878906,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.014972883276641369,
+      "rewards/margins": 0.2356092631816864,
+      "rewards/rejected": -0.22063639760017395,
+      "step": 410
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 32.68488619173213,
+      "learning_rate": 6.875296018047809e-07,
+      "logits/chosen": -1.5021634101867676,
+      "logits/rejected": -1.4257535934448242,
+      "logps/chosen": -349.61553955078125,
+      "logps/rejected": -340.30975341796875,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.10846195369958878,
+      "rewards/margins": 0.26752999424934387,
+      "rewards/rejected": -0.37599191069602966,
+      "step": 420
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 29.283057753738706,
+      "learning_rate": 6.704364923285857e-07,
+      "logits/chosen": -1.3049055337905884,
+      "logits/rejected": -1.4615538120269775,
+      "logps/chosen": -387.3922119140625,
+      "logps/rejected": -299.621337890625,
+      "loss": 0.6539,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06501199305057526,
+      "rewards/margins": 0.30248281359672546,
+      "rewards/rejected": -0.3674947917461395,
+      "step": 430
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 30.29029590057011,
+      "learning_rate": 6.531149075630796e-07,
+      "logits/chosen": -1.2950228452682495,
+      "logits/rejected": -1.3765883445739746,
+      "logps/chosen": -359.27691650390625,
+      "logps/rejected": -293.13104248046875,
+      "loss": 0.6508,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.05934730917215347,
+      "rewards/margins": 0.2306128442287445,
+      "rewards/rejected": -0.28996017575263977,
+      "step": 440
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 31.040736224709473,
+      "learning_rate": 6.355880676182085e-07,
+      "logits/chosen": -1.4344260692596436,
+      "logits/rejected": -1.3737527132034302,
+      "logps/chosen": -357.5885314941406,
+      "logps/rejected": -332.05499267578125,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04119770601391792,
+      "rewards/margins": 0.2782004475593567,
+      "rewards/rejected": -0.31939810514450073,
+      "step": 450
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 31.5228211898507,
+      "learning_rate": 6.178794677547137e-07,
+      "logits/chosen": -1.2749378681182861,
+      "logits/rejected": -1.4267973899841309,
+      "logps/chosen": -346.6052551269531,
+      "logps/rejected": -290.1454162597656,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.026611875742673874,
+      "rewards/margins": 0.3765586316585541,
+      "rewards/rejected": -0.3499467074871063,
+      "step": 460
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 31.660312290338712,
+      "learning_rate": 6.000128468880222e-07,
+      "logits/chosen": -1.3701679706573486,
+      "logits/rejected": -1.390453815460205,
+      "logps/chosen": -369.3509826660156,
+      "logps/rejected": -323.2626037597656,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04081202298402786,
+      "rewards/margins": 0.33853498101234436,
+      "rewards/rejected": -0.37934696674346924,
+      "step": 470
+    },
+    {
+      "epoch": 1.0049725202826485,
+      "grad_norm": 27.48266116042723,
+      "learning_rate": 5.820121557655108e-07,
+      "logits/chosen": -1.2848116159439087,
+      "logits/rejected": -1.3119057416915894,
+      "logps/chosen": -359.86102294921875,
+      "logps/rejected": -313.0788269042969,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.048723649233579636,
+      "rewards/margins": 0.27731940150260925,
+      "rewards/rejected": -0.3260430693626404,
+      "step": 480
+    },
+    {
+      "epoch": 1.025909447788537,
+      "grad_norm": 31.66810532106133,
+      "learning_rate": 5.639015248598023e-07,
+      "logits/chosen": -1.34721839427948,
+      "logits/rejected": -1.449410080909729,
+      "logps/chosen": -361.23046875,
+      "logps/rejected": -296.3492126464844,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09297003597021103,
+      "rewards/margins": 0.209228515625,
+      "rewards/rejected": -0.302198588848114,
+      "step": 490
+    },
+    {
+      "epoch": 1.0468463752944255,
+      "grad_norm": 31.533076924353868,
+      "learning_rate": 5.457052320211339e-07,
+      "logits/chosen": -1.3988004922866821,
+      "logits/rejected": -1.4485900402069092,
+      "logps/chosen": -370.951904296875,
+      "logps/rejected": -314.5948181152344,
+      "loss": 0.6456,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10979924350976944,
+      "rewards/margins": 0.23507650196552277,
+      "rewards/rejected": -0.3448757231235504,
+      "step": 500
+    },
+    {
+      "epoch": 1.0468463752944255,
+      "eval_logits/chosen": -1.636523962020874,
+      "eval_logits/rejected": -1.6931382417678833,
+      "eval_logps/chosen": -375.5513916015625,
+      "eval_logps/rejected": -323.6509704589844,
+      "eval_loss": 0.6429626941680908,
+      "eval_rewards/accuracies": 0.6666666865348816,
+      "eval_rewards/chosen": 0.01327058020979166,
+      "eval_rewards/margins": 0.36985495686531067,
+      "eval_rewards/rejected": -0.356584370136261,
+      "eval_runtime": 22.4458,
+      "eval_samples_per_second": 89.104,
+      "eval_steps_per_second": 2.807,
+      "step": 500
+    },
+    {
+      "epoch": 1.067783302800314,
+      "grad_norm": 30.307455199305462,
+      "learning_rate": 5.274476699321637e-07,
+      "logits/chosen": -1.4101717472076416,
+      "logits/rejected": -1.420190453529358,
+      "logps/chosen": -327.46893310546875,
+      "logps/rejected": -302.391357421875,
+      "loss": 0.6345,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04776526987552643,
+      "rewards/margins": 0.2989855408668518,
+      "rewards/rejected": -0.3467508554458618,
+      "step": 510
+    },
+    {
+      "epoch": 1.0887202303062025,
+      "grad_norm": 29.722389429702496,
+      "learning_rate": 5.091533134088387e-07,
+      "logits/chosen": -1.4273402690887451,
+      "logits/rejected": -1.4231868982315063,
+      "logps/chosen": -323.42376708984375,
+      "logps/rejected": -304.4673767089844,
+      "loss": 0.6284,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0829726904630661,
+      "rewards/margins": 0.24710920453071594,
+      "rewards/rejected": -0.33008190989494324,
+      "step": 520
+    },
+    {
+      "epoch": 1.109657157812091,
+      "grad_norm": 28.885339598508185,
+      "learning_rate": 4.908466865911614e-07,
+      "logits/chosen": -1.4464526176452637,
+      "logits/rejected": -1.5408886671066284,
+      "logps/chosen": -344.06689453125,
+      "logps/rejected": -292.1816101074219,
+      "loss": 0.6328,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.03225661441683769,
+      "rewards/margins": 0.38520506024360657,
+      "rewards/rejected": -0.41746169328689575,
+      "step": 530
+    },
+    {
+      "epoch": 1.1305940853179797,
+      "grad_norm": 29.441675570164428,
+      "learning_rate": 4.7255233006783624e-07,
+      "logits/chosen": -1.3947404623031616,
+      "logits/rejected": -1.328045129776001,
+      "logps/chosen": -318.3937072753906,
+      "logps/rejected": -280.65435791015625,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03985299542546272,
+      "rewards/margins": 0.33030450344085693,
+      "rewards/rejected": -0.37015751004219055,
+      "step": 540
+    },
+    {
+      "epoch": 1.151531012823868,
+      "grad_norm": 29.84505193266847,
+      "learning_rate": 4.5429476797886617e-07,
+      "logits/chosen": -1.3157615661621094,
+      "logits/rejected": -1.463357925415039,
+      "logps/chosen": -367.59564208984375,
+      "logps/rejected": -284.3589172363281,
+      "loss": 0.6256,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04548700898885727,
+      "rewards/margins": 0.312529981136322,
+      "rewards/rejected": -0.3580169677734375,
+      "step": 550
+    },
+    {
+      "epoch": 1.1724679403297567,
+      "grad_norm": 33.58402259379722,
+      "learning_rate": 4.3609847514019763e-07,
+      "logits/chosen": -1.3491549491882324,
+      "logits/rejected": -1.3917362689971924,
+      "logps/chosen": -355.3868408203125,
+      "logps/rejected": -311.2879943847656,
+      "loss": 0.6294,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.005126100964844227,
+      "rewards/margins": 0.33152928948402405,
+      "rewards/rejected": -0.3264032006263733,
+      "step": 560
+    },
+    {
+      "epoch": 1.193404867835645,
+      "grad_norm": 29.80795991678476,
+      "learning_rate": 4.179878442344892e-07,
+      "logits/chosen": -1.4194284677505493,
+      "logits/rejected": -1.371366262435913,
+      "logps/chosen": -326.87066650390625,
+      "logps/rejected": -321.23785400390625,
+      "loss": 0.6441,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.0237637497484684,
+      "rewards/margins": 0.33783239126205444,
+      "rewards/rejected": -0.31406864523887634,
+      "step": 570
+    },
+    {
+      "epoch": 1.2143417953415336,
+      "grad_norm": 30.101844117489865,
+      "learning_rate": 3.9998715311197783e-07,
+      "logits/chosen": -1.452893853187561,
+      "logits/rejected": -1.460272192955017,
+      "logps/chosen": -346.24932861328125,
+      "logps/rejected": -292.27667236328125,
+      "loss": 0.6258,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.029639745131134987,
+      "rewards/margins": 0.4036487638950348,
+      "rewards/rejected": -0.43328848481178284,
+      "step": 580
+    },
+    {
+      "epoch": 1.235278722847422,
+      "grad_norm": 28.648910640416673,
+      "learning_rate": 3.821205322452863e-07,
+      "logits/chosen": -1.2438939809799194,
+      "logits/rejected": -1.3377447128295898,
+      "logps/chosen": -380.3300476074219,
+      "logps/rejected": -317.4806823730469,
+      "loss": 0.6249,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0937061458826065,
+      "rewards/margins": 0.34427446126937866,
+      "rewards/rejected": -0.43798065185546875,
+      "step": 590
+    },
+    {
+      "epoch": 1.2562156503533106,
+      "grad_norm": 34.234491670474995,
+      "learning_rate": 3.6441193238179146e-07,
+      "logits/chosen": -1.4130090475082397,
+      "logits/rejected": -1.340057611465454,
+      "logps/chosen": -377.28228759765625,
+      "logps/rejected": -362.57879638671875,
+      "loss": 0.6054,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08141253888607025,
+      "rewards/margins": 0.26778295636177063,
+      "rewards/rejected": -0.3491954803466797,
+      "step": 600
+    },
+    {
+      "epoch": 1.2562156503533106,
+      "eval_logits/chosen": -1.6430615186691284,
+      "eval_logits/rejected": -1.6991180181503296,
+      "eval_logps/chosen": -375.643798828125,
+      "eval_logps/rejected": -323.71673583984375,
+      "eval_loss": 0.6423465609550476,
+      "eval_rewards/accuracies": 0.6349206566810608,
+      "eval_rewards/chosen": -0.032912444323301315,
+      "eval_rewards/margins": 0.3565501272678375,
+      "eval_rewards/rejected": -0.38946259021759033,
+      "eval_runtime": 22.0916,
+      "eval_samples_per_second": 90.532,
+      "eval_steps_per_second": 2.852,
+      "step": 600
+    },
+    {
+      "epoch": 1.2771525778591992,
+      "grad_norm": 31.96916362076228,
+      "learning_rate": 3.4688509243692034e-07,
+      "logits/chosen": -1.3404086828231812,
+      "logits/rejected": -1.401881217956543,
+      "logps/chosen": -348.0643005371094,
+      "logps/rejected": -274.1236267089844,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.05336350202560425,
+      "rewards/margins": 0.5003852248191833,
+      "rewards/rejected": -0.4470217823982239,
+      "step": 610
+    },
+    {
+      "epoch": 1.2980895053650876,
+      "grad_norm": 27.687942045456108,
+      "learning_rate": 3.295635076714144e-07,
+      "logits/chosen": -1.1929457187652588,
+      "logits/rejected": -1.3312275409698486,
+      "logps/chosen": -339.0008544921875,
+      "logps/rejected": -282.86273193359375,
+      "loss": 0.6173,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04629793018102646,
+      "rewards/margins": 0.3611483573913574,
+      "rewards/rejected": -0.4074462950229645,
+      "step": 620
+    },
+    {
+      "epoch": 1.3190264328709762,
+      "grad_norm": 30.364814060074504,
+      "learning_rate": 3.12470398195219e-07,
+      "logits/chosen": -1.4581263065338135,
+      "logits/rejected": -1.4359983205795288,
+      "logps/chosen": -352.0130310058594,
+      "logps/rejected": -319.5924987792969,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.0860348716378212,
+      "rewards/margins": 0.4933602213859558,
+      "rewards/rejected": -0.4073253571987152,
+      "step": 630
+    },
+    {
+      "epoch": 1.3399633603768648,
+      "grad_norm": 28.207562684445328,
+      "learning_rate": 2.956286778402226e-07,
+      "logits/chosen": -1.3643245697021484,
+      "logits/rejected": -1.3367928266525269,
+      "logps/chosen": -329.8095397949219,
+      "logps/rejected": -325.6297912597656,
+      "loss": 0.6194,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.03843813017010689,
+      "rewards/margins": 0.4010621905326843,
+      "rewards/rejected": -0.4395003318786621,
+      "step": 640
+    },
+    {
+      "epoch": 1.3609002878827532,
+      "grad_norm": 31.931835373695627,
+      "learning_rate": 2.7906092344356826e-07,
+      "logits/chosen": -1.4707378149032593,
+      "logits/rejected": -1.4725669622421265,
+      "logps/chosen": -321.7577819824219,
+      "logps/rejected": -296.30072021484375,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.20691867172718048,
+      "rewards/margins": 0.24088874459266663,
+      "rewards/rejected": -0.4478074014186859,
+      "step": 650
+    },
+    {
+      "epoch": 1.3818372153886418,
+      "grad_norm": 33.26417313890622,
+      "learning_rate": 2.6278934458271996e-07,
+      "logits/chosen": -1.280631422996521,
+      "logits/rejected": -1.3918514251708984,
+      "logps/chosen": -319.20391845703125,
+      "logps/rejected": -294.9554138183594,
+      "loss": 0.6281,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.15276700258255005,
+      "rewards/margins": 0.14049580693244934,
+      "rewards/rejected": -0.293262779712677,
+      "step": 660
+    },
+    {
+      "epoch": 1.4027741428945302,
+      "grad_norm": 27.692247842703622,
+      "learning_rate": 2.468357538028487e-07,
+      "logits/chosen": -1.3904974460601807,
+      "logits/rejected": -1.4726046323776245,
+      "logps/chosen": -351.35614013671875,
+      "logps/rejected": -299.1191101074219,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.012430467642843723,
+      "rewards/margins": 0.4077607989311218,
+      "rewards/rejected": -0.39533036947250366,
+      "step": 670
+    },
+    {
+      "epoch": 1.4237110704004188,
+      "grad_norm": 32.32896888329579,
+      "learning_rate": 2.312215373764551e-07,
+      "logits/chosen": -1.3210734128952026,
+      "logits/rejected": -1.2972867488861084,
+      "logps/chosen": -357.87725830078125,
+      "logps/rejected": -346.83172607421875,
+      "loss": 0.612,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.030674666166305542,
+      "rewards/margins": 0.2985823154449463,
+      "rewards/rejected": -0.32925695180892944,
+      "step": 680
+    },
+    {
+      "epoch": 1.4446479979063072,
+      "grad_norm": 26.559915261204598,
+      "learning_rate": 2.1596762663442213e-07,
+      "logits/chosen": -1.353945016860962,
+      "logits/rejected": -1.331405758857727,
+      "logps/chosen": -359.0257263183594,
+      "logps/rejected": -302.020751953125,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09165488183498383,
+      "rewards/margins": 0.32221120595932007,
+      "rewards/rejected": -0.4138661324977875,
+      "step": 690
+    },
+    {
+      "epoch": 1.4655849254121958,
+      "grad_norm": 28.923951135790823,
+      "learning_rate": 2.0109446990692963e-07,
+      "logits/chosen": -1.3177390098571777,
+      "logits/rejected": -1.2676212787628174,
+      "logps/chosen": -382.9945373535156,
+      "logps/rejected": -382.7451171875,
+      "loss": 0.6129,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.00983230210840702,
+      "rewards/margins": 0.44890522956848145,
+      "rewards/rejected": -0.43907293677330017,
+      "step": 700
+    },
+    {
+      "epoch": 1.4655849254121958,
+      "eval_logits/chosen": -1.6413902044296265,
+      "eval_logits/rejected": -1.6979249715805054,
+      "eval_logps/chosen": -375.667724609375,
+      "eval_logps/rejected": -323.7745361328125,
+      "eval_loss": 0.6431064009666443,
+      "eval_rewards/accuracies": 0.6349206566810608,
+      "eval_rewards/chosen": -0.044887345284223557,
+      "eval_rewards/margins": 0.373458594083786,
+      "eval_rewards/rejected": -0.41834595799446106,
+      "eval_runtime": 22.4693,
+      "eval_samples_per_second": 89.01,
+      "eval_steps_per_second": 2.804,
+      "step": 700
+    },
+    {
+      "epoch": 1.4865218529180844,
+      "grad_norm": 29.793015750302608,
+      "learning_rate": 1.8662200511184872e-07,
+      "logits/chosen": -1.5391782522201538,
+      "logits/rejected": -1.5692270994186401,
+      "logps/chosen": -353.60504150390625,
+      "logps/rejected": -327.60333251953125,
+      "loss": 0.6413,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08452965319156647,
+      "rewards/margins": 0.3078867793083191,
+      "rewards/rejected": -0.39241647720336914,
+      "step": 710
+    },
+    {
+      "epoch": 1.5074587804239727,
+      "grad_norm": 30.154215075312383,
+      "learning_rate": 1.725696330273575e-07,
+      "logits/chosen": -1.4344627857208252,
+      "logits/rejected": -1.4114643335342407,
+      "logps/chosen": -367.28668212890625,
+      "logps/rejected": -330.210205078125,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0991826206445694,
+      "rewards/margins": 0.3690223693847656,
+      "rewards/rejected": -0.46820497512817383,
+      "step": 720
+    },
+    {
+      "epoch": 1.5283957079298613,
+      "grad_norm": 32.49028184373073,
+      "learning_rate": 1.589561912846089e-07,
+      "logits/chosen": -1.3968040943145752,
+      "logits/rejected": -1.4468437433242798,
+      "logps/chosen": -339.4754943847656,
+      "logps/rejected": -298.7085266113281,
+      "loss": 0.6139,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04794057086110115,
+      "rewards/margins": 0.3862064480781555,
+      "rewards/rejected": -0.43414703011512756,
+      "step": 730
+    },
+    {
+      "epoch": 1.54933263543575,
+      "grad_norm": 29.612038431579933,
+      "learning_rate": 1.4579992911531496e-07,
+      "logits/chosen": -1.2432403564453125,
+      "logits/rejected": -1.2390873432159424,
+      "logps/chosen": -379.5582275390625,
+      "logps/rejected": -332.1794128417969,
+      "loss": 0.6205,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.019659820944070816,
+      "rewards/margins": 0.3802977204322815,
+      "rewards/rejected": -0.360637903213501,
+      "step": 740
+    },
+    {
+      "epoch": 1.5702695629416383,
+      "grad_norm": 28.119711308052274,
+      "learning_rate": 1.3311848288809813e-07,
+      "logits/chosen": -1.4195433855056763,
+      "logits/rejected": -1.490561842918396,
+      "logps/chosen": -359.9119873046875,
+      "logps/rejected": -322.4441833496094,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0712754875421524,
+      "rewards/margins": 0.2602451741695404,
+      "rewards/rejected": -0.3315206468105316,
+      "step": 750
+    },
+    {
+      "epoch": 1.5912064904475267,
+      "grad_norm": 29.268690125436255,
+      "learning_rate": 1.209288524664029e-07,
+      "logits/chosen": -1.229116439819336,
+      "logits/rejected": -1.2306039333343506,
+      "logps/chosen": -435.7142028808594,
+      "logps/rejected": -397.52044677734375,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.11046306788921356,
+      "rewards/margins": 0.4868060052394867,
+      "rewards/rejected": -0.3763429522514343,
+      "step": 760
+    },
+    {
+      "epoch": 1.6121434179534153,
+      "grad_norm": 28.29047720088543,
+      "learning_rate": 1.0924737841966497e-07,
+      "logits/chosen": -1.220485806465149,
+      "logits/rejected": -1.4240710735321045,
+      "logps/chosen": -396.7095947265625,
+      "logps/rejected": -319.75958251953125,
+      "loss": 0.6247,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.055124759674072266,
+      "rewards/margins": 0.39260005950927734,
+      "rewards/rejected": -0.44772475957870483,
+      "step": 770
+    },
+    {
+      "epoch": 1.633080345459304,
+      "grad_norm": 29.17224056970089,
+      "learning_rate": 9.808972011828054e-08,
+      "logits/chosen": -1.2811758518218994,
+      "logits/rejected": -1.3020236492156982,
+      "logps/chosen": -384.5751647949219,
+      "logps/rejected": -329.9205017089844,
+      "loss": 0.626,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.14039060473442078,
+      "rewards/margins": 0.46962863206863403,
+      "rewards/rejected": -0.32923805713653564,
+      "step": 780
+    },
+    {
+      "epoch": 1.6540172729651923,
+      "grad_norm": 33.75145778470617,
+      "learning_rate": 8.747083474174527e-08,
+      "logits/chosen": -1.3885407447814941,
+      "logits/rejected": -1.4309613704681396,
+      "logps/chosen": -365.4181823730469,
+      "logps/rejected": -322.75054931640625,
+      "loss": 0.6123,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04203175753355026,
+      "rewards/margins": 0.3509315550327301,
+      "rewards/rejected": -0.39296332001686096,
+      "step": 790
+    },
+    {
+      "epoch": 1.674954200471081,
+      "grad_norm": 25.187206733853984,
+      "learning_rate": 7.740495722810269e-08,
+      "logits/chosen": -1.2112213373184204,
+      "logits/rejected": -1.3428361415863037,
+      "logps/chosen": -420.00830078125,
+      "logps/rejected": -358.3287353515625,
+      "loss": 0.5972,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.01959555223584175,
+      "rewards/margins": 0.5057805776596069,
+      "rewards/rejected": -0.4861849844455719,
+      "step": 800
+    },
+    {
+      "epoch": 1.674954200471081,
+      "eval_logits/chosen": -1.639930009841919,
+      "eval_logits/rejected": -1.6965380907058716,
+      "eval_logps/chosen": -375.7169494628906,
+      "eval_logps/rejected": -323.76556396484375,
+      "eval_loss": 0.6384284496307373,
+      "eval_rewards/accuracies": 0.6428571343421936,
+      "eval_rewards/chosen": -0.06950785219669342,
+      "eval_rewards/margins": 0.34436169266700745,
+      "eval_rewards/rejected": -0.41386955976486206,
+      "eval_runtime": 21.8454,
+      "eval_samples_per_second": 91.552,
+      "eval_steps_per_second": 2.884,
+      "step": 800
+    },
+    {
+      "epoch": 1.6958911279769695,
+      "grad_norm": 30.26944195604407,
+      "learning_rate": 6.790558119157597e-08,
+      "logits/chosen": -1.3412492275238037,
+      "logits/rejected": -1.3922173976898193,
+      "logps/chosen": -375.84429931640625,
+      "logps/rejected": -326.71417236328125,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.015008327551186085,
+      "rewards/margins": 0.4259034991264343,
+      "rewards/rejected": -0.44091176986694336,
+      "step": 810
+    },
+    {
+      "epoch": 1.7168280554828579,
+      "grad_norm": 30.885265100170944,
+      "learning_rate": 5.898544083397e-08,
+      "logits/chosen": -1.4171245098114014,
+      "logits/rejected": -1.5250798463821411,
+      "logps/chosen": -388.115478515625,
+      "logps/rejected": -320.20819091796875,
+      "loss": 0.6055,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.05425529554486275,
+      "rewards/margins": 0.4603540003299713,
+      "rewards/rejected": -0.5146092176437378,
+      "step": 820
+    },
+    {
+      "epoch": 1.7377649829887463,
+      "grad_norm": 32.971600431805705,
+      "learning_rate": 5.065649387408705e-08,
+      "logits/chosen": -1.3083207607269287,
+      "logits/rejected": -1.2884520292282104,
+      "logps/chosen": -338.13909912109375,
+      "logps/rejected": -327.7836608886719,
+      "loss": 0.6083,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.11997182667255402,
+      "rewards/margins": 0.3927108347415924,
+      "rewards/rejected": -0.5126826167106628,
+      "step": 830
+    },
+    {
+      "epoch": 1.7587019104946349,
+      "grad_norm": 29.20392566444622,
+      "learning_rate": 4.292990551804171e-08,
+      "logits/chosen": -1.5097484588623047,
+      "logits/rejected": -1.4066120386123657,
+      "logps/chosen": -314.53338623046875,
+      "logps/rejected": -309.9684143066406,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06001155450940132,
+      "rewards/margins": 0.30454209446907043,
+      "rewards/rejected": -0.36455363035202026,
+      "step": 840
+    },
+    {
+      "epoch": 1.7796388380005235,
+      "grad_norm": 30.280553572438347,
+      "learning_rate": 3.581603349196371e-08,
+      "logits/chosen": -1.3757822513580322,
+      "logits/rejected": -1.3954650163650513,
+      "logps/chosen": -331.03826904296875,
+      "logps/rejected": -323.1974182128906,
+      "loss": 0.6233,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10794013738632202,
+      "rewards/margins": 0.2812719941139221,
+      "rewards/rejected": -0.38921213150024414,
+      "step": 850
+    },
+    {
+      "epoch": 1.8005757655064119,
+      "grad_norm": 28.100566523523845,
+      "learning_rate": 2.9324414157151367e-08,
+      "logits/chosen": -1.3457200527191162,
+      "logits/rejected": -1.4598686695098877,
+      "logps/chosen": -356.97216796875,
+      "logps/rejected": -286.18865966796875,
+      "loss": 0.6215,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.05811784416437149,
+      "rewards/margins": 0.35237258672714233,
+      "rewards/rejected": -0.4104904234409332,
+      "step": 860
+    },
+    {
+      "epoch": 1.8215126930123005,
+      "grad_norm": 26.46890964804442,
+      "learning_rate": 2.3463749726290284e-08,
+      "logits/chosen": -1.4204833507537842,
+      "logits/rejected": -1.5046513080596924,
+      "logps/chosen": -404.6219177246094,
+      "logps/rejected": -336.10150146484375,
+      "loss": 0.6261,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02932531200349331,
+      "rewards/margins": 0.39560848474502563,
+      "rewards/rejected": -0.366283118724823,
+      "step": 870
+    },
+    {
+      "epoch": 1.842449620518189,
+      "grad_norm": 31.701066186197266,
+      "learning_rate": 1.824189659787284e-08,
+      "logits/chosen": -1.3159443140029907,
+      "logits/rejected": -1.3131808042526245,
+      "logps/chosen": -328.07098388671875,
+      "logps/rejected": -308.87701416015625,
+      "loss": 0.6266,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0714392215013504,
+      "rewards/margins": 0.3882763683795929,
+      "rewards/rejected": -0.45971551537513733,
+      "step": 880
+    },
+    {
+      "epoch": 1.8633865480240774,
+      "grad_norm": 31.12356616666238,
+      "learning_rate": 1.3665854824458035e-08,
+      "logits/chosen": -1.3845950365066528,
+      "logits/rejected": -1.434497594833374,
+      "logps/chosen": -376.0118103027344,
+      "logps/rejected": -331.4889831542969,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.061675846576690674,
+      "rewards/margins": 0.3783223330974579,
+      "rewards/rejected": -0.43999820947647095,
+      "step": 890
+    },
+    {
+      "epoch": 1.8843234755299658,
+      "grad_norm": 34.05071913262595,
+      "learning_rate": 9.741758728888217e-09,
+      "logits/chosen": -1.2828714847564697,
+      "logits/rejected": -1.3924423456192017,
+      "logps/chosen": -401.4546203613281,
+      "logps/rejected": -317.35443115234375,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.004245785064995289,
+      "rewards/margins": 0.4051188826560974,
+      "rewards/rejected": -0.40936464071273804,
+      "step": 900
+    },
+    {
+      "epoch": 1.8843234755299658,
+      "eval_logits/chosen": -1.6407170295715332,
+      "eval_logits/rejected": -1.6976144313812256,
+      "eval_logps/chosen": -375.70330810546875,
+      "eval_logps/rejected": -323.78216552734375,
+      "eval_loss": 0.6362326145172119,
+      "eval_rewards/accuracies": 0.6785714030265808,
+      "eval_rewards/chosen": -0.06266693770885468,
+      "eval_rewards/margins": 0.3595210015773773,
+      "eval_rewards/rejected": -0.4221878945827484,
+      "eval_runtime": 22.2411,
+      "eval_samples_per_second": 89.923,
+      "eval_steps_per_second": 2.833,
+      "step": 900
+    },
+    {
+      "epoch": 1.9052604030358546,
+      "grad_norm": 28.324746122618627,
+      "learning_rate": 6.474868681043577e-09,
+      "logits/chosen": -1.232722282409668,
+      "logits/rejected": -1.3370740413665771,
+      "logps/chosen": -330.0971984863281,
+      "logps/rejected": -274.8087463378906,
+      "loss": 0.6315,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04480002075433731,
+      "rewards/margins": 0.4823727011680603,
+      "rewards/rejected": -0.5271727442741394,
+      "step": 910
+    },
+    {
+      "epoch": 1.926197330541743,
+      "grad_norm": 32.98699943344935,
+      "learning_rate": 3.869564046156459e-09,
+      "logits/chosen": -1.3717395067214966,
+      "logits/rejected": -1.4535510540008545,
+      "logps/chosen": -373.8208923339844,
+      "logps/rejected": -311.4999084472656,
+      "loss": 0.6198,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06762387603521347,
+      "rewards/margins": 0.35773369669914246,
+      "rewards/rejected": -0.4253575801849365,
+      "step": 920
+    },
+    {
+      "epoch": 1.9471342580476314,
+      "grad_norm": 27.5401666731791,
+      "learning_rate": 1.929337314139412e-09,
+      "logits/chosen": -1.2926667928695679,
+      "logits/rejected": -1.3644298315048218,
+      "logps/chosen": -362.7403869628906,
+      "logps/rejected": -317.42303466796875,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.01872597262263298,
+      "rewards/margins": 0.4235456883907318,
+      "rewards/rejected": -0.40481966733932495,
+      "step": 930
+    },
+    {
+      "epoch": 1.96807118555352,
+      "grad_norm": 31.615150149296902,
+      "learning_rate": 6.567894177967325e-10,
+      "logits/chosen": -1.3021799325942993,
+      "logits/rejected": -1.3374966382980347,
+      "logps/chosen": -324.93255615234375,
+      "logps/rejected": -273.21929931640625,
+      "loss": 0.6255,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.05069868639111519,
+      "rewards/margins": 0.33984681963920593,
+      "rewards/rejected": -0.390545517206192,
+      "step": 940
+    },
+    {
+      "epoch": 1.9890081130594086,
+      "grad_norm": 29.09184931904493,
+      "learning_rate": 5.3626246194704575e-11,
+      "logits/chosen": -1.3507764339447021,
+      "logits/rejected": -1.4648717641830444,
+      "logps/chosen": -361.6529235839844,
+      "logps/rejected": -296.1907043457031,
+      "loss": 0.6169,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08508863300085068,
+      "rewards/margins": 0.3246951401233673,
+      "rewards/rejected": -0.40978384017944336,
+      "step": 950
+    },
+    {
+      "epoch": 1.9973828840617638,
+      "step": 954,
+      "total_flos": 0.0,
+      "train_loss": 0.6516540072998911,
+      "train_runtime": 5944.7081,
+      "train_samples_per_second": 20.568,
+      "train_steps_per_second": 0.16
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 954,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b7e0f2e2a3074b496c74cd65dd47e8da90de5ccf808bc233cb1420827c7d68f
+size 6520