diff --git "a/checkpoint-162/trainer_state.json" "b/checkpoint-162/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-162/trainer_state.json" @@ -0,0 +1,3275 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 82944, + "epoch": 0.35531185743660043, + "eval_steps": 500, + "global_step": 162, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 512, + "epoch": 0.0021932830705962986, + "eps": 6, + "loss/policy_avg": 0.15268513560295105, + "lr": 3e-06, + "objective/entropy": -45.407432556152344, + "objective/kl": 0.3935524821281433, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 4.679054260253906, + "objective/scores": 4.679054260253906, + "policy/approxkl_avg": 0.11690396815538406, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.8878304958343506, + "step": 0, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2729, + "val/ratio": 1.000101089477539, + "val/ratio_var": 1.6687282595739816e-06 + }, + { + "episode": 1024, + "epoch": 0.004386566141192597, + "eps": 6, + "loss/policy_avg": 0.10589599609375, + "lr": 2.9882812500000002e-06, + "objective/entropy": -51.535125732421875, + "objective/kl": 0.6498137712478638, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 4.541956901550293, + "objective/scores": 4.541956901550293, + "policy/approxkl_avg": 0.10543511807918549, + "policy/clipfrac_avg": 0.3203125, + "policy/entropy_avg": 0.8243188858032227, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3325, + "val/ratio": 1.0002458095550537, + "val/ratio_var": 1.5084623328220914e-06 + }, + { + "episode": 1536, + "epoch": 0.006579849211788897, + "eps": 6, + "loss/policy_avg": 0.13810233771800995, + "lr": 2.9765625e-06, + "objective/entropy": -55.89725112915039, + "objective/kl": 1.300195574760437, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 4.745310306549072, + "objective/scores": 4.745310306549072, + "policy/approxkl_avg": 0.11424587666988373, + "policy/clipfrac_avg": 0.302734375, + "policy/entropy_avg": 0.760991096496582, + "step": 2, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3801, + "val/ratio": 0.9999536275863647, + "val/ratio_var": 1.5804212125658523e-06 + }, + { + "episode": 2048, + "epoch": 0.008773132282385195, + "eps": 6, + "loss/policy_avg": 0.25361043214797974, + "lr": 2.96484375e-06, + "objective/entropy": -54.02920150756836, + "objective/kl": 1.721588373184204, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 4.8178606033325195, + "objective/scores": 4.8178606033325195, + "policy/approxkl_avg": 0.12672686576843262, + "policy/clipfrac_avg": 0.310546875, + "policy/entropy_avg": 0.7554011344909668, + "step": 3, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4010, + "val/ratio": 0.9999880790710449, + "val/ratio_var": 1.6228774484261521e-06 + }, + { + "episode": 2560, + "epoch": 0.010966415352981495, + "eps": 6, + "loss/policy_avg": 0.17445668578147888, + "lr": 2.953125e-06, + "objective/entropy": -60.463539123535156, + "objective/kl": 2.6447153091430664, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 4.877408981323242, + "objective/scores": 4.877408981323242, + "policy/approxkl_avg": 0.13307543098926544, + "policy/clipfrac_avg": 0.275390625, + "policy/entropy_avg": 0.690821647644043, + "step": 4, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4589, + "val/ratio": 1.0001537799835205, + "val/ratio_var": 1.283996084566752e-06 + }, + { + "episode": 3072, + "epoch": 0.013159698423577794, + "eps": 7, + "loss/policy_avg": 0.18251696228981018, + "lr": 2.94140625e-06, + "objective/entropy": -60.38776397705078, + "objective/kl": 3.3835220336914062, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 5.029230117797852, + "objective/scores": 5.029230117797852, + "policy/approxkl_avg": 0.11075231432914734, + "policy/clipfrac_avg": 0.30859375, + "policy/entropy_avg": 0.6673464775085449, + "step": 5, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4933, + "val/ratio": 1.0000388622283936, + "val/ratio_var": 1.886573841147765e-06 + }, + { + "episode": 3584, + "epoch": 0.015352981494174092, + "eps": 7, + "loss/policy_avg": 0.0862584114074707, + "lr": 2.9296875e-06, + "objective/entropy": -64.953857421875, + "objective/kl": 3.801461696624756, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 5.2262468338012695, + "objective/scores": 5.2262468338012695, + "policy/approxkl_avg": 0.0921158567070961, + "policy/clipfrac_avg": 0.31640625, + "policy/entropy_avg": 0.6317387819290161, + "step": 6, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 5224, + "val/ratio": 1.0000205039978027, + "val/ratio_var": 1.223428625962697e-06 + }, + { + "episode": 4096, + "epoch": 0.01754626456477039, + "eps": 7, + "loss/policy_avg": 0.10839146375656128, + "lr": 2.91796875e-06, + "objective/entropy": -64.41149139404297, + "objective/kl": 4.314542770385742, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 5.495314121246338, + "objective/scores": 5.495314121246338, + "policy/approxkl_avg": 0.11755571514368057, + "policy/clipfrac_avg": 0.28515625, + "policy/entropy_avg": 0.6287920475006104, + "step": 7, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 5463, + "val/ratio": 0.9998381733894348, + "val/ratio_var": 1.6926536545724957e-06 + }, + { + "episode": 4608, + "epoch": 0.01973954763536669, + "eps": 7, + "loss/policy_avg": 0.16045916080474854, + "lr": 2.90625e-06, + "objective/entropy": -65.33601379394531, + "objective/kl": 4.707500457763672, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 5.608126640319824, + "objective/scores": 5.608126640319824, + "policy/approxkl_avg": 0.10813738405704498, + "policy/clipfrac_avg": 0.2890625, + "policy/entropy_avg": 0.6515289545059204, + "step": 8, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 5590, + "val/ratio": 1.0000832080841064, + "val/ratio_var": 1.661756641624379e-06 + }, + { + "episode": 5120, + "epoch": 0.02193283070596299, + "eps": 7, + "loss/policy_avg": 0.16000822186470032, + "lr": 2.89453125e-06, + "objective/entropy": -64.82266235351562, + "objective/kl": 5.513130187988281, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 5.976546764373779, + "objective/scores": 5.976546764373779, + "policy/approxkl_avg": 0.12319613248109818, + "policy/clipfrac_avg": 0.29296875, + "policy/entropy_avg": 0.6444021463394165, + "step": 9, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 5330, + "val/ratio": 0.9997374415397644, + "val/ratio_var": 1.3334141613086103e-06 + }, + { + "episode": 5632, + "epoch": 0.024126113776559287, + "eps": 7, + "loss/policy_avg": 0.22769010066986084, + "lr": 2.8828125e-06, + "objective/entropy": -62.54362487792969, + "objective/kl": 6.375988960266113, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 6.329231262207031, + "objective/scores": 6.329231262207031, + "policy/approxkl_avg": 0.133827343583107, + "policy/clipfrac_avg": 0.30078125, + "policy/entropy_avg": 0.6720603704452515, + "step": 10, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4956, + "val/ratio": 0.9999898672103882, + "val/ratio_var": 2.208806336057023e-06 + }, + { + "episode": 6144, + "epoch": 0.026319396847155587, + "eps": 7, + "loss/policy_avg": 0.16269627213478088, + "lr": 2.87109375e-06, + "objective/entropy": -56.140655517578125, + "objective/kl": 7.935493469238281, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 6.335080146789551, + "objective/scores": 6.335080146789551, + "policy/approxkl_avg": 0.13188697397708893, + "policy/clipfrac_avg": 0.35546875, + "policy/entropy_avg": 0.6935639381408691, + "step": 11, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4904, + "val/ratio": 0.999968409538269, + "val/ratio_var": 1.8624212998474832e-06 + }, + { + "episode": 6656, + "epoch": 0.028512679917751884, + "eps": 7, + "loss/policy_avg": 0.1726413369178772, + "lr": 2.859375e-06, + "objective/entropy": -56.71982955932617, + "objective/kl": 10.505476951599121, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.327908515930176, + "objective/scores": 7.327908515930176, + "policy/approxkl_avg": 0.13946227729320526, + "policy/clipfrac_avg": 0.306640625, + "policy/entropy_avg": 0.6527446508407593, + "step": 12, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4210, + "val/ratio": 1.0002007484436035, + "val/ratio_var": 1.9585920654208167e-06 + }, + { + "episode": 7168, + "epoch": 0.030705962988348184, + "eps": 7, + "loss/policy_avg": 0.1594904661178589, + "lr": 2.84765625e-06, + "objective/entropy": -57.271263122558594, + "objective/kl": 12.708869934082031, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.788185119628906, + "objective/scores": 7.788185119628906, + "policy/approxkl_avg": 0.14631032943725586, + "policy/clipfrac_avg": 0.33203125, + "policy/entropy_avg": 0.6090783476829529, + "step": 13, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3340, + "val/ratio": 0.9998956918716431, + "val/ratio_var": 2.45040769186744e-06 + }, + { + "episode": 7680, + "epoch": 0.03289924605894448, + "eps": 7, + "loss/policy_avg": 0.15880730748176575, + "lr": 2.8359375e-06, + "objective/entropy": -53.81092071533203, + "objective/kl": 15.003689765930176, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.487188339233398, + "objective/scores": 7.487188339233398, + "policy/approxkl_avg": 0.17456671595573425, + "policy/clipfrac_avg": 0.33203125, + "policy/entropy_avg": 0.5235726237297058, + "step": 14, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2483, + "val/ratio": 1.0001764297485352, + "val/ratio_var": 1.739899175845494e-06 + }, + { + "episode": 8192, + "epoch": 0.03509252912954078, + "eps": 7, + "loss/policy_avg": 0.16226203739643097, + "lr": 2.82421875e-06, + "objective/entropy": -55.564083099365234, + "objective/kl": 14.753921508789062, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.572874069213867, + "objective/scores": 7.572874069213867, + "policy/approxkl_avg": 0.16666629910469055, + "policy/clipfrac_avg": 0.302734375, + "policy/entropy_avg": 0.5176196098327637, + "step": 15, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2405, + "val/ratio": 0.9999848008155823, + "val/ratio_var": 2.589519908724469e-06 + }, + { + "episode": 8704, + "epoch": 0.03728581220013708, + "eps": 7, + "loss/policy_avg": 0.21661071479320526, + "lr": 2.8125e-06, + "objective/entropy": -51.922306060791016, + "objective/kl": 16.144550323486328, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.781338691711426, + "objective/scores": 7.781338691711426, + "policy/approxkl_avg": 0.2002604901790619, + "policy/clipfrac_avg": 0.337890625, + "policy/entropy_avg": 0.5265668630599976, + "step": 16, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2439, + "val/ratio": 0.9997420907020569, + "val/ratio_var": 2.6209752377326367e-06 + }, + { + "episode": 9216, + "epoch": 0.03947909527073338, + "eps": 7, + "loss/policy_avg": 0.26599276065826416, + "lr": 2.80078125e-06, + "objective/entropy": -50.82372283935547, + "objective/kl": 17.150707244873047, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.842694282531738, + "objective/scores": 7.842694282531738, + "policy/approxkl_avg": 0.19617530703544617, + "policy/clipfrac_avg": 0.345703125, + "policy/entropy_avg": 0.5609679222106934, + "step": 17, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2719, + "val/ratio": 1.000065803527832, + "val/ratio_var": 2.653461933732615e-06 + }, + { + "episode": 9728, + "epoch": 0.041672378341329676, + "eps": 7, + "loss/policy_avg": 0.18615023791790009, + "lr": 2.7890625e-06, + "objective/entropy": -50.224365234375, + "objective/kl": 18.064285278320312, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.09566879272461, + "objective/scores": 8.09566879272461, + "policy/approxkl_avg": 0.2271636724472046, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5856865048408508, + "step": 18, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2943, + "val/ratio": 1.0001342296600342, + "val/ratio_var": 3.7938905279588653e-06 + }, + { + "episode": 10240, + "epoch": 0.04386566141192598, + "eps": 7, + "loss/policy_avg": 0.20427988469600677, + "lr": 2.77734375e-06, + "objective/entropy": -50.77484893798828, + "objective/kl": 17.688308715820312, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.970284461975098, + "objective/scores": 7.970284461975098, + "policy/approxkl_avg": 0.23018702864646912, + "policy/clipfrac_avg": 0.341796875, + "policy/entropy_avg": 0.5944638252258301, + "step": 19, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3183, + "val/ratio": 1.000162124633789, + "val/ratio_var": 3.043409833480837e-06 + }, + { + "episode": 10752, + "epoch": 0.04605894448252228, + "eps": 7, + "loss/policy_avg": 0.282001793384552, + "lr": 2.765625e-06, + "objective/entropy": -44.80044937133789, + "objective/kl": 19.763790130615234, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.012289047241211, + "objective/scores": 8.012289047241211, + "policy/approxkl_avg": 0.24563264846801758, + "policy/clipfrac_avg": 0.3359375, + "policy/entropy_avg": 0.6678606271743774, + "step": 20, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3309, + "val/ratio": 0.9997632503509521, + "val/ratio_var": 3.618298023866373e-06 + }, + { + "episode": 11264, + "epoch": 0.048252227553118573, + "eps": 7, + "loss/policy_avg": 0.15888847410678864, + "lr": 2.75390625e-06, + "objective/entropy": -45.08258056640625, + "objective/kl": 19.886899948120117, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.057466506958008, + "objective/scores": 8.057466506958008, + "policy/approxkl_avg": 0.2164287269115448, + "policy/clipfrac_avg": 0.3671875, + "policy/entropy_avg": 0.6539000272750854, + "step": 21, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3399, + "val/ratio": 0.9997671842575073, + "val/ratio_var": 3.0791625249548815e-06 + }, + { + "episode": 11776, + "epoch": 0.05044551062371487, + "eps": 7, + "loss/policy_avg": 0.276736855506897, + "lr": 2.7421875e-06, + "objective/entropy": -40.99183654785156, + "objective/kl": 22.14881134033203, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.060696601867676, + "objective/scores": 8.060696601867676, + "policy/approxkl_avg": 0.31746962666511536, + "policy/clipfrac_avg": 0.349609375, + "policy/entropy_avg": 0.7243172526359558, + "step": 22, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3907, + "val/ratio": 0.9996672868728638, + "val/ratio_var": 4.220366463414393e-06 + }, + { + "episode": 12288, + "epoch": 0.052638793694311174, + "eps": 7, + "loss/policy_avg": 0.4665297865867615, + "lr": 2.73046875e-06, + "objective/entropy": -35.2369384765625, + "objective/kl": 23.011898040771484, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 7.889699935913086, + "objective/scores": 7.889699935913086, + "policy/approxkl_avg": 0.3784940838813782, + "policy/clipfrac_avg": 0.3515625, + "policy/entropy_avg": 0.7655227184295654, + "step": 23, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3972, + "val/ratio": 0.9998047351837158, + "val/ratio_var": 4.525894382823026e-06 + }, + { + "episode": 12800, + "epoch": 0.05483207676490747, + "eps": 7, + "loss/policy_avg": 0.43642693758010864, + "lr": 2.71875e-06, + "objective/entropy": -36.217777252197266, + "objective/kl": 22.847183227539062, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.134607315063477, + "objective/scores": 8.134607315063477, + "policy/approxkl_avg": 0.330811470746994, + "policy/clipfrac_avg": 0.39453125, + "policy/entropy_avg": 0.7883225679397583, + "step": 24, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4373, + "val/ratio": 0.9999172687530518, + "val/ratio_var": 5.261089881969383e-06 + }, + { + "episode": 13312, + "epoch": 0.05702535983550377, + "eps": 7, + "loss/policy_avg": 0.3709805905818939, + "lr": 2.70703125e-06, + "objective/entropy": -28.68521499633789, + "objective/kl": 24.381669998168945, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.424071311950684, + "objective/scores": 8.424071311950684, + "policy/approxkl_avg": 0.26343879103660583, + "policy/clipfrac_avg": 0.359375, + "policy/entropy_avg": 0.7675451040267944, + "step": 25, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4221, + "val/ratio": 1.0003796815872192, + "val/ratio_var": 4.443480065674521e-06 + }, + { + "episode": 13824, + "epoch": 0.05921864290610007, + "eps": 7, + "loss/policy_avg": 0.32363858819007874, + "lr": 2.6953125e-06, + "objective/entropy": -28.342884063720703, + "objective/kl": 24.456398010253906, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.066158294677734, + "objective/scores": 8.066158294677734, + "policy/approxkl_avg": 0.2860070466995239, + "policy/clipfrac_avg": 0.40234375, + "policy/entropy_avg": 0.7560139298439026, + "step": 26, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3967, + "val/ratio": 0.999995231628418, + "val/ratio_var": 5.418739419837948e-06 + }, + { + "episode": 14336, + "epoch": 0.06141192597669637, + "eps": 7, + "loss/policy_avg": 0.27364417910575867, + "lr": 2.68359375e-06, + "objective/entropy": -21.591156005859375, + "objective/kl": 26.56311798095703, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.133939743041992, + "objective/scores": 8.133939743041992, + "policy/approxkl_avg": 0.34295791387557983, + "policy/clipfrac_avg": 0.3984375, + "policy/entropy_avg": 0.7555699944496155, + "step": 27, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3701, + "val/ratio": 0.9998049139976501, + "val/ratio_var": 5.146652256371453e-06 + }, + { + "episode": 14848, + "epoch": 0.06360520904729267, + "eps": 7, + "loss/policy_avg": 0.3071547746658325, + "lr": 2.671875e-06, + "objective/entropy": -23.85720443725586, + "objective/kl": 26.94635009765625, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.451851844787598, + "objective/scores": 8.451851844787598, + "policy/approxkl_avg": 0.301169216632843, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.7123795747756958, + "step": 28, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3446, + "val/ratio": 1.0001068115234375, + "val/ratio_var": 8.123261977743823e-06 + }, + { + "episode": 15360, + "epoch": 0.06579849211788896, + "eps": 7, + "loss/policy_avg": 0.24686866998672485, + "lr": 2.66015625e-06, + "objective/entropy": -23.746164321899414, + "objective/kl": 26.223037719726562, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.474257469177246, + "objective/scores": 8.474257469177246, + "policy/approxkl_avg": 0.2871531844139099, + "policy/clipfrac_avg": 0.384765625, + "policy/entropy_avg": 0.7285399436950684, + "step": 29, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3073, + "val/ratio": 0.9999381899833679, + "val/ratio_var": 4.340675786806969e-06 + }, + { + "episode": 15872, + "epoch": 0.06799177518848526, + "eps": 7, + "loss/policy_avg": 0.3226906359195709, + "lr": 2.6484375e-06, + "objective/entropy": -21.334320068359375, + "objective/kl": 27.262338638305664, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.353540420532227, + "objective/scores": 8.353540420532227, + "policy/approxkl_avg": 0.26032739877700806, + "policy/clipfrac_avg": 0.3671875, + "policy/entropy_avg": 0.7068374156951904, + "step": 30, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2797, + "val/ratio": 1.0000503063201904, + "val/ratio_var": 3.849579115922097e-06 + }, + { + "episode": 16384, + "epoch": 0.07018505825908156, + "eps": 7, + "loss/policy_avg": 0.8861868977546692, + "lr": 2.63671875e-06, + "objective/entropy": -16.9752197265625, + "objective/kl": 27.735870361328125, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.225103378295898, + "objective/scores": 8.225103378295898, + "policy/approxkl_avg": 0.3011099100112915, + "policy/clipfrac_avg": 0.373046875, + "policy/entropy_avg": 0.6992452144622803, + "step": 31, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2656, + "val/ratio": 1.0000051259994507, + "val/ratio_var": 3.2618629575154046e-06 + }, + { + "episode": 16896, + "epoch": 0.07237834132967787, + "eps": 7, + "loss/policy_avg": 0.42881882190704346, + "lr": 2.6250000000000003e-06, + "objective/entropy": -25.429481506347656, + "objective/kl": 26.258373260498047, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.536768913269043, + "objective/scores": 8.536768913269043, + "policy/approxkl_avg": 0.2783983647823334, + "policy/clipfrac_avg": 0.357421875, + "policy/entropy_avg": 0.6586651802062988, + "step": 32, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2739, + "val/ratio": 1.0000145435333252, + "val/ratio_var": 5.060162948211655e-06 + }, + { + "episode": 17408, + "epoch": 0.07457162440027416, + "eps": 7, + "loss/policy_avg": 0.3053734004497528, + "lr": 2.61328125e-06, + "objective/entropy": -27.58635711669922, + "objective/kl": 26.437305450439453, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.641759872436523, + "objective/scores": 8.641759872436523, + "policy/approxkl_avg": 0.27716416120529175, + "policy/clipfrac_avg": 0.384765625, + "policy/entropy_avg": 0.6354501843452454, + "step": 33, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2508, + "val/ratio": 1.0002658367156982, + "val/ratio_var": 5.2127916205790825e-06 + }, + { + "episode": 17920, + "epoch": 0.07676490747087046, + "eps": 7, + "loss/policy_avg": 0.29732874035835266, + "lr": 2.6015625e-06, + "objective/entropy": -21.862457275390625, + "objective/kl": 26.802410125732422, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.533287048339844, + "objective/scores": 8.533287048339844, + "policy/approxkl_avg": 0.2853066325187683, + "policy/clipfrac_avg": 0.34765625, + "policy/entropy_avg": 0.6434289813041687, + "step": 34, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2256, + "val/ratio": 1.0001918077468872, + "val/ratio_var": 3.2171374186873436e-06 + }, + { + "episode": 18432, + "epoch": 0.07895819054146676, + "eps": 7, + "loss/policy_avg": 0.29931700229644775, + "lr": 2.5898437500000003e-06, + "objective/entropy": -26.582080841064453, + "objective/kl": 26.952320098876953, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.649660110473633, + "objective/scores": 8.649660110473633, + "policy/approxkl_avg": 0.29774773120880127, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.6092206835746765, + "step": 35, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2189, + "val/ratio": 0.9998046159744263, + "val/ratio_var": 4.721144250652287e-06 + }, + { + "episode": 18944, + "epoch": 0.08115147361206305, + "eps": 7, + "loss/policy_avg": 0.29533323645591736, + "lr": 2.578125e-06, + "objective/entropy": -28.472558975219727, + "objective/kl": 26.34886932373047, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.683137893676758, + "objective/scores": 8.683137893676758, + "policy/approxkl_avg": 0.2845500707626343, + "policy/clipfrac_avg": 0.3515625, + "policy/entropy_avg": 0.6110947132110596, + "step": 36, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2113, + "val/ratio": 1.0004630088806152, + "val/ratio_var": 3.1590116122970358e-06 + }, + { + "episode": 19456, + "epoch": 0.08334475668265935, + "eps": 7, + "loss/policy_avg": 0.1853514462709427, + "lr": 2.56640625e-06, + "objective/entropy": -28.34747314453125, + "objective/kl": 26.15137481689453, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.642951965332031, + "objective/scores": 8.642951965332031, + "policy/approxkl_avg": 0.26775410771369934, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.5745651721954346, + "step": 37, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2008, + "val/ratio": 0.9999517202377319, + "val/ratio_var": 3.727831654032343e-06 + }, + { + "episode": 19968, + "epoch": 0.08553803975325565, + "eps": 7, + "loss/policy_avg": 0.43564116954803467, + "lr": 2.5546875000000003e-06, + "objective/entropy": -30.9276065826416, + "objective/kl": 26.764188766479492, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.747605323791504, + "objective/scores": 8.747605323791504, + "policy/approxkl_avg": 0.2727493345737457, + "policy/clipfrac_avg": 0.408203125, + "policy/entropy_avg": 0.5555662512779236, + "step": 38, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1949, + "val/ratio": 1.000280737876892, + "val/ratio_var": 4.035889560327632e-06 + }, + { + "episode": 20480, + "epoch": 0.08773132282385196, + "eps": 7, + "loss/policy_avg": 0.33555662631988525, + "lr": 2.54296875e-06, + "objective/entropy": -30.725311279296875, + "objective/kl": 26.652061462402344, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.656892776489258, + "objective/scores": 8.656892776489258, + "policy/approxkl_avg": 0.3008157014846802, + "policy/clipfrac_avg": 0.3515625, + "policy/entropy_avg": 0.5807157754898071, + "step": 39, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1964, + "val/ratio": 1.0001399517059326, + "val/ratio_var": 5.213231816014741e-06 + }, + { + "episode": 20992, + "epoch": 0.08992460589444826, + "eps": 7, + "loss/policy_avg": 0.251528263092041, + "lr": 2.53125e-06, + "objective/entropy": -31.469741821289062, + "objective/kl": 27.247440338134766, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.747458457946777, + "objective/scores": 8.747458457946777, + "policy/approxkl_avg": 0.31611520051956177, + "policy/clipfrac_avg": 0.39453125, + "policy/entropy_avg": 0.5639370679855347, + "step": 40, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1938, + "val/ratio": 0.9993410110473633, + "val/ratio_var": 3.2128821203514235e-06 + }, + { + "episode": 21504, + "epoch": 0.09211788896504455, + "eps": 7, + "loss/policy_avg": 0.2754300832748413, + "lr": 2.5195312500000003e-06, + "objective/entropy": -32.1821403503418, + "objective/kl": 26.554561614990234, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.68823528289795, + "objective/scores": 8.68823528289795, + "policy/approxkl_avg": 0.2688625454902649, + "policy/clipfrac_avg": 0.3984375, + "policy/entropy_avg": 0.5564634203910828, + "step": 41, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1861, + "val/ratio": 1.0001417398452759, + "val/ratio_var": 3.7918011912552174e-06 + }, + { + "episode": 22016, + "epoch": 0.09431117203564085, + "eps": 7, + "loss/policy_avg": 0.32364964485168457, + "lr": 2.5078125e-06, + "objective/entropy": -31.65809440612793, + "objective/kl": 26.914566040039062, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.63361930847168, + "objective/scores": 8.63361930847168, + "policy/approxkl_avg": 0.319877952337265, + "policy/clipfrac_avg": 0.357421875, + "policy/entropy_avg": 0.5548701882362366, + "step": 42, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1894, + "val/ratio": 0.9996398687362671, + "val/ratio_var": 3.310229885755689e-06 + }, + { + "episode": 22528, + "epoch": 0.09650445510623715, + "eps": 7, + "loss/policy_avg": 0.2156452238559723, + "lr": 2.49609375e-06, + "objective/entropy": -33.42183303833008, + "objective/kl": 25.768882751464844, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.491684913635254, + "objective/scores": 8.491684913635254, + "policy/approxkl_avg": 0.37833964824676514, + "policy/clipfrac_avg": 0.345703125, + "policy/entropy_avg": 0.5088037252426147, + "step": 43, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1638, + "val/ratio": 0.9993675947189331, + "val/ratio_var": 4.396941676532151e-06 + }, + { + "episode": 23040, + "epoch": 0.09869773817683344, + "eps": 7, + "loss/policy_avg": 0.283500611782074, + "lr": 2.4843750000000002e-06, + "objective/entropy": -38.62915802001953, + "objective/kl": 24.288204193115234, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.610681533813477, + "objective/scores": 8.610681533813477, + "policy/approxkl_avg": 0.26353907585144043, + "policy/clipfrac_avg": 0.341796875, + "policy/entropy_avg": 0.524039626121521, + "step": 44, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1721, + "val/ratio": 1.000080943107605, + "val/ratio_var": 5.167307790543418e-06 + }, + { + "episode": 23552, + "epoch": 0.10089102124742974, + "eps": 7, + "loss/policy_avg": 0.26535850763320923, + "lr": 2.47265625e-06, + "objective/entropy": -38.197269439697266, + "objective/kl": 23.877710342407227, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.435002326965332, + "objective/scores": 8.435002326965332, + "policy/approxkl_avg": 0.3536168932914734, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5166954398155212, + "step": 45, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1756, + "val/ratio": 1.0001449584960938, + "val/ratio_var": 3.216674940631492e-06 + }, + { + "episode": 24064, + "epoch": 0.10308430431802605, + "eps": 7, + "loss/policy_avg": 0.31962326169013977, + "lr": 2.4609375e-06, + "objective/entropy": -42.57143020629883, + "objective/kl": 23.369226455688477, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.742530822753906, + "objective/scores": 8.742530822753906, + "policy/approxkl_avg": 0.2840489149093628, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.5359203815460205, + "step": 46, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1840, + "val/ratio": 1.0000276565551758, + "val/ratio_var": 3.076736902585253e-06 + }, + { + "episode": 24576, + "epoch": 0.10527758738862235, + "eps": 7, + "loss/policy_avg": 0.1421811878681183, + "lr": 2.4492187500000002e-06, + "objective/entropy": -45.13118362426758, + "objective/kl": 22.429969787597656, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.711127281188965, + "objective/scores": 8.711127281188965, + "policy/approxkl_avg": 0.2315264642238617, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5265815854072571, + "step": 47, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1829, + "val/ratio": 1.000090479850769, + "val/ratio_var": 2.3445722945325542e-06 + }, + { + "episode": 25088, + "epoch": 0.10747087045921865, + "eps": 7, + "loss/policy_avg": 0.29607272148132324, + "lr": 2.4375e-06, + "objective/entropy": -47.1614875793457, + "objective/kl": 21.755367279052734, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.467025756835938, + "objective/scores": 8.467025756835938, + "policy/approxkl_avg": 0.23295345902442932, + "policy/clipfrac_avg": 0.310546875, + "policy/entropy_avg": 0.5038943290710449, + "step": 48, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1841, + "val/ratio": 1.0003342628479004, + "val/ratio_var": 2.7536118523130426e-06 + }, + { + "episode": 25600, + "epoch": 0.10966415352981494, + "eps": 7, + "loss/policy_avg": 0.26943719387054443, + "lr": 2.42578125e-06, + "objective/entropy": -47.87194061279297, + "objective/kl": 22.272235870361328, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.529675483703613, + "objective/scores": 8.529675483703613, + "policy/approxkl_avg": 0.23642447590827942, + "policy/clipfrac_avg": 0.30078125, + "policy/entropy_avg": 0.48028820753097534, + "step": 49, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1737, + "val/ratio": 1.0001472234725952, + "val/ratio_var": 3.836444648186443e-06 + }, + { + "episode": 26112, + "epoch": 0.11185743660041124, + "eps": 7, + "loss/policy_avg": 0.19396540522575378, + "lr": 2.4140625000000002e-06, + "objective/entropy": -46.96308135986328, + "objective/kl": 21.81145477294922, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.142440795898438, + "objective/scores": 8.142440795898438, + "policy/approxkl_avg": 0.23152337968349457, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.546012282371521, + "step": 50, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1915, + "val/ratio": 0.9999135136604309, + "val/ratio_var": 4.054582859680522e-06 + }, + { + "episode": 26624, + "epoch": 0.11405071967100754, + "eps": 7, + "loss/policy_avg": 0.17617464065551758, + "lr": 2.40234375e-06, + "objective/entropy": -50.88390350341797, + "objective/kl": 21.53017807006836, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.507946014404297, + "objective/scores": 8.507946014404297, + "policy/approxkl_avg": 0.23789095878601074, + "policy/clipfrac_avg": 0.345703125, + "policy/entropy_avg": 0.5126174092292786, + "step": 51, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1825, + "val/ratio": 0.9999390840530396, + "val/ratio_var": 3.831887170235859e-06 + }, + { + "episode": 27136, + "epoch": 0.11624400274160383, + "eps": 7, + "loss/policy_avg": 0.2936260402202606, + "lr": 2.390625e-06, + "objective/entropy": -49.55311584472656, + "objective/kl": 21.252099990844727, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.27835464477539, + "objective/scores": 8.27835464477539, + "policy/approxkl_avg": 0.26326990127563477, + "policy/clipfrac_avg": 0.310546875, + "policy/entropy_avg": 0.5240695476531982, + "step": 52, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1999, + "val/ratio": 1.00055992603302, + "val/ratio_var": 3.646538061730098e-06 + }, + { + "episode": 27648, + "epoch": 0.11843728581220014, + "eps": 7, + "loss/policy_avg": 0.28315332531929016, + "lr": 2.3789062500000002e-06, + "objective/entropy": -50.70973205566406, + "objective/kl": 20.522171020507812, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.508468627929688, + "objective/scores": 8.508468627929688, + "policy/approxkl_avg": 0.27963656187057495, + "policy/clipfrac_avg": 0.32421875, + "policy/entropy_avg": 0.5086656808853149, + "step": 53, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1877, + "val/ratio": 0.9998332262039185, + "val/ratio_var": 3.3264000194321852e-06 + }, + { + "episode": 28160, + "epoch": 0.12063056888279644, + "eps": 7, + "loss/policy_avg": 0.1823330521583557, + "lr": 2.3671875e-06, + "objective/entropy": -51.118812561035156, + "objective/kl": 21.896831512451172, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.411741256713867, + "objective/scores": 8.411741256713867, + "policy/approxkl_avg": 0.22382813692092896, + "policy/clipfrac_avg": 0.357421875, + "policy/entropy_avg": 0.5047107934951782, + "step": 54, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1885, + "val/ratio": 0.999740481376648, + "val/ratio_var": 2.7290527668810682e-06 + }, + { + "episode": 28672, + "epoch": 0.12282385195339274, + "eps": 7, + "loss/policy_avg": 0.25143423676490784, + "lr": 2.35546875e-06, + "objective/entropy": -50.577789306640625, + "objective/kl": 21.04738998413086, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.411981582641602, + "objective/scores": 8.411981582641602, + "policy/approxkl_avg": 0.22654137015342712, + "policy/clipfrac_avg": 0.32421875, + "policy/entropy_avg": 0.5130290985107422, + "step": 55, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 1979, + "val/ratio": 1.0000386238098145, + "val/ratio_var": 2.045640940195881e-06 + }, + { + "episode": 29184, + "epoch": 0.12501713502398903, + "eps": 7, + "loss/policy_avg": 0.14284127950668335, + "lr": 2.3437500000000002e-06, + "objective/entropy": -51.72842788696289, + "objective/kl": 21.067766189575195, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.488981246948242, + "objective/scores": 8.488981246948242, + "policy/approxkl_avg": 0.2212221920490265, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.4959627687931061, + "step": 56, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2032, + "val/ratio": 1.0002572536468506, + "val/ratio_var": 3.1830661555432016e-06 + }, + { + "episode": 29696, + "epoch": 0.12721041809458533, + "eps": 7, + "loss/policy_avg": 0.2772967517375946, + "lr": 2.33203125e-06, + "objective/entropy": -50.611061096191406, + "objective/kl": 20.819595336914062, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.417530059814453, + "objective/scores": 8.417530059814453, + "policy/approxkl_avg": 0.20757423341274261, + "policy/clipfrac_avg": 0.345703125, + "policy/entropy_avg": 0.5225619077682495, + "step": 57, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2159, + "val/ratio": 1.0001440048217773, + "val/ratio_var": 3.0312132821563864e-06 + }, + { + "episode": 30208, + "epoch": 0.12940370116518163, + "eps": 7, + "loss/policy_avg": 0.24927854537963867, + "lr": 2.3203125e-06, + "objective/entropy": -52.76747512817383, + "objective/kl": 20.694204330444336, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.423263549804688, + "objective/scores": 8.423263549804688, + "policy/approxkl_avg": 0.2698250412940979, + "policy/clipfrac_avg": 0.318359375, + "policy/entropy_avg": 0.5205322504043579, + "step": 58, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2188, + "val/ratio": 1.0001823902130127, + "val/ratio_var": 3.7059455735288793e-06 + }, + { + "episode": 30720, + "epoch": 0.13159698423577793, + "eps": 7, + "loss/policy_avg": 0.32080554962158203, + "lr": 2.30859375e-06, + "objective/entropy": -47.49811553955078, + "objective/kl": 21.551021575927734, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.264215469360352, + "objective/scores": 8.264215469360352, + "policy/approxkl_avg": 0.25660836696624756, + "policy/clipfrac_avg": 0.322265625, + "policy/entropy_avg": 0.5375617146492004, + "step": 59, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2128, + "val/ratio": 1.0002331733703613, + "val/ratio_var": 2.6993873234459898e-06 + }, + { + "episode": 31232, + "epoch": 0.13379026730637422, + "eps": 7, + "loss/policy_avg": 0.14086592197418213, + "lr": 2.296875e-06, + "objective/entropy": -51.65747833251953, + "objective/kl": 20.77255630493164, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.408650398254395, + "objective/scores": 8.408650398254395, + "policy/approxkl_avg": 0.23100361227989197, + "policy/clipfrac_avg": 0.3515625, + "policy/entropy_avg": 0.5363867878913879, + "step": 60, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2336, + "val/ratio": 1.000304102897644, + "val/ratio_var": 2.394616331002908e-06 + }, + { + "episode": 31744, + "epoch": 0.13598355037697052, + "eps": 7, + "loss/policy_avg": 0.2954641282558441, + "lr": 2.28515625e-06, + "objective/entropy": -50.37486267089844, + "objective/kl": 21.061641693115234, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.459826469421387, + "objective/scores": 8.459826469421387, + "policy/approxkl_avg": 0.24728071689605713, + "policy/clipfrac_avg": 0.3203125, + "policy/entropy_avg": 0.5351447463035583, + "step": 61, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2212, + "val/ratio": 1.000131607055664, + "val/ratio_var": 2.8336792183836224e-06 + }, + { + "episode": 32256, + "epoch": 0.13817683344756682, + "eps": 7, + "loss/policy_avg": 0.2441791594028473, + "lr": 2.2734375e-06, + "objective/entropy": -51.75057601928711, + "objective/kl": 21.086687088012695, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.422032356262207, + "objective/scores": 8.422032356262207, + "policy/approxkl_avg": 0.23406898975372314, + "policy/clipfrac_avg": 0.310546875, + "policy/entropy_avg": 0.5176072120666504, + "step": 62, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2191, + "val/ratio": 1.0005054473876953, + "val/ratio_var": 3.65329015039606e-06 + }, + { + "episode": 32768, + "epoch": 0.1403701165181631, + "eps": 7, + "loss/policy_avg": 0.22965137660503387, + "lr": 2.26171875e-06, + "objective/entropy": -51.704444885253906, + "objective/kl": 21.60610008239746, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.439580917358398, + "objective/scores": 8.439580917358398, + "policy/approxkl_avg": 0.21967627108097076, + "policy/clipfrac_avg": 0.333984375, + "policy/entropy_avg": 0.524364173412323, + "step": 63, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2332, + "val/ratio": 0.9997615814208984, + "val/ratio_var": 3.444454250711715e-06 + }, + { + "episode": 33280, + "epoch": 0.14256339958875944, + "eps": 7, + "loss/policy_avg": 0.11451836675405502, + "lr": 2.25e-06, + "objective/entropy": -52.000946044921875, + "objective/kl": 21.42214012145996, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.531951904296875, + "objective/scores": 8.531951904296875, + "policy/approxkl_avg": 0.21284940838813782, + "policy/clipfrac_avg": 0.3359375, + "policy/entropy_avg": 0.551526665687561, + "step": 64, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2463, + "val/ratio": 0.9998893141746521, + "val/ratio_var": 2.1311489035724662e-06 + }, + { + "episode": 33792, + "epoch": 0.14475668265935573, + "eps": 7, + "loss/policy_avg": 0.1996244490146637, + "lr": 2.23828125e-06, + "objective/entropy": -49.46868896484375, + "objective/kl": 22.10938262939453, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.431674003601074, + "objective/scores": 8.431674003601074, + "policy/approxkl_avg": 0.2345934510231018, + "policy/clipfrac_avg": 0.3671875, + "policy/entropy_avg": 0.5430195331573486, + "step": 65, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2527, + "val/ratio": 1.000130534172058, + "val/ratio_var": 2.9720413294853643e-06 + }, + { + "episode": 34304, + "epoch": 0.14694996572995203, + "eps": 7, + "loss/policy_avg": 0.24282732605934143, + "lr": 2.2265625e-06, + "objective/entropy": -49.502906799316406, + "objective/kl": 21.639503479003906, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.574268341064453, + "objective/scores": 8.574268341064453, + "policy/approxkl_avg": 0.27672481536865234, + "policy/clipfrac_avg": 0.373046875, + "policy/entropy_avg": 0.5653039216995239, + "step": 66, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2667, + "val/ratio": 1.000170111656189, + "val/ratio_var": 3.4806193980330136e-06 + }, + { + "episode": 34816, + "epoch": 0.14914324880054833, + "eps": 7, + "loss/policy_avg": 0.2232193946838379, + "lr": 2.21484375e-06, + "objective/entropy": -49.555030822753906, + "objective/kl": 22.374631881713867, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.583580017089844, + "objective/scores": 8.583580017089844, + "policy/approxkl_avg": 0.3105822801589966, + "policy/clipfrac_avg": 0.365234375, + "policy/entropy_avg": 0.5452811121940613, + "step": 67, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2650, + "val/ratio": 1.0001208782196045, + "val/ratio_var": 8.155854629876558e-06 + }, + { + "episode": 35328, + "epoch": 0.15133653187114463, + "eps": 7, + "loss/policy_avg": 0.23417699337005615, + "lr": 2.203125e-06, + "objective/entropy": -46.87506866455078, + "objective/kl": 23.203367233276367, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.507232666015625, + "objective/scores": 8.507232666015625, + "policy/approxkl_avg": 0.3590773344039917, + "policy/clipfrac_avg": 0.349609375, + "policy/entropy_avg": 0.5592520833015442, + "step": 68, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2704, + "val/ratio": 0.9996556043624878, + "val/ratio_var": 3.690880021167686e-06 + }, + { + "episode": 35840, + "epoch": 0.15352981494174092, + "eps": 7, + "loss/policy_avg": 0.4602639079093933, + "lr": 2.19140625e-06, + "objective/entropy": -46.802467346191406, + "objective/kl": 23.537273406982422, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.526100158691406, + "objective/scores": 8.526100158691406, + "policy/approxkl_avg": 0.2938518822193146, + "policy/clipfrac_avg": 0.32421875, + "policy/entropy_avg": 0.562473714351654, + "step": 69, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2694, + "val/ratio": 1.0002529621124268, + "val/ratio_var": 4.539975634543225e-06 + }, + { + "episode": 36352, + "epoch": 0.15572309801233722, + "eps": 7, + "loss/policy_avg": 0.2672228515148163, + "lr": 2.1796875e-06, + "objective/entropy": -45.053016662597656, + "objective/kl": 23.575054168701172, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.515979766845703, + "objective/scores": 8.515979766845703, + "policy/approxkl_avg": 0.30573177337646484, + "policy/clipfrac_avg": 0.322265625, + "policy/entropy_avg": 0.5777955055236816, + "step": 70, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2750, + "val/ratio": 1.0002713203430176, + "val/ratio_var": 5.217795205680886e-06 + }, + { + "episode": 36864, + "epoch": 0.15791638108293352, + "eps": 7, + "loss/policy_avg": 0.3402022123336792, + "lr": 2.16796875e-06, + "objective/entropy": -42.40455627441406, + "objective/kl": 25.648136138916016, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.605457305908203, + "objective/scores": 8.605457305908203, + "policy/approxkl_avg": 0.35311511158943176, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.5561670064926147, + "step": 71, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2687, + "val/ratio": 0.9998939037322998, + "val/ratio_var": 4.396093117975397e-06 + }, + { + "episode": 37376, + "epoch": 0.1601096641535298, + "eps": 7, + "loss/policy_avg": 0.3977116346359253, + "lr": 2.15625e-06, + "objective/entropy": -42.49177169799805, + "objective/kl": 26.084217071533203, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.741865158081055, + "objective/scores": 8.741865158081055, + "policy/approxkl_avg": 0.43490105867385864, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5506061315536499, + "step": 72, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2755, + "val/ratio": 0.999513566493988, + "val/ratio_var": 5.121092272020178e-06 + }, + { + "episode": 37888, + "epoch": 0.1623029472241261, + "eps": 7, + "loss/policy_avg": 0.26749086380004883, + "lr": 2.14453125e-06, + "objective/entropy": -43.96216583251953, + "objective/kl": 25.2254638671875, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.43868637084961, + "objective/scores": 8.43868637084961, + "policy/approxkl_avg": 0.37626683712005615, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5635879039764404, + "step": 73, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2632, + "val/ratio": 1.0003223419189453, + "val/ratio_var": 4.566423740470782e-06 + }, + { + "episode": 38400, + "epoch": 0.1644962302947224, + "eps": 7, + "loss/policy_avg": 0.2991521656513214, + "lr": 2.1328125e-06, + "objective/entropy": -43.968116760253906, + "objective/kl": 26.172449111938477, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.619085311889648, + "objective/scores": 8.619085311889648, + "policy/approxkl_avg": 0.36614692211151123, + "policy/clipfrac_avg": 0.361328125, + "policy/entropy_avg": 0.5751455426216125, + "step": 74, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2762, + "val/ratio": 0.9997708797454834, + "val/ratio_var": 5.6084491006913595e-06 + }, + { + "episode": 38912, + "epoch": 0.1666895133653187, + "eps": 7, + "loss/policy_avg": 0.31739309430122375, + "lr": 2.12109375e-06, + "objective/entropy": -44.440555572509766, + "objective/kl": 27.502099990844727, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.551536560058594, + "objective/scores": 8.551536560058594, + "policy/approxkl_avg": 0.4447302520275116, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5544260740280151, + "step": 75, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2669, + "val/ratio": 0.9993975162506104, + "val/ratio_var": 4.979194272891618e-06 + }, + { + "episode": 39424, + "epoch": 0.168882796435915, + "eps": 7, + "loss/policy_avg": 0.2567436695098877, + "lr": 2.109375e-06, + "objective/entropy": -43.823219299316406, + "objective/kl": 26.728805541992188, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.652971267700195, + "objective/scores": 8.652971267700195, + "policy/approxkl_avg": 0.3301650285720825, + "policy/clipfrac_avg": 0.36328125, + "policy/entropy_avg": 0.5270683765411377, + "step": 76, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2688, + "val/ratio": 1.0002834796905518, + "val/ratio_var": 4.673919193010079e-06 + }, + { + "episode": 39936, + "epoch": 0.1710760795065113, + "eps": 7, + "loss/policy_avg": 0.4975939393043518, + "lr": 2.09765625e-06, + "objective/entropy": -40.26116943359375, + "objective/kl": 28.93381690979004, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.432289123535156, + "objective/scores": 8.432289123535156, + "policy/approxkl_avg": 0.5036437511444092, + "policy/clipfrac_avg": 0.380859375, + "policy/entropy_avg": 0.5689441561698914, + "step": 77, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2716, + "val/ratio": 0.9997487664222717, + "val/ratio_var": 5.711378889827756e-06 + }, + { + "episode": 40448, + "epoch": 0.17326936257710762, + "eps": 7, + "loss/policy_avg": 0.2763368487358093, + "lr": 2.0859375e-06, + "objective/entropy": -45.21563720703125, + "objective/kl": 27.39434242248535, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.653337478637695, + "objective/scores": 8.653337478637695, + "policy/approxkl_avg": 0.2975656986236572, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5368204116821289, + "step": 78, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2757, + "val/ratio": 1.0003888607025146, + "val/ratio_var": 3.7864729165448807e-06 + }, + { + "episode": 40960, + "epoch": 0.17546264564770392, + "eps": 7, + "loss/policy_avg": 0.4642253518104553, + "lr": 2.07421875e-06, + "objective/entropy": -41.11317825317383, + "objective/kl": 27.240459442138672, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.210575103759766, + "objective/scores": 8.210575103759766, + "policy/approxkl_avg": 0.4027579426765442, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.5595495700836182, + "step": 79, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2787, + "val/ratio": 1.0000183582305908, + "val/ratio_var": 3.993025529780425e-06 + }, + { + "episode": 41472, + "epoch": 0.17765592871830022, + "eps": 7, + "loss/policy_avg": 0.24956947565078735, + "lr": 2.0625e-06, + "objective/entropy": -40.720481872558594, + "objective/kl": 28.056228637695312, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.479925155639648, + "objective/scores": 8.479925155639648, + "policy/approxkl_avg": 0.31501132249832153, + "policy/clipfrac_avg": 0.36328125, + "policy/entropy_avg": 0.5648977756500244, + "step": 80, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2724, + "val/ratio": 0.9998843669891357, + "val/ratio_var": 4.445482773007825e-06 + }, + { + "episode": 41984, + "epoch": 0.1798492117888965, + "eps": 7, + "loss/policy_avg": 0.29195988178253174, + "lr": 2.05078125e-06, + "objective/entropy": -43.005149841308594, + "objective/kl": 26.689315795898438, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.50869083404541, + "objective/scores": 8.50869083404541, + "policy/approxkl_avg": 0.3312462866306305, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.5605291724205017, + "step": 81, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2891, + "val/ratio": 0.9999593496322632, + "val/ratio_var": 3.4600634535308927e-06 + }, + { + "episode": 42496, + "epoch": 0.1820424948594928, + "eps": 7, + "loss/policy_avg": 0.22648774087429047, + "lr": 2.0390625e-06, + "objective/entropy": -44.053279876708984, + "objective/kl": 26.523632049560547, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.40163516998291, + "objective/scores": 8.40163516998291, + "policy/approxkl_avg": 0.33716368675231934, + "policy/clipfrac_avg": 0.33984375, + "policy/entropy_avg": 0.5656260251998901, + "step": 82, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2995, + "val/ratio": 0.9996054172515869, + "val/ratio_var": 4.090476068085991e-06 + }, + { + "episode": 43008, + "epoch": 0.1842357779300891, + "eps": 7, + "loss/policy_avg": 0.4572581648826599, + "lr": 2.02734375e-06, + "objective/entropy": -40.501346588134766, + "objective/kl": 26.860363006591797, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.303882598876953, + "objective/scores": 8.303882598876953, + "policy/approxkl_avg": 0.350925087928772, + "policy/clipfrac_avg": 0.373046875, + "policy/entropy_avg": 0.557375431060791, + "step": 83, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3036, + "val/ratio": 0.9999963641166687, + "val/ratio_var": 4.392126811580965e-06 + }, + { + "episode": 43520, + "epoch": 0.1864290610006854, + "eps": 7, + "loss/policy_avg": 0.2703511714935303, + "lr": 2.015625e-06, + "objective/entropy": -38.586124420166016, + "objective/kl": 27.219919204711914, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.605122566223145, + "objective/scores": 8.605122566223145, + "policy/approxkl_avg": 0.32834964990615845, + "policy/clipfrac_avg": 0.390625, + "policy/entropy_avg": 0.5444257259368896, + "step": 84, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2940, + "val/ratio": 1.0001357793807983, + "val/ratio_var": 4.64254026155686e-06 + }, + { + "episode": 44032, + "epoch": 0.1886223440712817, + "eps": 7, + "loss/policy_avg": 0.2800593972206116, + "lr": 2.00390625e-06, + "objective/entropy": -46.36981964111328, + "objective/kl": 25.693588256835938, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.409098625183105, + "objective/scores": 8.409098625183105, + "policy/approxkl_avg": 0.3142682909965515, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5546380281448364, + "step": 85, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3223, + "val/ratio": 1.0001696348190308, + "val/ratio_var": 4.388642537378473e-06 + }, + { + "episode": 44544, + "epoch": 0.190815627141878, + "eps": 7, + "loss/policy_avg": 0.2657047212123871, + "lr": 1.9921875e-06, + "objective/entropy": -45.28251647949219, + "objective/kl": 25.583412170410156, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.682401657104492, + "objective/scores": 8.682401657104492, + "policy/approxkl_avg": 0.2970484495162964, + "policy/clipfrac_avg": 0.341796875, + "policy/entropy_avg": 0.5628992319107056, + "step": 86, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3049, + "val/ratio": 1.0001099109649658, + "val/ratio_var": 3.3909404919540975e-06 + }, + { + "episode": 45056, + "epoch": 0.1930089102124743, + "eps": 7, + "loss/policy_avg": 0.31868281960487366, + "lr": 1.98046875e-06, + "objective/entropy": -44.419532775878906, + "objective/kl": 25.884258270263672, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.452953338623047, + "objective/scores": 8.452953338623047, + "policy/approxkl_avg": 0.2683827877044678, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.5685138702392578, + "step": 87, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3113, + "val/ratio": 1.0001598596572876, + "val/ratio_var": 3.914261469617486e-06 + }, + { + "episode": 45568, + "epoch": 0.1952021932830706, + "eps": 7, + "loss/policy_avg": 0.224508136510849, + "lr": 1.96875e-06, + "objective/entropy": -48.27018737792969, + "objective/kl": 24.89946174621582, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.871210098266602, + "objective/scores": 8.871210098266602, + "policy/approxkl_avg": 0.20233646035194397, + "policy/clipfrac_avg": 0.361328125, + "policy/entropy_avg": 0.5470612645149231, + "step": 88, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3254, + "val/ratio": 1.0006496906280518, + "val/ratio_var": 3.585116246540565e-06 + }, + { + "episode": 46080, + "epoch": 0.1973954763536669, + "eps": 7, + "loss/policy_avg": 0.17881432175636292, + "lr": 1.95703125e-06, + "objective/entropy": -39.60390853881836, + "objective/kl": 26.777809143066406, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.489433288574219, + "objective/scores": 8.489433288574219, + "policy/approxkl_avg": 0.30483895540237427, + "policy/clipfrac_avg": 0.3828125, + "policy/entropy_avg": 0.5878047943115234, + "step": 89, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3105, + "val/ratio": 1.0000858306884766, + "val/ratio_var": 4.914289092994295e-06 + }, + { + "episode": 46592, + "epoch": 0.19958875942426318, + "eps": 7, + "loss/policy_avg": 0.33269819617271423, + "lr": 1.9453125e-06, + "objective/entropy": -42.723388671875, + "objective/kl": 26.804061889648438, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.605770111083984, + "objective/scores": 8.605770111083984, + "policy/approxkl_avg": 0.30788183212280273, + "policy/clipfrac_avg": 0.34765625, + "policy/entropy_avg": 0.557858407497406, + "step": 90, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3024, + "val/ratio": 1.0000172853469849, + "val/ratio_var": 3.867803116008872e-06 + }, + { + "episode": 47104, + "epoch": 0.20178204249485948, + "eps": 7, + "loss/policy_avg": 0.2373349964618683, + "lr": 1.93359375e-06, + "objective/entropy": -41.628440856933594, + "objective/kl": 26.94710350036621, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.717164039611816, + "objective/scores": 8.717164039611816, + "policy/approxkl_avg": 0.28812965750694275, + "policy/clipfrac_avg": 0.38671875, + "policy/entropy_avg": 0.5584580302238464, + "step": 91, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3233, + "val/ratio": 0.9999223351478577, + "val/ratio_var": 3.552548605512129e-06 + }, + { + "episode": 47616, + "epoch": 0.20397532556545578, + "eps": 7, + "loss/policy_avg": 0.46636855602264404, + "lr": 1.921875e-06, + "objective/entropy": -42.61951446533203, + "objective/kl": 27.169050216674805, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.583017349243164, + "objective/scores": 8.583017349243164, + "policy/approxkl_avg": 0.3408641815185547, + "policy/clipfrac_avg": 0.33984375, + "policy/entropy_avg": 0.56282639503479, + "step": 92, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3154, + "val/ratio": 0.9996733069419861, + "val/ratio_var": 2.5412805371161085e-06 + }, + { + "episode": 48128, + "epoch": 0.2061686086360521, + "eps": 7, + "loss/policy_avg": 0.33910614252090454, + "lr": 1.91015625e-06, + "objective/entropy": -43.72690200805664, + "objective/kl": 27.0749568939209, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.784333229064941, + "objective/scores": 8.784333229064941, + "policy/approxkl_avg": 0.3085598945617676, + "policy/clipfrac_avg": 0.35546875, + "policy/entropy_avg": 0.5553750991821289, + "step": 93, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3186, + "val/ratio": 0.9999817609786987, + "val/ratio_var": 3.2739924336055992e-06 + }, + { + "episode": 48640, + "epoch": 0.2083618917066484, + "eps": 7, + "loss/policy_avg": 0.3540273904800415, + "lr": 1.8984375e-06, + "objective/entropy": -41.448604583740234, + "objective/kl": 27.778303146362305, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.456501007080078, + "objective/scores": 8.456501007080078, + "policy/approxkl_avg": 0.33497947454452515, + "policy/clipfrac_avg": 0.345703125, + "policy/entropy_avg": 0.5644186735153198, + "step": 94, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3182, + "val/ratio": 0.9999628663063049, + "val/ratio_var": 4.411289864947321e-06 + }, + { + "episode": 49152, + "epoch": 0.2105551747772447, + "eps": 7, + "loss/policy_avg": 0.4336338937282562, + "lr": 1.8867187500000001e-06, + "objective/entropy": -41.28203582763672, + "objective/kl": 27.56344985961914, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.71638011932373, + "objective/scores": 8.71638011932373, + "policy/approxkl_avg": 0.32576072216033936, + "policy/clipfrac_avg": 0.365234375, + "policy/entropy_avg": 0.5579572916030884, + "step": 95, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3167, + "val/ratio": 0.9997182488441467, + "val/ratio_var": 3.5568716612033313e-06 + }, + { + "episode": 49664, + "epoch": 0.212748457847841, + "eps": 7, + "loss/policy_avg": 0.37640368938446045, + "lr": 1.875e-06, + "objective/entropy": -39.93994140625, + "objective/kl": 28.659408569335938, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.587747573852539, + "objective/scores": 8.587747573852539, + "policy/approxkl_avg": 0.2930169403553009, + "policy/clipfrac_avg": 0.38671875, + "policy/entropy_avg": 0.5753656625747681, + "step": 96, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3320, + "val/ratio": 0.9999222755432129, + "val/ratio_var": 2.8392282729328144e-06 + }, + { + "episode": 50176, + "epoch": 0.2149417409184373, + "eps": 7, + "loss/policy_avg": 0.3704265356063843, + "lr": 1.86328125e-06, + "objective/entropy": -39.154541015625, + "objective/kl": 28.530101776123047, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.800564765930176, + "objective/scores": 8.800564765930176, + "policy/approxkl_avg": 0.30916929244995117, + "policy/clipfrac_avg": 0.333984375, + "policy/entropy_avg": 0.5570034384727478, + "step": 97, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3072, + "val/ratio": 1.0001652240753174, + "val/ratio_var": 4.666152108256938e-06 + }, + { + "episode": 50688, + "epoch": 0.2171350239890336, + "eps": 7, + "loss/policy_avg": 0.33340394496917725, + "lr": 1.8515625000000001e-06, + "objective/entropy": -42.39177703857422, + "objective/kl": 26.74437141418457, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.953100204467773, + "objective/scores": 8.953100204467773, + "policy/approxkl_avg": 0.25182923674583435, + "policy/clipfrac_avg": 0.34765625, + "policy/entropy_avg": 0.5791052579879761, + "step": 98, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3253, + "val/ratio": 1.0006077289581299, + "val/ratio_var": 4.077707671967801e-06 + }, + { + "episode": 51200, + "epoch": 0.21932830705962988, + "eps": 7, + "loss/policy_avg": 0.3429536521434784, + "lr": 1.83984375e-06, + "objective/entropy": -42.56659698486328, + "objective/kl": 27.317535400390625, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.745502471923828, + "objective/scores": 8.745502471923828, + "policy/approxkl_avg": 0.3268476724624634, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.5513091087341309, + "step": 99, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3308, + "val/ratio": 1.0001206398010254, + "val/ratio_var": 6.498310995084466e-06 + }, + { + "episode": 51712, + "epoch": 0.22152159013022618, + "eps": 7, + "loss/policy_avg": 0.18512360751628876, + "lr": 1.828125e-06, + "objective/entropy": -39.8660888671875, + "objective/kl": 28.532289505004883, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.849493026733398, + "objective/scores": 8.849493026733398, + "policy/approxkl_avg": 0.28483685851097107, + "policy/clipfrac_avg": 0.40625, + "policy/entropy_avg": 0.567420244216919, + "step": 100, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3124, + "val/ratio": 1.0002485513687134, + "val/ratio_var": 3.6835951959801605e-06 + }, + { + "episode": 52224, + "epoch": 0.22371487320082248, + "eps": 7, + "loss/policy_avg": 0.28863823413848877, + "lr": 1.81640625e-06, + "objective/entropy": -43.047508239746094, + "objective/kl": 27.43829345703125, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.060672760009766, + "objective/scores": 9.060672760009766, + "policy/approxkl_avg": 0.3050522208213806, + "policy/clipfrac_avg": 0.33984375, + "policy/entropy_avg": 0.5633636713027954, + "step": 101, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3185, + "val/ratio": 1.0002562999725342, + "val/ratio_var": 4.66149003841565e-06 + }, + { + "episode": 52736, + "epoch": 0.22590815627141878, + "eps": 7, + "loss/policy_avg": 0.27026695013046265, + "lr": 1.8046875e-06, + "objective/entropy": -44.638450622558594, + "objective/kl": 26.97273826599121, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.926921844482422, + "objective/scores": 8.926921844482422, + "policy/approxkl_avg": 0.26449355483055115, + "policy/clipfrac_avg": 0.337890625, + "policy/entropy_avg": 0.5377113223075867, + "step": 102, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2916, + "val/ratio": 0.9999263882637024, + "val/ratio_var": 2.791050519590499e-06 + }, + { + "episode": 53248, + "epoch": 0.22810143934201507, + "eps": 7, + "loss/policy_avg": 0.31279855966567993, + "lr": 1.79296875e-06, + "objective/entropy": -39.981178283691406, + "objective/kl": 28.416162490844727, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.903759956359863, + "objective/scores": 8.903759956359863, + "policy/approxkl_avg": 0.3121040463447571, + "policy/clipfrac_avg": 0.35546875, + "policy/entropy_avg": 0.5868556499481201, + "step": 103, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3269, + "val/ratio": 0.9997861385345459, + "val/ratio_var": 4.62020989289158e-06 + }, + { + "episode": 53760, + "epoch": 0.23029472241261137, + "eps": 7, + "loss/policy_avg": 0.20185165107250214, + "lr": 1.78125e-06, + "objective/entropy": -38.68260955810547, + "objective/kl": 28.745805740356445, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.793964385986328, + "objective/scores": 8.793964385986328, + "policy/approxkl_avg": 0.3357839286327362, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5680782198905945, + "step": 104, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3203, + "val/ratio": 0.9997376203536987, + "val/ratio_var": 3.0809435429546284e-06 + }, + { + "episode": 54272, + "epoch": 0.23248800548320767, + "eps": 7, + "loss/policy_avg": 0.27374643087387085, + "lr": 1.76953125e-06, + "objective/entropy": -41.97970199584961, + "objective/kl": 27.869659423828125, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.844221115112305, + "objective/scores": 8.844221115112305, + "policy/approxkl_avg": 0.2983897030353546, + "policy/clipfrac_avg": 0.388671875, + "policy/entropy_avg": 0.569934606552124, + "step": 105, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3125, + "val/ratio": 0.9998430013656616, + "val/ratio_var": 4.563687980407849e-06 + }, + { + "episode": 54784, + "epoch": 0.23468128855380396, + "eps": 7, + "loss/policy_avg": 0.28290292620658875, + "lr": 1.7578125e-06, + "objective/entropy": -41.56986618041992, + "objective/kl": 27.95354652404785, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.726874351501465, + "objective/scores": 8.726874351501465, + "policy/approxkl_avg": 0.35987648367881775, + "policy/clipfrac_avg": 0.37109375, + "policy/entropy_avg": 0.5299930572509766, + "step": 106, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2941, + "val/ratio": 0.9999336004257202, + "val/ratio_var": 5.3527737691183574e-06 + }, + { + "episode": 55296, + "epoch": 0.2368745716244003, + "eps": 7, + "loss/policy_avg": 0.2681512236595154, + "lr": 1.74609375e-06, + "objective/entropy": -43.27360916137695, + "objective/kl": 27.097171783447266, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.131484031677246, + "objective/scores": 9.131484031677246, + "policy/approxkl_avg": 0.27127525210380554, + "policy/clipfrac_avg": 0.34765625, + "policy/entropy_avg": 0.5349740982055664, + "step": 107, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3110, + "val/ratio": 1.0000276565551758, + "val/ratio_var": 3.7708189211116405e-06 + }, + { + "episode": 55808, + "epoch": 0.23906785469499658, + "eps": 7, + "loss/policy_avg": 0.3031000792980194, + "lr": 1.734375e-06, + "objective/entropy": -41.32918930053711, + "objective/kl": 28.28663444519043, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.978072166442871, + "objective/scores": 8.978072166442871, + "policy/approxkl_avg": 0.28883644938468933, + "policy/clipfrac_avg": 0.384765625, + "policy/entropy_avg": 0.5501591563224792, + "step": 108, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3257, + "val/ratio": 1.0002058744430542, + "val/ratio_var": 2.6081083888129797e-06 + }, + { + "episode": 56320, + "epoch": 0.24126113776559288, + "eps": 7, + "loss/policy_avg": 0.22930988669395447, + "lr": 1.72265625e-06, + "objective/entropy": -44.260066986083984, + "objective/kl": 27.051563262939453, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.064173698425293, + "objective/scores": 9.064173698425293, + "policy/approxkl_avg": 0.25021883845329285, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5275313854217529, + "step": 109, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3274, + "val/ratio": 1.0002585649490356, + "val/ratio_var": 4.10879465562175e-06 + }, + { + "episode": 56832, + "epoch": 0.24345442083618918, + "eps": 7, + "loss/policy_avg": 0.32286208868026733, + "lr": 1.7109375e-06, + "objective/entropy": -44.09670639038086, + "objective/kl": 27.300418853759766, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.928081512451172, + "objective/scores": 8.928081512451172, + "policy/approxkl_avg": 0.2881784737110138, + "policy/clipfrac_avg": 0.337890625, + "policy/entropy_avg": 0.5374830961227417, + "step": 110, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3310, + "val/ratio": 1.000059962272644, + "val/ratio_var": 4.859982709604083e-06 + }, + { + "episode": 57344, + "epoch": 0.24564770390678548, + "eps": 7, + "loss/policy_avg": 0.1695113182067871, + "lr": 1.69921875e-06, + "objective/entropy": -42.695960998535156, + "objective/kl": 26.841167449951172, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.010541915893555, + "objective/scores": 9.010541915893555, + "policy/approxkl_avg": 0.3248569369316101, + "policy/clipfrac_avg": 0.3828125, + "policy/entropy_avg": 0.557858943939209, + "step": 111, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3280, + "val/ratio": 0.9998301267623901, + "val/ratio_var": 3.137585736112669e-06 + }, + { + "episode": 57856, + "epoch": 0.24784098697738177, + "eps": 7, + "loss/policy_avg": 0.23961672186851501, + "lr": 1.6875e-06, + "objective/entropy": -41.781715393066406, + "objective/kl": 27.928836822509766, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.928047180175781, + "objective/scores": 8.928047180175781, + "policy/approxkl_avg": 0.2984079420566559, + "policy/clipfrac_avg": 0.37109375, + "policy/entropy_avg": 0.5693182945251465, + "step": 112, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3228, + "val/ratio": 0.9999939203262329, + "val/ratio_var": 3.22428149956977e-06 + }, + { + "episode": 58368, + "epoch": 0.25003427004797807, + "eps": 7, + "loss/policy_avg": 0.3170594573020935, + "lr": 1.67578125e-06, + "objective/entropy": -44.25893783569336, + "objective/kl": 27.417301177978516, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.042917251586914, + "objective/scores": 9.042917251586914, + "policy/approxkl_avg": 0.28602951765060425, + "policy/clipfrac_avg": 0.33984375, + "policy/entropy_avg": 0.5440371632575989, + "step": 113, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3082, + "val/ratio": 1.0005700588226318, + "val/ratio_var": 1.94922395166941e-05 + }, + { + "episode": 58880, + "epoch": 0.25222755311857437, + "eps": 7, + "loss/policy_avg": 0.40132516622543335, + "lr": 1.6640625e-06, + "objective/entropy": -40.10181427001953, + "objective/kl": 27.927541732788086, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.806680679321289, + "objective/scores": 8.806680679321289, + "policy/approxkl_avg": 0.3599626421928406, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5521677732467651, + "step": 114, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3332, + "val/ratio": 0.9998911023139954, + "val/ratio_var": 4.160545358899981e-06 + }, + { + "episode": 59392, + "epoch": 0.25442083618917066, + "eps": 7, + "loss/policy_avg": 0.3721383512020111, + "lr": 1.6523437500000001e-06, + "objective/entropy": -40.903629302978516, + "objective/kl": 28.625869750976562, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.888092994689941, + "objective/scores": 8.888092994689941, + "policy/approxkl_avg": 0.3267340660095215, + "policy/clipfrac_avg": 0.34765625, + "policy/entropy_avg": 0.5643830299377441, + "step": 115, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3251, + "val/ratio": 1.0002787113189697, + "val/ratio_var": 4.806289325642865e-06 + }, + { + "episode": 59904, + "epoch": 0.25661411925976696, + "eps": 7, + "loss/policy_avg": 0.27686673402786255, + "lr": 1.640625e-06, + "objective/entropy": -44.158592224121094, + "objective/kl": 26.43977928161621, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.87618637084961, + "objective/scores": 8.87618637084961, + "policy/approxkl_avg": 0.2663460671901703, + "policy/clipfrac_avg": 0.318359375, + "policy/entropy_avg": 0.5564490556716919, + "step": 116, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3377, + "val/ratio": 0.9997222423553467, + "val/ratio_var": 4.367329438537126e-06 + }, + { + "episode": 60416, + "epoch": 0.25880740233036326, + "eps": 7, + "loss/policy_avg": 0.25908106565475464, + "lr": 1.62890625e-06, + "objective/entropy": -41.13191223144531, + "objective/kl": 27.067249298095703, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.76617431640625, + "objective/scores": 8.76617431640625, + "policy/approxkl_avg": 0.2690734267234802, + "policy/clipfrac_avg": 0.3359375, + "policy/entropy_avg": 0.5597757697105408, + "step": 117, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3447, + "val/ratio": 1.0001375675201416, + "val/ratio_var": 3.1488286822423106e-06 + }, + { + "episode": 60928, + "epoch": 0.26100068540095955, + "eps": 7, + "loss/policy_avg": 0.3060078024864197, + "lr": 1.6171875000000001e-06, + "objective/entropy": -43.39402770996094, + "objective/kl": 26.568906784057617, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.188919067382812, + "objective/scores": 9.188919067382812, + "policy/approxkl_avg": 0.30087438225746155, + "policy/clipfrac_avg": 0.345703125, + "policy/entropy_avg": 0.543894350528717, + "step": 118, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3462, + "val/ratio": 1.0000953674316406, + "val/ratio_var": 3.770735020225402e-06 + }, + { + "episode": 61440, + "epoch": 0.26319396847155585, + "eps": 7, + "loss/policy_avg": 0.22503231465816498, + "lr": 1.60546875e-06, + "objective/entropy": -41.05299758911133, + "objective/kl": 27.94589614868164, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.894856452941895, + "objective/scores": 8.894856452941895, + "policy/approxkl_avg": 0.2811037003993988, + "policy/clipfrac_avg": 0.359375, + "policy/entropy_avg": 0.5578025579452515, + "step": 119, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3312, + "val/ratio": 0.9998151063919067, + "val/ratio_var": 3.168901230310439e-06 + }, + { + "episode": 61952, + "epoch": 0.26538725154215215, + "eps": 7, + "loss/policy_avg": 0.18785294890403748, + "lr": 1.59375e-06, + "objective/entropy": -44.082115173339844, + "objective/kl": 26.886157989501953, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.022890090942383, + "objective/scores": 9.022890090942383, + "policy/approxkl_avg": 0.23531067371368408, + "policy/clipfrac_avg": 0.330078125, + "policy/entropy_avg": 0.5334745049476624, + "step": 120, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3293, + "val/ratio": 1.0002555847167969, + "val/ratio_var": 3.1270933504856657e-06 + }, + { + "episode": 62464, + "epoch": 0.26758053461274844, + "eps": 7, + "loss/policy_avg": 0.2825835943222046, + "lr": 1.5820312500000001e-06, + "objective/entropy": -41.7612190246582, + "objective/kl": 28.106294631958008, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.90147590637207, + "objective/scores": 8.90147590637207, + "policy/approxkl_avg": 0.28883862495422363, + "policy/clipfrac_avg": 0.353515625, + "policy/entropy_avg": 0.5609524846076965, + "step": 121, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3501, + "val/ratio": 0.9996764659881592, + "val/ratio_var": 3.0721889743290376e-06 + }, + { + "episode": 62976, + "epoch": 0.26977381768334474, + "eps": 7, + "loss/policy_avg": 0.42201805114746094, + "lr": 1.5703125e-06, + "objective/entropy": -40.16608428955078, + "objective/kl": 29.224674224853516, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.230374336242676, + "objective/scores": 9.230374336242676, + "policy/approxkl_avg": 0.3132435083389282, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.545167863368988, + "step": 122, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3192, + "val/ratio": 1.000016689300537, + "val/ratio_var": 5.478911134559894e-06 + }, + { + "episode": 63488, + "epoch": 0.27196710075394104, + "eps": 7, + "loss/policy_avg": 0.13765352964401245, + "lr": 1.55859375e-06, + "objective/entropy": -38.40263366699219, + "objective/kl": 28.912315368652344, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.914356231689453, + "objective/scores": 8.914356231689453, + "policy/approxkl_avg": 0.26425743103027344, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5654212236404419, + "step": 123, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3085, + "val/ratio": 0.9999741315841675, + "val/ratio_var": 3.0958819934312487e-06 + }, + { + "episode": 64000, + "epoch": 0.27416038382453733, + "eps": 7, + "loss/policy_avg": 0.281406968832016, + "lr": 1.5468750000000001e-06, + "objective/entropy": -41.14397430419922, + "objective/kl": 27.943584442138672, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.850857734680176, + "objective/scores": 8.850857734680176, + "policy/approxkl_avg": 0.26425665616989136, + "policy/clipfrac_avg": 0.341796875, + "policy/entropy_avg": 0.5333235263824463, + "step": 124, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2966, + "val/ratio": 1.0001806020736694, + "val/ratio_var": 4.1297735151601955e-06 + }, + { + "episode": 64512, + "epoch": 0.27635366689513363, + "eps": 7, + "loss/policy_avg": 0.3354412317276001, + "lr": 1.53515625e-06, + "objective/entropy": -40.96007537841797, + "objective/kl": 28.996366500854492, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.931587219238281, + "objective/scores": 8.931587219238281, + "policy/approxkl_avg": 0.2954847514629364, + "policy/clipfrac_avg": 0.359375, + "policy/entropy_avg": 0.5366266965866089, + "step": 125, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2979, + "val/ratio": 1.000289797782898, + "val/ratio_var": 3.790634536926518e-06 + }, + { + "episode": 65024, + "epoch": 0.27854694996572993, + "eps": 7, + "loss/policy_avg": 0.2738310396671295, + "lr": 1.5234375e-06, + "objective/entropy": -40.16575622558594, + "objective/kl": 29.01610565185547, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.160589218139648, + "objective/scores": 9.160589218139648, + "policy/approxkl_avg": 0.30494195222854614, + "policy/clipfrac_avg": 0.33984375, + "policy/entropy_avg": 0.5441122055053711, + "step": 126, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3075, + "val/ratio": 0.9997348785400391, + "val/ratio_var": 3.715499815371004e-06 + }, + { + "episode": 65536, + "epoch": 0.2807402330363262, + "eps": 7, + "loss/policy_avg": 0.3204410672187805, + "lr": 1.5117187500000001e-06, + "objective/entropy": -41.28175354003906, + "objective/kl": 28.728578567504883, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.17586612701416, + "objective/scores": 9.17586612701416, + "policy/approxkl_avg": 0.25520649552345276, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.5370270013809204, + "step": 127, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2918, + "val/ratio": 1.0001736879348755, + "val/ratio_var": 3.415733999645454e-06 + }, + { + "episode": 66048, + "epoch": 0.2829335161069225, + "eps": 7, + "loss/policy_avg": 0.23167520761489868, + "lr": 1.5e-06, + "objective/entropy": -40.13172912597656, + "objective/kl": 29.34070587158203, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.950373649597168, + "objective/scores": 8.950373649597168, + "policy/approxkl_avg": 0.2691437304019928, + "policy/clipfrac_avg": 0.36328125, + "policy/entropy_avg": 0.534875750541687, + "step": 128, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2974, + "val/ratio": 0.9997478723526001, + "val/ratio_var": 3.2285590805258835e-06 + }, + { + "episode": 66560, + "epoch": 0.2851267991775189, + "eps": 7, + "loss/policy_avg": 0.2088831514120102, + "lr": 1.48828125e-06, + "objective/entropy": -38.12837600708008, + "objective/kl": 28.746782302856445, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.036067962646484, + "objective/scores": 9.036067962646484, + "policy/approxkl_avg": 0.3211039900779724, + "policy/clipfrac_avg": 0.361328125, + "policy/entropy_avg": 0.5321791172027588, + "step": 129, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3088, + "val/ratio": 1.0001753568649292, + "val/ratio_var": 5.1817132771248e-06 + }, + { + "episode": 67072, + "epoch": 0.28732008224811517, + "eps": 7, + "loss/policy_avg": 0.2225157618522644, + "lr": 1.4765625e-06, + "objective/entropy": -38.43605041503906, + "objective/kl": 30.095699310302734, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.020343780517578, + "objective/scores": 9.020343780517578, + "policy/approxkl_avg": 0.29494479298591614, + "policy/clipfrac_avg": 0.365234375, + "policy/entropy_avg": 0.528764009475708, + "step": 130, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2872, + "val/ratio": 0.9999305009841919, + "val/ratio_var": 4.7066237129911315e-06 + }, + { + "episode": 67584, + "epoch": 0.28951336531871147, + "eps": 7, + "loss/policy_avg": 0.18626652657985687, + "lr": 1.46484375e-06, + "objective/entropy": -38.2808837890625, + "objective/kl": 29.611225128173828, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.177016258239746, + "objective/scores": 9.177016258239746, + "policy/approxkl_avg": 0.24960750341415405, + "policy/clipfrac_avg": 0.37890625, + "policy/entropy_avg": 0.5560537576675415, + "step": 131, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3039, + "val/ratio": 0.9996904134750366, + "val/ratio_var": 2.8264887532714056e-06 + }, + { + "episode": 68096, + "epoch": 0.29170664838930777, + "eps": 7, + "loss/policy_avg": 0.34811753034591675, + "lr": 1.453125e-06, + "objective/entropy": -37.813743591308594, + "objective/kl": 30.11647605895996, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.143380165100098, + "objective/scores": 9.143380165100098, + "policy/approxkl_avg": 0.24777096509933472, + "policy/clipfrac_avg": 0.322265625, + "policy/entropy_avg": 0.5264837741851807, + "step": 132, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2881, + "val/ratio": 1.0001630783081055, + "val/ratio_var": 2.7501182557898574e-06 + }, + { + "episode": 68608, + "epoch": 0.29389993145990406, + "eps": 7, + "loss/policy_avg": 1.7297974824905396, + "lr": 1.44140625e-06, + "objective/entropy": -38.04670715332031, + "objective/kl": 29.923572540283203, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.23299789428711, + "objective/scores": 9.23299789428711, + "policy/approxkl_avg": 0.3208100199699402, + "policy/clipfrac_avg": 0.349609375, + "policy/entropy_avg": 0.5299798250198364, + "step": 133, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2948, + "val/ratio": 1.0002143383026123, + "val/ratio_var": 6.060246505512623e-06 + }, + { + "episode": 69120, + "epoch": 0.29609321453050036, + "eps": 7, + "loss/policy_avg": 0.2908250093460083, + "lr": 1.4296875e-06, + "objective/entropy": -39.67091751098633, + "objective/kl": 29.891416549682617, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.22230339050293, + "objective/scores": 9.22230339050293, + "policy/approxkl_avg": 0.2994619607925415, + "policy/clipfrac_avg": 0.361328125, + "policy/entropy_avg": 0.5484848618507385, + "step": 134, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 2992, + "val/ratio": 0.9998666048049927, + "val/ratio_var": 4.691268713941099e-06 + }, + { + "episode": 69632, + "epoch": 0.29828649760109666, + "eps": 6, + "loss/policy_avg": 0.16967931389808655, + "lr": 1.41796875e-06, + "objective/entropy": -40.47759246826172, + "objective/kl": 28.60110855102539, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.249778747558594, + "objective/scores": 9.249778747558594, + "policy/approxkl_avg": 0.30396944284439087, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.5294628739356995, + "step": 135, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3158, + "val/ratio": 0.9999102354049683, + "val/ratio_var": 4.847398031415651e-06 + }, + { + "episode": 70144, + "epoch": 0.30047978067169295, + "eps": 6, + "loss/policy_avg": 0.30670228600502014, + "lr": 1.40625e-06, + "objective/entropy": -41.60497283935547, + "objective/kl": 28.722915649414062, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.269282341003418, + "objective/scores": 9.269282341003418, + "policy/approxkl_avg": 0.27743324637413025, + "policy/clipfrac_avg": 0.35546875, + "policy/entropy_avg": 0.5208898782730103, + "step": 136, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3202, + "val/ratio": 1.0001318454742432, + "val/ratio_var": 4.43136786998366e-06 + }, + { + "episode": 70656, + "epoch": 0.30267306374228925, + "eps": 6, + "loss/policy_avg": 0.24920354783535004, + "lr": 1.39453125e-06, + "objective/entropy": -42.12297439575195, + "objective/kl": 28.017452239990234, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.922027587890625, + "objective/scores": 8.922027587890625, + "policy/approxkl_avg": 0.23385344445705414, + "policy/clipfrac_avg": 0.365234375, + "policy/entropy_avg": 0.5616132616996765, + "step": 137, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3543, + "val/ratio": 0.9999379515647888, + "val/ratio_var": 3.0522689939971315e-06 + }, + { + "episode": 71168, + "epoch": 0.30486634681288555, + "eps": 6, + "loss/policy_avg": 0.18219150602817535, + "lr": 1.3828125e-06, + "objective/entropy": -41.68804168701172, + "objective/kl": 28.044918060302734, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.24627685546875, + "objective/scores": 9.24627685546875, + "policy/approxkl_avg": 0.32243162393569946, + "policy/clipfrac_avg": 0.41796875, + "policy/entropy_avg": 0.5281212329864502, + "step": 138, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3289, + "val/ratio": 0.9998571872711182, + "val/ratio_var": 4.768267899635248e-06 + }, + { + "episode": 71680, + "epoch": 0.30705962988348184, + "eps": 6, + "loss/policy_avg": 0.252814918756485, + "lr": 1.37109375e-06, + "objective/entropy": -43.992950439453125, + "objective/kl": 27.172204971313477, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.964099884033203, + "objective/scores": 8.964099884033203, + "policy/approxkl_avg": 0.25193220376968384, + "policy/clipfrac_avg": 0.357421875, + "policy/entropy_avg": 0.5601909160614014, + "step": 139, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3426, + "val/ratio": 1.0002691745758057, + "val/ratio_var": 4.502016508922679e-06 + }, + { + "episode": 72192, + "epoch": 0.30925291295407814, + "eps": 6, + "loss/policy_avg": 0.19190388917922974, + "lr": 1.359375e-06, + "objective/entropy": -42.40169143676758, + "objective/kl": 28.326745986938477, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.860151290893555, + "objective/scores": 8.860151290893555, + "policy/approxkl_avg": 0.2868598699569702, + "policy/clipfrac_avg": 0.3671875, + "policy/entropy_avg": 0.5509663820266724, + "step": 140, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3451, + "val/ratio": 0.9999006390571594, + "val/ratio_var": 3.2771320093161194e-06 + }, + { + "episode": 72704, + "epoch": 0.31144619602467444, + "eps": 6, + "loss/policy_avg": 0.41269177198410034, + "lr": 1.34765625e-06, + "objective/entropy": -43.28689193725586, + "objective/kl": 27.77793312072754, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.921087265014648, + "objective/scores": 8.921087265014648, + "policy/approxkl_avg": 0.27822810411453247, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.5682795643806458, + "step": 141, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3703, + "val/ratio": 1.0001779794692993, + "val/ratio_var": 5.943717042100616e-06 + }, + { + "episode": 73216, + "epoch": 0.31363947909527073, + "eps": 6, + "loss/policy_avg": 0.22461438179016113, + "lr": 1.3359375e-06, + "objective/entropy": -41.46437072753906, + "objective/kl": 28.51311492919922, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.918792724609375, + "objective/scores": 8.918792724609375, + "policy/approxkl_avg": 0.2884824275970459, + "policy/clipfrac_avg": 0.392578125, + "policy/entropy_avg": 0.5668504238128662, + "step": 142, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3600, + "val/ratio": 0.9997174143791199, + "val/ratio_var": 3.1224142276187195e-06 + }, + { + "episode": 73728, + "epoch": 0.31583276216586703, + "eps": 6, + "loss/policy_avg": 0.26002606749534607, + "lr": 1.32421875e-06, + "objective/entropy": -40.683143615722656, + "objective/kl": 29.9025936126709, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.003212928771973, + "objective/scores": 9.003212928771973, + "policy/approxkl_avg": 0.41950270533561707, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.5366673469543457, + "step": 143, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3528, + "val/ratio": 1.0001637935638428, + "val/ratio_var": 4.806850483873859e-06 + }, + { + "episode": 74240, + "epoch": 0.31802604523646333, + "eps": 6, + "loss/policy_avg": 0.2675803303718567, + "lr": 1.3125000000000001e-06, + "objective/entropy": -41.83632278442383, + "objective/kl": 28.70281982421875, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.011963844299316, + "objective/scores": 9.011963844299316, + "policy/approxkl_avg": 0.23804181814193726, + "policy/clipfrac_avg": 0.38671875, + "policy/entropy_avg": 0.5452957153320312, + "step": 144, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3428, + "val/ratio": 1.0001215934753418, + "val/ratio_var": 3.4354568470007507e-06 + }, + { + "episode": 74752, + "epoch": 0.3202193283070596, + "eps": 6, + "loss/policy_avg": 0.3060368299484253, + "lr": 1.30078125e-06, + "objective/entropy": -40.88287353515625, + "objective/kl": 30.03162384033203, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.887903213500977, + "objective/scores": 8.887903213500977, + "policy/approxkl_avg": 0.2737124264240265, + "policy/clipfrac_avg": 0.376953125, + "policy/entropy_avg": 0.5587130784988403, + "step": 145, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3376, + "val/ratio": 0.9998096823692322, + "val/ratio_var": 3.0338110263983253e-06 + }, + { + "episode": 75264, + "epoch": 0.3224126113776559, + "eps": 6, + "loss/policy_avg": 0.3782508075237274, + "lr": 1.2890625e-06, + "objective/entropy": -41.50593185424805, + "objective/kl": 28.35731315612793, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.796640396118164, + "objective/scores": 8.796640396118164, + "policy/approxkl_avg": 0.26787978410720825, + "policy/clipfrac_avg": 0.34375, + "policy/entropy_avg": 0.5557115077972412, + "step": 146, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3456, + "val/ratio": 1.0003128051757812, + "val/ratio_var": 3.1068911994225346e-06 + }, + { + "episode": 75776, + "epoch": 0.3246058944482522, + "eps": 6, + "loss/policy_avg": 0.24699635803699493, + "lr": 1.2773437500000001e-06, + "objective/entropy": -38.084693908691406, + "objective/kl": 30.745229721069336, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.165950775146484, + "objective/scores": 9.165950775146484, + "policy/approxkl_avg": 0.30605587363243103, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.550534725189209, + "step": 147, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3354, + "val/ratio": 0.9999103546142578, + "val/ratio_var": 3.188604523529648e-06 + }, + { + "episode": 76288, + "epoch": 0.3267991775188485, + "eps": 6, + "loss/policy_avg": 0.21277828514575958, + "lr": 1.265625e-06, + "objective/entropy": -39.392494201660156, + "objective/kl": 31.08953094482422, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.794751167297363, + "objective/scores": 8.794751167297363, + "policy/approxkl_avg": 0.2738240957260132, + "policy/clipfrac_avg": 0.375, + "policy/entropy_avg": 0.5848383903503418, + "step": 148, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3624, + "val/ratio": 1.0000360012054443, + "val/ratio_var": 3.766557256312808e-06 + }, + { + "episode": 76800, + "epoch": 0.3289924605894448, + "eps": 6, + "loss/policy_avg": 0.4244789481163025, + "lr": 1.25390625e-06, + "objective/entropy": -39.247779846191406, + "objective/kl": 30.732606887817383, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 9.010191917419434, + "objective/scores": 9.010191917419434, + "policy/approxkl_avg": 0.29289907217025757, + "policy/clipfrac_avg": 0.35546875, + "policy/entropy_avg": 0.5503936409950256, + "step": 149, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3473, + "val/ratio": 0.9998835325241089, + "val/ratio_var": 4.186950718576554e-06 + }, + { + "episode": 77312, + "epoch": 0.3311857436600411, + "eps": 6, + "loss/policy_avg": 0.2913992404937744, + "lr": 1.2421875000000001e-06, + "objective/entropy": -36.79509735107422, + "objective/kl": 31.21529769897461, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.547361373901367, + "objective/scores": 8.547361373901367, + "policy/approxkl_avg": 0.2886246144771576, + "policy/clipfrac_avg": 0.40234375, + "policy/entropy_avg": 0.5547722578048706, + "step": 150, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3487, + "val/ratio": 1.000207543373108, + "val/ratio_var": 5.082135885459138e-06 + }, + { + "episode": 77824, + "epoch": 0.3333790267306374, + "eps": 6, + "loss/policy_avg": 0.26813989877700806, + "lr": 1.23046875e-06, + "objective/entropy": -37.272186279296875, + "objective/kl": 31.253156661987305, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.79878044128418, + "objective/scores": 8.79878044128418, + "policy/approxkl_avg": 0.30616575479507446, + "policy/clipfrac_avg": 0.380859375, + "policy/entropy_avg": 0.5631169676780701, + "step": 151, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3643, + "val/ratio": 1.000023365020752, + "val/ratio_var": 4.361940682429122e-06 + }, + { + "episode": 78336, + "epoch": 0.3355723098012337, + "eps": 6, + "loss/policy_avg": 0.23403358459472656, + "lr": 1.21875e-06, + "objective/entropy": -40.19910430908203, + "objective/kl": 30.405378341674805, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.710663795471191, + "objective/scores": 8.710663795471191, + "policy/approxkl_avg": 0.2954765856266022, + "policy/clipfrac_avg": 0.380859375, + "policy/entropy_avg": 0.5614674091339111, + "step": 152, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3742, + "val/ratio": 0.9998055100440979, + "val/ratio_var": 3.049866336368723e-06 + }, + { + "episode": 78848, + "epoch": 0.33776559287183, + "eps": 6, + "loss/policy_avg": 0.23645663261413574, + "lr": 1.2070312500000001e-06, + "objective/entropy": -39.029476165771484, + "objective/kl": 31.432140350341797, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.8472318649292, + "objective/scores": 8.8472318649292, + "policy/approxkl_avg": 0.3671955168247223, + "policy/clipfrac_avg": 0.41015625, + "policy/entropy_avg": 0.5521472096443176, + "step": 153, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3732, + "val/ratio": 0.9998006820678711, + "val/ratio_var": 4.156634986429708e-06 + }, + { + "episode": 79360, + "epoch": 0.3399588759424263, + "eps": 6, + "loss/policy_avg": 0.2862551510334015, + "lr": 1.1953125e-06, + "objective/entropy": -38.52696228027344, + "objective/kl": 31.745351791381836, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.84429931640625, + "objective/scores": 8.84429931640625, + "policy/approxkl_avg": 0.25862056016921997, + "policy/clipfrac_avg": 0.34765625, + "policy/entropy_avg": 0.5486101508140564, + "step": 154, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3528, + "val/ratio": 0.9999098777770996, + "val/ratio_var": 2.299218749612919e-06 + }, + { + "episode": 79872, + "epoch": 0.3421521590130226, + "eps": 6, + "loss/policy_avg": 0.14416684210300446, + "lr": 1.18359375e-06, + "objective/entropy": -42.237709045410156, + "objective/kl": 29.907611846923828, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.855605125427246, + "objective/scores": 8.855605125427246, + "policy/approxkl_avg": 0.21241676807403564, + "policy/clipfrac_avg": 0.380859375, + "policy/entropy_avg": 0.536536455154419, + "step": 155, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3724, + "val/ratio": 1.0001240968704224, + "val/ratio_var": 1.9381159290787764e-06 + }, + { + "episode": 80384, + "epoch": 0.3443454420836189, + "eps": 6, + "loss/policy_avg": 0.2458827793598175, + "lr": 1.1718750000000001e-06, + "objective/entropy": -43.01675033569336, + "objective/kl": 28.34514617919922, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.800366401672363, + "objective/scores": 8.800366401672363, + "policy/approxkl_avg": 0.2694607973098755, + "policy/clipfrac_avg": 0.384765625, + "policy/entropy_avg": 0.517730712890625, + "step": 156, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3589, + "val/ratio": 0.9997525811195374, + "val/ratio_var": 4.13375164498575e-06 + }, + { + "episode": 80896, + "epoch": 0.34653872515421524, + "eps": 6, + "loss/policy_avg": 0.2523859441280365, + "lr": 1.16015625e-06, + "objective/entropy": -39.49822998046875, + "objective/kl": 31.242511749267578, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.724842071533203, + "objective/scores": 8.724842071533203, + "policy/approxkl_avg": 0.25573909282684326, + "policy/clipfrac_avg": 0.3515625, + "policy/entropy_avg": 0.5711158514022827, + "step": 157, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3754, + "val/ratio": 0.9999100565910339, + "val/ratio_var": 2.5960848688555416e-06 + }, + { + "episode": 81408, + "epoch": 0.34873200822481154, + "eps": 6, + "loss/policy_avg": 0.325059175491333, + "lr": 1.1484375e-06, + "objective/entropy": -41.47160339355469, + "objective/kl": 29.920059204101562, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.936443328857422, + "objective/scores": 8.936443328857422, + "policy/approxkl_avg": 0.2773388624191284, + "policy/clipfrac_avg": 0.365234375, + "policy/entropy_avg": 0.5378261804580688, + "step": 158, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3784, + "val/ratio": 1.0000555515289307, + "val/ratio_var": 4.364408596302383e-06 + }, + { + "episode": 81920, + "epoch": 0.35092529129540784, + "eps": 6, + "loss/policy_avg": 0.21814444661140442, + "lr": 1.13671875e-06, + "objective/entropy": -38.8851318359375, + "objective/kl": 30.688180923461914, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.690467834472656, + "objective/scores": 8.690467834472656, + "policy/approxkl_avg": 0.2513526678085327, + "policy/clipfrac_avg": 0.373046875, + "policy/entropy_avg": 0.5520586967468262, + "step": 159, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3535, + "val/ratio": 1.0000085830688477, + "val/ratio_var": 3.6695571452582953e-06 + }, + { + "episode": 82432, + "epoch": 0.35311857436600413, + "eps": 6, + "loss/policy_avg": 0.2753603160381317, + "lr": 1.125e-06, + "objective/entropy": -38.512779235839844, + "objective/kl": 32.16954040527344, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.46960735321045, + "objective/scores": 8.46960735321045, + "policy/approxkl_avg": 0.3181775212287903, + "policy/clipfrac_avg": 0.380859375, + "policy/entropy_avg": 0.556164562702179, + "step": 160, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3743, + "val/ratio": 1.0000784397125244, + "val/ratio_var": 3.5829389162245207e-06 + }, + { + "episode": 82944, + "epoch": 0.35531185743660043, + "eps": 6, + "loss/policy_avg": 0.40143609046936035, + "lr": 1.11328125e-06, + "objective/entropy": -36.5264778137207, + "objective/kl": 32.430667877197266, + "objective/non_score_reward": 0.0, + "objective/rlhf_reward": 8.716072082519531, + "objective/scores": 8.716072082519531, + "policy/approxkl_avg": 0.31610533595085144, + "policy/clipfrac_avg": 0.369140625, + "policy/entropy_avg": 0.5550679564476013, + "step": 161, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3525, + "val/ratio": 1.0001959800720215, + "val/ratio_var": 3.588993422454223e-06 + } + ], + "logging_steps": 1, + "max_steps": 128, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.122960932145305, + "save_steps": 26, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": true, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +}