| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "episode": 77824, | |
| "epoch": 0.3333790267306374, | |
| "eval_steps": 500, | |
| "global_step": 152, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "episode": 512, | |
| "epoch": 0.0021932830705962986, | |
| "eps": 6, | |
| "loss/policy_avg": 0.15268513560295105, | |
| "lr": 3e-06, | |
| "objective/entropy": -45.407432556152344, | |
| "objective/kl": 0.3935524821281433, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.679054260253906, | |
| "objective/scores": 4.679054260253906, | |
| "policy/approxkl_avg": 0.11690396815538406, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.8878304958343506, | |
| "step": 0, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2729, | |
| "val/ratio": 1.000101089477539, | |
| "val/ratio_var": 1.6687282595739816e-06 | |
| }, | |
| { | |
| "episode": 1024, | |
| "epoch": 0.004386566141192597, | |
| "eps": 6, | |
| "loss/policy_avg": 0.10589599609375, | |
| "lr": 2.9882812500000002e-06, | |
| "objective/entropy": -51.535125732421875, | |
| "objective/kl": 0.6498137712478638, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.541956901550293, | |
| "objective/scores": 4.541956901550293, | |
| "policy/approxkl_avg": 0.10543511807918549, | |
| "policy/clipfrac_avg": 0.3203125, | |
| "policy/entropy_avg": 0.8243188858032227, | |
| "step": 1, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3325, | |
| "val/ratio": 1.0002458095550537, | |
| "val/ratio_var": 1.5084623328220914e-06 | |
| }, | |
| { | |
| "episode": 1536, | |
| "epoch": 0.006579849211788897, | |
| "eps": 6, | |
| "loss/policy_avg": 0.13810233771800995, | |
| "lr": 2.9765625e-06, | |
| "objective/entropy": -55.89725112915039, | |
| "objective/kl": 1.300195574760437, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.745310306549072, | |
| "objective/scores": 4.745310306549072, | |
| "policy/approxkl_avg": 0.11424587666988373, | |
| "policy/clipfrac_avg": 0.302734375, | |
| "policy/entropy_avg": 0.760991096496582, | |
| "step": 2, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3801, | |
| "val/ratio": 0.9999536275863647, | |
| "val/ratio_var": 1.5804212125658523e-06 | |
| }, | |
| { | |
| "episode": 2048, | |
| "epoch": 0.008773132282385195, | |
| "eps": 6, | |
| "loss/policy_avg": 0.25361043214797974, | |
| "lr": 2.96484375e-06, | |
| "objective/entropy": -54.02920150756836, | |
| "objective/kl": 1.721588373184204, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.8178606033325195, | |
| "objective/scores": 4.8178606033325195, | |
| "policy/approxkl_avg": 0.12672686576843262, | |
| "policy/clipfrac_avg": 0.310546875, | |
| "policy/entropy_avg": 0.7554011344909668, | |
| "step": 3, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4010, | |
| "val/ratio": 0.9999880790710449, | |
| "val/ratio_var": 1.6228774484261521e-06 | |
| }, | |
| { | |
| "episode": 2560, | |
| "epoch": 0.010966415352981495, | |
| "eps": 6, | |
| "loss/policy_avg": 0.17445668578147888, | |
| "lr": 2.953125e-06, | |
| "objective/entropy": -60.463539123535156, | |
| "objective/kl": 2.6447153091430664, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.877408981323242, | |
| "objective/scores": 4.877408981323242, | |
| "policy/approxkl_avg": 0.13307543098926544, | |
| "policy/clipfrac_avg": 0.275390625, | |
| "policy/entropy_avg": 0.690821647644043, | |
| "step": 4, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4589, | |
| "val/ratio": 1.0001537799835205, | |
| "val/ratio_var": 1.283996084566752e-06 | |
| }, | |
| { | |
| "episode": 3072, | |
| "epoch": 0.013159698423577794, | |
| "eps": 7, | |
| "loss/policy_avg": 0.18251696228981018, | |
| "lr": 2.94140625e-06, | |
| "objective/entropy": -60.38776397705078, | |
| "objective/kl": 3.3835220336914062, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.029230117797852, | |
| "objective/scores": 5.029230117797852, | |
| "policy/approxkl_avg": 0.11075231432914734, | |
| "policy/clipfrac_avg": 0.30859375, | |
| "policy/entropy_avg": 0.6673464775085449, | |
| "step": 5, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4933, | |
| "val/ratio": 1.0000388622283936, | |
| "val/ratio_var": 1.886573841147765e-06 | |
| }, | |
| { | |
| "episode": 3584, | |
| "epoch": 0.015352981494174092, | |
| "eps": 7, | |
| "loss/policy_avg": 0.0862584114074707, | |
| "lr": 2.9296875e-06, | |
| "objective/entropy": -64.953857421875, | |
| "objective/kl": 3.801461696624756, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.2262468338012695, | |
| "objective/scores": 5.2262468338012695, | |
| "policy/approxkl_avg": 0.0921158567070961, | |
| "policy/clipfrac_avg": 0.31640625, | |
| "policy/entropy_avg": 0.6317387819290161, | |
| "step": 6, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 5224, | |
| "val/ratio": 1.0000205039978027, | |
| "val/ratio_var": 1.223428625962697e-06 | |
| }, | |
| { | |
| "episode": 4096, | |
| "epoch": 0.01754626456477039, | |
| "eps": 7, | |
| "loss/policy_avg": 0.10839146375656128, | |
| "lr": 2.91796875e-06, | |
| "objective/entropy": -64.41149139404297, | |
| "objective/kl": 4.314542770385742, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.495314121246338, | |
| "objective/scores": 5.495314121246338, | |
| "policy/approxkl_avg": 0.11755571514368057, | |
| "policy/clipfrac_avg": 0.28515625, | |
| "policy/entropy_avg": 0.6287920475006104, | |
| "step": 7, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 5463, | |
| "val/ratio": 0.9998381733894348, | |
| "val/ratio_var": 1.6926536545724957e-06 | |
| }, | |
| { | |
| "episode": 4608, | |
| "epoch": 0.01973954763536669, | |
| "eps": 7, | |
| "loss/policy_avg": 0.16045916080474854, | |
| "lr": 2.90625e-06, | |
| "objective/entropy": -65.33601379394531, | |
| "objective/kl": 4.707500457763672, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.608126640319824, | |
| "objective/scores": 5.608126640319824, | |
| "policy/approxkl_avg": 0.10813738405704498, | |
| "policy/clipfrac_avg": 0.2890625, | |
| "policy/entropy_avg": 0.6515289545059204, | |
| "step": 8, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 5590, | |
| "val/ratio": 1.0000832080841064, | |
| "val/ratio_var": 1.661756641624379e-06 | |
| }, | |
| { | |
| "episode": 5120, | |
| "epoch": 0.02193283070596299, | |
| "eps": 7, | |
| "loss/policy_avg": 0.16000822186470032, | |
| "lr": 2.89453125e-06, | |
| "objective/entropy": -64.82266235351562, | |
| "objective/kl": 5.513130187988281, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.976546764373779, | |
| "objective/scores": 5.976546764373779, | |
| "policy/approxkl_avg": 0.12319613248109818, | |
| "policy/clipfrac_avg": 0.29296875, | |
| "policy/entropy_avg": 0.6444021463394165, | |
| "step": 9, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 5330, | |
| "val/ratio": 0.9997374415397644, | |
| "val/ratio_var": 1.3334141613086103e-06 | |
| }, | |
| { | |
| "episode": 5632, | |
| "epoch": 0.024126113776559287, | |
| "eps": 7, | |
| "loss/policy_avg": 0.22769010066986084, | |
| "lr": 2.8828125e-06, | |
| "objective/entropy": -62.54362487792969, | |
| "objective/kl": 6.375988960266113, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.329231262207031, | |
| "objective/scores": 6.329231262207031, | |
| "policy/approxkl_avg": 0.133827343583107, | |
| "policy/clipfrac_avg": 0.30078125, | |
| "policy/entropy_avg": 0.6720603704452515, | |
| "step": 10, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4956, | |
| "val/ratio": 0.9999898672103882, | |
| "val/ratio_var": 2.208806336057023e-06 | |
| }, | |
| { | |
| "episode": 6144, | |
| "epoch": 0.026319396847155587, | |
| "eps": 7, | |
| "loss/policy_avg": 0.16269627213478088, | |
| "lr": 2.87109375e-06, | |
| "objective/entropy": -56.140655517578125, | |
| "objective/kl": 7.935493469238281, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.335080146789551, | |
| "objective/scores": 6.335080146789551, | |
| "policy/approxkl_avg": 0.13188697397708893, | |
| "policy/clipfrac_avg": 0.35546875, | |
| "policy/entropy_avg": 0.6935639381408691, | |
| "step": 11, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4904, | |
| "val/ratio": 0.999968409538269, | |
| "val/ratio_var": 1.8624212998474832e-06 | |
| }, | |
| { | |
| "episode": 6656, | |
| "epoch": 0.028512679917751884, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1726413369178772, | |
| "lr": 2.859375e-06, | |
| "objective/entropy": -56.71982955932617, | |
| "objective/kl": 10.505476951599121, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.327908515930176, | |
| "objective/scores": 7.327908515930176, | |
| "policy/approxkl_avg": 0.13946227729320526, | |
| "policy/clipfrac_avg": 0.306640625, | |
| "policy/entropy_avg": 0.6527446508407593, | |
| "step": 12, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4210, | |
| "val/ratio": 1.0002007484436035, | |
| "val/ratio_var": 1.9585920654208167e-06 | |
| }, | |
| { | |
| "episode": 7168, | |
| "epoch": 0.030705962988348184, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1594904661178589, | |
| "lr": 2.84765625e-06, | |
| "objective/entropy": -57.271263122558594, | |
| "objective/kl": 12.708869934082031, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.788185119628906, | |
| "objective/scores": 7.788185119628906, | |
| "policy/approxkl_avg": 0.14631032943725586, | |
| "policy/clipfrac_avg": 0.33203125, | |
| "policy/entropy_avg": 0.6090783476829529, | |
| "step": 13, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3340, | |
| "val/ratio": 0.9998956918716431, | |
| "val/ratio_var": 2.45040769186744e-06 | |
| }, | |
| { | |
| "episode": 7680, | |
| "epoch": 0.03289924605894448, | |
| "eps": 7, | |
| "loss/policy_avg": 0.15880730748176575, | |
| "lr": 2.8359375e-06, | |
| "objective/entropy": -53.81092071533203, | |
| "objective/kl": 15.003689765930176, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.487188339233398, | |
| "objective/scores": 7.487188339233398, | |
| "policy/approxkl_avg": 0.17456671595573425, | |
| "policy/clipfrac_avg": 0.33203125, | |
| "policy/entropy_avg": 0.5235726237297058, | |
| "step": 14, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2483, | |
| "val/ratio": 1.0001764297485352, | |
| "val/ratio_var": 1.739899175845494e-06 | |
| }, | |
| { | |
| "episode": 8192, | |
| "epoch": 0.03509252912954078, | |
| "eps": 7, | |
| "loss/policy_avg": 0.16226203739643097, | |
| "lr": 2.82421875e-06, | |
| "objective/entropy": -55.564083099365234, | |
| "objective/kl": 14.753921508789062, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.572874069213867, | |
| "objective/scores": 7.572874069213867, | |
| "policy/approxkl_avg": 0.16666629910469055, | |
| "policy/clipfrac_avg": 0.302734375, | |
| "policy/entropy_avg": 0.5176196098327637, | |
| "step": 15, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2405, | |
| "val/ratio": 0.9999848008155823, | |
| "val/ratio_var": 2.589519908724469e-06 | |
| }, | |
| { | |
| "episode": 8704, | |
| "epoch": 0.03728581220013708, | |
| "eps": 7, | |
| "loss/policy_avg": 0.21661071479320526, | |
| "lr": 2.8125e-06, | |
| "objective/entropy": -51.922306060791016, | |
| "objective/kl": 16.144550323486328, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.781338691711426, | |
| "objective/scores": 7.781338691711426, | |
| "policy/approxkl_avg": 0.2002604901790619, | |
| "policy/clipfrac_avg": 0.337890625, | |
| "policy/entropy_avg": 0.5265668630599976, | |
| "step": 16, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2439, | |
| "val/ratio": 0.9997420907020569, | |
| "val/ratio_var": 2.6209752377326367e-06 | |
| }, | |
| { | |
| "episode": 9216, | |
| "epoch": 0.03947909527073338, | |
| "eps": 7, | |
| "loss/policy_avg": 0.26599276065826416, | |
| "lr": 2.80078125e-06, | |
| "objective/entropy": -50.82372283935547, | |
| "objective/kl": 17.150707244873047, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.842694282531738, | |
| "objective/scores": 7.842694282531738, | |
| "policy/approxkl_avg": 0.19617530703544617, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.5609679222106934, | |
| "step": 17, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2719, | |
| "val/ratio": 1.000065803527832, | |
| "val/ratio_var": 2.653461933732615e-06 | |
| }, | |
| { | |
| "episode": 9728, | |
| "epoch": 0.041672378341329676, | |
| "eps": 7, | |
| "loss/policy_avg": 0.18615023791790009, | |
| "lr": 2.7890625e-06, | |
| "objective/entropy": -50.224365234375, | |
| "objective/kl": 18.064285278320312, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.09566879272461, | |
| "objective/scores": 8.09566879272461, | |
| "policy/approxkl_avg": 0.2271636724472046, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5856865048408508, | |
| "step": 18, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2943, | |
| "val/ratio": 1.0001342296600342, | |
| "val/ratio_var": 3.7938905279588653e-06 | |
| }, | |
| { | |
| "episode": 10240, | |
| "epoch": 0.04386566141192598, | |
| "eps": 7, | |
| "loss/policy_avg": 0.20427988469600677, | |
| "lr": 2.77734375e-06, | |
| "objective/entropy": -50.77484893798828, | |
| "objective/kl": 17.688308715820312, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.970284461975098, | |
| "objective/scores": 7.970284461975098, | |
| "policy/approxkl_avg": 0.23018702864646912, | |
| "policy/clipfrac_avg": 0.341796875, | |
| "policy/entropy_avg": 0.5944638252258301, | |
| "step": 19, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3183, | |
| "val/ratio": 1.000162124633789, | |
| "val/ratio_var": 3.043409833480837e-06 | |
| }, | |
| { | |
| "episode": 10752, | |
| "epoch": 0.04605894448252228, | |
| "eps": 7, | |
| "loss/policy_avg": 0.282001793384552, | |
| "lr": 2.765625e-06, | |
| "objective/entropy": -44.80044937133789, | |
| "objective/kl": 19.763790130615234, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.012289047241211, | |
| "objective/scores": 8.012289047241211, | |
| "policy/approxkl_avg": 0.24563264846801758, | |
| "policy/clipfrac_avg": 0.3359375, | |
| "policy/entropy_avg": 0.6678606271743774, | |
| "step": 20, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3309, | |
| "val/ratio": 0.9997632503509521, | |
| "val/ratio_var": 3.618298023866373e-06 | |
| }, | |
| { | |
| "episode": 11264, | |
| "epoch": 0.048252227553118573, | |
| "eps": 7, | |
| "loss/policy_avg": 0.15888847410678864, | |
| "lr": 2.75390625e-06, | |
| "objective/entropy": -45.08258056640625, | |
| "objective/kl": 19.886899948120117, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.057466506958008, | |
| "objective/scores": 8.057466506958008, | |
| "policy/approxkl_avg": 0.2164287269115448, | |
| "policy/clipfrac_avg": 0.3671875, | |
| "policy/entropy_avg": 0.6539000272750854, | |
| "step": 21, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3399, | |
| "val/ratio": 0.9997671842575073, | |
| "val/ratio_var": 3.0791625249548815e-06 | |
| }, | |
| { | |
| "episode": 11776, | |
| "epoch": 0.05044551062371487, | |
| "eps": 7, | |
| "loss/policy_avg": 0.276736855506897, | |
| "lr": 2.7421875e-06, | |
| "objective/entropy": -40.99183654785156, | |
| "objective/kl": 22.14881134033203, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.060696601867676, | |
| "objective/scores": 8.060696601867676, | |
| "policy/approxkl_avg": 0.31746962666511536, | |
| "policy/clipfrac_avg": 0.349609375, | |
| "policy/entropy_avg": 0.7243172526359558, | |
| "step": 22, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3907, | |
| "val/ratio": 0.9996672868728638, | |
| "val/ratio_var": 4.220366463414393e-06 | |
| }, | |
| { | |
| "episode": 12288, | |
| "epoch": 0.052638793694311174, | |
| "eps": 7, | |
| "loss/policy_avg": 0.4665297865867615, | |
| "lr": 2.73046875e-06, | |
| "objective/entropy": -35.2369384765625, | |
| "objective/kl": 23.011898040771484, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.889699935913086, | |
| "objective/scores": 7.889699935913086, | |
| "policy/approxkl_avg": 0.3784940838813782, | |
| "policy/clipfrac_avg": 0.3515625, | |
| "policy/entropy_avg": 0.7655227184295654, | |
| "step": 23, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3972, | |
| "val/ratio": 0.9998047351837158, | |
| "val/ratio_var": 4.525894382823026e-06 | |
| }, | |
| { | |
| "episode": 12800, | |
| "epoch": 0.05483207676490747, | |
| "eps": 7, | |
| "loss/policy_avg": 0.43642693758010864, | |
| "lr": 2.71875e-06, | |
| "objective/entropy": -36.217777252197266, | |
| "objective/kl": 22.847183227539062, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.134607315063477, | |
| "objective/scores": 8.134607315063477, | |
| "policy/approxkl_avg": 0.330811470746994, | |
| "policy/clipfrac_avg": 0.39453125, | |
| "policy/entropy_avg": 0.7883225679397583, | |
| "step": 24, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4373, | |
| "val/ratio": 0.9999172687530518, | |
| "val/ratio_var": 5.261089881969383e-06 | |
| }, | |
| { | |
| "episode": 13312, | |
| "epoch": 0.05702535983550377, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3709805905818939, | |
| "lr": 2.70703125e-06, | |
| "objective/entropy": -28.68521499633789, | |
| "objective/kl": 24.381669998168945, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.424071311950684, | |
| "objective/scores": 8.424071311950684, | |
| "policy/approxkl_avg": 0.26343879103660583, | |
| "policy/clipfrac_avg": 0.359375, | |
| "policy/entropy_avg": 0.7675451040267944, | |
| "step": 25, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 4221, | |
| "val/ratio": 1.0003796815872192, | |
| "val/ratio_var": 4.443480065674521e-06 | |
| }, | |
| { | |
| "episode": 13824, | |
| "epoch": 0.05921864290610007, | |
| "eps": 7, | |
| "loss/policy_avg": 0.32363858819007874, | |
| "lr": 2.6953125e-06, | |
| "objective/entropy": -28.342884063720703, | |
| "objective/kl": 24.456398010253906, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.066158294677734, | |
| "objective/scores": 8.066158294677734, | |
| "policy/approxkl_avg": 0.2860070466995239, | |
| "policy/clipfrac_avg": 0.40234375, | |
| "policy/entropy_avg": 0.7560139298439026, | |
| "step": 26, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3967, | |
| "val/ratio": 0.999995231628418, | |
| "val/ratio_var": 5.418739419837948e-06 | |
| }, | |
| { | |
| "episode": 14336, | |
| "epoch": 0.06141192597669637, | |
| "eps": 7, | |
| "loss/policy_avg": 0.27364417910575867, | |
| "lr": 2.68359375e-06, | |
| "objective/entropy": -21.591156005859375, | |
| "objective/kl": 26.56311798095703, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.133939743041992, | |
| "objective/scores": 8.133939743041992, | |
| "policy/approxkl_avg": 0.34295791387557983, | |
| "policy/clipfrac_avg": 0.3984375, | |
| "policy/entropy_avg": 0.7555699944496155, | |
| "step": 27, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3701, | |
| "val/ratio": 0.9998049139976501, | |
| "val/ratio_var": 5.146652256371453e-06 | |
| }, | |
| { | |
| "episode": 14848, | |
| "epoch": 0.06360520904729267, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3071547746658325, | |
| "lr": 2.671875e-06, | |
| "objective/entropy": -23.85720443725586, | |
| "objective/kl": 26.94635009765625, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.451851844787598, | |
| "objective/scores": 8.451851844787598, | |
| "policy/approxkl_avg": 0.301169216632843, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.7123795747756958, | |
| "step": 28, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3446, | |
| "val/ratio": 1.0001068115234375, | |
| "val/ratio_var": 8.123261977743823e-06 | |
| }, | |
| { | |
| "episode": 15360, | |
| "epoch": 0.06579849211788896, | |
| "eps": 7, | |
| "loss/policy_avg": 0.24686866998672485, | |
| "lr": 2.66015625e-06, | |
| "objective/entropy": -23.746164321899414, | |
| "objective/kl": 26.223037719726562, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.474257469177246, | |
| "objective/scores": 8.474257469177246, | |
| "policy/approxkl_avg": 0.2871531844139099, | |
| "policy/clipfrac_avg": 0.384765625, | |
| "policy/entropy_avg": 0.7285399436950684, | |
| "step": 29, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3073, | |
| "val/ratio": 0.9999381899833679, | |
| "val/ratio_var": 4.340675786806969e-06 | |
| }, | |
| { | |
| "episode": 15872, | |
| "epoch": 0.06799177518848526, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3226906359195709, | |
| "lr": 2.6484375e-06, | |
| "objective/entropy": -21.334320068359375, | |
| "objective/kl": 27.262338638305664, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.353540420532227, | |
| "objective/scores": 8.353540420532227, | |
| "policy/approxkl_avg": 0.26032739877700806, | |
| "policy/clipfrac_avg": 0.3671875, | |
| "policy/entropy_avg": 0.7068374156951904, | |
| "step": 30, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2797, | |
| "val/ratio": 1.0000503063201904, | |
| "val/ratio_var": 3.849579115922097e-06 | |
| }, | |
| { | |
| "episode": 16384, | |
| "epoch": 0.07018505825908156, | |
| "eps": 7, | |
| "loss/policy_avg": 0.8861868977546692, | |
| "lr": 2.63671875e-06, | |
| "objective/entropy": -16.9752197265625, | |
| "objective/kl": 27.735870361328125, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.225103378295898, | |
| "objective/scores": 8.225103378295898, | |
| "policy/approxkl_avg": 0.3011099100112915, | |
| "policy/clipfrac_avg": 0.373046875, | |
| "policy/entropy_avg": 0.6992452144622803, | |
| "step": 31, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2656, | |
| "val/ratio": 1.0000051259994507, | |
| "val/ratio_var": 3.2618629575154046e-06 | |
| }, | |
| { | |
| "episode": 16896, | |
| "epoch": 0.07237834132967787, | |
| "eps": 7, | |
| "loss/policy_avg": 0.42881882190704346, | |
| "lr": 2.6250000000000003e-06, | |
| "objective/entropy": -25.429481506347656, | |
| "objective/kl": 26.258373260498047, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.536768913269043, | |
| "objective/scores": 8.536768913269043, | |
| "policy/approxkl_avg": 0.2783983647823334, | |
| "policy/clipfrac_avg": 0.357421875, | |
| "policy/entropy_avg": 0.6586651802062988, | |
| "step": 32, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2739, | |
| "val/ratio": 1.0000145435333252, | |
| "val/ratio_var": 5.060162948211655e-06 | |
| }, | |
| { | |
| "episode": 17408, | |
| "epoch": 0.07457162440027416, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3053734004497528, | |
| "lr": 2.61328125e-06, | |
| "objective/entropy": -27.58635711669922, | |
| "objective/kl": 26.437305450439453, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.641759872436523, | |
| "objective/scores": 8.641759872436523, | |
| "policy/approxkl_avg": 0.27716416120529175, | |
| "policy/clipfrac_avg": 0.384765625, | |
| "policy/entropy_avg": 0.6354501843452454, | |
| "step": 33, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2508, | |
| "val/ratio": 1.0002658367156982, | |
| "val/ratio_var": 5.2127916205790825e-06 | |
| }, | |
| { | |
| "episode": 17920, | |
| "epoch": 0.07676490747087046, | |
| "eps": 7, | |
| "loss/policy_avg": 0.29732874035835266, | |
| "lr": 2.6015625e-06, | |
| "objective/entropy": -21.862457275390625, | |
| "objective/kl": 26.802410125732422, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.533287048339844, | |
| "objective/scores": 8.533287048339844, | |
| "policy/approxkl_avg": 0.2853066325187683, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.6434289813041687, | |
| "step": 34, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2256, | |
| "val/ratio": 1.0001918077468872, | |
| "val/ratio_var": 3.2171374186873436e-06 | |
| }, | |
| { | |
| "episode": 18432, | |
| "epoch": 0.07895819054146676, | |
| "eps": 7, | |
| "loss/policy_avg": 0.29931700229644775, | |
| "lr": 2.5898437500000003e-06, | |
| "objective/entropy": -26.582080841064453, | |
| "objective/kl": 26.952320098876953, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.649660110473633, | |
| "objective/scores": 8.649660110473633, | |
| "policy/approxkl_avg": 0.29774773120880127, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.6092206835746765, | |
| "step": 35, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2189, | |
| "val/ratio": 0.9998046159744263, | |
| "val/ratio_var": 4.721144250652287e-06 | |
| }, | |
| { | |
| "episode": 18944, | |
| "epoch": 0.08115147361206305, | |
| "eps": 7, | |
| "loss/policy_avg": 0.29533323645591736, | |
| "lr": 2.578125e-06, | |
| "objective/entropy": -28.472558975219727, | |
| "objective/kl": 26.34886932373047, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.683137893676758, | |
| "objective/scores": 8.683137893676758, | |
| "policy/approxkl_avg": 0.2845500707626343, | |
| "policy/clipfrac_avg": 0.3515625, | |
| "policy/entropy_avg": 0.6110947132110596, | |
| "step": 36, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2113, | |
| "val/ratio": 1.0004630088806152, | |
| "val/ratio_var": 3.1590116122970358e-06 | |
| }, | |
| { | |
| "episode": 19456, | |
| "epoch": 0.08334475668265935, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1853514462709427, | |
| "lr": 2.56640625e-06, | |
| "objective/entropy": -28.34747314453125, | |
| "objective/kl": 26.15137481689453, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.642951965332031, | |
| "objective/scores": 8.642951965332031, | |
| "policy/approxkl_avg": 0.26775410771369934, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.5745651721954346, | |
| "step": 37, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2008, | |
| "val/ratio": 0.9999517202377319, | |
| "val/ratio_var": 3.727831654032343e-06 | |
| }, | |
| { | |
| "episode": 19968, | |
| "epoch": 0.08553803975325565, | |
| "eps": 7, | |
| "loss/policy_avg": 0.43564116954803467, | |
| "lr": 2.5546875000000003e-06, | |
| "objective/entropy": -30.9276065826416, | |
| "objective/kl": 26.764188766479492, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.747605323791504, | |
| "objective/scores": 8.747605323791504, | |
| "policy/approxkl_avg": 0.2727493345737457, | |
| "policy/clipfrac_avg": 0.408203125, | |
| "policy/entropy_avg": 0.5555662512779236, | |
| "step": 38, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1949, | |
| "val/ratio": 1.000280737876892, | |
| "val/ratio_var": 4.035889560327632e-06 | |
| }, | |
| { | |
| "episode": 20480, | |
| "epoch": 0.08773132282385196, | |
| "eps": 7, | |
| "loss/policy_avg": 0.33555662631988525, | |
| "lr": 2.54296875e-06, | |
| "objective/entropy": -30.725311279296875, | |
| "objective/kl": 26.652061462402344, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.656892776489258, | |
| "objective/scores": 8.656892776489258, | |
| "policy/approxkl_avg": 0.3008157014846802, | |
| "policy/clipfrac_avg": 0.3515625, | |
| "policy/entropy_avg": 0.5807157754898071, | |
| "step": 39, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1964, | |
| "val/ratio": 1.0001399517059326, | |
| "val/ratio_var": 5.213231816014741e-06 | |
| }, | |
| { | |
| "episode": 20992, | |
| "epoch": 0.08992460589444826, | |
| "eps": 7, | |
| "loss/policy_avg": 0.251528263092041, | |
| "lr": 2.53125e-06, | |
| "objective/entropy": -31.469741821289062, | |
| "objective/kl": 27.247440338134766, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.747458457946777, | |
| "objective/scores": 8.747458457946777, | |
| "policy/approxkl_avg": 0.31611520051956177, | |
| "policy/clipfrac_avg": 0.39453125, | |
| "policy/entropy_avg": 0.5639370679855347, | |
| "step": 40, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1938, | |
| "val/ratio": 0.9993410110473633, | |
| "val/ratio_var": 3.2128821203514235e-06 | |
| }, | |
| { | |
| "episode": 21504, | |
| "epoch": 0.09211788896504455, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2754300832748413, | |
| "lr": 2.5195312500000003e-06, | |
| "objective/entropy": -32.1821403503418, | |
| "objective/kl": 26.554561614990234, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.68823528289795, | |
| "objective/scores": 8.68823528289795, | |
| "policy/approxkl_avg": 0.2688625454902649, | |
| "policy/clipfrac_avg": 0.3984375, | |
| "policy/entropy_avg": 0.5564634203910828, | |
| "step": 41, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1861, | |
| "val/ratio": 1.0001417398452759, | |
| "val/ratio_var": 3.7918011912552174e-06 | |
| }, | |
| { | |
| "episode": 22016, | |
| "epoch": 0.09431117203564085, | |
| "eps": 7, | |
| "loss/policy_avg": 0.32364964485168457, | |
| "lr": 2.5078125e-06, | |
| "objective/entropy": -31.65809440612793, | |
| "objective/kl": 26.914566040039062, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.63361930847168, | |
| "objective/scores": 8.63361930847168, | |
| "policy/approxkl_avg": 0.319877952337265, | |
| "policy/clipfrac_avg": 0.357421875, | |
| "policy/entropy_avg": 0.5548701882362366, | |
| "step": 42, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1894, | |
| "val/ratio": 0.9996398687362671, | |
| "val/ratio_var": 3.310229885755689e-06 | |
| }, | |
| { | |
| "episode": 22528, | |
| "epoch": 0.09650445510623715, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2156452238559723, | |
| "lr": 2.49609375e-06, | |
| "objective/entropy": -33.42183303833008, | |
| "objective/kl": 25.768882751464844, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.491684913635254, | |
| "objective/scores": 8.491684913635254, | |
| "policy/approxkl_avg": 0.37833964824676514, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.5088037252426147, | |
| "step": 43, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1638, | |
| "val/ratio": 0.9993675947189331, | |
| "val/ratio_var": 4.396941676532151e-06 | |
| }, | |
| { | |
| "episode": 23040, | |
| "epoch": 0.09869773817683344, | |
| "eps": 7, | |
| "loss/policy_avg": 0.283500611782074, | |
| "lr": 2.4843750000000002e-06, | |
| "objective/entropy": -38.62915802001953, | |
| "objective/kl": 24.288204193115234, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.610681533813477, | |
| "objective/scores": 8.610681533813477, | |
| "policy/approxkl_avg": 0.26353907585144043, | |
| "policy/clipfrac_avg": 0.341796875, | |
| "policy/entropy_avg": 0.524039626121521, | |
| "step": 44, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1721, | |
| "val/ratio": 1.000080943107605, | |
| "val/ratio_var": 5.167307790543418e-06 | |
| }, | |
| { | |
| "episode": 23552, | |
| "epoch": 0.10089102124742974, | |
| "eps": 7, | |
| "loss/policy_avg": 0.26535850763320923, | |
| "lr": 2.47265625e-06, | |
| "objective/entropy": -38.197269439697266, | |
| "objective/kl": 23.877710342407227, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.435002326965332, | |
| "objective/scores": 8.435002326965332, | |
| "policy/approxkl_avg": 0.3536168932914734, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5166954398155212, | |
| "step": 45, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1756, | |
| "val/ratio": 1.0001449584960938, | |
| "val/ratio_var": 3.216674940631492e-06 | |
| }, | |
| { | |
| "episode": 24064, | |
| "epoch": 0.10308430431802605, | |
| "eps": 7, | |
| "loss/policy_avg": 0.31962326169013977, | |
| "lr": 2.4609375e-06, | |
| "objective/entropy": -42.57143020629883, | |
| "objective/kl": 23.369226455688477, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.742530822753906, | |
| "objective/scores": 8.742530822753906, | |
| "policy/approxkl_avg": 0.2840489149093628, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5359203815460205, | |
| "step": 46, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1840, | |
| "val/ratio": 1.0000276565551758, | |
| "val/ratio_var": 3.076736902585253e-06 | |
| }, | |
| { | |
| "episode": 24576, | |
| "epoch": 0.10527758738862235, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1421811878681183, | |
| "lr": 2.4492187500000002e-06, | |
| "objective/entropy": -45.13118362426758, | |
| "objective/kl": 22.429969787597656, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.711127281188965, | |
| "objective/scores": 8.711127281188965, | |
| "policy/approxkl_avg": 0.2315264642238617, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5265815854072571, | |
| "step": 47, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1829, | |
| "val/ratio": 1.000090479850769, | |
| "val/ratio_var": 2.3445722945325542e-06 | |
| }, | |
| { | |
| "episode": 25088, | |
| "epoch": 0.10747087045921865, | |
| "eps": 7, | |
| "loss/policy_avg": 0.29607272148132324, | |
| "lr": 2.4375e-06, | |
| "objective/entropy": -47.1614875793457, | |
| "objective/kl": 21.755367279052734, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.467025756835938, | |
| "objective/scores": 8.467025756835938, | |
| "policy/approxkl_avg": 0.23295345902442932, | |
| "policy/clipfrac_avg": 0.310546875, | |
| "policy/entropy_avg": 0.5038943290710449, | |
| "step": 48, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1841, | |
| "val/ratio": 1.0003342628479004, | |
| "val/ratio_var": 2.7536118523130426e-06 | |
| }, | |
| { | |
| "episode": 25600, | |
| "epoch": 0.10966415352981494, | |
| "eps": 7, | |
| "loss/policy_avg": 0.26943719387054443, | |
| "lr": 2.42578125e-06, | |
| "objective/entropy": -47.87194061279297, | |
| "objective/kl": 22.272235870361328, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.529675483703613, | |
| "objective/scores": 8.529675483703613, | |
| "policy/approxkl_avg": 0.23642447590827942, | |
| "policy/clipfrac_avg": 0.30078125, | |
| "policy/entropy_avg": 0.48028820753097534, | |
| "step": 49, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1737, | |
| "val/ratio": 1.0001472234725952, | |
| "val/ratio_var": 3.836444648186443e-06 | |
| }, | |
| { | |
| "episode": 26112, | |
| "epoch": 0.11185743660041124, | |
| "eps": 7, | |
| "loss/policy_avg": 0.19396540522575378, | |
| "lr": 2.4140625000000002e-06, | |
| "objective/entropy": -46.96308135986328, | |
| "objective/kl": 21.81145477294922, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.142440795898438, | |
| "objective/scores": 8.142440795898438, | |
| "policy/approxkl_avg": 0.23152337968349457, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.546012282371521, | |
| "step": 50, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1915, | |
| "val/ratio": 0.9999135136604309, | |
| "val/ratio_var": 4.054582859680522e-06 | |
| }, | |
| { | |
| "episode": 26624, | |
| "epoch": 0.11405071967100754, | |
| "eps": 7, | |
| "loss/policy_avg": 0.17617464065551758, | |
| "lr": 2.40234375e-06, | |
| "objective/entropy": -50.88390350341797, | |
| "objective/kl": 21.53017807006836, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.507946014404297, | |
| "objective/scores": 8.507946014404297, | |
| "policy/approxkl_avg": 0.23789095878601074, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.5126174092292786, | |
| "step": 51, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1825, | |
| "val/ratio": 0.9999390840530396, | |
| "val/ratio_var": 3.831887170235859e-06 | |
| }, | |
| { | |
| "episode": 27136, | |
| "epoch": 0.11624400274160383, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2936260402202606, | |
| "lr": 2.390625e-06, | |
| "objective/entropy": -49.55311584472656, | |
| "objective/kl": 21.252099990844727, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.27835464477539, | |
| "objective/scores": 8.27835464477539, | |
| "policy/approxkl_avg": 0.26326990127563477, | |
| "policy/clipfrac_avg": 0.310546875, | |
| "policy/entropy_avg": 0.5240695476531982, | |
| "step": 52, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1999, | |
| "val/ratio": 1.00055992603302, | |
| "val/ratio_var": 3.646538061730098e-06 | |
| }, | |
| { | |
| "episode": 27648, | |
| "epoch": 0.11843728581220014, | |
| "eps": 7, | |
| "loss/policy_avg": 0.28315332531929016, | |
| "lr": 2.3789062500000002e-06, | |
| "objective/entropy": -50.70973205566406, | |
| "objective/kl": 20.522171020507812, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.508468627929688, | |
| "objective/scores": 8.508468627929688, | |
| "policy/approxkl_avg": 0.27963656187057495, | |
| "policy/clipfrac_avg": 0.32421875, | |
| "policy/entropy_avg": 0.5086656808853149, | |
| "step": 53, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1877, | |
| "val/ratio": 0.9998332262039185, | |
| "val/ratio_var": 3.3264000194321852e-06 | |
| }, | |
| { | |
| "episode": 28160, | |
| "epoch": 0.12063056888279644, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1823330521583557, | |
| "lr": 2.3671875e-06, | |
| "objective/entropy": -51.118812561035156, | |
| "objective/kl": 21.896831512451172, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.411741256713867, | |
| "objective/scores": 8.411741256713867, | |
| "policy/approxkl_avg": 0.22382813692092896, | |
| "policy/clipfrac_avg": 0.357421875, | |
| "policy/entropy_avg": 0.5047107934951782, | |
| "step": 54, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1885, | |
| "val/ratio": 0.999740481376648, | |
| "val/ratio_var": 2.7290527668810682e-06 | |
| }, | |
| { | |
| "episode": 28672, | |
| "epoch": 0.12282385195339274, | |
| "eps": 7, | |
| "loss/policy_avg": 0.25143423676490784, | |
| "lr": 2.35546875e-06, | |
| "objective/entropy": -50.577789306640625, | |
| "objective/kl": 21.04738998413086, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.411981582641602, | |
| "objective/scores": 8.411981582641602, | |
| "policy/approxkl_avg": 0.22654137015342712, | |
| "policy/clipfrac_avg": 0.32421875, | |
| "policy/entropy_avg": 0.5130290985107422, | |
| "step": 55, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1979, | |
| "val/ratio": 1.0000386238098145, | |
| "val/ratio_var": 2.045640940195881e-06 | |
| }, | |
| { | |
| "episode": 29184, | |
| "epoch": 0.12501713502398903, | |
| "eps": 7, | |
| "loss/policy_avg": 0.14284127950668335, | |
| "lr": 2.3437500000000002e-06, | |
| "objective/entropy": -51.72842788696289, | |
| "objective/kl": 21.067766189575195, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.488981246948242, | |
| "objective/scores": 8.488981246948242, | |
| "policy/approxkl_avg": 0.2212221920490265, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.4959627687931061, | |
| "step": 56, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2032, | |
| "val/ratio": 1.0002572536468506, | |
| "val/ratio_var": 3.1830661555432016e-06 | |
| }, | |
| { | |
| "episode": 29696, | |
| "epoch": 0.12721041809458533, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2772967517375946, | |
| "lr": 2.33203125e-06, | |
| "objective/entropy": -50.611061096191406, | |
| "objective/kl": 20.819595336914062, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.417530059814453, | |
| "objective/scores": 8.417530059814453, | |
| "policy/approxkl_avg": 0.20757423341274261, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.5225619077682495, | |
| "step": 57, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2159, | |
| "val/ratio": 1.0001440048217773, | |
| "val/ratio_var": 3.0312132821563864e-06 | |
| }, | |
| { | |
| "episode": 30208, | |
| "epoch": 0.12940370116518163, | |
| "eps": 7, | |
| "loss/policy_avg": 0.24927854537963867, | |
| "lr": 2.3203125e-06, | |
| "objective/entropy": -52.76747512817383, | |
| "objective/kl": 20.694204330444336, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.423263549804688, | |
| "objective/scores": 8.423263549804688, | |
| "policy/approxkl_avg": 0.2698250412940979, | |
| "policy/clipfrac_avg": 0.318359375, | |
| "policy/entropy_avg": 0.5205322504043579, | |
| "step": 58, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2188, | |
| "val/ratio": 1.0001823902130127, | |
| "val/ratio_var": 3.7059455735288793e-06 | |
| }, | |
| { | |
| "episode": 30720, | |
| "epoch": 0.13159698423577793, | |
| "eps": 7, | |
| "loss/policy_avg": 0.32080554962158203, | |
| "lr": 2.30859375e-06, | |
| "objective/entropy": -47.49811553955078, | |
| "objective/kl": 21.551021575927734, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.264215469360352, | |
| "objective/scores": 8.264215469360352, | |
| "policy/approxkl_avg": 0.25660836696624756, | |
| "policy/clipfrac_avg": 0.322265625, | |
| "policy/entropy_avg": 0.5375617146492004, | |
| "step": 59, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2128, | |
| "val/ratio": 1.0002331733703613, | |
| "val/ratio_var": 2.6993873234459898e-06 | |
| }, | |
| { | |
| "episode": 31232, | |
| "epoch": 0.13379026730637422, | |
| "eps": 7, | |
| "loss/policy_avg": 0.14086592197418213, | |
| "lr": 2.296875e-06, | |
| "objective/entropy": -51.65747833251953, | |
| "objective/kl": 20.77255630493164, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.408650398254395, | |
| "objective/scores": 8.408650398254395, | |
| "policy/approxkl_avg": 0.23100361227989197, | |
| "policy/clipfrac_avg": 0.3515625, | |
| "policy/entropy_avg": 0.5363867878913879, | |
| "step": 60, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2336, | |
| "val/ratio": 1.000304102897644, | |
| "val/ratio_var": 2.394616331002908e-06 | |
| }, | |
| { | |
| "episode": 31744, | |
| "epoch": 0.13598355037697052, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2954641282558441, | |
| "lr": 2.28515625e-06, | |
| "objective/entropy": -50.37486267089844, | |
| "objective/kl": 21.061641693115234, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.459826469421387, | |
| "objective/scores": 8.459826469421387, | |
| "policy/approxkl_avg": 0.24728071689605713, | |
| "policy/clipfrac_avg": 0.3203125, | |
| "policy/entropy_avg": 0.5351447463035583, | |
| "step": 61, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2212, | |
| "val/ratio": 1.000131607055664, | |
| "val/ratio_var": 2.8336792183836224e-06 | |
| }, | |
| { | |
| "episode": 32256, | |
| "epoch": 0.13817683344756682, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2441791594028473, | |
| "lr": 2.2734375e-06, | |
| "objective/entropy": -51.75057601928711, | |
| "objective/kl": 21.086687088012695, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.422032356262207, | |
| "objective/scores": 8.422032356262207, | |
| "policy/approxkl_avg": 0.23406898975372314, | |
| "policy/clipfrac_avg": 0.310546875, | |
| "policy/entropy_avg": 0.5176072120666504, | |
| "step": 62, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2191, | |
| "val/ratio": 1.0005054473876953, | |
| "val/ratio_var": 3.65329015039606e-06 | |
| }, | |
| { | |
| "episode": 32768, | |
| "epoch": 0.1403701165181631, | |
| "eps": 7, | |
| "loss/policy_avg": 0.22965137660503387, | |
| "lr": 2.26171875e-06, | |
| "objective/entropy": -51.704444885253906, | |
| "objective/kl": 21.60610008239746, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.439580917358398, | |
| "objective/scores": 8.439580917358398, | |
| "policy/approxkl_avg": 0.21967627108097076, | |
| "policy/clipfrac_avg": 0.333984375, | |
| "policy/entropy_avg": 0.524364173412323, | |
| "step": 63, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2332, | |
| "val/ratio": 0.9997615814208984, | |
| "val/ratio_var": 3.444454250711715e-06 | |
| }, | |
| { | |
| "episode": 33280, | |
| "epoch": 0.14256339958875944, | |
| "eps": 7, | |
| "loss/policy_avg": 0.11451836675405502, | |
| "lr": 2.25e-06, | |
| "objective/entropy": -52.000946044921875, | |
| "objective/kl": 21.42214012145996, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.531951904296875, | |
| "objective/scores": 8.531951904296875, | |
| "policy/approxkl_avg": 0.21284940838813782, | |
| "policy/clipfrac_avg": 0.3359375, | |
| "policy/entropy_avg": 0.551526665687561, | |
| "step": 64, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2463, | |
| "val/ratio": 0.9998893141746521, | |
| "val/ratio_var": 2.1311489035724662e-06 | |
| }, | |
| { | |
| "episode": 33792, | |
| "epoch": 0.14475668265935573, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1996244490146637, | |
| "lr": 2.23828125e-06, | |
| "objective/entropy": -49.46868896484375, | |
| "objective/kl": 22.10938262939453, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.431674003601074, | |
| "objective/scores": 8.431674003601074, | |
| "policy/approxkl_avg": 0.2345934510231018, | |
| "policy/clipfrac_avg": 0.3671875, | |
| "policy/entropy_avg": 0.5430195331573486, | |
| "step": 65, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2527, | |
| "val/ratio": 1.000130534172058, | |
| "val/ratio_var": 2.9720413294853643e-06 | |
| }, | |
| { | |
| "episode": 34304, | |
| "epoch": 0.14694996572995203, | |
| "eps": 7, | |
| "loss/policy_avg": 0.24282732605934143, | |
| "lr": 2.2265625e-06, | |
| "objective/entropy": -49.502906799316406, | |
| "objective/kl": 21.639503479003906, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.574268341064453, | |
| "objective/scores": 8.574268341064453, | |
| "policy/approxkl_avg": 0.27672481536865234, | |
| "policy/clipfrac_avg": 0.373046875, | |
| "policy/entropy_avg": 0.5653039216995239, | |
| "step": 66, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2667, | |
| "val/ratio": 1.000170111656189, | |
| "val/ratio_var": 3.4806193980330136e-06 | |
| }, | |
| { | |
| "episode": 34816, | |
| "epoch": 0.14914324880054833, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2232193946838379, | |
| "lr": 2.21484375e-06, | |
| "objective/entropy": -49.555030822753906, | |
| "objective/kl": 22.374631881713867, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.583580017089844, | |
| "objective/scores": 8.583580017089844, | |
| "policy/approxkl_avg": 0.3105822801589966, | |
| "policy/clipfrac_avg": 0.365234375, | |
| "policy/entropy_avg": 0.5452811121940613, | |
| "step": 67, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2650, | |
| "val/ratio": 1.0001208782196045, | |
| "val/ratio_var": 8.155854629876558e-06 | |
| }, | |
| { | |
| "episode": 35328, | |
| "epoch": 0.15133653187114463, | |
| "eps": 7, | |
| "loss/policy_avg": 0.23417699337005615, | |
| "lr": 2.203125e-06, | |
| "objective/entropy": -46.87506866455078, | |
| "objective/kl": 23.203367233276367, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.507232666015625, | |
| "objective/scores": 8.507232666015625, | |
| "policy/approxkl_avg": 0.3590773344039917, | |
| "policy/clipfrac_avg": 0.349609375, | |
| "policy/entropy_avg": 0.5592520833015442, | |
| "step": 68, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2704, | |
| "val/ratio": 0.9996556043624878, | |
| "val/ratio_var": 3.690880021167686e-06 | |
| }, | |
| { | |
| "episode": 35840, | |
| "epoch": 0.15352981494174092, | |
| "eps": 7, | |
| "loss/policy_avg": 0.4602639079093933, | |
| "lr": 2.19140625e-06, | |
| "objective/entropy": -46.802467346191406, | |
| "objective/kl": 23.537273406982422, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.526100158691406, | |
| "objective/scores": 8.526100158691406, | |
| "policy/approxkl_avg": 0.2938518822193146, | |
| "policy/clipfrac_avg": 0.32421875, | |
| "policy/entropy_avg": 0.562473714351654, | |
| "step": 69, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2694, | |
| "val/ratio": 1.0002529621124268, | |
| "val/ratio_var": 4.539975634543225e-06 | |
| }, | |
| { | |
| "episode": 36352, | |
| "epoch": 0.15572309801233722, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2672228515148163, | |
| "lr": 2.1796875e-06, | |
| "objective/entropy": -45.053016662597656, | |
| "objective/kl": 23.575054168701172, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.515979766845703, | |
| "objective/scores": 8.515979766845703, | |
| "policy/approxkl_avg": 0.30573177337646484, | |
| "policy/clipfrac_avg": 0.322265625, | |
| "policy/entropy_avg": 0.5777955055236816, | |
| "step": 70, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2750, | |
| "val/ratio": 1.0002713203430176, | |
| "val/ratio_var": 5.217795205680886e-06 | |
| }, | |
| { | |
| "episode": 36864, | |
| "epoch": 0.15791638108293352, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3402022123336792, | |
| "lr": 2.16796875e-06, | |
| "objective/entropy": -42.40455627441406, | |
| "objective/kl": 25.648136138916016, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.605457305908203, | |
| "objective/scores": 8.605457305908203, | |
| "policy/approxkl_avg": 0.35311511158943176, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.5561670064926147, | |
| "step": 71, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2687, | |
| "val/ratio": 0.9998939037322998, | |
| "val/ratio_var": 4.396093117975397e-06 | |
| }, | |
| { | |
| "episode": 37376, | |
| "epoch": 0.1601096641535298, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3977116346359253, | |
| "lr": 2.15625e-06, | |
| "objective/entropy": -42.49177169799805, | |
| "objective/kl": 26.084217071533203, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.741865158081055, | |
| "objective/scores": 8.741865158081055, | |
| "policy/approxkl_avg": 0.43490105867385864, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5506061315536499, | |
| "step": 72, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2755, | |
| "val/ratio": 0.999513566493988, | |
| "val/ratio_var": 5.121092272020178e-06 | |
| }, | |
| { | |
| "episode": 37888, | |
| "epoch": 0.1623029472241261, | |
| "eps": 7, | |
| "loss/policy_avg": 0.26749086380004883, | |
| "lr": 2.14453125e-06, | |
| "objective/entropy": -43.96216583251953, | |
| "objective/kl": 25.2254638671875, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.43868637084961, | |
| "objective/scores": 8.43868637084961, | |
| "policy/approxkl_avg": 0.37626683712005615, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5635879039764404, | |
| "step": 73, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2632, | |
| "val/ratio": 1.0003223419189453, | |
| "val/ratio_var": 4.566423740470782e-06 | |
| }, | |
| { | |
| "episode": 38400, | |
| "epoch": 0.1644962302947224, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2991521656513214, | |
| "lr": 2.1328125e-06, | |
| "objective/entropy": -43.968116760253906, | |
| "objective/kl": 26.172449111938477, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.619085311889648, | |
| "objective/scores": 8.619085311889648, | |
| "policy/approxkl_avg": 0.36614692211151123, | |
| "policy/clipfrac_avg": 0.361328125, | |
| "policy/entropy_avg": 0.5751455426216125, | |
| "step": 74, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2762, | |
| "val/ratio": 0.9997708797454834, | |
| "val/ratio_var": 5.6084491006913595e-06 | |
| }, | |
| { | |
| "episode": 38912, | |
| "epoch": 0.1666895133653187, | |
| "eps": 7, | |
| "loss/policy_avg": 0.31739309430122375, | |
| "lr": 2.12109375e-06, | |
| "objective/entropy": -44.440555572509766, | |
| "objective/kl": 27.502099990844727, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.551536560058594, | |
| "objective/scores": 8.551536560058594, | |
| "policy/approxkl_avg": 0.4447302520275116, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5544260740280151, | |
| "step": 75, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2669, | |
| "val/ratio": 0.9993975162506104, | |
| "val/ratio_var": 4.979194272891618e-06 | |
| }, | |
| { | |
| "episode": 39424, | |
| "epoch": 0.168882796435915, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2567436695098877, | |
| "lr": 2.109375e-06, | |
| "objective/entropy": -43.823219299316406, | |
| "objective/kl": 26.728805541992188, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.652971267700195, | |
| "objective/scores": 8.652971267700195, | |
| "policy/approxkl_avg": 0.3301650285720825, | |
| "policy/clipfrac_avg": 0.36328125, | |
| "policy/entropy_avg": 0.5270683765411377, | |
| "step": 76, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2688, | |
| "val/ratio": 1.0002834796905518, | |
| "val/ratio_var": 4.673919193010079e-06 | |
| }, | |
| { | |
| "episode": 39936, | |
| "epoch": 0.1710760795065113, | |
| "eps": 7, | |
| "loss/policy_avg": 0.4975939393043518, | |
| "lr": 2.09765625e-06, | |
| "objective/entropy": -40.26116943359375, | |
| "objective/kl": 28.93381690979004, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.432289123535156, | |
| "objective/scores": 8.432289123535156, | |
| "policy/approxkl_avg": 0.5036437511444092, | |
| "policy/clipfrac_avg": 0.380859375, | |
| "policy/entropy_avg": 0.5689441561698914, | |
| "step": 77, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2716, | |
| "val/ratio": 0.9997487664222717, | |
| "val/ratio_var": 5.711378889827756e-06 | |
| }, | |
| { | |
| "episode": 40448, | |
| "epoch": 0.17326936257710762, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2763368487358093, | |
| "lr": 2.0859375e-06, | |
| "objective/entropy": -45.21563720703125, | |
| "objective/kl": 27.39434242248535, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.653337478637695, | |
| "objective/scores": 8.653337478637695, | |
| "policy/approxkl_avg": 0.2975656986236572, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5368204116821289, | |
| "step": 78, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2757, | |
| "val/ratio": 1.0003888607025146, | |
| "val/ratio_var": 3.7864729165448807e-06 | |
| }, | |
| { | |
| "episode": 40960, | |
| "epoch": 0.17546264564770392, | |
| "eps": 7, | |
| "loss/policy_avg": 0.4642253518104553, | |
| "lr": 2.07421875e-06, | |
| "objective/entropy": -41.11317825317383, | |
| "objective/kl": 27.240459442138672, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.210575103759766, | |
| "objective/scores": 8.210575103759766, | |
| "policy/approxkl_avg": 0.4027579426765442, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.5595495700836182, | |
| "step": 79, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2787, | |
| "val/ratio": 1.0000183582305908, | |
| "val/ratio_var": 3.993025529780425e-06 | |
| }, | |
| { | |
| "episode": 41472, | |
| "epoch": 0.17765592871830022, | |
| "eps": 7, | |
| "loss/policy_avg": 0.24956947565078735, | |
| "lr": 2.0625e-06, | |
| "objective/entropy": -40.720481872558594, | |
| "objective/kl": 28.056228637695312, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.479925155639648, | |
| "objective/scores": 8.479925155639648, | |
| "policy/approxkl_avg": 0.31501132249832153, | |
| "policy/clipfrac_avg": 0.36328125, | |
| "policy/entropy_avg": 0.5648977756500244, | |
| "step": 80, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2724, | |
| "val/ratio": 0.9998843669891357, | |
| "val/ratio_var": 4.445482773007825e-06 | |
| }, | |
| { | |
| "episode": 41984, | |
| "epoch": 0.1798492117888965, | |
| "eps": 7, | |
| "loss/policy_avg": 0.29195988178253174, | |
| "lr": 2.05078125e-06, | |
| "objective/entropy": -43.005149841308594, | |
| "objective/kl": 26.689315795898438, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.50869083404541, | |
| "objective/scores": 8.50869083404541, | |
| "policy/approxkl_avg": 0.3312462866306305, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5605291724205017, | |
| "step": 81, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2891, | |
| "val/ratio": 0.9999593496322632, | |
| "val/ratio_var": 3.4600634535308927e-06 | |
| }, | |
| { | |
| "episode": 42496, | |
| "epoch": 0.1820424948594928, | |
| "eps": 7, | |
| "loss/policy_avg": 0.22648774087429047, | |
| "lr": 2.0390625e-06, | |
| "objective/entropy": -44.053279876708984, | |
| "objective/kl": 26.523632049560547, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.40163516998291, | |
| "objective/scores": 8.40163516998291, | |
| "policy/approxkl_avg": 0.33716368675231934, | |
| "policy/clipfrac_avg": 0.33984375, | |
| "policy/entropy_avg": 0.5656260251998901, | |
| "step": 82, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2995, | |
| "val/ratio": 0.9996054172515869, | |
| "val/ratio_var": 4.090476068085991e-06 | |
| }, | |
| { | |
| "episode": 43008, | |
| "epoch": 0.1842357779300891, | |
| "eps": 7, | |
| "loss/policy_avg": 0.4572581648826599, | |
| "lr": 2.02734375e-06, | |
| "objective/entropy": -40.501346588134766, | |
| "objective/kl": 26.860363006591797, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.303882598876953, | |
| "objective/scores": 8.303882598876953, | |
| "policy/approxkl_avg": 0.350925087928772, | |
| "policy/clipfrac_avg": 0.373046875, | |
| "policy/entropy_avg": 0.557375431060791, | |
| "step": 83, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3036, | |
| "val/ratio": 0.9999963641166687, | |
| "val/ratio_var": 4.392126811580965e-06 | |
| }, | |
| { | |
| "episode": 43520, | |
| "epoch": 0.1864290610006854, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2703511714935303, | |
| "lr": 2.015625e-06, | |
| "objective/entropy": -38.586124420166016, | |
| "objective/kl": 27.219919204711914, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.605122566223145, | |
| "objective/scores": 8.605122566223145, | |
| "policy/approxkl_avg": 0.32834964990615845, | |
| "policy/clipfrac_avg": 0.390625, | |
| "policy/entropy_avg": 0.5444257259368896, | |
| "step": 84, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2940, | |
| "val/ratio": 1.0001357793807983, | |
| "val/ratio_var": 4.64254026155686e-06 | |
| }, | |
| { | |
| "episode": 44032, | |
| "epoch": 0.1886223440712817, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2800593972206116, | |
| "lr": 2.00390625e-06, | |
| "objective/entropy": -46.36981964111328, | |
| "objective/kl": 25.693588256835938, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.409098625183105, | |
| "objective/scores": 8.409098625183105, | |
| "policy/approxkl_avg": 0.3142682909965515, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5546380281448364, | |
| "step": 85, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3223, | |
| "val/ratio": 1.0001696348190308, | |
| "val/ratio_var": 4.388642537378473e-06 | |
| }, | |
| { | |
| "episode": 44544, | |
| "epoch": 0.190815627141878, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2657047212123871, | |
| "lr": 1.9921875e-06, | |
| "objective/entropy": -45.28251647949219, | |
| "objective/kl": 25.583412170410156, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.682401657104492, | |
| "objective/scores": 8.682401657104492, | |
| "policy/approxkl_avg": 0.2970484495162964, | |
| "policy/clipfrac_avg": 0.341796875, | |
| "policy/entropy_avg": 0.5628992319107056, | |
| "step": 86, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3049, | |
| "val/ratio": 1.0001099109649658, | |
| "val/ratio_var": 3.3909404919540975e-06 | |
| }, | |
| { | |
| "episode": 45056, | |
| "epoch": 0.1930089102124743, | |
| "eps": 7, | |
| "loss/policy_avg": 0.31868281960487366, | |
| "lr": 1.98046875e-06, | |
| "objective/entropy": -44.419532775878906, | |
| "objective/kl": 25.884258270263672, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.452953338623047, | |
| "objective/scores": 8.452953338623047, | |
| "policy/approxkl_avg": 0.2683827877044678, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.5685138702392578, | |
| "step": 87, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3113, | |
| "val/ratio": 1.0001598596572876, | |
| "val/ratio_var": 3.914261469617486e-06 | |
| }, | |
| { | |
| "episode": 45568, | |
| "epoch": 0.1952021932830706, | |
| "eps": 7, | |
| "loss/policy_avg": 0.224508136510849, | |
| "lr": 1.96875e-06, | |
| "objective/entropy": -48.27018737792969, | |
| "objective/kl": 24.89946174621582, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.871210098266602, | |
| "objective/scores": 8.871210098266602, | |
| "policy/approxkl_avg": 0.20233646035194397, | |
| "policy/clipfrac_avg": 0.361328125, | |
| "policy/entropy_avg": 0.5470612645149231, | |
| "step": 88, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3254, | |
| "val/ratio": 1.0006496906280518, | |
| "val/ratio_var": 3.585116246540565e-06 | |
| }, | |
| { | |
| "episode": 46080, | |
| "epoch": 0.1973954763536669, | |
| "eps": 7, | |
| "loss/policy_avg": 0.17881432175636292, | |
| "lr": 1.95703125e-06, | |
| "objective/entropy": -39.60390853881836, | |
| "objective/kl": 26.777809143066406, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.489433288574219, | |
| "objective/scores": 8.489433288574219, | |
| "policy/approxkl_avg": 0.30483895540237427, | |
| "policy/clipfrac_avg": 0.3828125, | |
| "policy/entropy_avg": 0.5878047943115234, | |
| "step": 89, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3105, | |
| "val/ratio": 1.0000858306884766, | |
| "val/ratio_var": 4.914289092994295e-06 | |
| }, | |
| { | |
| "episode": 46592, | |
| "epoch": 0.19958875942426318, | |
| "eps": 7, | |
| "loss/policy_avg": 0.33269819617271423, | |
| "lr": 1.9453125e-06, | |
| "objective/entropy": -42.723388671875, | |
| "objective/kl": 26.804061889648438, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.605770111083984, | |
| "objective/scores": 8.605770111083984, | |
| "policy/approxkl_avg": 0.30788183212280273, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.557858407497406, | |
| "step": 90, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3024, | |
| "val/ratio": 1.0000172853469849, | |
| "val/ratio_var": 3.867803116008872e-06 | |
| }, | |
| { | |
| "episode": 47104, | |
| "epoch": 0.20178204249485948, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2373349964618683, | |
| "lr": 1.93359375e-06, | |
| "objective/entropy": -41.628440856933594, | |
| "objective/kl": 26.94710350036621, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.717164039611816, | |
| "objective/scores": 8.717164039611816, | |
| "policy/approxkl_avg": 0.28812965750694275, | |
| "policy/clipfrac_avg": 0.38671875, | |
| "policy/entropy_avg": 0.5584580302238464, | |
| "step": 91, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3233, | |
| "val/ratio": 0.9999223351478577, | |
| "val/ratio_var": 3.552548605512129e-06 | |
| }, | |
| { | |
| "episode": 47616, | |
| "epoch": 0.20397532556545578, | |
| "eps": 7, | |
| "loss/policy_avg": 0.46636855602264404, | |
| "lr": 1.921875e-06, | |
| "objective/entropy": -42.61951446533203, | |
| "objective/kl": 27.169050216674805, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.583017349243164, | |
| "objective/scores": 8.583017349243164, | |
| "policy/approxkl_avg": 0.3408641815185547, | |
| "policy/clipfrac_avg": 0.33984375, | |
| "policy/entropy_avg": 0.56282639503479, | |
| "step": 92, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3154, | |
| "val/ratio": 0.9996733069419861, | |
| "val/ratio_var": 2.5412805371161085e-06 | |
| }, | |
| { | |
| "episode": 48128, | |
| "epoch": 0.2061686086360521, | |
| "eps": 7, | |
| "loss/policy_avg": 0.33910614252090454, | |
| "lr": 1.91015625e-06, | |
| "objective/entropy": -43.72690200805664, | |
| "objective/kl": 27.0749568939209, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.784333229064941, | |
| "objective/scores": 8.784333229064941, | |
| "policy/approxkl_avg": 0.3085598945617676, | |
| "policy/clipfrac_avg": 0.35546875, | |
| "policy/entropy_avg": 0.5553750991821289, | |
| "step": 93, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3186, | |
| "val/ratio": 0.9999817609786987, | |
| "val/ratio_var": 3.2739924336055992e-06 | |
| }, | |
| { | |
| "episode": 48640, | |
| "epoch": 0.2083618917066484, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3540273904800415, | |
| "lr": 1.8984375e-06, | |
| "objective/entropy": -41.448604583740234, | |
| "objective/kl": 27.778303146362305, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.456501007080078, | |
| "objective/scores": 8.456501007080078, | |
| "policy/approxkl_avg": 0.33497947454452515, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.5644186735153198, | |
| "step": 94, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3182, | |
| "val/ratio": 0.9999628663063049, | |
| "val/ratio_var": 4.411289864947321e-06 | |
| }, | |
| { | |
| "episode": 49152, | |
| "epoch": 0.2105551747772447, | |
| "eps": 7, | |
| "loss/policy_avg": 0.4336338937282562, | |
| "lr": 1.8867187500000001e-06, | |
| "objective/entropy": -41.28203582763672, | |
| "objective/kl": 27.56344985961914, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.71638011932373, | |
| "objective/scores": 8.71638011932373, | |
| "policy/approxkl_avg": 0.32576072216033936, | |
| "policy/clipfrac_avg": 0.365234375, | |
| "policy/entropy_avg": 0.5579572916030884, | |
| "step": 95, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3167, | |
| "val/ratio": 0.9997182488441467, | |
| "val/ratio_var": 3.5568716612033313e-06 | |
| }, | |
| { | |
| "episode": 49664, | |
| "epoch": 0.212748457847841, | |
| "eps": 7, | |
| "loss/policy_avg": 0.37640368938446045, | |
| "lr": 1.875e-06, | |
| "objective/entropy": -39.93994140625, | |
| "objective/kl": 28.659408569335938, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.587747573852539, | |
| "objective/scores": 8.587747573852539, | |
| "policy/approxkl_avg": 0.2930169403553009, | |
| "policy/clipfrac_avg": 0.38671875, | |
| "policy/entropy_avg": 0.5753656625747681, | |
| "step": 96, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3320, | |
| "val/ratio": 0.9999222755432129, | |
| "val/ratio_var": 2.8392282729328144e-06 | |
| }, | |
| { | |
| "episode": 50176, | |
| "epoch": 0.2149417409184373, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3704265356063843, | |
| "lr": 1.86328125e-06, | |
| "objective/entropy": -39.154541015625, | |
| "objective/kl": 28.530101776123047, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.800564765930176, | |
| "objective/scores": 8.800564765930176, | |
| "policy/approxkl_avg": 0.30916929244995117, | |
| "policy/clipfrac_avg": 0.333984375, | |
| "policy/entropy_avg": 0.5570034384727478, | |
| "step": 97, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3072, | |
| "val/ratio": 1.0001652240753174, | |
| "val/ratio_var": 4.666152108256938e-06 | |
| }, | |
| { | |
| "episode": 50688, | |
| "epoch": 0.2171350239890336, | |
| "eps": 7, | |
| "loss/policy_avg": 0.33340394496917725, | |
| "lr": 1.8515625000000001e-06, | |
| "objective/entropy": -42.39177703857422, | |
| "objective/kl": 26.74437141418457, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.953100204467773, | |
| "objective/scores": 8.953100204467773, | |
| "policy/approxkl_avg": 0.25182923674583435, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.5791052579879761, | |
| "step": 98, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3253, | |
| "val/ratio": 1.0006077289581299, | |
| "val/ratio_var": 4.077707671967801e-06 | |
| }, | |
| { | |
| "episode": 51200, | |
| "epoch": 0.21932830705962988, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3429536521434784, | |
| "lr": 1.83984375e-06, | |
| "objective/entropy": -42.56659698486328, | |
| "objective/kl": 27.317535400390625, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.745502471923828, | |
| "objective/scores": 8.745502471923828, | |
| "policy/approxkl_avg": 0.3268476724624634, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5513091087341309, | |
| "step": 99, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3308, | |
| "val/ratio": 1.0001206398010254, | |
| "val/ratio_var": 6.498310995084466e-06 | |
| }, | |
| { | |
| "episode": 51712, | |
| "epoch": 0.22152159013022618, | |
| "eps": 7, | |
| "loss/policy_avg": 0.18512360751628876, | |
| "lr": 1.828125e-06, | |
| "objective/entropy": -39.8660888671875, | |
| "objective/kl": 28.532289505004883, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.849493026733398, | |
| "objective/scores": 8.849493026733398, | |
| "policy/approxkl_avg": 0.28483685851097107, | |
| "policy/clipfrac_avg": 0.40625, | |
| "policy/entropy_avg": 0.567420244216919, | |
| "step": 100, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3124, | |
| "val/ratio": 1.0002485513687134, | |
| "val/ratio_var": 3.6835951959801605e-06 | |
| }, | |
| { | |
| "episode": 52224, | |
| "epoch": 0.22371487320082248, | |
| "eps": 7, | |
| "loss/policy_avg": 0.28863823413848877, | |
| "lr": 1.81640625e-06, | |
| "objective/entropy": -43.047508239746094, | |
| "objective/kl": 27.43829345703125, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.060672760009766, | |
| "objective/scores": 9.060672760009766, | |
| "policy/approxkl_avg": 0.3050522208213806, | |
| "policy/clipfrac_avg": 0.33984375, | |
| "policy/entropy_avg": 0.5633636713027954, | |
| "step": 101, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3185, | |
| "val/ratio": 1.0002562999725342, | |
| "val/ratio_var": 4.66149003841565e-06 | |
| }, | |
| { | |
| "episode": 52736, | |
| "epoch": 0.22590815627141878, | |
| "eps": 7, | |
| "loss/policy_avg": 0.27026695013046265, | |
| "lr": 1.8046875e-06, | |
| "objective/entropy": -44.638450622558594, | |
| "objective/kl": 26.97273826599121, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.926921844482422, | |
| "objective/scores": 8.926921844482422, | |
| "policy/approxkl_avg": 0.26449355483055115, | |
| "policy/clipfrac_avg": 0.337890625, | |
| "policy/entropy_avg": 0.5377113223075867, | |
| "step": 102, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2916, | |
| "val/ratio": 0.9999263882637024, | |
| "val/ratio_var": 2.791050519590499e-06 | |
| }, | |
| { | |
| "episode": 53248, | |
| "epoch": 0.22810143934201507, | |
| "eps": 7, | |
| "loss/policy_avg": 0.31279855966567993, | |
| "lr": 1.79296875e-06, | |
| "objective/entropy": -39.981178283691406, | |
| "objective/kl": 28.416162490844727, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.903759956359863, | |
| "objective/scores": 8.903759956359863, | |
| "policy/approxkl_avg": 0.3121040463447571, | |
| "policy/clipfrac_avg": 0.35546875, | |
| "policy/entropy_avg": 0.5868556499481201, | |
| "step": 103, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3269, | |
| "val/ratio": 0.9997861385345459, | |
| "val/ratio_var": 4.62020989289158e-06 | |
| }, | |
| { | |
| "episode": 53760, | |
| "epoch": 0.23029472241261137, | |
| "eps": 7, | |
| "loss/policy_avg": 0.20185165107250214, | |
| "lr": 1.78125e-06, | |
| "objective/entropy": -38.68260955810547, | |
| "objective/kl": 28.745805740356445, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.793964385986328, | |
| "objective/scores": 8.793964385986328, | |
| "policy/approxkl_avg": 0.3357839286327362, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5680782198905945, | |
| "step": 104, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3203, | |
| "val/ratio": 0.9997376203536987, | |
| "val/ratio_var": 3.0809435429546284e-06 | |
| }, | |
| { | |
| "episode": 54272, | |
| "epoch": 0.23248800548320767, | |
| "eps": 7, | |
| "loss/policy_avg": 0.27374643087387085, | |
| "lr": 1.76953125e-06, | |
| "objective/entropy": -41.97970199584961, | |
| "objective/kl": 27.869659423828125, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.844221115112305, | |
| "objective/scores": 8.844221115112305, | |
| "policy/approxkl_avg": 0.2983897030353546, | |
| "policy/clipfrac_avg": 0.388671875, | |
| "policy/entropy_avg": 0.569934606552124, | |
| "step": 105, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3125, | |
| "val/ratio": 0.9998430013656616, | |
| "val/ratio_var": 4.563687980407849e-06 | |
| }, | |
| { | |
| "episode": 54784, | |
| "epoch": 0.23468128855380396, | |
| "eps": 7, | |
| "loss/policy_avg": 0.28290292620658875, | |
| "lr": 1.7578125e-06, | |
| "objective/entropy": -41.56986618041992, | |
| "objective/kl": 27.95354652404785, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.726874351501465, | |
| "objective/scores": 8.726874351501465, | |
| "policy/approxkl_avg": 0.35987648367881775, | |
| "policy/clipfrac_avg": 0.37109375, | |
| "policy/entropy_avg": 0.5299930572509766, | |
| "step": 106, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2941, | |
| "val/ratio": 0.9999336004257202, | |
| "val/ratio_var": 5.3527737691183574e-06 | |
| }, | |
| { | |
| "episode": 55296, | |
| "epoch": 0.2368745716244003, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2681512236595154, | |
| "lr": 1.74609375e-06, | |
| "objective/entropy": -43.27360916137695, | |
| "objective/kl": 27.097171783447266, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.131484031677246, | |
| "objective/scores": 9.131484031677246, | |
| "policy/approxkl_avg": 0.27127525210380554, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.5349740982055664, | |
| "step": 107, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3110, | |
| "val/ratio": 1.0000276565551758, | |
| "val/ratio_var": 3.7708189211116405e-06 | |
| }, | |
| { | |
| "episode": 55808, | |
| "epoch": 0.23906785469499658, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3031000792980194, | |
| "lr": 1.734375e-06, | |
| "objective/entropy": -41.32918930053711, | |
| "objective/kl": 28.28663444519043, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.978072166442871, | |
| "objective/scores": 8.978072166442871, | |
| "policy/approxkl_avg": 0.28883644938468933, | |
| "policy/clipfrac_avg": 0.384765625, | |
| "policy/entropy_avg": 0.5501591563224792, | |
| "step": 108, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3257, | |
| "val/ratio": 1.0002058744430542, | |
| "val/ratio_var": 2.6081083888129797e-06 | |
| }, | |
| { | |
| "episode": 56320, | |
| "epoch": 0.24126113776559288, | |
| "eps": 7, | |
| "loss/policy_avg": 0.22930988669395447, | |
| "lr": 1.72265625e-06, | |
| "objective/entropy": -44.260066986083984, | |
| "objective/kl": 27.051563262939453, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.064173698425293, | |
| "objective/scores": 9.064173698425293, | |
| "policy/approxkl_avg": 0.25021883845329285, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5275313854217529, | |
| "step": 109, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3274, | |
| "val/ratio": 1.0002585649490356, | |
| "val/ratio_var": 4.10879465562175e-06 | |
| }, | |
| { | |
| "episode": 56832, | |
| "epoch": 0.24345442083618918, | |
| "eps": 7, | |
| "loss/policy_avg": 0.32286208868026733, | |
| "lr": 1.7109375e-06, | |
| "objective/entropy": -44.09670639038086, | |
| "objective/kl": 27.300418853759766, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.928081512451172, | |
| "objective/scores": 8.928081512451172, | |
| "policy/approxkl_avg": 0.2881784737110138, | |
| "policy/clipfrac_avg": 0.337890625, | |
| "policy/entropy_avg": 0.5374830961227417, | |
| "step": 110, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3310, | |
| "val/ratio": 1.000059962272644, | |
| "val/ratio_var": 4.859982709604083e-06 | |
| }, | |
| { | |
| "episode": 57344, | |
| "epoch": 0.24564770390678548, | |
| "eps": 7, | |
| "loss/policy_avg": 0.1695113182067871, | |
| "lr": 1.69921875e-06, | |
| "objective/entropy": -42.695960998535156, | |
| "objective/kl": 26.841167449951172, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.010541915893555, | |
| "objective/scores": 9.010541915893555, | |
| "policy/approxkl_avg": 0.3248569369316101, | |
| "policy/clipfrac_avg": 0.3828125, | |
| "policy/entropy_avg": 0.557858943939209, | |
| "step": 111, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3280, | |
| "val/ratio": 0.9998301267623901, | |
| "val/ratio_var": 3.137585736112669e-06 | |
| }, | |
| { | |
| "episode": 57856, | |
| "epoch": 0.24784098697738177, | |
| "eps": 7, | |
| "loss/policy_avg": 0.23961672186851501, | |
| "lr": 1.6875e-06, | |
| "objective/entropy": -41.781715393066406, | |
| "objective/kl": 27.928836822509766, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.928047180175781, | |
| "objective/scores": 8.928047180175781, | |
| "policy/approxkl_avg": 0.2984079420566559, | |
| "policy/clipfrac_avg": 0.37109375, | |
| "policy/entropy_avg": 0.5693182945251465, | |
| "step": 112, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3228, | |
| "val/ratio": 0.9999939203262329, | |
| "val/ratio_var": 3.22428149956977e-06 | |
| }, | |
| { | |
| "episode": 58368, | |
| "epoch": 0.25003427004797807, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3170594573020935, | |
| "lr": 1.67578125e-06, | |
| "objective/entropy": -44.25893783569336, | |
| "objective/kl": 27.417301177978516, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.042917251586914, | |
| "objective/scores": 9.042917251586914, | |
| "policy/approxkl_avg": 0.28602951765060425, | |
| "policy/clipfrac_avg": 0.33984375, | |
| "policy/entropy_avg": 0.5440371632575989, | |
| "step": 113, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3082, | |
| "val/ratio": 1.0005700588226318, | |
| "val/ratio_var": 1.94922395166941e-05 | |
| }, | |
| { | |
| "episode": 58880, | |
| "epoch": 0.25222755311857437, | |
| "eps": 7, | |
| "loss/policy_avg": 0.40132516622543335, | |
| "lr": 1.6640625e-06, | |
| "objective/entropy": -40.10181427001953, | |
| "objective/kl": 27.927541732788086, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.806680679321289, | |
| "objective/scores": 8.806680679321289, | |
| "policy/approxkl_avg": 0.3599626421928406, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5521677732467651, | |
| "step": 114, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3332, | |
| "val/ratio": 0.9998911023139954, | |
| "val/ratio_var": 4.160545358899981e-06 | |
| }, | |
| { | |
| "episode": 59392, | |
| "epoch": 0.25442083618917066, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3721383512020111, | |
| "lr": 1.6523437500000001e-06, | |
| "objective/entropy": -40.903629302978516, | |
| "objective/kl": 28.625869750976562, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.888092994689941, | |
| "objective/scores": 8.888092994689941, | |
| "policy/approxkl_avg": 0.3267340660095215, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.5643830299377441, | |
| "step": 115, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3251, | |
| "val/ratio": 1.0002787113189697, | |
| "val/ratio_var": 4.806289325642865e-06 | |
| }, | |
| { | |
| "episode": 59904, | |
| "epoch": 0.25661411925976696, | |
| "eps": 7, | |
| "loss/policy_avg": 0.27686673402786255, | |
| "lr": 1.640625e-06, | |
| "objective/entropy": -44.158592224121094, | |
| "objective/kl": 26.43977928161621, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.87618637084961, | |
| "objective/scores": 8.87618637084961, | |
| "policy/approxkl_avg": 0.2663460671901703, | |
| "policy/clipfrac_avg": 0.318359375, | |
| "policy/entropy_avg": 0.5564490556716919, | |
| "step": 116, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3377, | |
| "val/ratio": 0.9997222423553467, | |
| "val/ratio_var": 4.367329438537126e-06 | |
| }, | |
| { | |
| "episode": 60416, | |
| "epoch": 0.25880740233036326, | |
| "eps": 7, | |
| "loss/policy_avg": 0.25908106565475464, | |
| "lr": 1.62890625e-06, | |
| "objective/entropy": -41.13191223144531, | |
| "objective/kl": 27.067249298095703, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.76617431640625, | |
| "objective/scores": 8.76617431640625, | |
| "policy/approxkl_avg": 0.2690734267234802, | |
| "policy/clipfrac_avg": 0.3359375, | |
| "policy/entropy_avg": 0.5597757697105408, | |
| "step": 117, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3447, | |
| "val/ratio": 1.0001375675201416, | |
| "val/ratio_var": 3.1488286822423106e-06 | |
| }, | |
| { | |
| "episode": 60928, | |
| "epoch": 0.26100068540095955, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3060078024864197, | |
| "lr": 1.6171875000000001e-06, | |
| "objective/entropy": -43.39402770996094, | |
| "objective/kl": 26.568906784057617, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.188919067382812, | |
| "objective/scores": 9.188919067382812, | |
| "policy/approxkl_avg": 0.30087438225746155, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.543894350528717, | |
| "step": 118, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3462, | |
| "val/ratio": 1.0000953674316406, | |
| "val/ratio_var": 3.770735020225402e-06 | |
| }, | |
| { | |
| "episode": 61440, | |
| "epoch": 0.26319396847155585, | |
| "eps": 7, | |
| "loss/policy_avg": 0.22503231465816498, | |
| "lr": 1.60546875e-06, | |
| "objective/entropy": -41.05299758911133, | |
| "objective/kl": 27.94589614868164, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.894856452941895, | |
| "objective/scores": 8.894856452941895, | |
| "policy/approxkl_avg": 0.2811037003993988, | |
| "policy/clipfrac_avg": 0.359375, | |
| "policy/entropy_avg": 0.5578025579452515, | |
| "step": 119, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3312, | |
| "val/ratio": 0.9998151063919067, | |
| "val/ratio_var": 3.168901230310439e-06 | |
| }, | |
| { | |
| "episode": 61952, | |
| "epoch": 0.26538725154215215, | |
| "eps": 7, | |
| "loss/policy_avg": 0.18785294890403748, | |
| "lr": 1.59375e-06, | |
| "objective/entropy": -44.082115173339844, | |
| "objective/kl": 26.886157989501953, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.022890090942383, | |
| "objective/scores": 9.022890090942383, | |
| "policy/approxkl_avg": 0.23531067371368408, | |
| "policy/clipfrac_avg": 0.330078125, | |
| "policy/entropy_avg": 0.5334745049476624, | |
| "step": 120, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3293, | |
| "val/ratio": 1.0002555847167969, | |
| "val/ratio_var": 3.1270933504856657e-06 | |
| }, | |
| { | |
| "episode": 62464, | |
| "epoch": 0.26758053461274844, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2825835943222046, | |
| "lr": 1.5820312500000001e-06, | |
| "objective/entropy": -41.7612190246582, | |
| "objective/kl": 28.106294631958008, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.90147590637207, | |
| "objective/scores": 8.90147590637207, | |
| "policy/approxkl_avg": 0.28883862495422363, | |
| "policy/clipfrac_avg": 0.353515625, | |
| "policy/entropy_avg": 0.5609524846076965, | |
| "step": 121, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3501, | |
| "val/ratio": 0.9996764659881592, | |
| "val/ratio_var": 3.0721889743290376e-06 | |
| }, | |
| { | |
| "episode": 62976, | |
| "epoch": 0.26977381768334474, | |
| "eps": 7, | |
| "loss/policy_avg": 0.42201805114746094, | |
| "lr": 1.5703125e-06, | |
| "objective/entropy": -40.16608428955078, | |
| "objective/kl": 29.224674224853516, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.230374336242676, | |
| "objective/scores": 9.230374336242676, | |
| "policy/approxkl_avg": 0.3132435083389282, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.545167863368988, | |
| "step": 122, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3192, | |
| "val/ratio": 1.000016689300537, | |
| "val/ratio_var": 5.478911134559894e-06 | |
| }, | |
| { | |
| "episode": 63488, | |
| "epoch": 0.27196710075394104, | |
| "eps": 7, | |
| "loss/policy_avg": 0.13765352964401245, | |
| "lr": 1.55859375e-06, | |
| "objective/entropy": -38.40263366699219, | |
| "objective/kl": 28.912315368652344, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.914356231689453, | |
| "objective/scores": 8.914356231689453, | |
| "policy/approxkl_avg": 0.26425743103027344, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5654212236404419, | |
| "step": 123, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3085, | |
| "val/ratio": 0.9999741315841675, | |
| "val/ratio_var": 3.0958819934312487e-06 | |
| }, | |
| { | |
| "episode": 64000, | |
| "epoch": 0.27416038382453733, | |
| "eps": 7, | |
| "loss/policy_avg": 0.281406968832016, | |
| "lr": 1.5468750000000001e-06, | |
| "objective/entropy": -41.14397430419922, | |
| "objective/kl": 27.943584442138672, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.850857734680176, | |
| "objective/scores": 8.850857734680176, | |
| "policy/approxkl_avg": 0.26425665616989136, | |
| "policy/clipfrac_avg": 0.341796875, | |
| "policy/entropy_avg": 0.5333235263824463, | |
| "step": 124, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2966, | |
| "val/ratio": 1.0001806020736694, | |
| "val/ratio_var": 4.1297735151601955e-06 | |
| }, | |
| { | |
| "episode": 64512, | |
| "epoch": 0.27635366689513363, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3354412317276001, | |
| "lr": 1.53515625e-06, | |
| "objective/entropy": -40.96007537841797, | |
| "objective/kl": 28.996366500854492, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.931587219238281, | |
| "objective/scores": 8.931587219238281, | |
| "policy/approxkl_avg": 0.2954847514629364, | |
| "policy/clipfrac_avg": 0.359375, | |
| "policy/entropy_avg": 0.5366266965866089, | |
| "step": 125, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2979, | |
| "val/ratio": 1.000289797782898, | |
| "val/ratio_var": 3.790634536926518e-06 | |
| }, | |
| { | |
| "episode": 65024, | |
| "epoch": 0.27854694996572993, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2738310396671295, | |
| "lr": 1.5234375e-06, | |
| "objective/entropy": -40.16575622558594, | |
| "objective/kl": 29.01610565185547, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.160589218139648, | |
| "objective/scores": 9.160589218139648, | |
| "policy/approxkl_avg": 0.30494195222854614, | |
| "policy/clipfrac_avg": 0.33984375, | |
| "policy/entropy_avg": 0.5441122055053711, | |
| "step": 126, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3075, | |
| "val/ratio": 0.9997348785400391, | |
| "val/ratio_var": 3.715499815371004e-06 | |
| }, | |
| { | |
| "episode": 65536, | |
| "epoch": 0.2807402330363262, | |
| "eps": 7, | |
| "loss/policy_avg": 0.3204410672187805, | |
| "lr": 1.5117187500000001e-06, | |
| "objective/entropy": -41.28175354003906, | |
| "objective/kl": 28.728578567504883, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.17586612701416, | |
| "objective/scores": 9.17586612701416, | |
| "policy/approxkl_avg": 0.25520649552345276, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.5370270013809204, | |
| "step": 127, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2918, | |
| "val/ratio": 1.0001736879348755, | |
| "val/ratio_var": 3.415733999645454e-06 | |
| }, | |
| { | |
| "episode": 66048, | |
| "epoch": 0.2829335161069225, | |
| "eps": 7, | |
| "loss/policy_avg": 0.23167520761489868, | |
| "lr": 1.5e-06, | |
| "objective/entropy": -40.13172912597656, | |
| "objective/kl": 29.34070587158203, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.950373649597168, | |
| "objective/scores": 8.950373649597168, | |
| "policy/approxkl_avg": 0.2691437304019928, | |
| "policy/clipfrac_avg": 0.36328125, | |
| "policy/entropy_avg": 0.534875750541687, | |
| "step": 128, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2974, | |
| "val/ratio": 0.9997478723526001, | |
| "val/ratio_var": 3.2285590805258835e-06 | |
| }, | |
| { | |
| "episode": 66560, | |
| "epoch": 0.2851267991775189, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2088831514120102, | |
| "lr": 1.48828125e-06, | |
| "objective/entropy": -38.12837600708008, | |
| "objective/kl": 28.746782302856445, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.036067962646484, | |
| "objective/scores": 9.036067962646484, | |
| "policy/approxkl_avg": 0.3211039900779724, | |
| "policy/clipfrac_avg": 0.361328125, | |
| "policy/entropy_avg": 0.5321791172027588, | |
| "step": 129, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3088, | |
| "val/ratio": 1.0001753568649292, | |
| "val/ratio_var": 5.1817132771248e-06 | |
| }, | |
| { | |
| "episode": 67072, | |
| "epoch": 0.28732008224811517, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2225157618522644, | |
| "lr": 1.4765625e-06, | |
| "objective/entropy": -38.43605041503906, | |
| "objective/kl": 30.095699310302734, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.020343780517578, | |
| "objective/scores": 9.020343780517578, | |
| "policy/approxkl_avg": 0.29494479298591614, | |
| "policy/clipfrac_avg": 0.365234375, | |
| "policy/entropy_avg": 0.528764009475708, | |
| "step": 130, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2872, | |
| "val/ratio": 0.9999305009841919, | |
| "val/ratio_var": 4.7066237129911315e-06 | |
| }, | |
| { | |
| "episode": 67584, | |
| "epoch": 0.28951336531871147, | |
| "eps": 7, | |
| "loss/policy_avg": 0.18626652657985687, | |
| "lr": 1.46484375e-06, | |
| "objective/entropy": -38.2808837890625, | |
| "objective/kl": 29.611225128173828, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.177016258239746, | |
| "objective/scores": 9.177016258239746, | |
| "policy/approxkl_avg": 0.24960750341415405, | |
| "policy/clipfrac_avg": 0.37890625, | |
| "policy/entropy_avg": 0.5560537576675415, | |
| "step": 131, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3039, | |
| "val/ratio": 0.9996904134750366, | |
| "val/ratio_var": 2.8264887532714056e-06 | |
| }, | |
| { | |
| "episode": 68096, | |
| "epoch": 0.29170664838930777, | |
| "eps": 7, | |
| "loss/policy_avg": 0.34811753034591675, | |
| "lr": 1.453125e-06, | |
| "objective/entropy": -37.813743591308594, | |
| "objective/kl": 30.11647605895996, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.143380165100098, | |
| "objective/scores": 9.143380165100098, | |
| "policy/approxkl_avg": 0.24777096509933472, | |
| "policy/clipfrac_avg": 0.322265625, | |
| "policy/entropy_avg": 0.5264837741851807, | |
| "step": 132, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2881, | |
| "val/ratio": 1.0001630783081055, | |
| "val/ratio_var": 2.7501182557898574e-06 | |
| }, | |
| { | |
| "episode": 68608, | |
| "epoch": 0.29389993145990406, | |
| "eps": 7, | |
| "loss/policy_avg": 1.7297974824905396, | |
| "lr": 1.44140625e-06, | |
| "objective/entropy": -38.04670715332031, | |
| "objective/kl": 29.923572540283203, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.23299789428711, | |
| "objective/scores": 9.23299789428711, | |
| "policy/approxkl_avg": 0.3208100199699402, | |
| "policy/clipfrac_avg": 0.349609375, | |
| "policy/entropy_avg": 0.5299798250198364, | |
| "step": 133, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2948, | |
| "val/ratio": 1.0002143383026123, | |
| "val/ratio_var": 6.060246505512623e-06 | |
| }, | |
| { | |
| "episode": 69120, | |
| "epoch": 0.29609321453050036, | |
| "eps": 7, | |
| "loss/policy_avg": 0.2908250093460083, | |
| "lr": 1.4296875e-06, | |
| "objective/entropy": -39.67091751098633, | |
| "objective/kl": 29.891416549682617, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.22230339050293, | |
| "objective/scores": 9.22230339050293, | |
| "policy/approxkl_avg": 0.2994619607925415, | |
| "policy/clipfrac_avg": 0.361328125, | |
| "policy/entropy_avg": 0.5484848618507385, | |
| "step": 134, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2992, | |
| "val/ratio": 0.9998666048049927, | |
| "val/ratio_var": 4.691268713941099e-06 | |
| }, | |
| { | |
| "episode": 69632, | |
| "epoch": 0.29828649760109666, | |
| "eps": 6, | |
| "loss/policy_avg": 0.16967931389808655, | |
| "lr": 1.41796875e-06, | |
| "objective/entropy": -40.47759246826172, | |
| "objective/kl": 28.60110855102539, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.249778747558594, | |
| "objective/scores": 9.249778747558594, | |
| "policy/approxkl_avg": 0.30396944284439087, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.5294628739356995, | |
| "step": 135, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3158, | |
| "val/ratio": 0.9999102354049683, | |
| "val/ratio_var": 4.847398031415651e-06 | |
| }, | |
| { | |
| "episode": 70144, | |
| "epoch": 0.30047978067169295, | |
| "eps": 6, | |
| "loss/policy_avg": 0.30670228600502014, | |
| "lr": 1.40625e-06, | |
| "objective/entropy": -41.60497283935547, | |
| "objective/kl": 28.722915649414062, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.269282341003418, | |
| "objective/scores": 9.269282341003418, | |
| "policy/approxkl_avg": 0.27743324637413025, | |
| "policy/clipfrac_avg": 0.35546875, | |
| "policy/entropy_avg": 0.5208898782730103, | |
| "step": 136, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3202, | |
| "val/ratio": 1.0001318454742432, | |
| "val/ratio_var": 4.43136786998366e-06 | |
| }, | |
| { | |
| "episode": 70656, | |
| "epoch": 0.30267306374228925, | |
| "eps": 6, | |
| "loss/policy_avg": 0.24920354783535004, | |
| "lr": 1.39453125e-06, | |
| "objective/entropy": -42.12297439575195, | |
| "objective/kl": 28.017452239990234, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.922027587890625, | |
| "objective/scores": 8.922027587890625, | |
| "policy/approxkl_avg": 0.23385344445705414, | |
| "policy/clipfrac_avg": 0.365234375, | |
| "policy/entropy_avg": 0.5616132616996765, | |
| "step": 137, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3543, | |
| "val/ratio": 0.9999379515647888, | |
| "val/ratio_var": 3.0522689939971315e-06 | |
| }, | |
| { | |
| "episode": 71168, | |
| "epoch": 0.30486634681288555, | |
| "eps": 6, | |
| "loss/policy_avg": 0.18219150602817535, | |
| "lr": 1.3828125e-06, | |
| "objective/entropy": -41.68804168701172, | |
| "objective/kl": 28.044918060302734, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.24627685546875, | |
| "objective/scores": 9.24627685546875, | |
| "policy/approxkl_avg": 0.32243162393569946, | |
| "policy/clipfrac_avg": 0.41796875, | |
| "policy/entropy_avg": 0.5281212329864502, | |
| "step": 138, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3289, | |
| "val/ratio": 0.9998571872711182, | |
| "val/ratio_var": 4.768267899635248e-06 | |
| }, | |
| { | |
| "episode": 71680, | |
| "epoch": 0.30705962988348184, | |
| "eps": 6, | |
| "loss/policy_avg": 0.252814918756485, | |
| "lr": 1.37109375e-06, | |
| "objective/entropy": -43.992950439453125, | |
| "objective/kl": 27.172204971313477, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.964099884033203, | |
| "objective/scores": 8.964099884033203, | |
| "policy/approxkl_avg": 0.25193220376968384, | |
| "policy/clipfrac_avg": 0.357421875, | |
| "policy/entropy_avg": 0.5601909160614014, | |
| "step": 139, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3426, | |
| "val/ratio": 1.0002691745758057, | |
| "val/ratio_var": 4.502016508922679e-06 | |
| }, | |
| { | |
| "episode": 72192, | |
| "epoch": 0.30925291295407814, | |
| "eps": 6, | |
| "loss/policy_avg": 0.19190388917922974, | |
| "lr": 1.359375e-06, | |
| "objective/entropy": -42.40169143676758, | |
| "objective/kl": 28.326745986938477, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.860151290893555, | |
| "objective/scores": 8.860151290893555, | |
| "policy/approxkl_avg": 0.2868598699569702, | |
| "policy/clipfrac_avg": 0.3671875, | |
| "policy/entropy_avg": 0.5509663820266724, | |
| "step": 140, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3451, | |
| "val/ratio": 0.9999006390571594, | |
| "val/ratio_var": 3.2771320093161194e-06 | |
| }, | |
| { | |
| "episode": 72704, | |
| "epoch": 0.31144619602467444, | |
| "eps": 6, | |
| "loss/policy_avg": 0.41269177198410034, | |
| "lr": 1.34765625e-06, | |
| "objective/entropy": -43.28689193725586, | |
| "objective/kl": 27.77793312072754, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.921087265014648, | |
| "objective/scores": 8.921087265014648, | |
| "policy/approxkl_avg": 0.27822810411453247, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5682795643806458, | |
| "step": 141, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3703, | |
| "val/ratio": 1.0001779794692993, | |
| "val/ratio_var": 5.943717042100616e-06 | |
| }, | |
| { | |
| "episode": 73216, | |
| "epoch": 0.31363947909527073, | |
| "eps": 6, | |
| "loss/policy_avg": 0.22461438179016113, | |
| "lr": 1.3359375e-06, | |
| "objective/entropy": -41.46437072753906, | |
| "objective/kl": 28.51311492919922, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.918792724609375, | |
| "objective/scores": 8.918792724609375, | |
| "policy/approxkl_avg": 0.2884824275970459, | |
| "policy/clipfrac_avg": 0.392578125, | |
| "policy/entropy_avg": 0.5668504238128662, | |
| "step": 142, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3600, | |
| "val/ratio": 0.9997174143791199, | |
| "val/ratio_var": 3.1224142276187195e-06 | |
| }, | |
| { | |
| "episode": 73728, | |
| "epoch": 0.31583276216586703, | |
| "eps": 6, | |
| "loss/policy_avg": 0.26002606749534607, | |
| "lr": 1.32421875e-06, | |
| "objective/entropy": -40.683143615722656, | |
| "objective/kl": 29.9025936126709, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.003212928771973, | |
| "objective/scores": 9.003212928771973, | |
| "policy/approxkl_avg": 0.41950270533561707, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.5366673469543457, | |
| "step": 143, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3528, | |
| "val/ratio": 1.0001637935638428, | |
| "val/ratio_var": 4.806850483873859e-06 | |
| }, | |
| { | |
| "episode": 74240, | |
| "epoch": 0.31802604523646333, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2675803303718567, | |
| "lr": 1.3125000000000001e-06, | |
| "objective/entropy": -41.83632278442383, | |
| "objective/kl": 28.70281982421875, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.011963844299316, | |
| "objective/scores": 9.011963844299316, | |
| "policy/approxkl_avg": 0.23804181814193726, | |
| "policy/clipfrac_avg": 0.38671875, | |
| "policy/entropy_avg": 0.5452957153320312, | |
| "step": 144, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3428, | |
| "val/ratio": 1.0001215934753418, | |
| "val/ratio_var": 3.4354568470007507e-06 | |
| }, | |
| { | |
| "episode": 74752, | |
| "epoch": 0.3202193283070596, | |
| "eps": 6, | |
| "loss/policy_avg": 0.3060368299484253, | |
| "lr": 1.30078125e-06, | |
| "objective/entropy": -40.88287353515625, | |
| "objective/kl": 30.03162384033203, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.887903213500977, | |
| "objective/scores": 8.887903213500977, | |
| "policy/approxkl_avg": 0.2737124264240265, | |
| "policy/clipfrac_avg": 0.376953125, | |
| "policy/entropy_avg": 0.5587130784988403, | |
| "step": 145, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3376, | |
| "val/ratio": 0.9998096823692322, | |
| "val/ratio_var": 3.0338110263983253e-06 | |
| }, | |
| { | |
| "episode": 75264, | |
| "epoch": 0.3224126113776559, | |
| "eps": 6, | |
| "loss/policy_avg": 0.3782508075237274, | |
| "lr": 1.2890625e-06, | |
| "objective/entropy": -41.50593185424805, | |
| "objective/kl": 28.35731315612793, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.796640396118164, | |
| "objective/scores": 8.796640396118164, | |
| "policy/approxkl_avg": 0.26787978410720825, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5557115077972412, | |
| "step": 146, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3456, | |
| "val/ratio": 1.0003128051757812, | |
| "val/ratio_var": 3.1068911994225346e-06 | |
| }, | |
| { | |
| "episode": 75776, | |
| "epoch": 0.3246058944482522, | |
| "eps": 6, | |
| "loss/policy_avg": 0.24699635803699493, | |
| "lr": 1.2773437500000001e-06, | |
| "objective/entropy": -38.084693908691406, | |
| "objective/kl": 30.745229721069336, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.165950775146484, | |
| "objective/scores": 9.165950775146484, | |
| "policy/approxkl_avg": 0.30605587363243103, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.550534725189209, | |
| "step": 147, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3354, | |
| "val/ratio": 0.9999103546142578, | |
| "val/ratio_var": 3.188604523529648e-06 | |
| }, | |
| { | |
| "episode": 76288, | |
| "epoch": 0.3267991775188485, | |
| "eps": 6, | |
| "loss/policy_avg": 0.21277828514575958, | |
| "lr": 1.265625e-06, | |
| "objective/entropy": -39.392494201660156, | |
| "objective/kl": 31.08953094482422, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.794751167297363, | |
| "objective/scores": 8.794751167297363, | |
| "policy/approxkl_avg": 0.2738240957260132, | |
| "policy/clipfrac_avg": 0.375, | |
| "policy/entropy_avg": 0.5848383903503418, | |
| "step": 148, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3624, | |
| "val/ratio": 1.0000360012054443, | |
| "val/ratio_var": 3.766557256312808e-06 | |
| }, | |
| { | |
| "episode": 76800, | |
| "epoch": 0.3289924605894448, | |
| "eps": 6, | |
| "loss/policy_avg": 0.4244789481163025, | |
| "lr": 1.25390625e-06, | |
| "objective/entropy": -39.247779846191406, | |
| "objective/kl": 30.732606887817383, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.010191917419434, | |
| "objective/scores": 9.010191917419434, | |
| "policy/approxkl_avg": 0.29289907217025757, | |
| "policy/clipfrac_avg": 0.35546875, | |
| "policy/entropy_avg": 0.5503936409950256, | |
| "step": 149, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3473, | |
| "val/ratio": 0.9998835325241089, | |
| "val/ratio_var": 4.186950718576554e-06 | |
| }, | |
| { | |
| "episode": 77312, | |
| "epoch": 0.3311857436600411, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2913992404937744, | |
| "lr": 1.2421875000000001e-06, | |
| "objective/entropy": -36.79509735107422, | |
| "objective/kl": 31.21529769897461, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.547361373901367, | |
| "objective/scores": 8.547361373901367, | |
| "policy/approxkl_avg": 0.2886246144771576, | |
| "policy/clipfrac_avg": 0.40234375, | |
| "policy/entropy_avg": 0.5547722578048706, | |
| "step": 150, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3487, | |
| "val/ratio": 1.000207543373108, | |
| "val/ratio_var": 5.082135885459138e-06 | |
| }, | |
| { | |
| "episode": 77824, | |
| "epoch": 0.3333790267306374, | |
| "eps": 6, | |
| "loss/policy_avg": 0.26813989877700806, | |
| "lr": 1.23046875e-06, | |
| "objective/entropy": -37.272186279296875, | |
| "objective/kl": 31.253156661987305, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.79878044128418, | |
| "objective/scores": 8.79878044128418, | |
| "policy/approxkl_avg": 0.30616575479507446, | |
| "policy/clipfrac_avg": 0.380859375, | |
| "policy/entropy_avg": 0.5631169676780701, | |
| "step": 151, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3643, | |
| "val/ratio": 1.000023365020752, | |
| "val/ratio_var": 4.361940682429122e-06 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 128, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1.122960932145305, | |
| "save_steps": 26, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": true, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0, | |
| "train_batch_size": null, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |