| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.992, | |
| "eval_steps": 500, | |
| "global_step": 125, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 178.3541717529297, | |
| "epoch": 0.016, | |
| "grad_norm": 0.7053135404726241, | |
| "kl": 0.0, | |
| "learning_rate": 1.25e-07, | |
| "loss": 0.0, | |
| "reward": 0.5438157171010971, | |
| "reward_std": 0.4464171230792999, | |
| "rewards/correct_code_reward_func": 0.2500000074505806, | |
| "rewards/len_reward_func": 0.2938157171010971, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 176.89584350585938, | |
| "epoch": 0.032, | |
| "grad_norm": 2.3417507999605864, | |
| "kl": 0.0, | |
| "learning_rate": 2.5e-07, | |
| "loss": -0.0, | |
| "reward": 0.4928237199783325, | |
| "reward_std": 0.3440842181444168, | |
| "rewards/correct_code_reward_func": 0.1458333432674408, | |
| "rewards/len_reward_func": 0.34699034690856934, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 192.25, | |
| "epoch": 0.048, | |
| "grad_norm": 0.48071888167986226, | |
| "kl": -3.4809112548828125e-05, | |
| "learning_rate": 3.75e-07, | |
| "loss": -0.0, | |
| "reward": 0.449026882648468, | |
| "reward_std": 0.47666139900684357, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.2823602259159088, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 270.7291717529297, | |
| "epoch": 0.064, | |
| "grad_norm": 0.3569537292235904, | |
| "kl": 4.172325134277344e-07, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 0.45451289415359497, | |
| "reward_std": 0.3296015188097954, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.3503462225198746, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 232.58334350585938, | |
| "epoch": 0.08, | |
| "grad_norm": 0.4439935569873824, | |
| "kl": -4.214048385620117e-05, | |
| "learning_rate": 4.999157413258781e-07, | |
| "loss": -0.0, | |
| "reward": 0.2809216380119324, | |
| "reward_std": 0.37323758006095886, | |
| "rewards/correct_code_reward_func": 0.0, | |
| "rewards/len_reward_func": 0.2809216380119324, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 261.9375, | |
| "epoch": 0.096, | |
| "grad_norm": 0.3541024748340305, | |
| "kl": -3.1054019927978516e-05, | |
| "learning_rate": 4.996630220997057e-07, | |
| "loss": -0.0, | |
| "reward": 0.4689294695854187, | |
| "reward_std": 0.4829525202512741, | |
| "rewards/correct_code_reward_func": 0.1875, | |
| "rewards/len_reward_func": 0.2814294695854187, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 306.12501525878906, | |
| "epoch": 0.112, | |
| "grad_norm": 0.3947267586718111, | |
| "kl": -2.968311309814453e-05, | |
| "learning_rate": 4.992420126717784e-07, | |
| "loss": -0.0, | |
| "reward": 0.3299577385187149, | |
| "reward_std": 0.30695800483226776, | |
| "rewards/correct_code_reward_func": 0.0, | |
| "rewards/len_reward_func": 0.3299577236175537, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 238.85417938232422, | |
| "epoch": 0.128, | |
| "grad_norm": 0.5118206169412205, | |
| "kl": -4.1604042053222656e-05, | |
| "learning_rate": 4.986529968316653e-07, | |
| "loss": 0.0, | |
| "reward": 0.4368641823530197, | |
| "reward_std": 0.2371533028781414, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.3535308539867401, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 304.7708435058594, | |
| "epoch": 0.144, | |
| "grad_norm": 0.3493288643723073, | |
| "kl": -5.060434341430664e-05, | |
| "learning_rate": 4.978963716169165e-07, | |
| "loss": -0.0, | |
| "reward": 0.4636567234992981, | |
| "reward_std": 0.46522316336631775, | |
| "rewards/correct_code_reward_func": 0.1666666679084301, | |
| "rewards/len_reward_func": 0.2969900518655777, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 264.4791717529297, | |
| "epoch": 0.16, | |
| "grad_norm": 0.8959331411324142, | |
| "kl": -7.329881191253662e-05, | |
| "learning_rate": 4.969726470454313e-07, | |
| "loss": -0.0, | |
| "reward": 0.5069085508584976, | |
| "reward_std": 0.37757958471775055, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.3402418941259384, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 201.3541717529297, | |
| "epoch": 0.176, | |
| "grad_norm": 0.4500851901199424, | |
| "kl": -4.0531158447265625e-05, | |
| "learning_rate": 4.958824457716706e-07, | |
| "loss": -0.0, | |
| "reward": 0.5010976195335388, | |
| "reward_std": 0.5051684230566025, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.334430992603302, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 297.37500762939453, | |
| "epoch": 0.192, | |
| "grad_norm": 0.43567877016900936, | |
| "kl": -4.696846008300781e-05, | |
| "learning_rate": 4.946265026669454e-07, | |
| "loss": -0.0, | |
| "reward": 0.3353367894887924, | |
| "reward_std": 0.3315615653991699, | |
| "rewards/correct_code_reward_func": 0.02083333395421505, | |
| "rewards/len_reward_func": 0.3145034611225128, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 201.75, | |
| "epoch": 0.208, | |
| "grad_norm": 0.5760425272270367, | |
| "kl": 2.5451183319091797e-05, | |
| "learning_rate": 4.932056643240618e-07, | |
| "loss": 0.0, | |
| "reward": 0.6330613493919373, | |
| "reward_std": 0.5388243198394775, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.27889466285705566, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 288.0416717529297, | |
| "epoch": 0.224, | |
| "grad_norm": 0.6576192766028296, | |
| "kl": 7.191300392150879e-05, | |
| "learning_rate": 4.916208884866592e-07, | |
| "loss": 0.0, | |
| "reward": 0.39246678352355957, | |
| "reward_std": 0.44181104004383087, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.3091334402561188, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 183.75000762939453, | |
| "epoch": 0.24, | |
| "grad_norm": 0.6134701851816485, | |
| "kl": 0.00014084577560424805, | |
| "learning_rate": 4.898732434036243e-07, | |
| "loss": 0.0, | |
| "reward": 0.42976467311382294, | |
| "reward_std": 0.3796156942844391, | |
| "rewards/correct_code_reward_func": 0.1250000037252903, | |
| "rewards/len_reward_func": 0.30476468801498413, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 207.77083587646484, | |
| "epoch": 0.256, | |
| "grad_norm": 0.7064422469677513, | |
| "kl": 0.000293731689453125, | |
| "learning_rate": 4.879639071090173e-07, | |
| "loss": 0.0, | |
| "reward": 0.35718174278736115, | |
| "reward_std": 0.408719003200531, | |
| "rewards/correct_code_reward_func": 0.0625, | |
| "rewards/len_reward_func": 0.29468175768852234, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 194.00000762939453, | |
| "epoch": 0.272, | |
| "grad_norm": 0.4827506793012771, | |
| "kl": 2.288818359375e-05, | |
| "learning_rate": 4.858941666279955e-07, | |
| "loss": -0.0, | |
| "reward": 0.5703845322132111, | |
| "reward_std": 0.39449170231819153, | |
| "rewards/correct_code_reward_func": 0.2500000111758709, | |
| "rewards/len_reward_func": 0.32038453221321106, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 238.18750762939453, | |
| "epoch": 0.288, | |
| "grad_norm": 0.41996129426983214, | |
| "kl": 0.00011217594146728516, | |
| "learning_rate": 4.836654171092682e-07, | |
| "loss": 0.0, | |
| "reward": 0.6033974885940552, | |
| "reward_std": 0.43378083407878876, | |
| "rewards/correct_code_reward_func": 0.2708333358168602, | |
| "rewards/len_reward_func": 0.33256417512893677, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 233.95834350585938, | |
| "epoch": 0.304, | |
| "grad_norm": 0.5123314162893023, | |
| "kl": 0.00013169646263122559, | |
| "learning_rate": 4.812791608846709e-07, | |
| "loss": 0.0, | |
| "reward": 0.40910618007183075, | |
| "reward_std": 0.3389303684234619, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.32577285170555115, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 316.3125, | |
| "epoch": 0.32, | |
| "grad_norm": 0.47220846544005757, | |
| "kl": 0.0002579689025878906, | |
| "learning_rate": 4.787370064564882e-07, | |
| "loss": 0.0, | |
| "reward": 0.41953830420970917, | |
| "reward_std": 0.4723764508962631, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.2737049460411072, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 227.7916717529297, | |
| "epoch": 0.336, | |
| "grad_norm": 0.6008188696097762, | |
| "kl": 0.0004267692565917969, | |
| "learning_rate": 4.7604066741321253e-07, | |
| "loss": 0.0, | |
| "reward": 0.6169633865356445, | |
| "reward_std": 0.48675188422203064, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.28363004326820374, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 252.27084350585938, | |
| "epoch": 0.352, | |
| "grad_norm": 0.5160855324353004, | |
| "kl": 0.0001907944679260254, | |
| "learning_rate": 4.731919612744659e-07, | |
| "loss": 0.0, | |
| "reward": 0.5648790299892426, | |
| "reward_std": 0.31752249598503113, | |
| "rewards/correct_code_reward_func": 0.18750000558793545, | |
| "rewards/len_reward_func": 0.37737900018692017, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 199.27083587646484, | |
| "epoch": 0.368, | |
| "grad_norm": 0.4296758598414565, | |
| "kl": 0.000273287296295166, | |
| "learning_rate": 4.7019280826586604e-07, | |
| "loss": 0.0, | |
| "reward": 0.45347318053245544, | |
| "reward_std": 0.3857118636369705, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.34930650889873505, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 221.77083587646484, | |
| "epoch": 0.384, | |
| "grad_norm": 1.2892124765727169, | |
| "kl": 0.0007638931274414062, | |
| "learning_rate": 4.6704523002466094e-07, | |
| "loss": 0.0, | |
| "reward": 0.4506646543741226, | |
| "reward_std": 0.20875498466193676, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.40899796783924103, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 266.8333435058594, | |
| "epoch": 0.4, | |
| "grad_norm": 1.4838265312357841, | |
| "kl": 0.0009393692016601562, | |
| "learning_rate": 4.6375134823700503e-07, | |
| "loss": 0.0, | |
| "reward": 0.4240592122077942, | |
| "reward_std": 0.36766907572746277, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.3198925405740738, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 225.20834350585938, | |
| "epoch": 0.416, | |
| "grad_norm": 0.45923128114507034, | |
| "kl": 0.0007772445678710938, | |
| "learning_rate": 4.603133832077953e-07, | |
| "loss": 0.0, | |
| "reward": 0.48156437277793884, | |
| "reward_std": 0.4315789043903351, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.33573102951049805, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 127.33333587646484, | |
| "epoch": 0.432, | |
| "grad_norm": 0.5853476018540942, | |
| "kl": 0.00115966796875, | |
| "learning_rate": 4.5673365236403216e-07, | |
| "loss": 0.0, | |
| "reward": 0.5744044184684753, | |
| "reward_std": 0.43071019649505615, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.30357107520103455, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 273.1666717529297, | |
| "epoch": 0.448, | |
| "grad_norm": 0.4925160720854453, | |
| "kl": 0.0010128021240234375, | |
| "learning_rate": 4.530145686927125e-07, | |
| "loss": 0.0, | |
| "reward": 0.5949445962905884, | |
| "reward_std": 0.42838945984840393, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.3241112679243088, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 178.75000762939453, | |
| "epoch": 0.464, | |
| "grad_norm": 1.1463419828713735, | |
| "kl": 0.0027008056640625, | |
| "learning_rate": 4.4915863911430897e-07, | |
| "loss": 0.0, | |
| "reward": 0.3950851857662201, | |
| "reward_std": 0.4124990254640579, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.3117518424987793, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 210.0, | |
| "epoch": 0.48, | |
| "grad_norm": 0.38472151178254205, | |
| "kl": 0.0011115074157714844, | |
| "learning_rate": 4.45168462792932e-07, | |
| "loss": 0.0, | |
| "reward": 0.48962171375751495, | |
| "reward_std": 0.443721666932106, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.32295504212379456, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 204.95833587646484, | |
| "epoch": 0.496, | |
| "grad_norm": 1.5507179471403478, | |
| "kl": 0.00183868408203125, | |
| "learning_rate": 4.4104672938431223e-07, | |
| "loss": 0.0, | |
| "reward": 0.6203064322471619, | |
| "reward_std": 0.39608660340309143, | |
| "rewards/correct_code_reward_func": 0.2916666865348816, | |
| "rewards/len_reward_func": 0.32863976061344147, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 159.83333587646484, | |
| "epoch": 0.512, | |
| "grad_norm": 0.9492121763047169, | |
| "kl": 0.001636505126953125, | |
| "learning_rate": 4.367962172227866e-07, | |
| "loss": 0.0, | |
| "reward": 0.5581120550632477, | |
| "reward_std": 0.4106949418783188, | |
| "rewards/correct_code_reward_func": 0.25, | |
| "rewards/len_reward_func": 0.3081120699644089, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 214.52083587646484, | |
| "epoch": 0.528, | |
| "grad_norm": 1.1197088117962448, | |
| "kl": 0.005927085876464844, | |
| "learning_rate": 4.324197914485075e-07, | |
| "loss": 0.0, | |
| "reward": 0.4833623319864273, | |
| "reward_std": 0.39532361924648285, | |
| "rewards/correct_code_reward_func": 0.1250000037252903, | |
| "rewards/len_reward_func": 0.3583623170852661, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 255.39583587646484, | |
| "epoch": 0.544, | |
| "grad_norm": 0.4856761942711223, | |
| "kl": 0.00038909912109375, | |
| "learning_rate": 4.2792040207614e-07, | |
| "loss": 0.0, | |
| "reward": 0.3419315367937088, | |
| "reward_std": 0.23230206966400146, | |
| "rewards/correct_code_reward_func": 0.02083333395421505, | |
| "rewards/len_reward_func": 0.3210982233285904, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 215.93750762939453, | |
| "epoch": 0.56, | |
| "grad_norm": 0.6317607844858494, | |
| "kl": 0.00643157958984375, | |
| "learning_rate": 4.2330108200634723e-07, | |
| "loss": 0.0, | |
| "reward": 0.48397234082221985, | |
| "reward_std": 0.36393553018569946, | |
| "rewards/correct_code_reward_func": 0.1458333432674408, | |
| "rewards/len_reward_func": 0.33813901245594025, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 170.52084350585938, | |
| "epoch": 0.576, | |
| "grad_norm": 0.47983643730367526, | |
| "kl": 0.0009899139404296875, | |
| "learning_rate": 4.185649449814045e-07, | |
| "loss": 0.0, | |
| "reward": 0.5446349233388901, | |
| "reward_std": 0.475721538066864, | |
| "rewards/correct_code_reward_func": 0.229166679084301, | |
| "rewards/len_reward_func": 0.3154682517051697, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 155.41666793823242, | |
| "epoch": 0.592, | |
| "grad_norm": 0.7182646382516522, | |
| "kl": 0.00705718994140625, | |
| "learning_rate": 4.137151834863213e-07, | |
| "loss": 0.0, | |
| "reward": 0.659955769777298, | |
| "reward_std": 0.4905228465795517, | |
| "rewards/correct_code_reward_func": 0.2916666865348816, | |
| "rewards/len_reward_func": 0.3682890981435776, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 234.35416793823242, | |
| "epoch": 0.608, | |
| "grad_norm": 0.5385402773601645, | |
| "kl": 0.001377105712890625, | |
| "learning_rate": 4.087550665968846e-07, | |
| "loss": 0.0, | |
| "reward": 0.34026579558849335, | |
| "reward_std": 0.42601510882377625, | |
| "rewards/correct_code_reward_func": 0.10416666977107525, | |
| "rewards/len_reward_func": 0.23609913140535355, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 150.02083587646484, | |
| "epoch": 0.624, | |
| "grad_norm": 0.8263933056203022, | |
| "kl": 0.0065155029296875, | |
| "learning_rate": 4.036879377760752e-07, | |
| "loss": 0.0, | |
| "reward": 0.6042845845222473, | |
| "reward_std": 0.4889480620622635, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.3334512561559677, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 168.77084350585938, | |
| "epoch": 0.64, | |
| "grad_norm": 0.6434934249206259, | |
| "kl": 0.00261688232421875, | |
| "learning_rate": 3.9851721262034157e-07, | |
| "loss": 0.0, | |
| "reward": 0.4757592976093292, | |
| "reward_std": 0.3624489903450012, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.3507593274116516, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 194.0833396911621, | |
| "epoch": 0.656, | |
| "grad_norm": 0.5493289883401115, | |
| "kl": 0.003704071044921875, | |
| "learning_rate": 3.932463765572505e-07, | |
| "loss": 0.0, | |
| "reward": 0.42005764693021774, | |
| "reward_std": 0.37090964615345, | |
| "rewards/correct_code_reward_func": 0.16666667722165585, | |
| "rewards/len_reward_func": 0.25339096784591675, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 223.87500762939453, | |
| "epoch": 0.672, | |
| "grad_norm": 0.43766199341669654, | |
| "kl": 0.003650665283203125, | |
| "learning_rate": 3.8787898249606767e-07, | |
| "loss": 0.0, | |
| "reward": 0.35570722818374634, | |
| "reward_std": 0.3742265850305557, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.31404057145118713, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 159.93750762939453, | |
| "epoch": 0.688, | |
| "grad_norm": 0.662510453229864, | |
| "kl": 0.004608154296875, | |
| "learning_rate": 3.8241864843284964e-07, | |
| "loss": 0.0, | |
| "reward": 0.6946325600147247, | |
| "reward_std": 0.4132131338119507, | |
| "rewards/correct_code_reward_func": 0.3125000074505806, | |
| "rewards/len_reward_func": 0.38213254511356354, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 202.43750762939453, | |
| "epoch": 0.704, | |
| "grad_norm": 0.4902137587205668, | |
| "kl": 0.006561279296875, | |
| "learning_rate": 3.768690550116639e-07, | |
| "loss": 0.0, | |
| "reward": 0.4793711006641388, | |
| "reward_std": 0.4331624209880829, | |
| "rewards/correct_code_reward_func": 0.1875000074505806, | |
| "rewards/len_reward_func": 0.2918711006641388, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 201.52084350585938, | |
| "epoch": 0.72, | |
| "grad_norm": 1.5244716072983422, | |
| "kl": 0.0045623779296875, | |
| "learning_rate": 3.712339430435792e-07, | |
| "loss": 0.0, | |
| "reward": 0.376946821808815, | |
| "reward_std": 0.36263740062713623, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.2936134934425354, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 211.87501525878906, | |
| "epoch": 0.736, | |
| "grad_norm": 0.47019285013227063, | |
| "kl": 0.00263214111328125, | |
| "learning_rate": 3.65517110985099e-07, | |
| "loss": 0.0, | |
| "reward": 0.4974285364151001, | |
| "reward_std": 0.3727869838476181, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.3724285215139389, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 128.29166793823242, | |
| "epoch": 0.752, | |
| "grad_norm": 0.5038135341546401, | |
| "kl": 0.00347900390625, | |
| "learning_rate": 3.597224123777389e-07, | |
| "loss": 0.0, | |
| "reward": 0.608631819486618, | |
| "reward_std": 0.4802524596452713, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.25446511805057526, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 161.8541717529297, | |
| "epoch": 0.768, | |
| "grad_norm": 0.5836224104866172, | |
| "kl": 0.0087890625, | |
| "learning_rate": 3.5385375325047163e-07, | |
| "loss": 0.0, | |
| "reward": 0.5433830618858337, | |
| "reward_std": 0.4709310829639435, | |
| "rewards/correct_code_reward_func": 0.2291666679084301, | |
| "rewards/len_reward_func": 0.31421639025211334, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 140.47916793823242, | |
| "epoch": 0.784, | |
| "grad_norm": 0.49678584086470384, | |
| "kl": 0.0029144287109375, | |
| "learning_rate": 3.479150894867926e-07, | |
| "loss": 0.0, | |
| "reward": 0.5527551472187042, | |
| "reward_std": 0.5054349154233932, | |
| "rewards/correct_code_reward_func": 0.22916667722165585, | |
| "rewards/len_reward_func": 0.3235885202884674, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 190.87500762939453, | |
| "epoch": 0.8, | |
| "grad_norm": 1.1393985621399754, | |
| "kl": 0.004364013671875, | |
| "learning_rate": 3.4191042415818e-07, | |
| "loss": 0.0, | |
| "reward": 0.4087870866060257, | |
| "reward_std": 0.35092872381210327, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.3046204149723053, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 189.95833587646484, | |
| "epoch": 0.816, | |
| "grad_norm": 1.1299817912717918, | |
| "kl": 0.01027679443359375, | |
| "learning_rate": 3.3584380482574717e-07, | |
| "loss": 0.0, | |
| "reward": 0.48216497898101807, | |
| "reward_std": 0.35060346126556396, | |
| "rewards/correct_code_reward_func": 0.1875, | |
| "rewards/len_reward_func": 0.29466497898101807, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 155.20833587646484, | |
| "epoch": 0.832, | |
| "grad_norm": 1.0099136669800142, | |
| "kl": 0.0089263916015625, | |
| "learning_rate": 3.297193208119047e-07, | |
| "loss": 0.0, | |
| "reward": 0.7047297656536102, | |
| "reward_std": 0.537945419549942, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.32972970604896545, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 119.43750381469727, | |
| "epoch": 0.848, | |
| "grad_norm": 0.6910875224957795, | |
| "kl": 0.00537109375, | |
| "learning_rate": 3.235411004438741e-07, | |
| "loss": 0.0, | |
| "reward": 0.37493598461151123, | |
| "reward_std": 0.32232099026441574, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.33326931297779083, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 239.00001525878906, | |
| "epoch": 0.864, | |
| "grad_norm": 0.9366579226902805, | |
| "kl": 0.01202392578125, | |
| "learning_rate": 3.173133082709086e-07, | |
| "loss": 0.0, | |
| "reward": 0.5343351364135742, | |
| "reward_std": 0.39477548003196716, | |
| "rewards/correct_code_reward_func": 0.2083333358168602, | |
| "rewards/len_reward_func": 0.32600177824497223, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 165.6458396911621, | |
| "epoch": 0.88, | |
| "grad_norm": 0.5543105836651998, | |
| "kl": 0.0074310302734375, | |
| "learning_rate": 3.1104014225709784e-07, | |
| "loss": 0.0, | |
| "reward": 0.5656554698944092, | |
| "reward_std": 0.4120694398880005, | |
| "rewards/correct_code_reward_func": 0.2500000111758709, | |
| "rewards/len_reward_func": 0.315655454993248, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 224.8541717529297, | |
| "epoch": 0.896, | |
| "grad_norm": 0.4221238954900066, | |
| "kl": 0.0029735565185546875, | |
| "learning_rate": 3.0472583095164873e-07, | |
| "loss": 0.0, | |
| "reward": 0.4594677835702896, | |
| "reward_std": 0.3312383443117142, | |
| "rewards/correct_code_reward_func": 0.10416666977107525, | |
| "rewards/len_reward_func": 0.3553011566400528, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 135.8541717529297, | |
| "epoch": 0.912, | |
| "grad_norm": 0.7883647712663692, | |
| "kl": 0.005859375, | |
| "learning_rate": 2.983746306385499e-07, | |
| "loss": 0.0, | |
| "reward": 0.5004815310239792, | |
| "reward_std": 0.4576799273490906, | |
| "rewards/correct_code_reward_func": 0.1458333395421505, | |
| "rewards/len_reward_func": 0.3546481877565384, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 177.625, | |
| "epoch": 0.928, | |
| "grad_norm": 0.823591617621225, | |
| "kl": 0.00780487060546875, | |
| "learning_rate": 2.919908224675412e-07, | |
| "loss": 0.0, | |
| "reward": 0.47625844180583954, | |
| "reward_std": 0.4644129127264023, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.30959178507328033, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 111.95833587646484, | |
| "epoch": 0.944, | |
| "grad_norm": 0.7895834198702301, | |
| "kl": 0.01739501953125, | |
| "learning_rate": 2.8557870956832133e-07, | |
| "loss": 0.0, | |
| "reward": 0.45759040117263794, | |
| "reward_std": 0.33291806280612946, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.3325904309749603, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 200.64583587646484, | |
| "epoch": 0.96, | |
| "grad_norm": 0.398008582749878, | |
| "kl": 0.00305938720703125, | |
| "learning_rate": 2.7914261414993976e-07, | |
| "loss": 0.0, | |
| "reward": 0.5845803320407867, | |
| "reward_std": 0.3288002014160156, | |
| "rewards/correct_code_reward_func": 0.2291666716337204, | |
| "rewards/len_reward_func": 0.35541366040706635, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 123.75000762939453, | |
| "epoch": 0.976, | |
| "grad_norm": 0.6258506341186686, | |
| "kl": 0.00853729248046875, | |
| "learning_rate": 2.726868745873286e-07, | |
| "loss": 0.0, | |
| "reward": 0.5053079277276993, | |
| "reward_std": 0.45688633620738983, | |
| "rewards/correct_code_reward_func": 0.16666667722165585, | |
| "rewards/len_reward_func": 0.3386412411928177, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 178.4166717529297, | |
| "epoch": 0.992, | |
| "grad_norm": 0.4226697044495165, | |
| "kl": 0.004604339599609375, | |
| "learning_rate": 2.662158424969357e-07, | |
| "loss": 0.0, | |
| "reward": 0.4181392341852188, | |
| "reward_std": 0.39034655690193176, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.3139725774526596, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 150.375, | |
| "epoch": 1.0, | |
| "grad_norm": 0.4226697044495165, | |
| "kl": 0.00823974609375, | |
| "learning_rate": 2.597338798034344e-07, | |
| "loss": 0.0, | |
| "reward": 0.7250348925590515, | |
| "reward_std": 0.3359350562095642, | |
| "rewards/correct_code_reward_func": 0.375, | |
| "rewards/len_reward_func": 0.3500348925590515, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 137.45833587646484, | |
| "epoch": 1.016, | |
| "grad_norm": 0.621781928712815, | |
| "kl": 0.0059356689453125, | |
| "learning_rate": 2.532453557994827e-07, | |
| "loss": 0.0, | |
| "reward": 0.5079408586025238, | |
| "reward_std": 0.43556541204452515, | |
| "rewards/correct_code_reward_func": 0.16666667722165585, | |
| "rewards/len_reward_func": 0.3412741720676422, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 237.45834350585938, | |
| "epoch": 1.032, | |
| "grad_norm": 0.4700296188876399, | |
| "kl": 0.00553131103515625, | |
| "learning_rate": 2.467546442005173e-07, | |
| "loss": 0.0, | |
| "reward": 0.3794639855623245, | |
| "reward_std": 0.3328210711479187, | |
| "rewards/correct_code_reward_func": 0.0625, | |
| "rewards/len_reward_func": 0.3169640153646469, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 217.33333587646484, | |
| "epoch": 1.048, | |
| "grad_norm": 0.47407157003975803, | |
| "kl": 0.01959228515625, | |
| "learning_rate": 2.4026612019656556e-07, | |
| "loss": 0.0, | |
| "reward": 0.5135317444801331, | |
| "reward_std": 0.3767416924238205, | |
| "rewards/correct_code_reward_func": 0.2083333395421505, | |
| "rewards/len_reward_func": 0.30519840121269226, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 159.9791717529297, | |
| "epoch": 1.064, | |
| "grad_norm": 1.33408581993753, | |
| "kl": 0.004638671875, | |
| "learning_rate": 2.337841575030642e-07, | |
| "loss": 0.0, | |
| "reward": 0.5043017268180847, | |
| "reward_std": 0.5010640621185303, | |
| "rewards/correct_code_reward_func": 0.1875000111758709, | |
| "rewards/len_reward_func": 0.3168017417192459, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 196.06250762939453, | |
| "epoch": 1.08, | |
| "grad_norm": 0.39338975551762256, | |
| "kl": 0.0045166015625, | |
| "learning_rate": 2.2731312541267143e-07, | |
| "loss": 0.0, | |
| "reward": 0.40096263587474823, | |
| "reward_std": 0.38369233906269073, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.29679596424102783, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 168.20833587646484, | |
| "epoch": 1.096, | |
| "grad_norm": 0.5733635012795789, | |
| "kl": 0.011474609375, | |
| "learning_rate": 2.2085738585006021e-07, | |
| "loss": 0.0, | |
| "reward": 0.5634751617908478, | |
| "reward_std": 0.42181093990802765, | |
| "rewards/correct_code_reward_func": 0.229166679084301, | |
| "rewards/len_reward_func": 0.3343084752559662, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 131.62500762939453, | |
| "epoch": 1.112, | |
| "grad_norm": 0.5768348851420809, | |
| "kl": 0.0180511474609375, | |
| "learning_rate": 2.1442129043167873e-07, | |
| "loss": 0.0, | |
| "reward": 0.44008754193782806, | |
| "reward_std": 0.4175649434328079, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.29425420612096786, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 178.1041717529297, | |
| "epoch": 1.1280000000000001, | |
| "grad_norm": 0.8562478784181595, | |
| "kl": 0.01507568359375, | |
| "learning_rate": 2.0800917753245875e-07, | |
| "loss": 0.0, | |
| "reward": 0.42609627544879913, | |
| "reward_std": 0.395632266998291, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.34276294708251953, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 142.43750381469727, | |
| "epoch": 1.144, | |
| "grad_norm": 0.5850086202761346, | |
| "kl": 0.007049560546875, | |
| "learning_rate": 2.0162536936145008e-07, | |
| "loss": 0.0, | |
| "reward": 0.6273844540119171, | |
| "reward_std": 0.37064287066459656, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.3565511107444763, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 173.4166717529297, | |
| "epoch": 1.16, | |
| "grad_norm": 1.0060077647295973, | |
| "kl": 0.01995849609375, | |
| "learning_rate": 1.9527416904835132e-07, | |
| "loss": 0.0, | |
| "reward": 0.6327731013298035, | |
| "reward_std": 0.40126484632492065, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.3202730864286423, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 196.5416717529297, | |
| "epoch": 1.176, | |
| "grad_norm": 0.5247248856581095, | |
| "kl": 0.004150390625, | |
| "learning_rate": 1.889598577429022e-07, | |
| "loss": 0.0, | |
| "reward": 0.4798154681921005, | |
| "reward_std": 0.4057523310184479, | |
| "rewards/correct_code_reward_func": 0.1458333395421505, | |
| "rewards/len_reward_func": 0.33398209512233734, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 163.37500762939453, | |
| "epoch": 1.192, | |
| "grad_norm": 0.44940299985092946, | |
| "kl": 0.00665283203125, | |
| "learning_rate": 1.8268669172909136e-07, | |
| "loss": 0.0, | |
| "reward": 0.6056158542633057, | |
| "reward_std": 0.4253086894750595, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.3347824960947037, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 127.60417175292969, | |
| "epoch": 1.208, | |
| "grad_norm": 0.5154656852815648, | |
| "kl": 0.0311279296875, | |
| "learning_rate": 1.7645889955612592e-07, | |
| "loss": 0.0, | |
| "reward": 0.5518045127391815, | |
| "reward_std": 0.46641653776168823, | |
| "rewards/correct_code_reward_func": 0.2083333358168602, | |
| "rewards/len_reward_func": 0.3434711843729019, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 126.47917175292969, | |
| "epoch": 1.224, | |
| "grad_norm": 0.7328993764471904, | |
| "kl": 0.01027679443359375, | |
| "learning_rate": 1.7028067918809535e-07, | |
| "loss": 0.0, | |
| "reward": 0.5090649425983429, | |
| "reward_std": 0.34914855659008026, | |
| "rewards/correct_code_reward_func": 0.20833333395421505, | |
| "rewards/len_reward_func": 0.3007315993309021, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 137.2291717529297, | |
| "epoch": 1.24, | |
| "grad_norm": 0.7418796199300344, | |
| "kl": 0.009124755859375, | |
| "learning_rate": 1.6415619517425294e-07, | |
| "loss": 0.0, | |
| "reward": 0.578598827123642, | |
| "reward_std": 0.4721776694059372, | |
| "rewards/correct_code_reward_func": 0.2500000074505806, | |
| "rewards/len_reward_func": 0.32859882712364197, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 158.95833587646484, | |
| "epoch": 1.256, | |
| "grad_norm": 2.068496131846233, | |
| "kl": 0.02191162109375, | |
| "learning_rate": 1.5808957584181994e-07, | |
| "loss": 0.0, | |
| "reward": 0.4869799315929413, | |
| "reward_std": 0.4040430933237076, | |
| "rewards/correct_code_reward_func": 0.1666666679084301, | |
| "rewards/len_reward_func": 0.3203132748603821, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 106.93750381469727, | |
| "epoch": 1.272, | |
| "grad_norm": 0.5762356445650323, | |
| "kl": 0.00872802734375, | |
| "learning_rate": 1.5208491051320744e-07, | |
| "loss": 0.0, | |
| "reward": 0.5087297856807709, | |
| "reward_std": 0.39580225944519043, | |
| "rewards/correct_code_reward_func": 0.1875000074505806, | |
| "rewards/len_reward_func": 0.3212297558784485, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 114.00000381469727, | |
| "epoch": 1.288, | |
| "grad_norm": 0.786665682978904, | |
| "kl": 0.013519287109375, | |
| "learning_rate": 1.461462467495284e-07, | |
| "loss": 0.0, | |
| "reward": 0.4894479066133499, | |
| "reward_std": 0.3821127265691757, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.3644479066133499, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 154.83333587646484, | |
| "epoch": 1.304, | |
| "grad_norm": 0.7107423115065932, | |
| "kl": 0.0141754150390625, | |
| "learning_rate": 1.4027758762226107e-07, | |
| "loss": 0.0, | |
| "reward": 0.4811897426843643, | |
| "reward_std": 0.3424055427312851, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.3770230710506439, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 235.70834350585938, | |
| "epoch": 1.32, | |
| "grad_norm": 0.43515005998860457, | |
| "kl": 0.0047760009765625, | |
| "learning_rate": 1.3448288901490092e-07, | |
| "loss": 0.0, | |
| "reward": 0.4617680013179779, | |
| "reward_std": 0.3315645009279251, | |
| "rewards/correct_code_reward_func": 0.1250000037252903, | |
| "rewards/len_reward_func": 0.3367680013179779, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 143.1666717529297, | |
| "epoch": 1.336, | |
| "grad_norm": 0.65180392684865, | |
| "kl": 0.0086669921875, | |
| "learning_rate": 1.2876605695642084e-07, | |
| "loss": 0.0, | |
| "reward": 0.4678248018026352, | |
| "reward_std": 0.3193260580301285, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.3636581301689148, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 131.8333396911621, | |
| "epoch": 1.3519999999999999, | |
| "grad_norm": 0.4416459826072822, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 1.231309449883361e-07, | |
| "loss": 0.0, | |
| "reward": 0.41201435029506683, | |
| "reward_std": 0.3407471626996994, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.37034766376018524, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 103.77083969116211, | |
| "epoch": 1.3679999999999999, | |
| "grad_norm": 0.43361682764693815, | |
| "kl": 0.0066986083984375, | |
| "learning_rate": 1.1758135156715041e-07, | |
| "loss": 0.0, | |
| "reward": 0.423152431845665, | |
| "reward_std": 0.3362526297569275, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.3189857602119446, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 176.375, | |
| "epoch": 1.384, | |
| "grad_norm": 1.352523990215239, | |
| "kl": 0.03180694580078125, | |
| "learning_rate": 1.1212101750393235e-07, | |
| "loss": 0.0, | |
| "reward": 0.4561140537261963, | |
| "reward_std": 0.39378371834754944, | |
| "rewards/correct_code_reward_func": 0.10416666977107525, | |
| "rewards/len_reward_func": 0.3519473969936371, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 79.97916793823242, | |
| "epoch": 1.4, | |
| "grad_norm": 2.141260004886994, | |
| "kl": 0.0297698974609375, | |
| "learning_rate": 1.0675362344274952e-07, | |
| "loss": 0.0, | |
| "reward": 0.42587006092071533, | |
| "reward_std": 0.4436161369085312, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.30087001621723175, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 107.37500381469727, | |
| "epoch": 1.416, | |
| "grad_norm": 0.4125343610721507, | |
| "kl": 0.017974853515625, | |
| "learning_rate": 1.0148278737965844e-07, | |
| "loss": 0.0, | |
| "reward": 0.5323555767536163, | |
| "reward_std": 0.4073493778705597, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.36568886041641235, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 87.83333587646484, | |
| "epoch": 1.432, | |
| "grad_norm": 1.3465092335562199, | |
| "kl": 0.0328369140625, | |
| "learning_rate": 9.631206222392479e-08, | |
| "loss": 0.0, | |
| "reward": 0.5186317265033722, | |
| "reward_std": 0.4107673317193985, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.3519650846719742, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 124.33333969116211, | |
| "epoch": 1.448, | |
| "grad_norm": 0.7479671719828264, | |
| "kl": 0.02008056640625, | |
| "learning_rate": 9.124493340311537e-08, | |
| "loss": 0.0, | |
| "reward": 0.5086182951927185, | |
| "reward_std": 0.12347583472728729, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.4252849221229553, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 167.79166793823242, | |
| "epoch": 1.464, | |
| "grad_norm": 1.1543274217148969, | |
| "kl": 0.0083770751953125, | |
| "learning_rate": 8.628481651367875e-08, | |
| "loss": 0.0, | |
| "reward": 0.6109435856342316, | |
| "reward_std": 0.422005370259285, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.34011024236679077, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 115.33333969116211, | |
| "epoch": 1.48, | |
| "grad_norm": 0.3931440618018204, | |
| "kl": 0.0080108642578125, | |
| "learning_rate": 8.143505501859551e-08, | |
| "loss": 0.0, | |
| "reward": 0.44855794310569763, | |
| "reward_std": 0.32760028541088104, | |
| "rewards/correct_code_reward_func": 0.1250000037252903, | |
| "rewards/len_reward_func": 0.3235579580068588, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 151.39583587646484, | |
| "epoch": 1.496, | |
| "grad_norm": 0.5894110998499853, | |
| "kl": 0.009307861328125, | |
| "learning_rate": 7.669891799365282e-08, | |
| "loss": 0.0, | |
| "reward": 0.36864979565143585, | |
| "reward_std": 0.36151623725891113, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.32698309421539307, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 147.7916717529297, | |
| "epoch": 1.512, | |
| "grad_norm": 0.5972288037218009, | |
| "kl": 0.019500732421875, | |
| "learning_rate": 7.207959792385998e-08, | |
| "loss": 0.0, | |
| "reward": 0.4600509703159332, | |
| "reward_std": 0.4011112302541733, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.3558843284845352, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 149.9791717529297, | |
| "epoch": 1.528, | |
| "grad_norm": 0.86174582631064, | |
| "kl": 0.00836181640625, | |
| "learning_rate": 6.758020855149249e-08, | |
| "loss": 0.0, | |
| "reward": 0.44805125892162323, | |
| "reward_std": 0.33948560059070587, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.32305125892162323, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 163.6041717529297, | |
| "epoch": 1.544, | |
| "grad_norm": 0.36337651628903067, | |
| "kl": 0.0181121826171875, | |
| "learning_rate": 6.320378277721342e-08, | |
| "loss": 0.0, | |
| "reward": 0.4679824113845825, | |
| "reward_std": 0.4093552529811859, | |
| "rewards/correct_code_reward_func": 0.10416666977107525, | |
| "rewards/len_reward_func": 0.3638157695531845, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 102.75000381469727, | |
| "epoch": 1.56, | |
| "grad_norm": 1.175706006208444, | |
| "kl": 0.023834228515625, | |
| "learning_rate": 5.895327061568775e-08, | |
| "loss": 0.0, | |
| "reward": 0.5549444258213043, | |
| "reward_std": 0.43229806423187256, | |
| "rewards/correct_code_reward_func": 0.2500000111758709, | |
| "rewards/len_reward_func": 0.3049444109201431, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 144.58333587646484, | |
| "epoch": 1.576, | |
| "grad_norm": 0.7970883391032116, | |
| "kl": 0.011474609375, | |
| "learning_rate": 5.483153720706798e-08, | |
| "loss": 0.0, | |
| "reward": 0.5082628130912781, | |
| "reward_std": 0.46417203545570374, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.2999294698238373, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 93.41667175292969, | |
| "epoch": 1.592, | |
| "grad_norm": 0.577298607650083, | |
| "kl": 0.0172119140625, | |
| "learning_rate": 5.0841360885690996e-08, | |
| "loss": 0.0, | |
| "reward": 0.43817608058452606, | |
| "reward_std": 0.4479677081108093, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.29234276711940765, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 112.79167175292969, | |
| "epoch": 1.608, | |
| "grad_norm": 0.6328737919939129, | |
| "kl": 0.0224151611328125, | |
| "learning_rate": 4.698543130728755e-08, | |
| "loss": 0.0, | |
| "reward": 0.6780606508255005, | |
| "reward_std": 0.37215377390384674, | |
| "rewards/correct_code_reward_func": 0.3333333358168602, | |
| "rewards/len_reward_func": 0.3447272926568985, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 125.33333587646484, | |
| "epoch": 1.624, | |
| "grad_norm": 0.8805964414937946, | |
| "kl": 0.007904052734375, | |
| "learning_rate": 4.326634763596784e-08, | |
| "loss": 0.0, | |
| "reward": 0.5231586992740631, | |
| "reward_std": 0.39199909567832947, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.3981587141752243, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 143.81250762939453, | |
| "epoch": 1.6400000000000001, | |
| "grad_norm": 0.4566354927971138, | |
| "kl": 0.0088348388671875, | |
| "learning_rate": 3.968661679220467e-08, | |
| "loss": 0.0, | |
| "reward": 0.5204833149909973, | |
| "reward_std": 0.25895993411540985, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.4163166582584381, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 118.27083587646484, | |
| "epoch": 1.6560000000000001, | |
| "grad_norm": 0.41571642382372326, | |
| "kl": 0.00982666015625, | |
| "learning_rate": 3.624865176299499e-08, | |
| "loss": 0.0, | |
| "reward": 0.6309479027986526, | |
| "reward_std": 0.4369208961725235, | |
| "rewards/correct_code_reward_func": 0.29166667722165585, | |
| "rewards/len_reward_func": 0.33928124606609344, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 127.50000381469727, | |
| "epoch": 1.6720000000000002, | |
| "grad_norm": 0.44810915489413017, | |
| "kl": 0.021331787109375, | |
| "learning_rate": 3.295476997533905e-08, | |
| "loss": 0.0, | |
| "reward": 0.43531325459480286, | |
| "reward_std": 0.2511162757873535, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.39364662766456604, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 179.75000762939453, | |
| "epoch": 1.688, | |
| "grad_norm": 0.5235689630320457, | |
| "kl": 0.029052734375, | |
| "learning_rate": 2.980719173413396e-08, | |
| "loss": 0.0, | |
| "reward": 0.354349747300148, | |
| "reward_std": 0.2833855152130127, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.3126830607652664, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 120.50000381469727, | |
| "epoch": 1.704, | |
| "grad_norm": 1.2523814931919, | |
| "kl": 0.0085601806640625, | |
| "learning_rate": 2.680803872553408e-08, | |
| "loss": 0.0, | |
| "reward": 0.48751458525657654, | |
| "reward_std": 0.4157916307449341, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.32084792852401733, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 151.16666793823242, | |
| "epoch": 1.72, | |
| "grad_norm": 0.6447932351262547, | |
| "kl": 0.00946807861328125, | |
| "learning_rate": 2.395933258678745e-08, | |
| "loss": 0.0, | |
| "reward": 0.6216467022895813, | |
| "reward_std": 0.44435153901576996, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.3508133441209793, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 123.4375, | |
| "epoch": 1.736, | |
| "grad_norm": 0.5052351271825365, | |
| "kl": 0.0091552734375, | |
| "learning_rate": 2.1262993543511715e-08, | |
| "loss": 0.0, | |
| "reward": 0.6645375192165375, | |
| "reward_std": 0.4121406674385071, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.37287086248397827, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 153.47916793823242, | |
| "epoch": 1.752, | |
| "grad_norm": 0.6369640119331228, | |
| "kl": 0.0086212158203125, | |
| "learning_rate": 1.872083911532907e-08, | |
| "loss": 0.0, | |
| "reward": 0.40650297701358795, | |
| "reward_std": 0.27610746026039124, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.36483629047870636, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 132.1041717529297, | |
| "epoch": 1.768, | |
| "grad_norm": 0.44311314663042145, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 1.6334582890731697e-08, | |
| "loss": 0.0, | |
| "reward": 0.7341299653053284, | |
| "reward_std": 0.504623532295227, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.29662999510765076, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 162.0625, | |
| "epoch": 1.784, | |
| "grad_norm": 0.6586562319299408, | |
| "kl": 0.0101318359375, | |
| "learning_rate": 1.4105833372004523e-08, | |
| "loss": 0.0, | |
| "reward": 0.4220695346593857, | |
| "reward_std": 0.2905489057302475, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.3804028630256653, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 106.89583587646484, | |
| "epoch": 1.8, | |
| "grad_norm": 0.7764527703399275, | |
| "kl": 0.019287109375, | |
| "learning_rate": 1.2036092890982619e-08, | |
| "loss": 0.0, | |
| "reward": 0.5292025506496429, | |
| "reward_std": 0.3711909055709839, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.38336920738220215, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 132.2916717529297, | |
| "epoch": 1.8159999999999998, | |
| "grad_norm": 0.6064692969361543, | |
| "kl": 0.009307861328125, | |
| "learning_rate": 1.0126756596375685e-08, | |
| "loss": 0.0, | |
| "reward": 0.48789724707603455, | |
| "reward_std": 0.3553258925676346, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.34206391870975494, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 120.22917175292969, | |
| "epoch": 1.8319999999999999, | |
| "grad_norm": 0.3843336433099292, | |
| "kl": 0.0174560546875, | |
| "learning_rate": 8.379111513340753e-09, | |
| "loss": 0.0, | |
| "reward": 0.47604209184646606, | |
| "reward_std": 0.3304741531610489, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.39270876348018646, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 145.31250762939453, | |
| "epoch": 1.8479999999999999, | |
| "grad_norm": 0.5532480666977985, | |
| "kl": 0.024169921875, | |
| "learning_rate": 6.7943356759381785e-09, | |
| "loss": 0.0, | |
| "reward": 0.46019650995731354, | |
| "reward_std": 0.3715391010046005, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.35602983832359314, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 149.93750381469727, | |
| "epoch": 1.8639999999999999, | |
| "grad_norm": 1.0372718715130624, | |
| "kl": 0.00921630859375, | |
| "learning_rate": 5.373497333054616e-09, | |
| "loss": 0.0, | |
| "reward": 0.4656513184309006, | |
| "reward_std": 0.44700081646442413, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.29898466169834137, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 144.89583587646484, | |
| "epoch": 1.88, | |
| "grad_norm": 0.4882633725665408, | |
| "kl": 0.014007568359375, | |
| "learning_rate": 4.117554228329406e-09, | |
| "loss": 0.0, | |
| "reward": 0.5118084400892258, | |
| "reward_std": 0.34739528596401215, | |
| "rewards/correct_code_reward_func": 0.1875000074505806, | |
| "rewards/len_reward_func": 0.32430844008922577, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 97.45833587646484, | |
| "epoch": 1.896, | |
| "grad_norm": 0.4820971241858704, | |
| "kl": 0.01251220703125, | |
| "learning_rate": 3.0273529545687125e-09, | |
| "loss": 0.0, | |
| "reward": 0.4883972406387329, | |
| "reward_std": 0.43138815462589264, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.3633972257375717, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 120.95833587646484, | |
| "epoch": 1.912, | |
| "grad_norm": 0.6000792918829387, | |
| "kl": 0.02130126953125, | |
| "learning_rate": 2.1036283830834224e-09, | |
| "loss": 0.0, | |
| "reward": 0.6599289178848267, | |
| "reward_std": 0.32202909141778946, | |
| "rewards/correct_code_reward_func": 0.2291666716337204, | |
| "rewards/len_reward_func": 0.4307622164487839, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 161.4791717529297, | |
| "epoch": 1.928, | |
| "grad_norm": 0.9160239570442854, | |
| "kl": 0.040740966796875, | |
| "learning_rate": 1.347003168334665e-09, | |
| "loss": 0.0, | |
| "reward": 0.6658598780632019, | |
| "reward_std": 0.45104770362377167, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.2908598631620407, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 197.33333587646484, | |
| "epoch": 1.944, | |
| "grad_norm": 1.2887565831293122, | |
| "kl": 0.0357666015625, | |
| "learning_rate": 7.579873282216598e-10, | |
| "loss": 0.0, | |
| "reward": 0.5509243905544281, | |
| "reward_std": 0.42525260150432587, | |
| "rewards/correct_code_reward_func": 0.229166679084301, | |
| "rewards/len_reward_func": 0.3217576891183853, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 238.39584350585938, | |
| "epoch": 1.96, | |
| "grad_norm": 0.36929428922486, | |
| "kl": 0.0052337646484375, | |
| "learning_rate": 3.3697790029424413e-10, | |
| "loss": 0.0, | |
| "reward": 0.5057644844055176, | |
| "reward_std": 0.4373111426830292, | |
| "rewards/correct_code_reward_func": 0.1458333395421505, | |
| "rewards/len_reward_func": 0.3599311411380768, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 126.41667175292969, | |
| "epoch": 1.976, | |
| "grad_norm": 0.4901359623088852, | |
| "kl": 0.007110595703125, | |
| "learning_rate": 8.425867412190091e-11, | |
| "loss": 0.0, | |
| "reward": 0.5390879511833191, | |
| "reward_std": 0.30444033443927765, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.3724212795495987, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 196.77083587646484, | |
| "epoch": 1.992, | |
| "grad_norm": 0.41890098929296765, | |
| "kl": 0.005767822265625, | |
| "learning_rate": 0.0, | |
| "loss": 0.0, | |
| "reward": 0.5094343423843384, | |
| "reward_std": 0.314863845705986, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.4052676856517792, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.992, | |
| "step": 125, | |
| "total_flos": 0.0, | |
| "train_loss": 8.671089399598486e-06, | |
| "train_runtime": 8102.573, | |
| "train_samples_per_second": 0.093, | |
| "train_steps_per_second": 0.015 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 125, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 25, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |