| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 375, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 376.9687805175781, | |
| "epoch": 0.0026666666666666666, | |
| "grad_norm": 0.8290089342545105, | |
| "kl": 0.00012969970703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1243, | |
| "reward": 0.26896461844444275, | |
| "reward_std": 0.2736624479293823, | |
| "rewards/length_reward": 0.026339290663599968, | |
| "rewards/similarity_reward": 0.24262532591819763, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 363.27679443359375, | |
| "epoch": 0.005333333333333333, | |
| "grad_norm": 0.8474061525539559, | |
| "kl": 0.00015354156494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1524, | |
| "reward": 0.1628378927707672, | |
| "reward_std": 0.21853798627853394, | |
| "rewards/length_reward": 0.0178571455180645, | |
| "rewards/similarity_reward": 0.14498072862625122, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 378.4732360839844, | |
| "epoch": 0.008, | |
| "grad_norm": 0.8192733588218007, | |
| "kl": 0.0001430511474609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1852, | |
| "reward": 0.27797675132751465, | |
| "reward_std": 0.23766961693763733, | |
| "rewards/length_reward": 0.02633928880095482, | |
| "rewards/similarity_reward": 0.25163745880126953, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 359.21429443359375, | |
| "epoch": 0.010666666666666666, | |
| "grad_norm": 0.845054880446133, | |
| "kl": 0.00015544891357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0616, | |
| "reward": 0.1538199633359909, | |
| "reward_std": 0.17373259365558624, | |
| "rewards/length_reward": 0.013839286752045155, | |
| "rewards/similarity_reward": 0.13998067378997803, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 336.15625, | |
| "epoch": 0.013333333333333334, | |
| "grad_norm": 0.8717553512231005, | |
| "kl": 0.00014972686767578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1209, | |
| "reward": 0.21149994432926178, | |
| "reward_std": 0.2120143473148346, | |
| "rewards/length_reward": 0.01830357313156128, | |
| "rewards/similarity_reward": 0.1931963711977005, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 373.6026916503906, | |
| "epoch": 0.016, | |
| "grad_norm": 0.7610025457969676, | |
| "kl": 0.00013446807861328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1513, | |
| "reward": 0.1827303022146225, | |
| "reward_std": 0.24866001307964325, | |
| "rewards/length_reward": 0.02187500335276127, | |
| "rewards/similarity_reward": 0.16085529327392578, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 341.7589416503906, | |
| "epoch": 0.018666666666666668, | |
| "grad_norm": 0.8626611704865026, | |
| "kl": 0.0001583099365234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1271, | |
| "reward": 0.19529196619987488, | |
| "reward_std": 0.2814559042453766, | |
| "rewards/length_reward": 0.021428575739264488, | |
| "rewards/similarity_reward": 0.17386338114738464, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 424.98663330078125, | |
| "epoch": 0.021333333333333333, | |
| "grad_norm": 0.7146043340468313, | |
| "kl": 0.00017452239990234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1881, | |
| "reward": 0.21610300242900848, | |
| "reward_std": 0.2689198851585388, | |
| "rewards/length_reward": 0.01830357313156128, | |
| "rewards/similarity_reward": 0.197799414396286, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 348.2321472167969, | |
| "epoch": 0.024, | |
| "grad_norm": 0.7606250954270842, | |
| "kl": 0.000186920166015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1276, | |
| "reward": 0.20473892986774445, | |
| "reward_std": 0.23727914690971375, | |
| "rewards/length_reward": 0.02008928917348385, | |
| "rewards/similarity_reward": 0.18464964628219604, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 355.54913330078125, | |
| "epoch": 0.02666666666666667, | |
| "grad_norm": 0.8173370703071066, | |
| "kl": 0.00018310546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0787, | |
| "reward": 0.227640300989151, | |
| "reward_std": 0.2539962828159332, | |
| "rewards/length_reward": 0.01741071790456772, | |
| "rewards/similarity_reward": 0.21022957563400269, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 343.7232360839844, | |
| "epoch": 0.029333333333333333, | |
| "grad_norm": 0.9104553890156574, | |
| "kl": 0.00019550323486328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0575, | |
| "reward": 0.25083568692207336, | |
| "reward_std": 0.2815442383289337, | |
| "rewards/length_reward": 0.02767857536673546, | |
| "rewards/similarity_reward": 0.22315707802772522, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 355.0535888671875, | |
| "epoch": 0.032, | |
| "grad_norm": 0.8007928014475878, | |
| "kl": 0.0003528594970703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1448, | |
| "reward": 0.2787685990333557, | |
| "reward_std": 0.25941261649131775, | |
| "rewards/length_reward": 0.025892863050103188, | |
| "rewards/similarity_reward": 0.25287577509880066, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 395.4107360839844, | |
| "epoch": 0.034666666666666665, | |
| "grad_norm": 0.7050603406845205, | |
| "kl": 0.000255584716796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1127, | |
| "reward": 0.31717172265052795, | |
| "reward_std": 0.2762907147407532, | |
| "rewards/length_reward": 0.02946428954601288, | |
| "rewards/similarity_reward": 0.28770744800567627, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 361.05804443359375, | |
| "epoch": 0.037333333333333336, | |
| "grad_norm": 0.9360978406768153, | |
| "kl": 0.0003376007080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.142, | |
| "reward": 0.24003884196281433, | |
| "reward_std": 0.27974435687065125, | |
| "rewards/length_reward": 0.021428575739264488, | |
| "rewards/similarity_reward": 0.2186102569103241, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 398.0044860839844, | |
| "epoch": 0.04, | |
| "grad_norm": 0.7389563411116621, | |
| "kl": 0.0003604888916015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1137, | |
| "reward": 0.23077349364757538, | |
| "reward_std": 0.24957218766212463, | |
| "rewards/length_reward": 0.0178571455180645, | |
| "rewards/similarity_reward": 0.21291638910770416, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 366.4196472167969, | |
| "epoch": 0.042666666666666665, | |
| "grad_norm": 0.7079515986093292, | |
| "kl": 0.0004177093505859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1596, | |
| "reward": 0.16758890450000763, | |
| "reward_std": 0.1997506469488144, | |
| "rewards/length_reward": 0.0178571455180645, | |
| "rewards/similarity_reward": 0.14973175525665283, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 385.87054443359375, | |
| "epoch": 0.04533333333333334, | |
| "grad_norm": 0.7793857856999354, | |
| "kl": 0.000385284423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1863, | |
| "reward": 0.2408275306224823, | |
| "reward_std": 0.28883570432662964, | |
| "rewards/length_reward": 0.022321434691548347, | |
| "rewards/similarity_reward": 0.2185060679912567, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 373.0446472167969, | |
| "epoch": 0.048, | |
| "grad_norm": 0.8181367418694138, | |
| "kl": 0.0005035400390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0998, | |
| "reward": 0.2590915262699127, | |
| "reward_std": 0.26667794585227966, | |
| "rewards/length_reward": 0.02678571827709675, | |
| "rewards/similarity_reward": 0.23230580985546112, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 314.1026916503906, | |
| "epoch": 0.050666666666666665, | |
| "grad_norm": 0.8473993736940718, | |
| "kl": 0.000438690185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0861, | |
| "reward": 0.34537845849990845, | |
| "reward_std": 0.26645439863204956, | |
| "rewards/length_reward": 0.03705357387661934, | |
| "rewards/similarity_reward": 0.3083249032497406, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 354.93304443359375, | |
| "epoch": 0.05333333333333334, | |
| "grad_norm": 0.8882148635006984, | |
| "kl": 0.00057220458984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1732, | |
| "reward": 0.3256897032260895, | |
| "reward_std": 0.25436195731163025, | |
| "rewards/length_reward": 0.025446433573961258, | |
| "rewards/similarity_reward": 0.3002432584762573, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 354.1250305175781, | |
| "epoch": 0.056, | |
| "grad_norm": 0.8096248306365297, | |
| "kl": 0.000713348388671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1809, | |
| "reward": 0.42166247963905334, | |
| "reward_std": 0.2462671995162964, | |
| "rewards/length_reward": 0.0401785746216774, | |
| "rewards/similarity_reward": 0.38148391246795654, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 379.8258972167969, | |
| "epoch": 0.058666666666666666, | |
| "grad_norm": 0.8324528992208251, | |
| "kl": 0.000713348388671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1888, | |
| "reward": 0.4674707353115082, | |
| "reward_std": 0.28602704405784607, | |
| "rewards/length_reward": 0.03883929178118706, | |
| "rewards/similarity_reward": 0.42863139510154724, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 336.1964416503906, | |
| "epoch": 0.06133333333333333, | |
| "grad_norm": 0.883659092626158, | |
| "kl": 0.001129150390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1224, | |
| "reward": 0.3572904169559479, | |
| "reward_std": 0.2817726135253906, | |
| "rewards/length_reward": 0.030803577974438667, | |
| "rewards/similarity_reward": 0.32648688554763794, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 379.8750305175781, | |
| "epoch": 0.064, | |
| "grad_norm": 0.7759947978320788, | |
| "kl": 0.00135040283203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.163, | |
| "reward": 0.34559932351112366, | |
| "reward_std": 0.26974016427993774, | |
| "rewards/length_reward": 0.03705357387661934, | |
| "rewards/similarity_reward": 0.3085457384586334, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 334.71875, | |
| "epoch": 0.06666666666666667, | |
| "grad_norm": 0.8358157724868338, | |
| "kl": 0.001068115234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1326, | |
| "reward": 0.3908008337020874, | |
| "reward_std": 0.3024666905403137, | |
| "rewards/length_reward": 0.03928571566939354, | |
| "rewards/similarity_reward": 0.35151511430740356, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 372.55804443359375, | |
| "epoch": 0.06933333333333333, | |
| "grad_norm": 0.7610834907565935, | |
| "kl": 0.0027008056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1688, | |
| "reward": 0.2864897847175598, | |
| "reward_std": 0.2402629852294922, | |
| "rewards/length_reward": 0.02500000409781933, | |
| "rewards/similarity_reward": 0.2614898085594177, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 332.1651916503906, | |
| "epoch": 0.072, | |
| "grad_norm": 0.9340372327621089, | |
| "kl": 0.00152587890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1936, | |
| "reward": 0.32003700733184814, | |
| "reward_std": 0.28589487075805664, | |
| "rewards/length_reward": 0.0334821492433548, | |
| "rewards/similarity_reward": 0.28655487298965454, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 421.08929443359375, | |
| "epoch": 0.07466666666666667, | |
| "grad_norm": 0.7506727526601732, | |
| "kl": 0.00148773193359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1452, | |
| "reward": 0.39091238379478455, | |
| "reward_std": 0.21896174550056458, | |
| "rewards/length_reward": 0.03482143208384514, | |
| "rewards/similarity_reward": 0.3560909032821655, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 381.3348388671875, | |
| "epoch": 0.07733333333333334, | |
| "grad_norm": 0.6952200929063435, | |
| "kl": 0.0017242431640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1113, | |
| "reward": 0.36936715245246887, | |
| "reward_std": 0.23492401838302612, | |
| "rewards/length_reward": 0.030357148498296738, | |
| "rewards/similarity_reward": 0.33900997042655945, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 334.6607360839844, | |
| "epoch": 0.08, | |
| "grad_norm": 0.9106612624614471, | |
| "kl": 0.00170135498046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0756, | |
| "reward": 0.4293070435523987, | |
| "reward_std": 0.2823811173439026, | |
| "rewards/length_reward": 0.0401785746216774, | |
| "rewards/similarity_reward": 0.3891284763813019, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 461.99554443359375, | |
| "epoch": 0.08266666666666667, | |
| "grad_norm": 0.661013366993289, | |
| "kl": 0.0020751953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2489, | |
| "reward": 0.3686121702194214, | |
| "reward_std": 0.2422313094139099, | |
| "rewards/length_reward": 0.028571434319019318, | |
| "rewards/similarity_reward": 0.3400407135486603, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 354.83038330078125, | |
| "epoch": 0.08533333333333333, | |
| "grad_norm": 0.9261837111640477, | |
| "kl": 0.0028533935546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.3225, | |
| "reward": 0.3788739740848541, | |
| "reward_std": 0.26962369680404663, | |
| "rewards/length_reward": 0.03482143208384514, | |
| "rewards/similarity_reward": 0.34405258297920227, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 355.5357360839844, | |
| "epoch": 0.088, | |
| "grad_norm": 0.8643961614615921, | |
| "kl": 0.0026397705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1893, | |
| "reward": 0.3799653649330139, | |
| "reward_std": 0.2525205910205841, | |
| "rewards/length_reward": 0.03303571790456772, | |
| "rewards/similarity_reward": 0.346929669380188, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 303.2901916503906, | |
| "epoch": 0.09066666666666667, | |
| "grad_norm": 0.9531070627714385, | |
| "kl": 0.002532958984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0718, | |
| "reward": 0.435234397649765, | |
| "reward_std": 0.2490427941083908, | |
| "rewards/length_reward": 0.03883928805589676, | |
| "rewards/similarity_reward": 0.39639511704444885, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 391.89288330078125, | |
| "epoch": 0.09333333333333334, | |
| "grad_norm": 0.8111465127450542, | |
| "kl": 0.0019683837890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1647, | |
| "reward": 0.4038808047771454, | |
| "reward_std": 0.31206637620925903, | |
| "rewards/length_reward": 0.03883928805589676, | |
| "rewards/similarity_reward": 0.3650415241718292, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 359.9196472167969, | |
| "epoch": 0.096, | |
| "grad_norm": 0.8281785424636492, | |
| "kl": 0.00335693359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1875, | |
| "reward": 0.4340634346008301, | |
| "reward_std": 0.27002009749412537, | |
| "rewards/length_reward": 0.04151785373687744, | |
| "rewards/similarity_reward": 0.39254552125930786, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 287.6875, | |
| "epoch": 0.09866666666666667, | |
| "grad_norm": 1.053364246590671, | |
| "kl": 0.00518798828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.085, | |
| "reward": 0.47383809089660645, | |
| "reward_std": 0.22637499868869781, | |
| "rewards/length_reward": 0.04374999925494194, | |
| "rewards/similarity_reward": 0.430088073015213, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 398.1160888671875, | |
| "epoch": 0.10133333333333333, | |
| "grad_norm": 0.7805356469167566, | |
| "kl": 0.00341796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2197, | |
| "reward": 0.4928036332130432, | |
| "reward_std": 0.24267539381980896, | |
| "rewards/length_reward": 0.050446417182683945, | |
| "rewards/similarity_reward": 0.4423570930957794, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 312.7276916503906, | |
| "epoch": 0.104, | |
| "grad_norm": 0.819085900048414, | |
| "kl": 0.00177001953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1426, | |
| "reward": 0.5058891177177429, | |
| "reward_std": 0.24202971160411835, | |
| "rewards/length_reward": 0.04776785522699356, | |
| "rewards/similarity_reward": 0.4581212103366852, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 386.8214416503906, | |
| "epoch": 0.10666666666666667, | |
| "grad_norm": 0.7650271991129984, | |
| "kl": 0.0030364990234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1882, | |
| "reward": 0.4022373855113983, | |
| "reward_std": 0.24781934916973114, | |
| "rewards/length_reward": 0.0401785746216774, | |
| "rewards/similarity_reward": 0.3620587885379791, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 250.4107208251953, | |
| "epoch": 0.10933333333333334, | |
| "grad_norm": 0.9803970599540968, | |
| "kl": 0.003875732421875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0008, | |
| "reward": 0.5008234977722168, | |
| "reward_std": 0.22121772170066833, | |
| "rewards/length_reward": 0.050446417182683945, | |
| "rewards/similarity_reward": 0.45037704706192017, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 317.1160888671875, | |
| "epoch": 0.112, | |
| "grad_norm": 0.8986374178737812, | |
| "kl": 0.00341796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2065, | |
| "reward": 0.45162686705589294, | |
| "reward_std": 0.27914097905158997, | |
| "rewards/length_reward": 0.04598214104771614, | |
| "rewards/similarity_reward": 0.4056447148323059, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 313.4821472167969, | |
| "epoch": 0.11466666666666667, | |
| "grad_norm": 0.9439169733302692, | |
| "kl": 0.0040283203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1189, | |
| "reward": 0.5015469789505005, | |
| "reward_std": 0.2071218341588974, | |
| "rewards/length_reward": 0.050892848521471024, | |
| "rewards/similarity_reward": 0.4506540596485138, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 308.23663330078125, | |
| "epoch": 0.11733333333333333, | |
| "grad_norm": 0.843848181524653, | |
| "kl": 0.004150390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1423, | |
| "reward": 0.49464964866638184, | |
| "reward_std": 0.19760312139987946, | |
| "rewards/length_reward": 0.05223213508725166, | |
| "rewards/similarity_reward": 0.4424174726009369, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 322.51788330078125, | |
| "epoch": 0.12, | |
| "grad_norm": 0.808670899867239, | |
| "kl": 0.004730224609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2162, | |
| "reward": 0.45207658410072327, | |
| "reward_std": 0.22255302965641022, | |
| "rewards/length_reward": 0.03883929178118706, | |
| "rewards/similarity_reward": 0.4132373034954071, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 306.9732360839844, | |
| "epoch": 0.12266666666666666, | |
| "grad_norm": 0.9215129819151354, | |
| "kl": 0.0038604736328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1492, | |
| "reward": 0.5217949151992798, | |
| "reward_std": 0.24197062849998474, | |
| "rewards/length_reward": 0.04598213732242584, | |
| "rewards/similarity_reward": 0.47581273317337036, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 305.45538330078125, | |
| "epoch": 0.12533333333333332, | |
| "grad_norm": 0.8829840145792894, | |
| "kl": 0.00518798828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2397, | |
| "reward": 0.471711665391922, | |
| "reward_std": 0.15981332957744598, | |
| "rewards/length_reward": 0.049553561955690384, | |
| "rewards/similarity_reward": 0.42215806245803833, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 294.4419860839844, | |
| "epoch": 0.128, | |
| "grad_norm": 0.8246638858466566, | |
| "kl": 0.004241943359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0913, | |
| "reward": 0.4828924238681793, | |
| "reward_std": 0.19750112295150757, | |
| "rewards/length_reward": 0.053124990314245224, | |
| "rewards/similarity_reward": 0.429767370223999, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 332.45538330078125, | |
| "epoch": 0.13066666666666665, | |
| "grad_norm": 0.7904388485187617, | |
| "kl": 0.004913330078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1529, | |
| "reward": 0.45801258087158203, | |
| "reward_std": 0.2542867660522461, | |
| "rewards/length_reward": 0.04196428507566452, | |
| "rewards/similarity_reward": 0.4160482585430145, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 281.77679443359375, | |
| "epoch": 0.13333333333333333, | |
| "grad_norm": 0.9845645428183626, | |
| "kl": 0.00433349609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0801, | |
| "reward": 0.5682670474052429, | |
| "reward_std": 0.23296673595905304, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5030884742736816, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 316.7410888671875, | |
| "epoch": 0.136, | |
| "grad_norm": 0.9300429591791828, | |
| "kl": 0.00592041015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.103, | |
| "reward": 0.44630467891693115, | |
| "reward_std": 0.12811601161956787, | |
| "rewards/length_reward": 0.04821427911520004, | |
| "rewards/similarity_reward": 0.3980904519557953, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 296.8973388671875, | |
| "epoch": 0.13866666666666666, | |
| "grad_norm": 0.8592422567531082, | |
| "kl": 0.005218505859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1159, | |
| "reward": 0.5130535960197449, | |
| "reward_std": 0.1873682290315628, | |
| "rewards/length_reward": 0.0491071380674839, | |
| "rewards/similarity_reward": 0.4639464318752289, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 284.0401916503906, | |
| "epoch": 0.14133333333333334, | |
| "grad_norm": 0.8593724590061699, | |
| "kl": 0.0048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0087, | |
| "reward": 0.5250208973884583, | |
| "reward_std": 0.21563619375228882, | |
| "rewards/length_reward": 0.057142842561006546, | |
| "rewards/similarity_reward": 0.46787798404693604, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 268.21429443359375, | |
| "epoch": 0.144, | |
| "grad_norm": 0.9220995505083402, | |
| "kl": 0.005645751953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0234, | |
| "reward": 0.6078009605407715, | |
| "reward_std": 0.18404294550418854, | |
| "rewards/length_reward": 0.04508928582072258, | |
| "rewards/similarity_reward": 0.5627117156982422, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 332.2232360839844, | |
| "epoch": 0.14666666666666667, | |
| "grad_norm": 0.8153244083746986, | |
| "kl": 0.004852294921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1089, | |
| "reward": 0.5709711313247681, | |
| "reward_std": 0.18112631142139435, | |
| "rewards/length_reward": 0.058035701513290405, | |
| "rewards/similarity_reward": 0.5129354596138, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 298.62054443359375, | |
| "epoch": 0.14933333333333335, | |
| "grad_norm": 0.861900790561817, | |
| "kl": 0.00567626953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1037, | |
| "reward": 0.5129757523536682, | |
| "reward_std": 0.21154648065567017, | |
| "rewards/length_reward": 0.05401784926652908, | |
| "rewards/similarity_reward": 0.45895785093307495, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 243.4241180419922, | |
| "epoch": 0.152, | |
| "grad_norm": 0.928994891862699, | |
| "kl": 0.004241943359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0328, | |
| "reward": 0.6340307593345642, | |
| "reward_std": 0.16285859048366547, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.56840580701828, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 293.0669860839844, | |
| "epoch": 0.15466666666666667, | |
| "grad_norm": 0.89777989101008, | |
| "kl": 0.00579833984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0313, | |
| "reward": 0.5502158999443054, | |
| "reward_std": 0.1914074867963791, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.48503735661506653, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 348.3973388671875, | |
| "epoch": 0.15733333333333333, | |
| "grad_norm": 0.830581459672137, | |
| "kl": 0.005706787109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1262, | |
| "reward": 0.5427281260490417, | |
| "reward_std": 0.18273915350437164, | |
| "rewards/length_reward": 0.04508928582072258, | |
| "rewards/similarity_reward": 0.4976387917995453, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 300.43304443359375, | |
| "epoch": 0.16, | |
| "grad_norm": 0.7841145484798535, | |
| "kl": 0.007080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0356, | |
| "reward": 0.6150097846984863, | |
| "reward_std": 0.15837538242340088, | |
| "rewards/length_reward": 0.06383926421403885, | |
| "rewards/similarity_reward": 0.5511705279350281, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 291.64288330078125, | |
| "epoch": 0.16266666666666665, | |
| "grad_norm": 0.9759946887460155, | |
| "kl": 0.005645751953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1347, | |
| "reward": 0.6720048785209656, | |
| "reward_std": 0.16562286019325256, | |
| "rewards/length_reward": 0.06428569555282593, | |
| "rewards/similarity_reward": 0.6077191233634949, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 300.3973388671875, | |
| "epoch": 0.16533333333333333, | |
| "grad_norm": 0.8353754395282778, | |
| "kl": 0.00592041015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1508, | |
| "reward": 0.6174642443656921, | |
| "reward_std": 0.1775916963815689, | |
| "rewards/length_reward": 0.05848212540149689, | |
| "rewards/similarity_reward": 0.5589820742607117, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 299.0, | |
| "epoch": 0.168, | |
| "grad_norm": 0.8434412806636016, | |
| "kl": 0.0068359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0787, | |
| "reward": 0.5795109272003174, | |
| "reward_std": 0.18212977051734924, | |
| "rewards/length_reward": 0.056249987334012985, | |
| "rewards/similarity_reward": 0.5232609510421753, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 314.3883972167969, | |
| "epoch": 0.17066666666666666, | |
| "grad_norm": 1.1818234014256608, | |
| "kl": 0.005706787109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2756, | |
| "reward": 0.5499185919761658, | |
| "reward_std": 0.22555634379386902, | |
| "rewards/length_reward": 0.04776785522699356, | |
| "rewards/similarity_reward": 0.5021507143974304, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 259.1785888671875, | |
| "epoch": 0.17333333333333334, | |
| "grad_norm": 0.9529921486665629, | |
| "kl": 0.006439208984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0704, | |
| "reward": 0.5430376529693604, | |
| "reward_std": 0.2042228877544403, | |
| "rewards/length_reward": 0.052232131361961365, | |
| "rewards/similarity_reward": 0.4908054769039154, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 258.0714416503906, | |
| "epoch": 0.176, | |
| "grad_norm": 1.1488507934693044, | |
| "kl": 0.006805419921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1057, | |
| "reward": 0.570214033126831, | |
| "reward_std": 0.160283625125885, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.505035400390625, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 330.90179443359375, | |
| "epoch": 0.17866666666666667, | |
| "grad_norm": 0.912379625363708, | |
| "kl": 0.0062255859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1291, | |
| "reward": 0.49484553933143616, | |
| "reward_std": 0.21234968304634094, | |
| "rewards/length_reward": 0.04553570970892906, | |
| "rewards/similarity_reward": 0.4493098556995392, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 266.46429443359375, | |
| "epoch": 0.18133333333333335, | |
| "grad_norm": 0.9382639131370187, | |
| "kl": 0.00909423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0685, | |
| "reward": 0.5787621736526489, | |
| "reward_std": 0.17865508794784546, | |
| "rewards/length_reward": 0.054464273154735565, | |
| "rewards/similarity_reward": 0.5242978930473328, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 285.25, | |
| "epoch": 0.184, | |
| "grad_norm": 0.8385679542137942, | |
| "kl": 0.00555419921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.046, | |
| "reward": 0.6689252257347107, | |
| "reward_std": 0.16466915607452393, | |
| "rewards/length_reward": 0.06651783734560013, | |
| "rewards/similarity_reward": 0.6024073958396912, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 247.68751525878906, | |
| "epoch": 0.18666666666666668, | |
| "grad_norm": 1.01200025847724, | |
| "kl": 0.00860595703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1382, | |
| "reward": 0.4780524969100952, | |
| "reward_std": 0.19645950198173523, | |
| "rewards/length_reward": 0.04196428507566452, | |
| "rewards/similarity_reward": 0.4360882043838501, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 307.24554443359375, | |
| "epoch": 0.18933333333333333, | |
| "grad_norm": 0.8185082695628789, | |
| "kl": 0.00787353515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0749, | |
| "reward": 0.5303549766540527, | |
| "reward_std": 0.1896388977766037, | |
| "rewards/length_reward": 0.056696414947509766, | |
| "rewards/similarity_reward": 0.4736584722995758, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 329.15179443359375, | |
| "epoch": 0.192, | |
| "grad_norm": 0.8562549539520792, | |
| "kl": 0.00823974609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1642, | |
| "reward": 0.5008990168571472, | |
| "reward_std": 0.17187656462192535, | |
| "rewards/length_reward": 0.050892848521471024, | |
| "rewards/similarity_reward": 0.4500061273574829, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 253.59376525878906, | |
| "epoch": 0.19466666666666665, | |
| "grad_norm": 0.8806238339574037, | |
| "kl": 0.006591796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1082, | |
| "reward": 0.7047773003578186, | |
| "reward_std": 0.12662379443645477, | |
| "rewards/length_reward": 0.07232140004634857, | |
| "rewards/similarity_reward": 0.6324558258056641, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 302.7321472167969, | |
| "epoch": 0.19733333333333333, | |
| "grad_norm": 0.888373625390179, | |
| "kl": 0.01300048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0384, | |
| "reward": 0.5046581625938416, | |
| "reward_std": 0.18071489036083221, | |
| "rewards/length_reward": 0.0491071380674839, | |
| "rewards/similarity_reward": 0.45555105805397034, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 309.0982360839844, | |
| "epoch": 0.2, | |
| "grad_norm": 0.8352994571315709, | |
| "kl": 0.0081787109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1895, | |
| "reward": 0.6111252903938293, | |
| "reward_std": 0.19863885641098022, | |
| "rewards/length_reward": 0.054464273154735565, | |
| "rewards/similarity_reward": 0.5566610097885132, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 255.25001525878906, | |
| "epoch": 0.20266666666666666, | |
| "grad_norm": 1.0786021298964794, | |
| "kl": 0.0089111328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0384, | |
| "reward": 0.5118966102600098, | |
| "reward_std": 0.1661101132631302, | |
| "rewards/length_reward": 0.05223213508725166, | |
| "rewards/similarity_reward": 0.4596644341945648, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 299.92413330078125, | |
| "epoch": 0.20533333333333334, | |
| "grad_norm": 0.887828324089484, | |
| "kl": 0.0128173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1341, | |
| "reward": 0.5058793425559998, | |
| "reward_std": 0.2038315385580063, | |
| "rewards/length_reward": 0.05044642463326454, | |
| "rewards/similarity_reward": 0.45543283224105835, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 300.26788330078125, | |
| "epoch": 0.208, | |
| "grad_norm": 0.960422578229874, | |
| "kl": 0.01080322265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2173, | |
| "reward": 0.5535677075386047, | |
| "reward_std": 0.16259299218654633, | |
| "rewards/length_reward": 0.06205355376005173, | |
| "rewards/similarity_reward": 0.49151411652565, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 256.21875, | |
| "epoch": 0.21066666666666667, | |
| "grad_norm": 0.9394611442130687, | |
| "kl": 0.01171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.031, | |
| "reward": 0.6236703991889954, | |
| "reward_std": 0.16783180832862854, | |
| "rewards/length_reward": 0.05982141196727753, | |
| "rewards/similarity_reward": 0.5638489127159119, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 279.9419860839844, | |
| "epoch": 0.21333333333333335, | |
| "grad_norm": 1.0860808591863038, | |
| "kl": 0.0096435546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2206, | |
| "reward": 0.5311146974563599, | |
| "reward_std": 0.20672693848609924, | |
| "rewards/length_reward": 0.052678562700748444, | |
| "rewards/similarity_reward": 0.47843608260154724, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 302.7008972167969, | |
| "epoch": 0.216, | |
| "grad_norm": 0.7695067843560371, | |
| "kl": 0.00653076171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0387, | |
| "reward": 0.5848192572593689, | |
| "reward_std": 0.20639710128307343, | |
| "rewards/length_reward": 0.060267843306064606, | |
| "rewards/similarity_reward": 0.5245514512062073, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 247.18751525878906, | |
| "epoch": 0.21866666666666668, | |
| "grad_norm": 1.0172061111487714, | |
| "kl": 0.009765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0528, | |
| "reward": 0.6127398610115051, | |
| "reward_std": 0.13182979822158813, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.5471147894859314, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 303.40625, | |
| "epoch": 0.22133333333333333, | |
| "grad_norm": 0.7610369219853271, | |
| "kl": 0.00921630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1034, | |
| "reward": 0.5650977492332458, | |
| "reward_std": 0.16646917164325714, | |
| "rewards/length_reward": 0.056696418672800064, | |
| "rewards/similarity_reward": 0.5084013342857361, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 296.625, | |
| "epoch": 0.224, | |
| "grad_norm": 0.8819878638905205, | |
| "kl": 0.00653076171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0509, | |
| "reward": 0.6824392676353455, | |
| "reward_std": 0.13198219239711761, | |
| "rewards/length_reward": 0.07678568363189697, | |
| "rewards/similarity_reward": 0.6056535243988037, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 264.4375, | |
| "epoch": 0.22666666666666666, | |
| "grad_norm": 0.9860703009968039, | |
| "kl": 0.0103759765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1082, | |
| "reward": 0.5452346205711365, | |
| "reward_std": 0.18002980947494507, | |
| "rewards/length_reward": 0.055357132107019424, | |
| "rewards/similarity_reward": 0.48987752199172974, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 275.1696472167969, | |
| "epoch": 0.22933333333333333, | |
| "grad_norm": 0.876007599982239, | |
| "kl": 0.007720947265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1075, | |
| "reward": 0.5654360055923462, | |
| "reward_std": 0.15497317910194397, | |
| "rewards/length_reward": 0.050892848521471024, | |
| "rewards/similarity_reward": 0.5145430564880371, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 268.3883972167969, | |
| "epoch": 0.232, | |
| "grad_norm": 0.9877196366166759, | |
| "kl": 0.00872802734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0664, | |
| "reward": 0.6144102811813354, | |
| "reward_std": 0.1374298632144928, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5469995141029358, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 339.67413330078125, | |
| "epoch": 0.23466666666666666, | |
| "grad_norm": 0.6868950012188707, | |
| "kl": 0.006072998046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0478, | |
| "reward": 0.6562062501907349, | |
| "reward_std": 0.1523490995168686, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5843312740325928, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 262.4598388671875, | |
| "epoch": 0.23733333333333334, | |
| "grad_norm": 0.8224596138062096, | |
| "kl": 0.006378173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.101, | |
| "reward": 0.6333271265029907, | |
| "reward_std": 0.170791357755661, | |
| "rewards/length_reward": 0.06785711646080017, | |
| "rewards/similarity_reward": 0.5654700398445129, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 256.42413330078125, | |
| "epoch": 0.24, | |
| "grad_norm": 0.9173507116779652, | |
| "kl": 0.00946044921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0437, | |
| "reward": 0.5871028900146484, | |
| "reward_std": 0.16378919780254364, | |
| "rewards/length_reward": 0.04732142388820648, | |
| "rewards/similarity_reward": 0.5397815108299255, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 350.6562805175781, | |
| "epoch": 0.24266666666666667, | |
| "grad_norm": 0.7989839039096065, | |
| "kl": 0.01251220703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0614, | |
| "reward": 0.5199065208435059, | |
| "reward_std": 0.20764127373695374, | |
| "rewards/length_reward": 0.05044642463326454, | |
| "rewards/similarity_reward": 0.4694600999355316, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 249.0044708251953, | |
| "epoch": 0.24533333333333332, | |
| "grad_norm": 0.9605121844841826, | |
| "kl": 0.01123046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.036, | |
| "reward": 0.6044343113899231, | |
| "reward_std": 0.164906844496727, | |
| "rewards/length_reward": 0.057589270174503326, | |
| "rewards/similarity_reward": 0.5468449592590332, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 312.2232360839844, | |
| "epoch": 0.248, | |
| "grad_norm": 0.836379585280954, | |
| "kl": 0.007781982421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0948, | |
| "reward": 0.5776776075363159, | |
| "reward_std": 0.15271225571632385, | |
| "rewards/length_reward": 0.055803555995225906, | |
| "rewards/similarity_reward": 0.5218740701675415, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 311.3482360839844, | |
| "epoch": 0.25066666666666665, | |
| "grad_norm": 0.7945534599494852, | |
| "kl": 0.00897216796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1441, | |
| "reward": 0.6111860275268555, | |
| "reward_std": 0.13805179297924042, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5442216992378235, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 234.93751525878906, | |
| "epoch": 0.25333333333333335, | |
| "grad_norm": 1.2028189171950667, | |
| "kl": 0.00909423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1766, | |
| "reward": 0.5961614847183228, | |
| "reward_std": 0.16394107043743134, | |
| "rewards/length_reward": 0.05982141196727753, | |
| "rewards/similarity_reward": 0.5363399982452393, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 305.6964416503906, | |
| "epoch": 0.256, | |
| "grad_norm": 0.8067577376172387, | |
| "kl": 0.00982666015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0671, | |
| "reward": 0.656367301940918, | |
| "reward_std": 0.12278923392295837, | |
| "rewards/length_reward": 0.06874997913837433, | |
| "rewards/similarity_reward": 0.5876173377037048, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 327.71429443359375, | |
| "epoch": 0.25866666666666666, | |
| "grad_norm": 0.770922327161602, | |
| "kl": 0.0064697265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0279, | |
| "reward": 0.5429174304008484, | |
| "reward_std": 0.16670171916484833, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.47729235887527466, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 256.62054443359375, | |
| "epoch": 0.2613333333333333, | |
| "grad_norm": 1.051628532134925, | |
| "kl": 0.0072021484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0872, | |
| "reward": 0.5515283346176147, | |
| "reward_std": 0.17894278466701508, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.48947471380233765, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 268.59375, | |
| "epoch": 0.264, | |
| "grad_norm": 0.8240697442290599, | |
| "kl": 0.008544921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0672, | |
| "reward": 0.6131307482719421, | |
| "reward_std": 0.17423538863658905, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.5510770678520203, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 257.37054443359375, | |
| "epoch": 0.26666666666666666, | |
| "grad_norm": 131.12733526048441, | |
| "kl": 0.1923828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0817, | |
| "reward": 0.6513006687164307, | |
| "reward_std": 0.14835356175899506, | |
| "rewards/length_reward": 0.06830354779958725, | |
| "rewards/similarity_reward": 0.5829971432685852, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 307.1473388671875, | |
| "epoch": 0.2693333333333333, | |
| "grad_norm": 0.837570841896231, | |
| "kl": 0.0054931640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0194, | |
| "reward": 0.6764991283416748, | |
| "reward_std": 0.1323472112417221, | |
| "rewards/length_reward": 0.07410712540149689, | |
| "rewards/similarity_reward": 0.6023918986320496, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 265.5044860839844, | |
| "epoch": 0.272, | |
| "grad_norm": 0.7713890189466205, | |
| "kl": 0.00830078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0212, | |
| "reward": 0.6779460310935974, | |
| "reward_std": 0.12496771663427353, | |
| "rewards/length_reward": 0.07276783138513565, | |
| "rewards/similarity_reward": 0.6051782369613647, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 280.46429443359375, | |
| "epoch": 0.27466666666666667, | |
| "grad_norm": 0.8825358886169125, | |
| "kl": 0.008056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1086, | |
| "reward": 0.5848525166511536, | |
| "reward_std": 0.11382713168859482, | |
| "rewards/length_reward": 0.06071426719427109, | |
| "rewards/similarity_reward": 0.5241381525993347, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 320.3571472167969, | |
| "epoch": 0.2773333333333333, | |
| "grad_norm": 0.8471202564701443, | |
| "kl": 0.0069580078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0388, | |
| "reward": 0.6058804988861084, | |
| "reward_std": 0.15757833421230316, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5407018661499023, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 287.0848388671875, | |
| "epoch": 0.28, | |
| "grad_norm": 0.885756988877436, | |
| "kl": 0.00860595703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.089, | |
| "reward": 0.6150888204574585, | |
| "reward_std": 0.1344638168811798, | |
| "rewards/length_reward": 0.057589273899793625, | |
| "rewards/similarity_reward": 0.5574995875358582, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 293.1339416503906, | |
| "epoch": 0.2826666666666667, | |
| "grad_norm": 0.9299759085944364, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0428, | |
| "reward": 0.560218334197998, | |
| "reward_std": 0.2031860500574112, | |
| "rewards/length_reward": 0.058035701513290405, | |
| "rewards/similarity_reward": 0.50218266248703, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 292.75, | |
| "epoch": 0.2853333333333333, | |
| "grad_norm": 0.8374882655316597, | |
| "kl": 0.00848388671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0683, | |
| "reward": 0.6374148726463318, | |
| "reward_std": 0.15000107884407043, | |
| "rewards/length_reward": 0.06919640302658081, | |
| "rewards/similarity_reward": 0.5682184100151062, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 263.3571472167969, | |
| "epoch": 0.288, | |
| "grad_norm": 1.0433586800088648, | |
| "kl": 0.0078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0913, | |
| "reward": 0.5456939935684204, | |
| "reward_std": 0.1411367952823639, | |
| "rewards/length_reward": 0.056696414947509766, | |
| "rewards/similarity_reward": 0.48899757862091064, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 263.9107360839844, | |
| "epoch": 0.2906666666666667, | |
| "grad_norm": 0.9650468316923807, | |
| "kl": 0.01129150390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.119, | |
| "reward": 0.6117041110992432, | |
| "reward_std": 0.13907021284103394, | |
| "rewards/length_reward": 0.06205355376005173, | |
| "rewards/similarity_reward": 0.5496505498886108, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 235.0848388671875, | |
| "epoch": 0.29333333333333333, | |
| "grad_norm": 0.9205848620805003, | |
| "kl": 0.009521484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.006, | |
| "reward": 0.5724084377288818, | |
| "reward_std": 0.12264589220285416, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.5027655959129333, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 280.9419860839844, | |
| "epoch": 0.296, | |
| "grad_norm": 0.8242814043162366, | |
| "kl": 0.00836181640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1358, | |
| "reward": 0.6025325059890747, | |
| "reward_std": 0.13276302814483643, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5351218581199646, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 261.55804443359375, | |
| "epoch": 0.2986666666666667, | |
| "grad_norm": 0.8979430693793525, | |
| "kl": 0.0145263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0898, | |
| "reward": 0.5723416805267334, | |
| "reward_std": 0.11434419453144073, | |
| "rewards/length_reward": 0.06428569555282593, | |
| "rewards/similarity_reward": 0.5080559253692627, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 273.8482360839844, | |
| "epoch": 0.30133333333333334, | |
| "grad_norm": 0.8994640436743108, | |
| "kl": 0.0084228515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0884, | |
| "reward": 0.6239952445030212, | |
| "reward_std": 0.15253794193267822, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5588168501853943, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 265.29913330078125, | |
| "epoch": 0.304, | |
| "grad_norm": 0.8511084352415984, | |
| "kl": 0.015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0217, | |
| "reward": 0.5796217322349548, | |
| "reward_std": 0.16319997608661652, | |
| "rewards/length_reward": 0.06785712391138077, | |
| "rewards/similarity_reward": 0.511764645576477, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 227.94644165039062, | |
| "epoch": 0.30666666666666664, | |
| "grad_norm": 1.0504493112285522, | |
| "kl": 0.013427734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.092, | |
| "reward": 0.611219584941864, | |
| "reward_std": 0.1473054587841034, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.5455944538116455, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 312.2008972167969, | |
| "epoch": 0.30933333333333335, | |
| "grad_norm": 0.8491847097599164, | |
| "kl": 0.009033203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0818, | |
| "reward": 0.5205245018005371, | |
| "reward_std": 0.18279042840003967, | |
| "rewards/length_reward": 0.055803555995225906, | |
| "rewards/similarity_reward": 0.46472102403640747, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 285.4151916503906, | |
| "epoch": 0.312, | |
| "grad_norm": 0.948136046714223, | |
| "kl": 0.01251220703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1402, | |
| "reward": 0.5244685411453247, | |
| "reward_std": 0.1221655011177063, | |
| "rewards/length_reward": 0.058482129126787186, | |
| "rewards/similarity_reward": 0.4659864008426666, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 291.8973388671875, | |
| "epoch": 0.31466666666666665, | |
| "grad_norm": 0.8327937599541795, | |
| "kl": 0.00823974609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0416, | |
| "reward": 0.6440633535385132, | |
| "reward_std": 0.14113157987594604, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5770990252494812, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 276.3035888671875, | |
| "epoch": 0.31733333333333336, | |
| "grad_norm": 1.0522569506493296, | |
| "kl": 0.01007080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2508, | |
| "reward": 0.501847505569458, | |
| "reward_std": 0.16830717027187347, | |
| "rewards/length_reward": 0.051785703748464584, | |
| "rewards/similarity_reward": 0.4500618278980255, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 231.46876525878906, | |
| "epoch": 0.32, | |
| "grad_norm": 1.0564887037389263, | |
| "kl": 0.01544189453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1258, | |
| "reward": 0.5212039351463318, | |
| "reward_std": 0.1660899519920349, | |
| "rewards/length_reward": 0.053124986588954926, | |
| "rewards/similarity_reward": 0.46807900071144104, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 274.55804443359375, | |
| "epoch": 0.32266666666666666, | |
| "grad_norm": 0.892927807825851, | |
| "kl": 0.006988525390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0887, | |
| "reward": 0.5594373941421509, | |
| "reward_std": 0.13949331641197205, | |
| "rewards/length_reward": 0.06651782989501953, | |
| "rewards/similarity_reward": 0.49291953444480896, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 309.36163330078125, | |
| "epoch": 0.3253333333333333, | |
| "grad_norm": 0.8555023561165935, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1512, | |
| "reward": 0.5939301252365112, | |
| "reward_std": 0.16705819964408875, | |
| "rewards/length_reward": 0.061160698533058167, | |
| "rewards/similarity_reward": 0.5327693819999695, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 264.3973388671875, | |
| "epoch": 0.328, | |
| "grad_norm": 0.9083757893001095, | |
| "kl": 0.00775146484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1449, | |
| "reward": 0.6276513934135437, | |
| "reward_std": 0.14763577282428741, | |
| "rewards/length_reward": 0.06160712614655495, | |
| "rewards/similarity_reward": 0.5660442113876343, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 267.96875, | |
| "epoch": 0.33066666666666666, | |
| "grad_norm": 0.9115344595637944, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0998, | |
| "reward": 0.6213053464889526, | |
| "reward_std": 0.16126255691051483, | |
| "rewards/length_reward": 0.05848212540149689, | |
| "rewards/similarity_reward": 0.5628232359886169, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 275.2276916503906, | |
| "epoch": 0.3333333333333333, | |
| "grad_norm": 0.8933631069209625, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0232, | |
| "reward": 0.6394702792167664, | |
| "reward_std": 0.17729975283145905, | |
| "rewards/length_reward": 0.061160698533058167, | |
| "rewards/similarity_reward": 0.5783094763755798, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 280.0848388671875, | |
| "epoch": 0.336, | |
| "grad_norm": 0.9959640208447441, | |
| "kl": 0.0203857421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.066, | |
| "reward": 0.5415524840354919, | |
| "reward_std": 0.18598264455795288, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.47414183616638184, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 294.37054443359375, | |
| "epoch": 0.33866666666666667, | |
| "grad_norm": 0.944172883238238, | |
| "kl": 0.0078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0825, | |
| "reward": 0.6250823736190796, | |
| "reward_std": 0.1783696711063385, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5581181049346924, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 252.44644165039062, | |
| "epoch": 0.3413333333333333, | |
| "grad_norm": 0.8622018142523461, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0136, | |
| "reward": 0.5941780209541321, | |
| "reward_std": 0.1297590583562851, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5289995074272156, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 275.1696472167969, | |
| "epoch": 0.344, | |
| "grad_norm": 0.997627840820869, | |
| "kl": 0.00933837890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1634, | |
| "reward": 0.5641010999679565, | |
| "reward_std": 0.13691328465938568, | |
| "rewards/length_reward": 0.057142842561006546, | |
| "rewards/similarity_reward": 0.5069582462310791, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 291.78125, | |
| "epoch": 0.3466666666666667, | |
| "grad_norm": 0.9141566771741596, | |
| "kl": 0.00885009765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.233, | |
| "reward": 0.5903910398483276, | |
| "reward_std": 0.14815300703048706, | |
| "rewards/length_reward": 0.06249998137354851, | |
| "rewards/similarity_reward": 0.5278909802436829, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 294.58929443359375, | |
| "epoch": 0.34933333333333333, | |
| "grad_norm": 0.9307314460988763, | |
| "kl": 0.01153564453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0979, | |
| "reward": 0.5972681045532227, | |
| "reward_std": 0.16272346675395966, | |
| "rewards/length_reward": 0.061160698533058167, | |
| "rewards/similarity_reward": 0.5361074805259705, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 240.2232208251953, | |
| "epoch": 0.352, | |
| "grad_norm": 0.9959808951952684, | |
| "kl": 0.01092529296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0428, | |
| "reward": 0.6472880244255066, | |
| "reward_std": 0.15316687524318695, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.5816629528999329, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 237.1607208251953, | |
| "epoch": 0.3546666666666667, | |
| "grad_norm": 0.8515521500324365, | |
| "kl": 0.01251220703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0683, | |
| "reward": 0.631507396697998, | |
| "reward_std": 0.15118519961833954, | |
| "rewards/length_reward": 0.06428569555282593, | |
| "rewards/similarity_reward": 0.5672216415405273, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 307.5, | |
| "epoch": 0.35733333333333334, | |
| "grad_norm": 0.719487956498844, | |
| "kl": 0.0059814453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0352, | |
| "reward": 0.6587818264961243, | |
| "reward_std": 0.14100806415081024, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5869067907333374, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 305.2589416503906, | |
| "epoch": 0.36, | |
| "grad_norm": 1.0877799003245066, | |
| "kl": 0.0169677734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0955, | |
| "reward": 0.5947835445404053, | |
| "reward_std": 0.12429028749465942, | |
| "rewards/length_reward": 0.06071426719427109, | |
| "rewards/similarity_reward": 0.5340692400932312, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 223.81251525878906, | |
| "epoch": 0.3626666666666667, | |
| "grad_norm": 1.056046449389469, | |
| "kl": 0.00665283203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0631, | |
| "reward": 0.6106573343276978, | |
| "reward_std": 0.13982126116752625, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5454786419868469, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 302.52679443359375, | |
| "epoch": 0.36533333333333334, | |
| "grad_norm": 0.8473080240759754, | |
| "kl": 0.01116943359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.121, | |
| "reward": 0.5767890810966492, | |
| "reward_std": 0.1566361039876938, | |
| "rewards/length_reward": 0.06071426719427109, | |
| "rewards/similarity_reward": 0.5160747766494751, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 294.9285888671875, | |
| "epoch": 0.368, | |
| "grad_norm": 0.8165111113975745, | |
| "kl": 0.00677490234375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0013, | |
| "reward": 0.6466237902641296, | |
| "reward_std": 0.11831733584403992, | |
| "rewards/length_reward": 0.07142855226993561, | |
| "rewards/similarity_reward": 0.5751951336860657, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 297.4107360839844, | |
| "epoch": 0.37066666666666664, | |
| "grad_norm": 0.8905760527062927, | |
| "kl": 0.00653076171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0894, | |
| "reward": 0.6628533601760864, | |
| "reward_std": 0.10040118545293808, | |
| "rewards/length_reward": 0.06339284032583237, | |
| "rewards/similarity_reward": 0.5994604229927063, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 282.9508972167969, | |
| "epoch": 0.37333333333333335, | |
| "grad_norm": 0.9489224311946435, | |
| "kl": 0.01007080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0747, | |
| "reward": 0.5422684550285339, | |
| "reward_std": 0.18701300024986267, | |
| "rewards/length_reward": 0.055357132107019424, | |
| "rewards/similarity_reward": 0.4869112968444824, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 286.9419860839844, | |
| "epoch": 0.376, | |
| "grad_norm": 1.5325112007084152, | |
| "kl": 0.0205078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0749, | |
| "reward": 0.6462909579277039, | |
| "reward_std": 0.1564369648694992, | |
| "rewards/length_reward": 0.06919640302658081, | |
| "rewards/similarity_reward": 0.5770944356918335, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 233.50001525878906, | |
| "epoch": 0.37866666666666665, | |
| "grad_norm": 1.1124358172264561, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1038, | |
| "reward": 0.6390895247459412, | |
| "reward_std": 0.11253345012664795, | |
| "rewards/length_reward": 0.05982141196727753, | |
| "rewards/similarity_reward": 0.5792680978775024, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 308.0401916503906, | |
| "epoch": 0.38133333333333336, | |
| "grad_norm": 0.8437782349764958, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0439, | |
| "reward": 0.6860373616218567, | |
| "reward_std": 0.08081385493278503, | |
| "rewards/length_reward": 0.07901783287525177, | |
| "rewards/similarity_reward": 0.6070196032524109, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 274.54913330078125, | |
| "epoch": 0.384, | |
| "grad_norm": 0.9174096594145076, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0938, | |
| "reward": 0.6485283970832825, | |
| "reward_std": 0.1347315013408661, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5739747881889343, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 257.5535888671875, | |
| "epoch": 0.38666666666666666, | |
| "grad_norm": 0.9244068415305253, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.056, | |
| "reward": 0.6137626767158508, | |
| "reward_std": 0.13303914666175842, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.5481376647949219, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 255.62501525878906, | |
| "epoch": 0.3893333333333333, | |
| "grad_norm": 0.8596867360926773, | |
| "kl": 0.011474609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0491, | |
| "reward": 0.6294366717338562, | |
| "reward_std": 0.13696229457855225, | |
| "rewards/length_reward": 0.0741071105003357, | |
| "rewards/similarity_reward": 0.5553295016288757, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 321.3258972167969, | |
| "epoch": 0.392, | |
| "grad_norm": 0.7611409673177786, | |
| "kl": 0.01171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0851, | |
| "reward": 0.6082260012626648, | |
| "reward_std": 0.11479248106479645, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.5385831594467163, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 256.5223388671875, | |
| "epoch": 0.39466666666666667, | |
| "grad_norm": 1.017158083005092, | |
| "kl": 0.01165771484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0583, | |
| "reward": 0.5373588800430298, | |
| "reward_std": 0.1524331271648407, | |
| "rewards/length_reward": 0.060267843306064606, | |
| "rewards/similarity_reward": 0.477090984582901, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 247.34376525878906, | |
| "epoch": 0.3973333333333333, | |
| "grad_norm": 1.100826516879252, | |
| "kl": 0.011474609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1543, | |
| "reward": 0.6250883340835571, | |
| "reward_std": 0.1562027931213379, | |
| "rewards/length_reward": 0.06785711646080017, | |
| "rewards/similarity_reward": 0.5572311878204346, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 271.61163330078125, | |
| "epoch": 0.4, | |
| "grad_norm": 0.85368826964619, | |
| "kl": 0.00897216796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0525, | |
| "reward": 0.5830017924308777, | |
| "reward_std": 0.1454438865184784, | |
| "rewards/length_reward": 0.06160712614655495, | |
| "rewards/similarity_reward": 0.5213946104049683, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 274.40179443359375, | |
| "epoch": 0.4026666666666667, | |
| "grad_norm": 0.9117887687890662, | |
| "kl": 0.014892578125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0237, | |
| "reward": 0.6028919219970703, | |
| "reward_std": 0.15602950751781464, | |
| "rewards/length_reward": 0.06651782989501953, | |
| "rewards/similarity_reward": 0.536374032497406, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 269.2008972167969, | |
| "epoch": 0.4053333333333333, | |
| "grad_norm": 0.8208276830838094, | |
| "kl": 0.010009765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.014, | |
| "reward": 0.665276825428009, | |
| "reward_std": 0.1257169246673584, | |
| "rewards/length_reward": 0.08035711199045181, | |
| "rewards/similarity_reward": 0.5849196910858154, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 264.4732360839844, | |
| "epoch": 0.408, | |
| "grad_norm": 0.9062154210012625, | |
| "kl": 0.013427734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0701, | |
| "reward": 0.6374659538269043, | |
| "reward_std": 0.1712835431098938, | |
| "rewards/length_reward": 0.06830354779958725, | |
| "rewards/similarity_reward": 0.5691623091697693, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 325.1071472167969, | |
| "epoch": 0.4106666666666667, | |
| "grad_norm": 0.8808738957904011, | |
| "kl": 0.0089111328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1136, | |
| "reward": 0.6423187255859375, | |
| "reward_std": 0.1033661887049675, | |
| "rewards/length_reward": 0.06383926421403885, | |
| "rewards/similarity_reward": 0.578479528427124, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 281.3035888671875, | |
| "epoch": 0.41333333333333333, | |
| "grad_norm": 0.8449149570191646, | |
| "kl": 0.012451171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0893, | |
| "reward": 0.6530374884605408, | |
| "reward_std": 0.12996266782283783, | |
| "rewards/length_reward": 0.06116069480776787, | |
| "rewards/similarity_reward": 0.5918766856193542, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 297.02679443359375, | |
| "epoch": 0.416, | |
| "grad_norm": 0.8274002453741087, | |
| "kl": 0.008056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0593, | |
| "reward": 0.7200801372528076, | |
| "reward_std": 0.12102329730987549, | |
| "rewards/length_reward": 0.07901783287525177, | |
| "rewards/similarity_reward": 0.6410622596740723, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 245.7991180419922, | |
| "epoch": 0.4186666666666667, | |
| "grad_norm": 1.0463728826517769, | |
| "kl": 0.0145263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1192, | |
| "reward": 0.6804168820381165, | |
| "reward_std": 0.1330643892288208, | |
| "rewards/length_reward": 0.0741071105003357, | |
| "rewards/similarity_reward": 0.6063097715377808, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 272.2410888671875, | |
| "epoch": 0.42133333333333334, | |
| "grad_norm": 0.8424445256337731, | |
| "kl": 0.0152587890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0411, | |
| "reward": 0.6152999401092529, | |
| "reward_std": 0.18344512581825256, | |
| "rewards/length_reward": 0.06428569555282593, | |
| "rewards/similarity_reward": 0.5510141849517822, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 283.90625, | |
| "epoch": 0.424, | |
| "grad_norm": 0.944378171141832, | |
| "kl": 0.0128173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0973, | |
| "reward": 0.6098131537437439, | |
| "reward_std": 0.14866778254508972, | |
| "rewards/length_reward": 0.05982141196727753, | |
| "rewards/similarity_reward": 0.5499916672706604, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 251.37054443359375, | |
| "epoch": 0.4266666666666667, | |
| "grad_norm": 0.867614538281579, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0069, | |
| "reward": 0.6304399371147156, | |
| "reward_std": 0.12713229656219482, | |
| "rewards/length_reward": 0.07276783138513565, | |
| "rewards/similarity_reward": 0.5576720237731934, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 254.6607208251953, | |
| "epoch": 0.42933333333333334, | |
| "grad_norm": 1.0006767726840313, | |
| "kl": 0.01226806640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0167, | |
| "reward": 0.6183627843856812, | |
| "reward_std": 0.12064019590616226, | |
| "rewards/length_reward": 0.057589273899793625, | |
| "rewards/similarity_reward": 0.560773491859436, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 278.9821472167969, | |
| "epoch": 0.432, | |
| "grad_norm": 0.7754115998151179, | |
| "kl": 0.0108642578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0624, | |
| "reward": 0.6279152035713196, | |
| "reward_std": 0.1739441603422165, | |
| "rewards/length_reward": 0.06651783734560013, | |
| "rewards/similarity_reward": 0.5613973140716553, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 261.51788330078125, | |
| "epoch": 0.43466666666666665, | |
| "grad_norm": 0.9381033539462706, | |
| "kl": 0.011962890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1291, | |
| "reward": 0.6449581384658813, | |
| "reward_std": 0.13920167088508606, | |
| "rewards/length_reward": 0.07589282840490341, | |
| "rewards/similarity_reward": 0.569065272808075, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 311.8169860839844, | |
| "epoch": 0.43733333333333335, | |
| "grad_norm": 0.959860639301872, | |
| "kl": 0.0084228515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0207, | |
| "reward": 0.6583375930786133, | |
| "reward_std": 0.1428201049566269, | |
| "rewards/length_reward": 0.07008926570415497, | |
| "rewards/similarity_reward": 0.5882483720779419, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 261.5758972167969, | |
| "epoch": 0.44, | |
| "grad_norm": 0.8574273981386299, | |
| "kl": 0.009033203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1177, | |
| "reward": 0.6945616006851196, | |
| "reward_std": 0.12529133260250092, | |
| "rewards/length_reward": 0.07812497019767761, | |
| "rewards/similarity_reward": 0.6164366006851196, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 281.45538330078125, | |
| "epoch": 0.44266666666666665, | |
| "grad_norm": 0.8588941157426009, | |
| "kl": 0.01220703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0615, | |
| "reward": 0.7305233478546143, | |
| "reward_std": 0.12011624127626419, | |
| "rewards/length_reward": 0.0808035358786583, | |
| "rewards/similarity_reward": 0.6497198343276978, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 259.64288330078125, | |
| "epoch": 0.44533333333333336, | |
| "grad_norm": 1.137909715907424, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2205, | |
| "reward": 0.5699202418327332, | |
| "reward_std": 0.1761079728603363, | |
| "rewards/length_reward": 0.06294640898704529, | |
| "rewards/similarity_reward": 0.5069737434387207, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 294.875, | |
| "epoch": 0.448, | |
| "grad_norm": 1.0016080727138688, | |
| "kl": 0.01171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1368, | |
| "reward": 0.6165792942047119, | |
| "reward_std": 0.12841306626796722, | |
| "rewards/length_reward": 0.064732126891613, | |
| "rewards/similarity_reward": 0.5518471002578735, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 308.37054443359375, | |
| "epoch": 0.45066666666666666, | |
| "grad_norm": 1.2557098703938632, | |
| "kl": 0.0162353515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0777, | |
| "reward": 0.6842705607414246, | |
| "reward_std": 0.09632124751806259, | |
| "rewards/length_reward": 0.08124997466802597, | |
| "rewards/similarity_reward": 0.6030204892158508, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 281.4821472167969, | |
| "epoch": 0.4533333333333333, | |
| "grad_norm": 0.8773655821391068, | |
| "kl": 0.009033203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0602, | |
| "reward": 0.6346014738082886, | |
| "reward_std": 0.14319205284118652, | |
| "rewards/length_reward": 0.06116069480776787, | |
| "rewards/similarity_reward": 0.5734407901763916, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 281.8125, | |
| "epoch": 0.456, | |
| "grad_norm": 0.9145529663215465, | |
| "kl": 0.01275634765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1688, | |
| "reward": 0.5963006019592285, | |
| "reward_std": 0.15331213176250458, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5293362736701965, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 280.8571472167969, | |
| "epoch": 0.45866666666666667, | |
| "grad_norm": 3.878703302716922, | |
| "kl": 0.0269775390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0219, | |
| "reward": 0.6730906963348389, | |
| "reward_std": 0.11424030363559723, | |
| "rewards/length_reward": 0.06874997913837433, | |
| "rewards/similarity_reward": 0.604340672492981, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 277.9910888671875, | |
| "epoch": 0.4613333333333333, | |
| "grad_norm": 1.0003205251640386, | |
| "kl": 0.01141357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1181, | |
| "reward": 0.6080780029296875, | |
| "reward_std": 0.14715011417865753, | |
| "rewards/length_reward": 0.06651782989501953, | |
| "rewards/similarity_reward": 0.541560173034668, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 257.0133972167969, | |
| "epoch": 0.464, | |
| "grad_norm": 0.8626427313272481, | |
| "kl": 0.00921630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0332, | |
| "reward": 0.6258962154388428, | |
| "reward_std": 0.13628825545310974, | |
| "rewards/length_reward": 0.07098211348056793, | |
| "rewards/similarity_reward": 0.554914116859436, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 267.1294860839844, | |
| "epoch": 0.4666666666666667, | |
| "grad_norm": 0.9448046232693003, | |
| "kl": 0.0128173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0686, | |
| "reward": 0.582805871963501, | |
| "reward_std": 0.13543623685836792, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5082523226737976, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 230.3169708251953, | |
| "epoch": 0.4693333333333333, | |
| "grad_norm": 1.0336066582105279, | |
| "kl": 0.01611328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0188, | |
| "reward": 0.5923266410827637, | |
| "reward_std": 0.15992802381515503, | |
| "rewards/length_reward": 0.06607140600681305, | |
| "rewards/similarity_reward": 0.5262552499771118, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 259.2857360839844, | |
| "epoch": 0.472, | |
| "grad_norm": 0.8515404437990851, | |
| "kl": 0.01422119140625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0031, | |
| "reward": 0.6366464495658875, | |
| "reward_std": 0.14244325459003448, | |
| "rewards/length_reward": 0.06785711646080017, | |
| "rewards/similarity_reward": 0.5687893033027649, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 269.0535888671875, | |
| "epoch": 0.4746666666666667, | |
| "grad_norm": 1.0508009846238586, | |
| "kl": 0.01202392578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1414, | |
| "reward": 0.6338518857955933, | |
| "reward_std": 0.13359463214874268, | |
| "rewards/length_reward": 0.06651783734560013, | |
| "rewards/similarity_reward": 0.5673341155052185, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 254.2053680419922, | |
| "epoch": 0.47733333333333333, | |
| "grad_norm": 1.0742338846656552, | |
| "kl": 0.00799560546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0226, | |
| "reward": 0.6352322697639465, | |
| "reward_std": 0.16355818510055542, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5633572936058044, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 258.8482360839844, | |
| "epoch": 0.48, | |
| "grad_norm": 1.1021168870169997, | |
| "kl": 0.0213623046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0656, | |
| "reward": 0.594104528427124, | |
| "reward_std": 0.1770821362733841, | |
| "rewards/length_reward": 0.060714274644851685, | |
| "rewards/similarity_reward": 0.5333902835845947, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 256.8125, | |
| "epoch": 0.4826666666666667, | |
| "grad_norm": 0.8501219854036921, | |
| "kl": 0.009033203125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0019, | |
| "reward": 0.6487245559692383, | |
| "reward_std": 0.13405689597129822, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5741709470748901, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 256.02679443359375, | |
| "epoch": 0.48533333333333334, | |
| "grad_norm": 1.0385629776489995, | |
| "kl": 0.012939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0721, | |
| "reward": 0.6005666255950928, | |
| "reward_std": 0.18559977412223816, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5331559181213379, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 245.2723388671875, | |
| "epoch": 0.488, | |
| "grad_norm": 0.9856929099072189, | |
| "kl": 0.0140380859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0559, | |
| "reward": 0.6080025434494019, | |
| "reward_std": 0.12059400230646133, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5428239703178406, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 275.40179443359375, | |
| "epoch": 0.49066666666666664, | |
| "grad_norm": 1.0764117312018395, | |
| "kl": 0.0147705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1746, | |
| "reward": 0.5906988382339478, | |
| "reward_std": 0.13717274367809296, | |
| "rewards/length_reward": 0.06339284032583237, | |
| "rewards/similarity_reward": 0.5273059606552124, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 235.33929443359375, | |
| "epoch": 0.49333333333333335, | |
| "grad_norm": 1.216465109274426, | |
| "kl": 0.015869140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0441, | |
| "reward": 0.6832688450813293, | |
| "reward_std": 0.12071473151445389, | |
| "rewards/length_reward": 0.07767853885889053, | |
| "rewards/similarity_reward": 0.6055901646614075, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 271.0401916503906, | |
| "epoch": 0.496, | |
| "grad_norm": 0.9417708264398014, | |
| "kl": 0.0113525390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0103, | |
| "reward": 0.7055503726005554, | |
| "reward_std": 0.09871623665094376, | |
| "rewards/length_reward": 0.07812497019767761, | |
| "rewards/similarity_reward": 0.6274253129959106, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 288.40179443359375, | |
| "epoch": 0.49866666666666665, | |
| "grad_norm": 0.7904664413572577, | |
| "kl": 0.0113525390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0943, | |
| "reward": 0.6588479280471802, | |
| "reward_std": 0.1498415172100067, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5883120894432068, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 262.2857360839844, | |
| "epoch": 0.5013333333333333, | |
| "grad_norm": 1.0107893927701763, | |
| "kl": 0.0113525390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0639, | |
| "reward": 0.5783969759941101, | |
| "reward_std": 0.1660866141319275, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5109862685203552, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 243.23214721679688, | |
| "epoch": 0.504, | |
| "grad_norm": 0.9914068826603122, | |
| "kl": 0.0263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0183, | |
| "reward": 0.5762468576431274, | |
| "reward_std": 0.1855197250843048, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5088360905647278, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 279.8973388671875, | |
| "epoch": 0.5066666666666667, | |
| "grad_norm": 0.8186202175206256, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0181, | |
| "reward": 0.6954742074012756, | |
| "reward_std": 0.08623984456062317, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.624938428401947, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 257.21429443359375, | |
| "epoch": 0.5093333333333333, | |
| "grad_norm": 0.877876828642467, | |
| "kl": 0.014892578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0328, | |
| "reward": 0.6462003588676453, | |
| "reward_std": 0.11538383364677429, | |
| "rewards/length_reward": 0.06830354779958725, | |
| "rewards/similarity_reward": 0.577896773815155, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 257.7946472167969, | |
| "epoch": 0.512, | |
| "grad_norm": 0.8857490639900779, | |
| "kl": 0.01214599609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0477, | |
| "reward": 0.6250221133232117, | |
| "reward_std": 0.15633754432201385, | |
| "rewards/length_reward": 0.06830354779958725, | |
| "rewards/similarity_reward": 0.5567185282707214, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 303.4464416503906, | |
| "epoch": 0.5146666666666667, | |
| "grad_norm": 0.8375649728004798, | |
| "kl": 0.00897216796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0444, | |
| "reward": 0.6938925981521606, | |
| "reward_std": 0.13664484024047852, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.6184461116790771, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 239.02679443359375, | |
| "epoch": 0.5173333333333333, | |
| "grad_norm": 0.9796244769392795, | |
| "kl": 0.0169677734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0277, | |
| "reward": 0.6308580636978149, | |
| "reward_std": 0.09844722598791122, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.5554116368293762, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 220.04019165039062, | |
| "epoch": 0.52, | |
| "grad_norm": 1.1138163852092198, | |
| "kl": 0.0211181640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.101, | |
| "reward": 0.5808507204055786, | |
| "reward_std": 0.14026090502738953, | |
| "rewards/length_reward": 0.056696418672800064, | |
| "rewards/similarity_reward": 0.5241542458534241, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 310.6294860839844, | |
| "epoch": 0.5226666666666666, | |
| "grad_norm": 0.7596161810526226, | |
| "kl": 0.012451171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0671, | |
| "reward": 0.6398200988769531, | |
| "reward_std": 0.16089944541454315, | |
| "rewards/length_reward": 0.07633925974369049, | |
| "rewards/similarity_reward": 0.5634807348251343, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 293.65179443359375, | |
| "epoch": 0.5253333333333333, | |
| "grad_norm": 1.3057397068251875, | |
| "kl": 0.0137939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0229, | |
| "reward": 0.69722580909729, | |
| "reward_std": 0.10665407031774521, | |
| "rewards/length_reward": 0.07901783287525177, | |
| "rewards/similarity_reward": 0.6182078719139099, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 271.8794860839844, | |
| "epoch": 0.528, | |
| "grad_norm": 0.9454287770215252, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0697, | |
| "reward": 0.6203178763389587, | |
| "reward_std": 0.1754215508699417, | |
| "rewards/length_reward": 0.06339284032583237, | |
| "rewards/similarity_reward": 0.5569249987602234, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 259.3571472167969, | |
| "epoch": 0.5306666666666666, | |
| "grad_norm": 0.8381899247069013, | |
| "kl": 0.011474609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0283, | |
| "reward": 0.6383811831474304, | |
| "reward_std": 0.11189709603786469, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.5629347562789917, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 258.0401916503906, | |
| "epoch": 0.5333333333333333, | |
| "grad_norm": 1.0128439289515407, | |
| "kl": 0.012451171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1465, | |
| "reward": 0.5912656784057617, | |
| "reward_std": 0.14550404250621796, | |
| "rewards/length_reward": 0.058482129126787186, | |
| "rewards/similarity_reward": 0.5327835083007812, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 311.27679443359375, | |
| "epoch": 0.536, | |
| "grad_norm": 0.86779018830801, | |
| "kl": 0.00909423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0985, | |
| "reward": 0.588790237903595, | |
| "reward_std": 0.140910342335701, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5169152021408081, | |
| "step": 201 | |
| }, | |
| { | |
| "completion_length": 223.9598388671875, | |
| "epoch": 0.5386666666666666, | |
| "grad_norm": 0.9896401672605407, | |
| "kl": 0.01263427734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0605, | |
| "reward": 0.5413497686386108, | |
| "reward_std": 0.13121715188026428, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.4761711657047272, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 245.66964721679688, | |
| "epoch": 0.5413333333333333, | |
| "grad_norm": 0.8825950131925253, | |
| "kl": 0.01300048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0168, | |
| "reward": 0.632973849773407, | |
| "reward_std": 0.15790660679340363, | |
| "rewards/length_reward": 0.07678568363189697, | |
| "rewards/similarity_reward": 0.5561880469322205, | |
| "step": 203 | |
| }, | |
| { | |
| "completion_length": 214.50894165039062, | |
| "epoch": 0.544, | |
| "grad_norm": 0.9691668766051184, | |
| "kl": 0.0133056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0135, | |
| "reward": 0.6077868938446045, | |
| "reward_std": 0.12028573453426361, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.5381439328193665, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 292.73663330078125, | |
| "epoch": 0.5466666666666666, | |
| "grad_norm": 0.7967760450327859, | |
| "kl": 0.00982666015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0266, | |
| "reward": 0.6232799887657166, | |
| "reward_std": 0.10140591114759445, | |
| "rewards/length_reward": 0.07410712540149689, | |
| "rewards/similarity_reward": 0.5491728186607361, | |
| "step": 205 | |
| }, | |
| { | |
| "completion_length": 221.25001525878906, | |
| "epoch": 0.5493333333333333, | |
| "grad_norm": 1.1638296703356164, | |
| "kl": 0.01385498046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1653, | |
| "reward": 0.5745998620986938, | |
| "reward_std": 0.1350637972354889, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5071890950202942, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 265.58929443359375, | |
| "epoch": 0.552, | |
| "grad_norm": 0.845270302637572, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0111, | |
| "reward": 0.6836676001548767, | |
| "reward_std": 0.10602893680334091, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.6131318211555481, | |
| "step": 207 | |
| }, | |
| { | |
| "completion_length": 295.58929443359375, | |
| "epoch": 0.5546666666666666, | |
| "grad_norm": 0.9323286808595849, | |
| "kl": 0.00921630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0749, | |
| "reward": 0.5889706611633301, | |
| "reward_std": 0.1236046850681305, | |
| "rewards/length_reward": 0.06517855823040009, | |
| "rewards/similarity_reward": 0.5237920880317688, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 293.58038330078125, | |
| "epoch": 0.5573333333333333, | |
| "grad_norm": 0.9233381319586115, | |
| "kl": 0.01055908203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2, | |
| "reward": 0.6363462805747986, | |
| "reward_std": 0.12041884660720825, | |
| "rewards/length_reward": 0.06874997913837433, | |
| "rewards/similarity_reward": 0.5675963759422302, | |
| "step": 209 | |
| }, | |
| { | |
| "completion_length": 266.3660888671875, | |
| "epoch": 0.56, | |
| "grad_norm": 0.9921663239986533, | |
| "kl": 0.01214599609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0664, | |
| "reward": 0.5890473127365112, | |
| "reward_std": 0.14096976816654205, | |
| "rewards/length_reward": 0.06339284032583237, | |
| "rewards/similarity_reward": 0.5256544351577759, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 257.46875, | |
| "epoch": 0.5626666666666666, | |
| "grad_norm": 0.9774355514560761, | |
| "kl": 0.01080322265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0824, | |
| "reward": 0.5761434435844421, | |
| "reward_std": 0.18852439522743225, | |
| "rewards/length_reward": 0.061160698533058167, | |
| "rewards/similarity_reward": 0.5149827003479004, | |
| "step": 211 | |
| }, | |
| { | |
| "completion_length": 214.66964721679688, | |
| "epoch": 0.5653333333333334, | |
| "grad_norm": 1.0333338701683394, | |
| "kl": 0.00994873046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1128, | |
| "reward": 0.6288223266601562, | |
| "reward_std": 0.1170286163687706, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.5591794848442078, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 284.0089416503906, | |
| "epoch": 0.568, | |
| "grad_norm": 0.9552749101564338, | |
| "kl": 0.01422119140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1971, | |
| "reward": 0.5316947102546692, | |
| "reward_std": 0.14774499833583832, | |
| "rewards/length_reward": 0.06607140600681305, | |
| "rewards/similarity_reward": 0.46562325954437256, | |
| "step": 213 | |
| }, | |
| { | |
| "completion_length": 264.84375, | |
| "epoch": 0.5706666666666667, | |
| "grad_norm": 0.9328609206359839, | |
| "kl": 0.0128173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0611, | |
| "reward": 0.61468505859375, | |
| "reward_std": 0.0943475142121315, | |
| "rewards/length_reward": 0.08214282244443893, | |
| "rewards/similarity_reward": 0.5325421690940857, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 288.65625, | |
| "epoch": 0.5733333333333334, | |
| "grad_norm": 0.9888552258575887, | |
| "kl": 0.0147705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1398, | |
| "reward": 0.5841876864433289, | |
| "reward_std": 0.10097295790910721, | |
| "rewards/length_reward": 0.06294640898704529, | |
| "rewards/similarity_reward": 0.5212411880493164, | |
| "step": 215 | |
| }, | |
| { | |
| "completion_length": 270.28125, | |
| "epoch": 0.576, | |
| "grad_norm": 0.8803342156226522, | |
| "kl": 0.014892578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0359, | |
| "reward": 0.6344039440155029, | |
| "reward_std": 0.17091530561447144, | |
| "rewards/length_reward": 0.06607141345739365, | |
| "rewards/similarity_reward": 0.5683325529098511, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 255.12501525878906, | |
| "epoch": 0.5786666666666667, | |
| "grad_norm": 0.8979196392383272, | |
| "kl": 0.027099609375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0078, | |
| "reward": 0.661945641040802, | |
| "reward_std": 0.1631477326154709, | |
| "rewards/length_reward": 0.07767854630947113, | |
| "rewards/similarity_reward": 0.5842669606208801, | |
| "step": 217 | |
| }, | |
| { | |
| "completion_length": 263.58038330078125, | |
| "epoch": 0.5813333333333334, | |
| "grad_norm": 1.180022786404114, | |
| "kl": 0.0191650390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1111, | |
| "reward": 0.5408477187156677, | |
| "reward_std": 0.11287137866020203, | |
| "rewards/length_reward": 0.060267843306064606, | |
| "rewards/similarity_reward": 0.4805798828601837, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 293.24554443359375, | |
| "epoch": 0.584, | |
| "grad_norm": 1.082130352994329, | |
| "kl": 0.0113525390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.166, | |
| "reward": 0.6229541301727295, | |
| "reward_std": 0.18498755991458893, | |
| "rewards/length_reward": 0.06294640898704529, | |
| "rewards/similarity_reward": 0.5600076913833618, | |
| "step": 219 | |
| }, | |
| { | |
| "completion_length": 255.9553680419922, | |
| "epoch": 0.5866666666666667, | |
| "grad_norm": 0.8518142779942337, | |
| "kl": 0.0216064453125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0418, | |
| "reward": 0.6351791620254517, | |
| "reward_std": 0.1440075933933258, | |
| "rewards/length_reward": 0.06919640302658081, | |
| "rewards/similarity_reward": 0.5659827589988708, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 268.2589416503906, | |
| "epoch": 0.5893333333333334, | |
| "grad_norm": 0.8005051959777295, | |
| "kl": 0.00799560546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.029, | |
| "reward": 0.6807352304458618, | |
| "reward_std": 0.11082387715578079, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.6061817407608032, | |
| "step": 221 | |
| }, | |
| { | |
| "completion_length": 267.5089416503906, | |
| "epoch": 0.592, | |
| "grad_norm": 0.8727360410582777, | |
| "kl": 0.00927734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0333, | |
| "reward": 0.6831346154212952, | |
| "reward_std": 0.09732881933450699, | |
| "rewards/length_reward": 0.07633925974369049, | |
| "rewards/similarity_reward": 0.6067953109741211, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 271.2008972167969, | |
| "epoch": 0.5946666666666667, | |
| "grad_norm": 0.7543972270797626, | |
| "kl": 0.00921630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0251, | |
| "reward": 0.7254729866981506, | |
| "reward_std": 0.13280263543128967, | |
| "rewards/length_reward": 0.07723211497068405, | |
| "rewards/similarity_reward": 0.6482407450675964, | |
| "step": 223 | |
| }, | |
| { | |
| "completion_length": 276.33038330078125, | |
| "epoch": 0.5973333333333334, | |
| "grad_norm": 0.8477057601765857, | |
| "kl": 0.01470947265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.093, | |
| "reward": 0.6127163171768188, | |
| "reward_std": 0.14752325415611267, | |
| "rewards/length_reward": 0.06160712614655495, | |
| "rewards/similarity_reward": 0.5511091351509094, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 313.02679443359375, | |
| "epoch": 0.6, | |
| "grad_norm": 0.9444843501933834, | |
| "kl": 0.01953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1058, | |
| "reward": 0.6097243428230286, | |
| "reward_std": 0.1704142987728119, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5427600741386414, | |
| "step": 225 | |
| }, | |
| { | |
| "completion_length": 288.37054443359375, | |
| "epoch": 0.6026666666666667, | |
| "grad_norm": 0.8579103953599808, | |
| "kl": 0.01373291015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1122, | |
| "reward": 0.6366080641746521, | |
| "reward_std": 0.12147609889507294, | |
| "rewards/length_reward": 0.07232140004634857, | |
| "rewards/similarity_reward": 0.5642866492271423, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 310.1607360839844, | |
| "epoch": 0.6053333333333333, | |
| "grad_norm": 0.759533963504491, | |
| "kl": 0.01312255859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0699, | |
| "reward": 0.6739456057548523, | |
| "reward_std": 0.10951042920351028, | |
| "rewards/length_reward": 0.07276783138513565, | |
| "rewards/similarity_reward": 0.6011778116226196, | |
| "step": 227 | |
| }, | |
| { | |
| "completion_length": 239.0848388671875, | |
| "epoch": 0.608, | |
| "grad_norm": 0.9025930213219101, | |
| "kl": 0.01104736328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0215, | |
| "reward": 0.6142429709434509, | |
| "reward_std": 0.08829416334629059, | |
| "rewards/length_reward": 0.07633925974369049, | |
| "rewards/similarity_reward": 0.5379037261009216, | |
| "step": 228 | |
| }, | |
| { | |
| "completion_length": 278.3125, | |
| "epoch": 0.6106666666666667, | |
| "grad_norm": 0.886301576249163, | |
| "kl": 0.01104736328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0334, | |
| "reward": 0.7060741186141968, | |
| "reward_std": 0.11311851441860199, | |
| "rewards/length_reward": 0.07723211497068405, | |
| "rewards/similarity_reward": 0.6288419961929321, | |
| "step": 229 | |
| }, | |
| { | |
| "completion_length": 251.18751525878906, | |
| "epoch": 0.6133333333333333, | |
| "grad_norm": 2.476356671041086, | |
| "kl": 0.0244140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0096, | |
| "reward": 0.5848217010498047, | |
| "reward_std": 0.11717528849840164, | |
| "rewards/length_reward": 0.061160698533058167, | |
| "rewards/similarity_reward": 0.5236610770225525, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 236.29019165039062, | |
| "epoch": 0.616, | |
| "grad_norm": 0.9163834681525471, | |
| "kl": 0.0108642578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0201, | |
| "reward": 0.6646043658256531, | |
| "reward_std": 0.11276809126138687, | |
| "rewards/length_reward": 0.07991068810224533, | |
| "rewards/similarity_reward": 0.5846936702728271, | |
| "step": 231 | |
| }, | |
| { | |
| "completion_length": 317.0089416503906, | |
| "epoch": 0.6186666666666667, | |
| "grad_norm": 0.8636877609886525, | |
| "kl": 0.013427734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.036, | |
| "reward": 0.5721753835678101, | |
| "reward_std": 0.16120396554470062, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.506996750831604, | |
| "step": 232 | |
| }, | |
| { | |
| "completion_length": 307.7008972167969, | |
| "epoch": 0.6213333333333333, | |
| "grad_norm": 1.3729033520790577, | |
| "kl": 0.0166015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0383, | |
| "reward": 0.6260521411895752, | |
| "reward_std": 0.11853621900081635, | |
| "rewards/length_reward": 0.07767853885889053, | |
| "rewards/similarity_reward": 0.5483735799789429, | |
| "step": 233 | |
| }, | |
| { | |
| "completion_length": 283.5714416503906, | |
| "epoch": 0.624, | |
| "grad_norm": 0.8530397710423918, | |
| "kl": 0.016357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0206, | |
| "reward": 0.6967118382453918, | |
| "reward_std": 0.1562497317790985, | |
| "rewards/length_reward": 0.07857140153646469, | |
| "rewards/similarity_reward": 0.618140459060669, | |
| "step": 234 | |
| }, | |
| { | |
| "completion_length": 272.6160888671875, | |
| "epoch": 0.6266666666666667, | |
| "grad_norm": 0.8799568084373302, | |
| "kl": 0.01153564453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0181, | |
| "reward": 0.6851814985275269, | |
| "reward_std": 0.10234292596578598, | |
| "rewards/length_reward": 0.07142854481935501, | |
| "rewards/similarity_reward": 0.6137529611587524, | |
| "step": 235 | |
| }, | |
| { | |
| "completion_length": 316.4776916503906, | |
| "epoch": 0.6293333333333333, | |
| "grad_norm": 0.8224273598135922, | |
| "kl": 0.01416015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1293, | |
| "reward": 0.5818712115287781, | |
| "reward_std": 0.1467462033033371, | |
| "rewards/length_reward": 0.059374988079071045, | |
| "rewards/similarity_reward": 0.5224961638450623, | |
| "step": 236 | |
| }, | |
| { | |
| "completion_length": 276.1651916503906, | |
| "epoch": 0.632, | |
| "grad_norm": 0.8214244182848573, | |
| "kl": 0.01373291015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0324, | |
| "reward": 0.6201799511909485, | |
| "reward_std": 0.14638349413871765, | |
| "rewards/length_reward": 0.07098212093114853, | |
| "rewards/similarity_reward": 0.5491978526115417, | |
| "step": 237 | |
| }, | |
| { | |
| "completion_length": 314.15179443359375, | |
| "epoch": 0.6346666666666667, | |
| "grad_norm": 0.8751337279602847, | |
| "kl": 0.011474609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1279, | |
| "reward": 0.6131877899169922, | |
| "reward_std": 0.15269529819488525, | |
| "rewards/length_reward": 0.06383927166461945, | |
| "rewards/similarity_reward": 0.5493485331535339, | |
| "step": 238 | |
| }, | |
| { | |
| "completion_length": 241.5982208251953, | |
| "epoch": 0.6373333333333333, | |
| "grad_norm": 0.9065686563133485, | |
| "kl": 0.01300048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.053, | |
| "reward": 0.6322412490844727, | |
| "reward_std": 0.13913773000240326, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5648305416107178, | |
| "step": 239 | |
| }, | |
| { | |
| "completion_length": 282.3973388671875, | |
| "epoch": 0.64, | |
| "grad_norm": 0.8759378979761268, | |
| "kl": 0.0130615234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0351, | |
| "reward": 0.6563798785209656, | |
| "reward_std": 0.11333189904689789, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5858440399169922, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 258.2946472167969, | |
| "epoch": 0.6426666666666667, | |
| "grad_norm": 0.8083399774508907, | |
| "kl": 0.0115966796875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0089, | |
| "reward": 0.7068819403648376, | |
| "reward_std": 0.10833070427179337, | |
| "rewards/length_reward": 0.08392854034900665, | |
| "rewards/similarity_reward": 0.6229532957077026, | |
| "step": 241 | |
| }, | |
| { | |
| "completion_length": 226.43751525878906, | |
| "epoch": 0.6453333333333333, | |
| "grad_norm": 1.113129598732782, | |
| "kl": 0.01708984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0985, | |
| "reward": 0.47489413619041443, | |
| "reward_std": 0.14593513309955597, | |
| "rewards/length_reward": 0.053124986588954926, | |
| "rewards/similarity_reward": 0.4217691719532013, | |
| "step": 242 | |
| }, | |
| { | |
| "completion_length": 254.5982208251953, | |
| "epoch": 0.648, | |
| "grad_norm": 1.0574663727866278, | |
| "kl": 0.01171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0655, | |
| "reward": 0.585192084312439, | |
| "reward_std": 0.1674540489912033, | |
| "rewards/length_reward": 0.05937498062849045, | |
| "rewards/similarity_reward": 0.5258170962333679, | |
| "step": 243 | |
| }, | |
| { | |
| "completion_length": 245.9866180419922, | |
| "epoch": 0.6506666666666666, | |
| "grad_norm": 0.9866678905813414, | |
| "kl": 0.01385498046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0992, | |
| "reward": 0.6301730871200562, | |
| "reward_std": 0.11110112071037292, | |
| "rewards/length_reward": 0.07142854481935501, | |
| "rewards/similarity_reward": 0.5587445497512817, | |
| "step": 244 | |
| }, | |
| { | |
| "completion_length": 276.9419860839844, | |
| "epoch": 0.6533333333333333, | |
| "grad_norm": 0.8334751875894263, | |
| "kl": 0.01226806640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0199, | |
| "reward": 0.734417736530304, | |
| "reward_std": 0.13278159499168396, | |
| "rewards/length_reward": 0.07767854630947113, | |
| "rewards/similarity_reward": 0.6567391157150269, | |
| "step": 245 | |
| }, | |
| { | |
| "completion_length": 224.4241180419922, | |
| "epoch": 0.656, | |
| "grad_norm": 1.039042606617133, | |
| "kl": 0.0211181640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1146, | |
| "reward": 0.6315779089927673, | |
| "reward_std": 0.1330062597990036, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5610421895980835, | |
| "step": 246 | |
| }, | |
| { | |
| "completion_length": 252.9732208251953, | |
| "epoch": 0.6586666666666666, | |
| "grad_norm": 0.8538008305966633, | |
| "kl": 0.01275634765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0224, | |
| "reward": 0.6646360754966736, | |
| "reward_std": 0.12239360809326172, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5900824666023254, | |
| "step": 247 | |
| }, | |
| { | |
| "completion_length": 218.7366180419922, | |
| "epoch": 0.6613333333333333, | |
| "grad_norm": 0.963162350651896, | |
| "kl": 0.01507568359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0349, | |
| "reward": 0.6808683276176453, | |
| "reward_std": 0.13527972996234894, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.6089933514595032, | |
| "step": 248 | |
| }, | |
| { | |
| "completion_length": 254.5982208251953, | |
| "epoch": 0.664, | |
| "grad_norm": 0.9385464548992294, | |
| "kl": 0.0179443359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0288, | |
| "reward": 0.631544828414917, | |
| "reward_std": 0.13531894981861115, | |
| "rewards/length_reward": 0.06874997913837433, | |
| "rewards/similarity_reward": 0.5627948045730591, | |
| "step": 249 | |
| }, | |
| { | |
| "completion_length": 291.37054443359375, | |
| "epoch": 0.6666666666666666, | |
| "grad_norm": 0.9140004673035673, | |
| "kl": 0.01708984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0837, | |
| "reward": 0.5439311861991882, | |
| "reward_std": 0.13576674461364746, | |
| "rewards/length_reward": 0.05848212540149689, | |
| "rewards/similarity_reward": 0.48544901609420776, | |
| "step": 250 | |
| }, | |
| { | |
| "completion_length": 302.15179443359375, | |
| "epoch": 0.6693333333333333, | |
| "grad_norm": 0.8827607842873424, | |
| "kl": 0.0137939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1511, | |
| "reward": 0.521554172039032, | |
| "reward_std": 0.13812950253486633, | |
| "rewards/length_reward": 0.06249998137354851, | |
| "rewards/similarity_reward": 0.459054172039032, | |
| "step": 251 | |
| }, | |
| { | |
| "completion_length": 256.67413330078125, | |
| "epoch": 0.672, | |
| "grad_norm": 0.9030494234496588, | |
| "kl": 0.015869140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0098, | |
| "reward": 0.6373765468597412, | |
| "reward_std": 0.16011908650398254, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5721979141235352, | |
| "step": 252 | |
| }, | |
| { | |
| "completion_length": 263.4464416503906, | |
| "epoch": 0.6746666666666666, | |
| "grad_norm": 0.9917515330394924, | |
| "kl": 0.0157470703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1071, | |
| "reward": 0.5835117697715759, | |
| "reward_std": 0.15427549183368683, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.5214581489562988, | |
| "step": 253 | |
| }, | |
| { | |
| "completion_length": 263.1875, | |
| "epoch": 0.6773333333333333, | |
| "grad_norm": 0.8527537949516601, | |
| "kl": 0.01007080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0704, | |
| "reward": 0.659292459487915, | |
| "reward_std": 0.11287476867437363, | |
| "rewards/length_reward": 0.07723211497068405, | |
| "rewards/similarity_reward": 0.5820602774620056, | |
| "step": 254 | |
| }, | |
| { | |
| "completion_length": 247.35714721679688, | |
| "epoch": 0.68, | |
| "grad_norm": 0.9306878660258886, | |
| "kl": 0.018798828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1071, | |
| "reward": 0.5428202748298645, | |
| "reward_std": 0.14576061069965363, | |
| "rewards/length_reward": 0.055357132107019424, | |
| "rewards/similarity_reward": 0.4874631464481354, | |
| "step": 255 | |
| }, | |
| { | |
| "completion_length": 304.77679443359375, | |
| "epoch": 0.6826666666666666, | |
| "grad_norm": 0.8122651765327112, | |
| "kl": 0.01123046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0722, | |
| "reward": 0.6534665822982788, | |
| "reward_std": 0.17661263048648834, | |
| "rewards/length_reward": 0.06651783734560013, | |
| "rewards/similarity_reward": 0.5869486927986145, | |
| "step": 256 | |
| }, | |
| { | |
| "completion_length": 316.0, | |
| "epoch": 0.6853333333333333, | |
| "grad_norm": 0.7637865876113388, | |
| "kl": 0.01153564453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0805, | |
| "reward": 0.6809090375900269, | |
| "reward_std": 0.12792253494262695, | |
| "rewards/length_reward": 0.07499997317790985, | |
| "rewards/similarity_reward": 0.6059090495109558, | |
| "step": 257 | |
| }, | |
| { | |
| "completion_length": 267.7410888671875, | |
| "epoch": 0.688, | |
| "grad_norm": 0.908951786987476, | |
| "kl": 0.0123291015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0919, | |
| "reward": 0.6498162150382996, | |
| "reward_std": 0.12118736654520035, | |
| "rewards/length_reward": 0.057589273899793625, | |
| "rewards/similarity_reward": 0.5922268629074097, | |
| "step": 258 | |
| }, | |
| { | |
| "completion_length": 255.13394165039062, | |
| "epoch": 0.6906666666666667, | |
| "grad_norm": 0.9868527698980504, | |
| "kl": 0.01116943359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0518, | |
| "reward": 0.6177918910980225, | |
| "reward_std": 0.1076013594865799, | |
| "rewards/length_reward": 0.07232140004634857, | |
| "rewards/similarity_reward": 0.5454704165458679, | |
| "step": 259 | |
| }, | |
| { | |
| "completion_length": 211.96429443359375, | |
| "epoch": 0.6933333333333334, | |
| "grad_norm": 1.1390879691759828, | |
| "kl": 0.01251220703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0376, | |
| "reward": 0.6133698225021362, | |
| "reward_std": 0.1394728273153305, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5414947867393494, | |
| "step": 260 | |
| }, | |
| { | |
| "completion_length": 226.0491180419922, | |
| "epoch": 0.696, | |
| "grad_norm": 0.9181631341556423, | |
| "kl": 0.01458740234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0146, | |
| "reward": 0.6710724234580994, | |
| "reward_std": 0.12615807354450226, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.6041080951690674, | |
| "step": 261 | |
| }, | |
| { | |
| "completion_length": 290.71875, | |
| "epoch": 0.6986666666666667, | |
| "grad_norm": 0.9242817884998483, | |
| "kl": 0.010986328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0492, | |
| "reward": 0.758361279964447, | |
| "reward_std": 0.0939282700419426, | |
| "rewards/length_reward": 0.07812497019767761, | |
| "rewards/similarity_reward": 0.680236279964447, | |
| "step": 262 | |
| }, | |
| { | |
| "completion_length": 295.2276916503906, | |
| "epoch": 0.7013333333333334, | |
| "grad_norm": 0.9268240432998979, | |
| "kl": 0.01434326171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0777, | |
| "reward": 0.5887910723686218, | |
| "reward_std": 0.15629605948925018, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5236124992370605, | |
| "step": 263 | |
| }, | |
| { | |
| "completion_length": 197.97769165039062, | |
| "epoch": 0.704, | |
| "grad_norm": 0.9100124092555839, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0176, | |
| "reward": 0.5667382478713989, | |
| "reward_std": 0.1061021164059639, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.4993274211883545, | |
| "step": 264 | |
| }, | |
| { | |
| "completion_length": 242.96429443359375, | |
| "epoch": 0.7066666666666667, | |
| "grad_norm": 0.9638821889698813, | |
| "kl": 0.0205078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0458, | |
| "reward": 0.6227900981903076, | |
| "reward_std": 0.1025083139538765, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.5571650862693787, | |
| "step": 265 | |
| }, | |
| { | |
| "completion_length": 223.3973388671875, | |
| "epoch": 0.7093333333333334, | |
| "grad_norm": 1.0581452656511359, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0641, | |
| "reward": 0.6723743081092834, | |
| "reward_std": 0.1159893348813057, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.6071956753730774, | |
| "step": 266 | |
| }, | |
| { | |
| "completion_length": 270.2098388671875, | |
| "epoch": 0.712, | |
| "grad_norm": 0.8609101277095984, | |
| "kl": 0.0146484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0419, | |
| "reward": 0.7352553009986877, | |
| "reward_std": 0.13158555328845978, | |
| "rewards/length_reward": 0.07499997317790985, | |
| "rewards/similarity_reward": 0.6602552533149719, | |
| "step": 267 | |
| }, | |
| { | |
| "completion_length": 286.65179443359375, | |
| "epoch": 0.7146666666666667, | |
| "grad_norm": 0.85187609520963, | |
| "kl": 0.01470947265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0801, | |
| "reward": 0.5860309600830078, | |
| "reward_std": 0.12361589819192886, | |
| "rewards/length_reward": 0.06562498211860657, | |
| "rewards/similarity_reward": 0.5204059481620789, | |
| "step": 268 | |
| }, | |
| { | |
| "completion_length": 258.5, | |
| "epoch": 0.7173333333333334, | |
| "grad_norm": 0.8462850933355499, | |
| "kl": 0.01080322265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1016, | |
| "reward": 0.7648903131484985, | |
| "reward_std": 0.08282845467329025, | |
| "rewards/length_reward": 0.08348211646080017, | |
| "rewards/similarity_reward": 0.6814082264900208, | |
| "step": 269 | |
| }, | |
| { | |
| "completion_length": 255.4598388671875, | |
| "epoch": 0.72, | |
| "grad_norm": 0.8789894818999019, | |
| "kl": 0.01507568359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1231, | |
| "reward": 0.6465427875518799, | |
| "reward_std": 0.12229768186807632, | |
| "rewards/length_reward": 0.06607140600681305, | |
| "rewards/similarity_reward": 0.5804713368415833, | |
| "step": 270 | |
| }, | |
| { | |
| "completion_length": 273.01788330078125, | |
| "epoch": 0.7226666666666667, | |
| "grad_norm": 0.8695715294946503, | |
| "kl": 0.0184326171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.087, | |
| "reward": 0.6258493661880493, | |
| "reward_std": 0.13294367492198944, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5553135871887207, | |
| "step": 271 | |
| }, | |
| { | |
| "completion_length": 260.8169860839844, | |
| "epoch": 0.7253333333333334, | |
| "grad_norm": 1.2217664906535957, | |
| "kl": 0.015380859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0431, | |
| "reward": 0.6779768466949463, | |
| "reward_std": 0.11835993826389313, | |
| "rewards/length_reward": 0.07678568363189697, | |
| "rewards/similarity_reward": 0.6011910438537598, | |
| "step": 272 | |
| }, | |
| { | |
| "completion_length": 284.625, | |
| "epoch": 0.728, | |
| "grad_norm": 0.70076206752431, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0623, | |
| "reward": 0.6957324147224426, | |
| "reward_std": 0.11651583760976791, | |
| "rewards/length_reward": 0.08348210901021957, | |
| "rewards/similarity_reward": 0.6122502088546753, | |
| "step": 273 | |
| }, | |
| { | |
| "completion_length": 255.76339721679688, | |
| "epoch": 0.7306666666666667, | |
| "grad_norm": 0.9105713339266117, | |
| "kl": 0.012939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0998, | |
| "reward": 0.6966086626052856, | |
| "reward_std": 0.08685937523841858, | |
| "rewards/length_reward": 0.07187496870756149, | |
| "rewards/similarity_reward": 0.624733567237854, | |
| "step": 274 | |
| }, | |
| { | |
| "completion_length": 296.76788330078125, | |
| "epoch": 0.7333333333333333, | |
| "grad_norm": 0.7414942240643484, | |
| "kl": 0.00921630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0451, | |
| "reward": 0.7098910212516785, | |
| "reward_std": 0.09432552009820938, | |
| "rewards/length_reward": 0.08482139557600021, | |
| "rewards/similarity_reward": 0.6250695586204529, | |
| "step": 275 | |
| }, | |
| { | |
| "completion_length": 270.0669860839844, | |
| "epoch": 0.736, | |
| "grad_norm": 0.898333119316704, | |
| "kl": 0.0137939453125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0225, | |
| "reward": 0.7013087868690491, | |
| "reward_std": 0.11285625398159027, | |
| "rewards/length_reward": 0.0741071105003357, | |
| "rewards/similarity_reward": 0.6272015571594238, | |
| "step": 276 | |
| }, | |
| { | |
| "completion_length": 198.93751525878906, | |
| "epoch": 0.7386666666666667, | |
| "grad_norm": 1.1511982372559852, | |
| "kl": 0.0205078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0183, | |
| "reward": 0.5090009570121765, | |
| "reward_std": 0.13011598587036133, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.4469473958015442, | |
| "step": 277 | |
| }, | |
| { | |
| "completion_length": 271.58038330078125, | |
| "epoch": 0.7413333333333333, | |
| "grad_norm": 0.8195827963319392, | |
| "kl": 0.01092529296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0099, | |
| "reward": 0.6257685422897339, | |
| "reward_std": 0.1082058921456337, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.5503220558166504, | |
| "step": 278 | |
| }, | |
| { | |
| "completion_length": 273.2232360839844, | |
| "epoch": 0.744, | |
| "grad_norm": 0.9459393517121532, | |
| "kl": 0.01422119140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0902, | |
| "reward": 0.6197928190231323, | |
| "reward_std": 0.15125982463359833, | |
| "rewards/length_reward": 0.06874997913837433, | |
| "rewards/similarity_reward": 0.551042914390564, | |
| "step": 279 | |
| }, | |
| { | |
| "completion_length": 279.65625, | |
| "epoch": 0.7466666666666667, | |
| "grad_norm": 0.9319904211339567, | |
| "kl": 0.0126953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0701, | |
| "reward": 0.6224436163902283, | |
| "reward_std": 0.11631693691015244, | |
| "rewards/length_reward": 0.07008926570415497, | |
| "rewards/similarity_reward": 0.5523543357849121, | |
| "step": 280 | |
| }, | |
| { | |
| "completion_length": 289.61163330078125, | |
| "epoch": 0.7493333333333333, | |
| "grad_norm": 1.071648613346095, | |
| "kl": 0.010009765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1723, | |
| "reward": 0.6961318850517273, | |
| "reward_std": 0.10897497087717056, | |
| "rewards/length_reward": 0.07723211497068405, | |
| "rewards/similarity_reward": 0.6188997030258179, | |
| "step": 281 | |
| }, | |
| { | |
| "completion_length": 243.8348388671875, | |
| "epoch": 0.752, | |
| "grad_norm": 0.994726741013797, | |
| "kl": 0.01495361328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0637, | |
| "reward": 0.6346572637557983, | |
| "reward_std": 0.10820147395133972, | |
| "rewards/length_reward": 0.07008925825357437, | |
| "rewards/similarity_reward": 0.5645679831504822, | |
| "step": 282 | |
| }, | |
| { | |
| "completion_length": 251.38394165039062, | |
| "epoch": 0.7546666666666667, | |
| "grad_norm": 0.976491323713793, | |
| "kl": 0.015380859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0921, | |
| "reward": 0.5713584423065186, | |
| "reward_std": 0.1460532546043396, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.5061798095703125, | |
| "step": 283 | |
| }, | |
| { | |
| "completion_length": 295.8348388671875, | |
| "epoch": 0.7573333333333333, | |
| "grad_norm": 0.8830201574560093, | |
| "kl": 0.0140380859375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0096, | |
| "reward": 0.5986955761909485, | |
| "reward_std": 0.15517514944076538, | |
| "rewards/length_reward": 0.07008926570415497, | |
| "rewards/similarity_reward": 0.5286062955856323, | |
| "step": 284 | |
| }, | |
| { | |
| "completion_length": 247.72769165039062, | |
| "epoch": 0.76, | |
| "grad_norm": 0.8557479345922782, | |
| "kl": 0.01385498046875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0017, | |
| "reward": 0.6696428656578064, | |
| "reward_std": 0.10514307767152786, | |
| "rewards/length_reward": 0.07857140153646469, | |
| "rewards/similarity_reward": 0.5910714268684387, | |
| "step": 285 | |
| }, | |
| { | |
| "completion_length": 264.9196472167969, | |
| "epoch": 0.7626666666666667, | |
| "grad_norm": 0.900029253879394, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.011, | |
| "reward": 0.6437191367149353, | |
| "reward_std": 0.12881356477737427, | |
| "rewards/length_reward": 0.07991068065166473, | |
| "rewards/similarity_reward": 0.5638083815574646, | |
| "step": 286 | |
| }, | |
| { | |
| "completion_length": 275.76788330078125, | |
| "epoch": 0.7653333333333333, | |
| "grad_norm": 0.8434547706339766, | |
| "kl": 0.01519775390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0667, | |
| "reward": 0.6621875762939453, | |
| "reward_std": 0.15463142096996307, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.5925447344779968, | |
| "step": 287 | |
| }, | |
| { | |
| "completion_length": 251.4866180419922, | |
| "epoch": 0.768, | |
| "grad_norm": 0.9853184231195431, | |
| "kl": 0.01507568359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0183, | |
| "reward": 0.5950483679771423, | |
| "reward_std": 0.1278952956199646, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.5196019411087036, | |
| "step": 288 | |
| }, | |
| { | |
| "completion_length": 279.9419860839844, | |
| "epoch": 0.7706666666666667, | |
| "grad_norm": 0.8374958728984951, | |
| "kl": 0.0126953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1275, | |
| "reward": 0.6563042402267456, | |
| "reward_std": 0.14691661298274994, | |
| "rewards/length_reward": 0.07098212093114853, | |
| "rewards/similarity_reward": 0.5853220820426941, | |
| "step": 289 | |
| }, | |
| { | |
| "completion_length": 283.12054443359375, | |
| "epoch": 0.7733333333333333, | |
| "grad_norm": 0.9945411525052974, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0685, | |
| "reward": 0.7337676882743835, | |
| "reward_std": 0.09462190419435501, | |
| "rewards/length_reward": 0.07633925974369049, | |
| "rewards/similarity_reward": 0.6574283838272095, | |
| "step": 290 | |
| }, | |
| { | |
| "completion_length": 286.1026916503906, | |
| "epoch": 0.776, | |
| "grad_norm": 0.7519820653879999, | |
| "kl": 0.01275634765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.069, | |
| "reward": 0.6660787463188171, | |
| "reward_std": 0.13770896196365356, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5915251970291138, | |
| "step": 291 | |
| }, | |
| { | |
| "completion_length": 300.4419860839844, | |
| "epoch": 0.7786666666666666, | |
| "grad_norm": 0.9216655286630218, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1068, | |
| "reward": 0.7210602164268494, | |
| "reward_std": 0.15427368879318237, | |
| "rewards/length_reward": 0.08035711199045181, | |
| "rewards/similarity_reward": 0.6407030820846558, | |
| "step": 292 | |
| }, | |
| { | |
| "completion_length": 252.0178680419922, | |
| "epoch": 0.7813333333333333, | |
| "grad_norm": 0.8654140719017164, | |
| "kl": 0.022705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1224, | |
| "reward": 0.6323553919792175, | |
| "reward_std": 0.1670098751783371, | |
| "rewards/length_reward": 0.06651784479618073, | |
| "rewards/similarity_reward": 0.5658375024795532, | |
| "step": 293 | |
| }, | |
| { | |
| "completion_length": 274.8839416503906, | |
| "epoch": 0.784, | |
| "grad_norm": 0.7786721291031314, | |
| "kl": 0.0142822265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0166, | |
| "reward": 0.7442488074302673, | |
| "reward_std": 0.10799020528793335, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.6696951389312744, | |
| "step": 294 | |
| }, | |
| { | |
| "completion_length": 279.1294860839844, | |
| "epoch": 0.7866666666666666, | |
| "grad_norm": 0.8056776017830058, | |
| "kl": 0.011962890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0048, | |
| "reward": 0.6635159850120544, | |
| "reward_std": 0.11003357172012329, | |
| "rewards/length_reward": 0.07008926570415497, | |
| "rewards/similarity_reward": 0.5934267044067383, | |
| "step": 295 | |
| }, | |
| { | |
| "completion_length": 254.96429443359375, | |
| "epoch": 0.7893333333333333, | |
| "grad_norm": 1.0354511241505295, | |
| "kl": 0.0137939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0919, | |
| "reward": 0.6252850294113159, | |
| "reward_std": 0.10695895552635193, | |
| "rewards/length_reward": 0.07142854481935501, | |
| "rewards/similarity_reward": 0.553856372833252, | |
| "step": 296 | |
| }, | |
| { | |
| "completion_length": 233.25894165039062, | |
| "epoch": 0.792, | |
| "grad_norm": 0.9468597047570717, | |
| "kl": 0.01275634765625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0101, | |
| "reward": 0.6699472665786743, | |
| "reward_std": 0.16828653216362, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.6003044247627258, | |
| "step": 297 | |
| }, | |
| { | |
| "completion_length": 249.1428680419922, | |
| "epoch": 0.7946666666666666, | |
| "grad_norm": 0.9140690517111535, | |
| "kl": 0.01141357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.072, | |
| "reward": 0.6824041604995728, | |
| "reward_std": 0.12977474927902222, | |
| "rewards/length_reward": 0.08035711199045181, | |
| "rewards/similarity_reward": 0.6020469069480896, | |
| "step": 298 | |
| }, | |
| { | |
| "completion_length": 276.3571472167969, | |
| "epoch": 0.7973333333333333, | |
| "grad_norm": 0.8259435738042828, | |
| "kl": 0.01141357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0105, | |
| "reward": 0.7071071863174438, | |
| "reward_std": 0.08347765356302261, | |
| "rewards/length_reward": 0.08169639110565186, | |
| "rewards/similarity_reward": 0.625410795211792, | |
| "step": 299 | |
| }, | |
| { | |
| "completion_length": 270.0758972167969, | |
| "epoch": 0.8, | |
| "grad_norm": 0.8981450111371676, | |
| "kl": 0.0126953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0594, | |
| "reward": 0.6137918829917908, | |
| "reward_std": 0.12631313502788544, | |
| "rewards/length_reward": 0.064732126891613, | |
| "rewards/similarity_reward": 0.5490598082542419, | |
| "step": 300 | |
| }, | |
| { | |
| "completion_length": 278.83038330078125, | |
| "epoch": 0.8026666666666666, | |
| "grad_norm": 0.8303352041330266, | |
| "kl": 0.01397705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0166, | |
| "reward": 0.6976829767227173, | |
| "reward_std": 0.11335788667201996, | |
| "rewards/length_reward": 0.08169639110565186, | |
| "rewards/similarity_reward": 0.6159866452217102, | |
| "step": 301 | |
| }, | |
| { | |
| "completion_length": 257.5357360839844, | |
| "epoch": 0.8053333333333333, | |
| "grad_norm": 0.8867998574709848, | |
| "kl": 0.0135498046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0754, | |
| "reward": 0.6026350855827332, | |
| "reward_std": 0.0930032953619957, | |
| "rewards/length_reward": 0.07589282840490341, | |
| "rewards/similarity_reward": 0.526742160320282, | |
| "step": 302 | |
| }, | |
| { | |
| "completion_length": 283.5669860839844, | |
| "epoch": 0.808, | |
| "grad_norm": 0.8168218668358965, | |
| "kl": 0.0120849609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0773, | |
| "reward": 0.5895494222640991, | |
| "reward_std": 0.1272886097431183, | |
| "rewards/length_reward": 0.06874997913837433, | |
| "rewards/similarity_reward": 0.5207993984222412, | |
| "step": 303 | |
| }, | |
| { | |
| "completion_length": 303.6026916503906, | |
| "epoch": 0.8106666666666666, | |
| "grad_norm": 0.8899545222480755, | |
| "kl": 0.012939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0372, | |
| "reward": 0.6067291498184204, | |
| "reward_std": 0.1214829757809639, | |
| "rewards/length_reward": 0.07008926570415497, | |
| "rewards/similarity_reward": 0.5366398692131042, | |
| "step": 304 | |
| }, | |
| { | |
| "completion_length": 294.3482360839844, | |
| "epoch": 0.8133333333333334, | |
| "grad_norm": 0.8194602682013028, | |
| "kl": 0.01153564453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0566, | |
| "reward": 0.7332960963249207, | |
| "reward_std": 0.08479318022727966, | |
| "rewards/length_reward": 0.07767854630947113, | |
| "rewards/similarity_reward": 0.6556174755096436, | |
| "step": 305 | |
| }, | |
| { | |
| "completion_length": 266.4375, | |
| "epoch": 0.816, | |
| "grad_norm": 0.987377079764631, | |
| "kl": 0.0133056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0409, | |
| "reward": 0.6382983326911926, | |
| "reward_std": 0.1240207627415657, | |
| "rewards/length_reward": 0.07366068661212921, | |
| "rewards/similarity_reward": 0.5646375417709351, | |
| "step": 306 | |
| }, | |
| { | |
| "completion_length": 267.95538330078125, | |
| "epoch": 0.8186666666666667, | |
| "grad_norm": 0.9217292474743543, | |
| "kl": 0.01263427734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0267, | |
| "reward": 0.6616266965866089, | |
| "reward_std": 0.1070173904299736, | |
| "rewards/length_reward": 0.06964283436536789, | |
| "rewards/similarity_reward": 0.5919837355613708, | |
| "step": 307 | |
| }, | |
| { | |
| "completion_length": 279.0535888671875, | |
| "epoch": 0.8213333333333334, | |
| "grad_norm": 0.8072891907153936, | |
| "kl": 0.01385498046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0943, | |
| "reward": 0.6182869672775269, | |
| "reward_std": 0.1361446976661682, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5464120507240295, | |
| "step": 308 | |
| }, | |
| { | |
| "completion_length": 287.90625, | |
| "epoch": 0.824, | |
| "grad_norm": 0.9054980085837359, | |
| "kl": 0.01153564453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.068, | |
| "reward": 0.7372510433197021, | |
| "reward_std": 0.0957195907831192, | |
| "rewards/length_reward": 0.07499997317790985, | |
| "rewards/similarity_reward": 0.6622509956359863, | |
| "step": 309 | |
| }, | |
| { | |
| "completion_length": 258.3973388671875, | |
| "epoch": 0.8266666666666667, | |
| "grad_norm": 0.9082585914171709, | |
| "kl": 0.01202392578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0061, | |
| "reward": 0.647227942943573, | |
| "reward_std": 0.1310262531042099, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5766921043395996, | |
| "step": 310 | |
| }, | |
| { | |
| "completion_length": 221.2991180419922, | |
| "epoch": 0.8293333333333334, | |
| "grad_norm": 0.9448119727952166, | |
| "kl": 0.021728515625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0416, | |
| "reward": 0.6708490252494812, | |
| "reward_std": 0.1254904717206955, | |
| "rewards/length_reward": 0.07812497019767761, | |
| "rewards/similarity_reward": 0.5927240252494812, | |
| "step": 311 | |
| }, | |
| { | |
| "completion_length": 249.3616180419922, | |
| "epoch": 0.832, | |
| "grad_norm": 0.8967865636957736, | |
| "kl": 0.010009765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0332, | |
| "reward": 0.7173448801040649, | |
| "reward_std": 0.10435692220926285, | |
| "rewards/length_reward": 0.08214282244443893, | |
| "rewards/similarity_reward": 0.6352020502090454, | |
| "step": 312 | |
| }, | |
| { | |
| "completion_length": 246.80804443359375, | |
| "epoch": 0.8346666666666667, | |
| "grad_norm": 0.9778150152016866, | |
| "kl": 0.01373291015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0158, | |
| "reward": 0.6462720036506653, | |
| "reward_std": 0.13069510459899902, | |
| "rewards/length_reward": 0.07499997317790985, | |
| "rewards/similarity_reward": 0.5712720155715942, | |
| "step": 313 | |
| }, | |
| { | |
| "completion_length": 244.13839721679688, | |
| "epoch": 0.8373333333333334, | |
| "grad_norm": 0.9354621189513169, | |
| "kl": 0.01214599609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.264, | |
| "reward": 0.6220008730888367, | |
| "reward_std": 0.12123651802539825, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.5599472522735596, | |
| "step": 314 | |
| }, | |
| { | |
| "completion_length": 269.5446472167969, | |
| "epoch": 0.84, | |
| "grad_norm": 0.9611408485021674, | |
| "kl": 0.01513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0553, | |
| "reward": 0.725965678691864, | |
| "reward_std": 0.09657153487205505, | |
| "rewards/length_reward": 0.07723211497068405, | |
| "rewards/similarity_reward": 0.6487335562705994, | |
| "step": 315 | |
| }, | |
| { | |
| "completion_length": 274.84375, | |
| "epoch": 0.8426666666666667, | |
| "grad_norm": 0.8797127493353065, | |
| "kl": 0.01129150390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.014, | |
| "reward": 0.6985806226730347, | |
| "reward_std": 0.11010481417179108, | |
| "rewards/length_reward": 0.0741071105003357, | |
| "rewards/similarity_reward": 0.6244734525680542, | |
| "step": 316 | |
| }, | |
| { | |
| "completion_length": 272.8348388671875, | |
| "epoch": 0.8453333333333334, | |
| "grad_norm": 0.779139564448991, | |
| "kl": 0.01055908203125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0161, | |
| "reward": 0.7422655820846558, | |
| "reward_std": 0.06264423578977585, | |
| "rewards/length_reward": 0.08124996721744537, | |
| "rewards/similarity_reward": 0.6610156297683716, | |
| "step": 317 | |
| }, | |
| { | |
| "completion_length": 229.7678680419922, | |
| "epoch": 0.848, | |
| "grad_norm": 4.278780421719415, | |
| "kl": 0.0390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0681, | |
| "reward": 0.6605138778686523, | |
| "reward_std": 0.14701789617538452, | |
| "rewards/length_reward": 0.07187496870756149, | |
| "rewards/similarity_reward": 0.5886389017105103, | |
| "step": 318 | |
| }, | |
| { | |
| "completion_length": 286.2723388671875, | |
| "epoch": 0.8506666666666667, | |
| "grad_norm": 1.0371097952967725, | |
| "kl": 0.01611328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1372, | |
| "reward": 0.6110987663269043, | |
| "reward_std": 0.18697677552700043, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.549045205116272, | |
| "step": 319 | |
| }, | |
| { | |
| "completion_length": 260.0625, | |
| "epoch": 0.8533333333333334, | |
| "grad_norm": 0.9056186985134533, | |
| "kl": 0.009521484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0351, | |
| "reward": 0.6572511196136475, | |
| "reward_std": 0.08565808087587357, | |
| "rewards/length_reward": 0.07812497019767761, | |
| "rewards/similarity_reward": 0.5791261196136475, | |
| "step": 320 | |
| }, | |
| { | |
| "completion_length": 281.71875, | |
| "epoch": 0.856, | |
| "grad_norm": 0.71560536286136, | |
| "kl": 0.00860595703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0613, | |
| "reward": 0.7018586993217468, | |
| "reward_std": 0.14483648538589478, | |
| "rewards/length_reward": 0.08169639110565186, | |
| "rewards/similarity_reward": 0.620162308216095, | |
| "step": 321 | |
| }, | |
| { | |
| "completion_length": 258.2276916503906, | |
| "epoch": 0.8586666666666667, | |
| "grad_norm": 0.9896625298898443, | |
| "kl": 0.0181884765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2194, | |
| "reward": 0.5979973077774048, | |
| "reward_std": 0.13931064307689667, | |
| "rewards/length_reward": 0.06249998137354851, | |
| "rewards/similarity_reward": 0.5354973077774048, | |
| "step": 322 | |
| }, | |
| { | |
| "completion_length": 237.1741180419922, | |
| "epoch": 0.8613333333333333, | |
| "grad_norm": 0.9609452774135127, | |
| "kl": 0.01513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0016, | |
| "reward": 0.6683059334754944, | |
| "reward_std": 0.10867080092430115, | |
| "rewards/length_reward": 0.07991068810224533, | |
| "rewards/similarity_reward": 0.5883952975273132, | |
| "step": 323 | |
| }, | |
| { | |
| "completion_length": 254.01339721679688, | |
| "epoch": 0.864, | |
| "grad_norm": 1.0568546085260409, | |
| "kl": 0.01531982421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0933, | |
| "reward": 0.6546286940574646, | |
| "reward_std": 0.11366698145866394, | |
| "rewards/length_reward": 0.07008926570415497, | |
| "rewards/similarity_reward": 0.5845393538475037, | |
| "step": 324 | |
| }, | |
| { | |
| "completion_length": 242.03126525878906, | |
| "epoch": 0.8666666666666667, | |
| "grad_norm": 0.926913110151395, | |
| "kl": 0.014892578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0865, | |
| "reward": 0.5440469980239868, | |
| "reward_std": 0.11594089865684509, | |
| "rewards/length_reward": 0.06383926421403885, | |
| "rewards/similarity_reward": 0.48020774126052856, | |
| "step": 325 | |
| }, | |
| { | |
| "completion_length": 228.21429443359375, | |
| "epoch": 0.8693333333333333, | |
| "grad_norm": 1.0043052790420461, | |
| "kl": 0.01385498046875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0272, | |
| "reward": 0.6953443884849548, | |
| "reward_std": 0.1263352483510971, | |
| "rewards/length_reward": 0.06785711646080017, | |
| "rewards/similarity_reward": 0.6274873614311218, | |
| "step": 326 | |
| }, | |
| { | |
| "completion_length": 307.3125, | |
| "epoch": 0.872, | |
| "grad_norm": 0.7734278067274474, | |
| "kl": 0.0089111328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0474, | |
| "reward": 0.689052402973175, | |
| "reward_std": 0.10326018929481506, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.6144987940788269, | |
| "step": 327 | |
| }, | |
| { | |
| "completion_length": 256.7589416503906, | |
| "epoch": 0.8746666666666667, | |
| "grad_norm": 0.8349456447351374, | |
| "kl": 0.01275634765625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0053, | |
| "reward": 0.6755567193031311, | |
| "reward_std": 0.12627391517162323, | |
| "rewards/length_reward": 0.07321426272392273, | |
| "rewards/similarity_reward": 0.6023423671722412, | |
| "step": 328 | |
| }, | |
| { | |
| "completion_length": 272.67413330078125, | |
| "epoch": 0.8773333333333333, | |
| "grad_norm": 0.9337813221087722, | |
| "kl": 0.013671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0863, | |
| "reward": 0.6436842083930969, | |
| "reward_std": 0.1272781938314438, | |
| "rewards/length_reward": 0.06339284032583237, | |
| "rewards/similarity_reward": 0.5802912712097168, | |
| "step": 329 | |
| }, | |
| { | |
| "completion_length": 268.5401916503906, | |
| "epoch": 0.88, | |
| "grad_norm": 0.7822000698940798, | |
| "kl": 0.014892578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0076, | |
| "reward": 0.6488507986068726, | |
| "reward_std": 0.1486339569091797, | |
| "rewards/length_reward": 0.07812497019767761, | |
| "rewards/similarity_reward": 0.5707257986068726, | |
| "step": 330 | |
| }, | |
| { | |
| "completion_length": 255.2991180419922, | |
| "epoch": 0.8826666666666667, | |
| "grad_norm": 0.9970416796611882, | |
| "kl": 0.01348876953125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0091, | |
| "reward": 0.623367190361023, | |
| "reward_std": 0.11435237526893616, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.547920823097229, | |
| "step": 331 | |
| }, | |
| { | |
| "completion_length": 281.2544860839844, | |
| "epoch": 0.8853333333333333, | |
| "grad_norm": 0.9393864746480084, | |
| "kl": 0.0177001953125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0269, | |
| "reward": 0.6835038661956787, | |
| "reward_std": 0.12119947373867035, | |
| "rewards/length_reward": 0.07366069406270981, | |
| "rewards/similarity_reward": 0.6098431348800659, | |
| "step": 332 | |
| }, | |
| { | |
| "completion_length": 277.3883972167969, | |
| "epoch": 0.888, | |
| "grad_norm": 0.8857739277618905, | |
| "kl": 0.0133056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1155, | |
| "reward": 0.6458525657653809, | |
| "reward_std": 0.10718663036823273, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5739776492118835, | |
| "step": 333 | |
| }, | |
| { | |
| "completion_length": 222.0491180419922, | |
| "epoch": 0.8906666666666667, | |
| "grad_norm": 0.9211484451974372, | |
| "kl": 0.0145263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0326, | |
| "reward": 0.6779581308364868, | |
| "reward_std": 0.09158685058355331, | |
| "rewards/length_reward": 0.07098212093114853, | |
| "rewards/similarity_reward": 0.6069758534431458, | |
| "step": 334 | |
| }, | |
| { | |
| "completion_length": 253.54464721679688, | |
| "epoch": 0.8933333333333333, | |
| "grad_norm": 1.0439727694024494, | |
| "kl": 0.01513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0938, | |
| "reward": 0.6808232069015503, | |
| "reward_std": 0.11848772317171097, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.6053767204284668, | |
| "step": 335 | |
| }, | |
| { | |
| "completion_length": 233.00001525878906, | |
| "epoch": 0.896, | |
| "grad_norm": 0.9838382076638493, | |
| "kl": 0.015869140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0451, | |
| "reward": 0.58315509557724, | |
| "reward_std": 0.11559745669364929, | |
| "rewards/length_reward": 0.06741069257259369, | |
| "rewards/similarity_reward": 0.5157443881034851, | |
| "step": 336 | |
| }, | |
| { | |
| "completion_length": 269.0133972167969, | |
| "epoch": 0.8986666666666666, | |
| "grad_norm": 0.836096159627277, | |
| "kl": 0.01361083984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0062, | |
| "reward": 0.6409623622894287, | |
| "reward_std": 0.12886659801006317, | |
| "rewards/length_reward": 0.07767854630947113, | |
| "rewards/similarity_reward": 0.5632836818695068, | |
| "step": 337 | |
| }, | |
| { | |
| "completion_length": 251.40179443359375, | |
| "epoch": 0.9013333333333333, | |
| "grad_norm": 0.8535550782434568, | |
| "kl": 0.01422119140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0089, | |
| "reward": 0.6296460032463074, | |
| "reward_std": 0.16196994483470917, | |
| "rewards/length_reward": 0.07187497615814209, | |
| "rewards/similarity_reward": 0.5577709674835205, | |
| "step": 338 | |
| }, | |
| { | |
| "completion_length": 244.80804443359375, | |
| "epoch": 0.904, | |
| "grad_norm": 1.0854210987310389, | |
| "kl": 0.0181884765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0595, | |
| "reward": 0.597773551940918, | |
| "reward_std": 0.10373269766569138, | |
| "rewards/length_reward": 0.06339284032583237, | |
| "rewards/similarity_reward": 0.5343807935714722, | |
| "step": 339 | |
| }, | |
| { | |
| "completion_length": 262.6875, | |
| "epoch": 0.9066666666666666, | |
| "grad_norm": 0.8356624214145669, | |
| "kl": 0.013916015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0101, | |
| "reward": 0.6685509085655212, | |
| "reward_std": 0.0960090234875679, | |
| "rewards/length_reward": 0.07946424931287766, | |
| "rewards/similarity_reward": 0.5890867114067078, | |
| "step": 340 | |
| }, | |
| { | |
| "completion_length": 262.5535888671875, | |
| "epoch": 0.9093333333333333, | |
| "grad_norm": 1.0702777784285404, | |
| "kl": 0.02783203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1024, | |
| "reward": 0.5271078944206238, | |
| "reward_std": 0.14886566996574402, | |
| "rewards/length_reward": 0.064732126891613, | |
| "rewards/similarity_reward": 0.46237578988075256, | |
| "step": 341 | |
| }, | |
| { | |
| "completion_length": 251.03126525878906, | |
| "epoch": 0.912, | |
| "grad_norm": 0.9235089036400403, | |
| "kl": 0.0172119140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0324, | |
| "reward": 0.6528847217559814, | |
| "reward_std": 0.11811169981956482, | |
| "rewards/length_reward": 0.06383927166461945, | |
| "rewards/similarity_reward": 0.5890454053878784, | |
| "step": 342 | |
| }, | |
| { | |
| "completion_length": 236.2857208251953, | |
| "epoch": 0.9146666666666666, | |
| "grad_norm": 0.9202873381431551, | |
| "kl": 0.01556396484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0497, | |
| "reward": 0.6048458814620972, | |
| "reward_std": 0.13773028552532196, | |
| "rewards/length_reward": 0.07544640451669693, | |
| "rewards/similarity_reward": 0.5293995141983032, | |
| "step": 343 | |
| }, | |
| { | |
| "completion_length": 309.99554443359375, | |
| "epoch": 0.9173333333333333, | |
| "grad_norm": 0.9489271796425148, | |
| "kl": 0.01226806640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0812, | |
| "reward": 0.6291395425796509, | |
| "reward_std": 0.15959399938583374, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5621752738952637, | |
| "step": 344 | |
| }, | |
| { | |
| "completion_length": 245.21429443359375, | |
| "epoch": 0.92, | |
| "grad_norm": 1.1363976997302783, | |
| "kl": 0.0213623046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1781, | |
| "reward": 0.5569170713424683, | |
| "reward_std": 0.14195482432842255, | |
| "rewards/length_reward": 0.06160712614655495, | |
| "rewards/similarity_reward": 0.49530985951423645, | |
| "step": 345 | |
| }, | |
| { | |
| "completion_length": 272.2232360839844, | |
| "epoch": 0.9226666666666666, | |
| "grad_norm": 0.7565012868381632, | |
| "kl": 0.0108642578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0695, | |
| "reward": 0.6887885928153992, | |
| "reward_std": 0.11395367234945297, | |
| "rewards/length_reward": 0.07857140153646469, | |
| "rewards/similarity_reward": 0.6102170348167419, | |
| "step": 346 | |
| }, | |
| { | |
| "completion_length": 310.28125, | |
| "epoch": 0.9253333333333333, | |
| "grad_norm": 0.782594647397142, | |
| "kl": 0.01202392578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0752, | |
| "reward": 0.6732801795005798, | |
| "reward_std": 0.12288369983434677, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5987265706062317, | |
| "step": 347 | |
| }, | |
| { | |
| "completion_length": 307.86163330078125, | |
| "epoch": 0.928, | |
| "grad_norm": 0.7087010173295871, | |
| "kl": 0.013671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.008, | |
| "reward": 0.6750614047050476, | |
| "reward_std": 0.09951343387365341, | |
| "rewards/length_reward": 0.07410712540149689, | |
| "rewards/similarity_reward": 0.6009542942047119, | |
| "step": 348 | |
| }, | |
| { | |
| "completion_length": 273.3883972167969, | |
| "epoch": 0.9306666666666666, | |
| "grad_norm": 1.0456544147832767, | |
| "kl": 0.01446533203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0404, | |
| "reward": 0.6893116235733032, | |
| "reward_std": 0.09679926186800003, | |
| "rewards/length_reward": 0.06517855077981949, | |
| "rewards/similarity_reward": 0.6241331100463867, | |
| "step": 349 | |
| }, | |
| { | |
| "completion_length": 264.1919860839844, | |
| "epoch": 0.9333333333333333, | |
| "grad_norm": 0.8991803002318822, | |
| "kl": 0.01397705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0129, | |
| "reward": 0.6719235181808472, | |
| "reward_std": 0.13838014006614685, | |
| "rewards/length_reward": 0.07321426272392273, | |
| "rewards/similarity_reward": 0.598709225654602, | |
| "step": 350 | |
| }, | |
| { | |
| "completion_length": 274.9508972167969, | |
| "epoch": 0.936, | |
| "grad_norm": 0.8591029679110356, | |
| "kl": 0.013916015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0312, | |
| "reward": 0.5572786331176758, | |
| "reward_std": 0.14849816262722015, | |
| "rewards/length_reward": 0.06785711646080017, | |
| "rewards/similarity_reward": 0.489421546459198, | |
| "step": 351 | |
| }, | |
| { | |
| "completion_length": 277.9508972167969, | |
| "epoch": 0.9386666666666666, | |
| "grad_norm": 0.8388349903111052, | |
| "kl": 0.01141357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0278, | |
| "reward": 0.681722104549408, | |
| "reward_std": 0.0899442657828331, | |
| "rewards/length_reward": 0.07991068065166473, | |
| "rewards/similarity_reward": 0.6018112897872925, | |
| "step": 352 | |
| }, | |
| { | |
| "completion_length": 240.3348388671875, | |
| "epoch": 0.9413333333333334, | |
| "grad_norm": 0.867198144640214, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0137, | |
| "reward": 0.6065589785575867, | |
| "reward_std": 0.11837570369243622, | |
| "rewards/length_reward": 0.07946424931287766, | |
| "rewards/similarity_reward": 0.5270946025848389, | |
| "step": 353 | |
| }, | |
| { | |
| "completion_length": 287.4151916503906, | |
| "epoch": 0.944, | |
| "grad_norm": 1.2363858371428533, | |
| "kl": 0.0157470703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0649, | |
| "reward": 0.6748880743980408, | |
| "reward_std": 0.13465073704719543, | |
| "rewards/length_reward": 0.07187496870756149, | |
| "rewards/similarity_reward": 0.6030132174491882, | |
| "step": 354 | |
| }, | |
| { | |
| "completion_length": 309.6919860839844, | |
| "epoch": 0.9466666666666667, | |
| "grad_norm": 0.6824545679415536, | |
| "kl": 0.00946044921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0881, | |
| "reward": 0.6755697727203369, | |
| "reward_std": 0.13920390605926514, | |
| "rewards/length_reward": 0.07901783287525177, | |
| "rewards/similarity_reward": 0.5965518355369568, | |
| "step": 355 | |
| }, | |
| { | |
| "completion_length": 304.0089416503906, | |
| "epoch": 0.9493333333333334, | |
| "grad_norm": 0.7612207527990814, | |
| "kl": 0.00860595703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0573, | |
| "reward": 0.6815410852432251, | |
| "reward_std": 0.11536341905593872, | |
| "rewards/length_reward": 0.07455354928970337, | |
| "rewards/similarity_reward": 0.6069875955581665, | |
| "step": 356 | |
| }, | |
| { | |
| "completion_length": 280.4151916503906, | |
| "epoch": 0.952, | |
| "grad_norm": 0.9197442279455559, | |
| "kl": 0.014892578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0976, | |
| "reward": 0.6213651895523071, | |
| "reward_std": 0.1452549546957016, | |
| "rewards/length_reward": 0.059374988079071045, | |
| "rewards/similarity_reward": 0.5619902014732361, | |
| "step": 357 | |
| }, | |
| { | |
| "completion_length": 311.9375, | |
| "epoch": 0.9546666666666667, | |
| "grad_norm": 0.7667231406265689, | |
| "kl": 0.0123291015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0949, | |
| "reward": 0.671852707862854, | |
| "reward_std": 0.12415429949760437, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.5972990989685059, | |
| "step": 358 | |
| }, | |
| { | |
| "completion_length": 296.4598388671875, | |
| "epoch": 0.9573333333333334, | |
| "grad_norm": 0.784755991417782, | |
| "kl": 0.0093994140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0284, | |
| "reward": 0.7474254965782166, | |
| "reward_std": 0.10959716141223907, | |
| "rewards/length_reward": 0.07901783287525177, | |
| "rewards/similarity_reward": 0.6684076189994812, | |
| "step": 359 | |
| }, | |
| { | |
| "completion_length": 253.15626525878906, | |
| "epoch": 0.96, | |
| "grad_norm": 0.8190677358549971, | |
| "kl": 0.0125732421875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0109, | |
| "reward": 0.6444076299667358, | |
| "reward_std": 0.1199827641248703, | |
| "rewards/length_reward": 0.07142855226993561, | |
| "rewards/similarity_reward": 0.5729790925979614, | |
| "step": 360 | |
| }, | |
| { | |
| "completion_length": 306.5848388671875, | |
| "epoch": 0.9626666666666667, | |
| "grad_norm": 0.8327047575723723, | |
| "kl": 0.0101318359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.044, | |
| "reward": 0.6085981726646423, | |
| "reward_std": 0.15067243576049805, | |
| "rewards/length_reward": 0.07366069406270981, | |
| "rewards/similarity_reward": 0.5349374413490295, | |
| "step": 361 | |
| }, | |
| { | |
| "completion_length": 254.7723388671875, | |
| "epoch": 0.9653333333333334, | |
| "grad_norm": 0.8753840404117226, | |
| "kl": 0.0155029296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0219, | |
| "reward": 0.6461009383201599, | |
| "reward_std": 0.11166159808635712, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5755651593208313, | |
| "step": 362 | |
| }, | |
| { | |
| "completion_length": 260.33929443359375, | |
| "epoch": 0.968, | |
| "grad_norm": 0.8487935107258318, | |
| "kl": 0.0135498046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0758, | |
| "reward": 0.6731547713279724, | |
| "reward_std": 0.0943944975733757, | |
| "rewards/length_reward": 0.07946424931287766, | |
| "rewards/similarity_reward": 0.5936905145645142, | |
| "step": 363 | |
| }, | |
| { | |
| "completion_length": 284.9107360839844, | |
| "epoch": 0.9706666666666667, | |
| "grad_norm": 0.9104348928736092, | |
| "kl": 0.0146484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0438, | |
| "reward": 0.6029422879219055, | |
| "reward_std": 0.13879723846912384, | |
| "rewards/length_reward": 0.07053568959236145, | |
| "rewards/similarity_reward": 0.5324065685272217, | |
| "step": 364 | |
| }, | |
| { | |
| "completion_length": 291.1339416503906, | |
| "epoch": 0.9733333333333334, | |
| "grad_norm": 0.8351250207880698, | |
| "kl": 0.01312255859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0447, | |
| "reward": 0.6295793056488037, | |
| "reward_std": 0.11455141007900238, | |
| "rewards/length_reward": 0.07321426272392273, | |
| "rewards/similarity_reward": 0.556364893913269, | |
| "step": 365 | |
| }, | |
| { | |
| "completion_length": 313.6160888671875, | |
| "epoch": 0.976, | |
| "grad_norm": 0.8331869500678173, | |
| "kl": 0.0137939453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1377, | |
| "reward": 0.6497610807418823, | |
| "reward_std": 0.13638634979724884, | |
| "rewards/length_reward": 0.064732126891613, | |
| "rewards/similarity_reward": 0.5850289463996887, | |
| "step": 366 | |
| }, | |
| { | |
| "completion_length": 260.46875, | |
| "epoch": 0.9786666666666667, | |
| "grad_norm": 0.9519334592833407, | |
| "kl": 0.0184326171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.108, | |
| "reward": 0.6215986609458923, | |
| "reward_std": 0.13745638728141785, | |
| "rewards/length_reward": 0.06696426123380661, | |
| "rewards/similarity_reward": 0.5546343326568604, | |
| "step": 367 | |
| }, | |
| { | |
| "completion_length": 261.9196472167969, | |
| "epoch": 0.9813333333333333, | |
| "grad_norm": 0.9206057200583376, | |
| "kl": 0.01361083984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1112, | |
| "reward": 0.5996190905570984, | |
| "reward_std": 0.13816344738006592, | |
| "rewards/length_reward": 0.0741071105003357, | |
| "rewards/similarity_reward": 0.5255119204521179, | |
| "step": 368 | |
| }, | |
| { | |
| "completion_length": 187.60269165039062, | |
| "epoch": 0.984, | |
| "grad_norm": 34.2971123341626, | |
| "kl": 0.0152587890625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0025, | |
| "reward": 0.6047165989875793, | |
| "reward_std": 0.11468542367219925, | |
| "rewards/length_reward": 0.07276783138513565, | |
| "rewards/similarity_reward": 0.5319487452507019, | |
| "step": 369 | |
| }, | |
| { | |
| "completion_length": 273.24554443359375, | |
| "epoch": 0.9866666666666667, | |
| "grad_norm": 1.0225592676611843, | |
| "kl": 0.0157470703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0974, | |
| "reward": 0.5848848819732666, | |
| "reward_std": 0.13747373223304749, | |
| "rewards/length_reward": 0.06205355003476143, | |
| "rewards/similarity_reward": 0.5228313207626343, | |
| "step": 370 | |
| }, | |
| { | |
| "completion_length": 242.43304443359375, | |
| "epoch": 0.9893333333333333, | |
| "grad_norm": 0.9519953414264258, | |
| "kl": 0.01556396484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1648, | |
| "reward": 0.6343554854393005, | |
| "reward_std": 0.14080199599266052, | |
| "rewards/length_reward": 0.07142855226993561, | |
| "rewards/similarity_reward": 0.5629268884658813, | |
| "step": 371 | |
| }, | |
| { | |
| "completion_length": 295.7633972167969, | |
| "epoch": 0.992, | |
| "grad_norm": 0.7534581065002547, | |
| "kl": 0.0140380859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0601, | |
| "reward": 0.6254644393920898, | |
| "reward_std": 0.14738810062408447, | |
| "rewards/length_reward": 0.07232140004634857, | |
| "rewards/similarity_reward": 0.5531430244445801, | |
| "step": 372 | |
| }, | |
| { | |
| "completion_length": 240.38394165039062, | |
| "epoch": 0.9946666666666667, | |
| "grad_norm": 0.9672618888481953, | |
| "kl": 0.01239013671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0627, | |
| "reward": 0.6286079287528992, | |
| "reward_std": 0.1355430781841278, | |
| "rewards/length_reward": 0.07232140004634857, | |
| "rewards/similarity_reward": 0.5562865734100342, | |
| "step": 373 | |
| }, | |
| { | |
| "completion_length": 296.3482360839844, | |
| "epoch": 0.9973333333333333, | |
| "grad_norm": 0.8090409603684708, | |
| "kl": 0.0118408203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0202, | |
| "reward": 0.7064945697784424, | |
| "reward_std": 0.08492975682020187, | |
| "rewards/length_reward": 0.07455354183912277, | |
| "rewards/similarity_reward": 0.631941020488739, | |
| "step": 374 | |
| }, | |
| { | |
| "completion_length": 207.4114227294922, | |
| "epoch": 1.0, | |
| "grad_norm": 0.9985931023470964, | |
| "kl": 0.01458740234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0328, | |
| "reward": 0.6288642883300781, | |
| "reward_std": 0.13037118315696716, | |
| "rewards/length_reward": 0.07276783138513565, | |
| "rewards/similarity_reward": 0.5560964941978455, | |
| "step": 375 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 375, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 32, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |