| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 8.0, |
| "eval_steps": 1, |
| "global_step": 472, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01694915254237288, |
| "grad_norm": 74.35959798227883, |
| "learning_rate": 1.0416666666666666e-08, |
| "logits/chosen": -1.0022015571594238, |
| "logits/rejected": -1.0571039915084839, |
| "logps/chosen": -26.953861236572266, |
| "logps/rejected": -41.69861602783203, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.03389830508474576, |
| "grad_norm": 71.81906743432833, |
| "learning_rate": 2.083333333333333e-08, |
| "logits/chosen": -0.9866722822189331, |
| "logits/rejected": -1.1117209196090698, |
| "logps/chosen": -33.70802307128906, |
| "logps/rejected": -37.13496398925781, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.05084745762711865, |
| "grad_norm": 75.40773138219605, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": -1.2932835817337036, |
| "logits/rejected": -1.2812855243682861, |
| "logps/chosen": -30.66956329345703, |
| "logps/rejected": -49.71609878540039, |
| "loss": 0.7141, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.015645474195480347, |
| "rewards/margins": -0.0929969847202301, |
| "rewards/rejected": 0.07735151052474976, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.06779661016949153, |
| "grad_norm": 74.47397381287325, |
| "learning_rate": 4.166666666666666e-08, |
| "logits/chosen": -1.0415635108947754, |
| "logits/rejected": -1.0745270252227783, |
| "logps/chosen": -26.642629623413086, |
| "logps/rejected": -38.44277572631836, |
| "loss": 0.7026, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.048020511865615845, |
| "rewards/margins": -0.018185943365097046, |
| "rewards/rejected": 0.06620645523071289, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.0847457627118644, |
| "grad_norm": 76.30048141954929, |
| "learning_rate": 5.208333333333333e-08, |
| "logits/chosen": -1.2889574766159058, |
| "logits/rejected": -1.219961404800415, |
| "logps/chosen": -31.46270751953125, |
| "logps/rejected": -28.84333610534668, |
| "loss": 0.6924, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.04683685302734375, |
| "rewards/margins": -0.016847282648086548, |
| "rewards/rejected": -0.029989570379257202, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.1016949152542373, |
| "grad_norm": 71.45564125608341, |
| "learning_rate": 6.25e-08, |
| "logits/chosen": -1.2035868167877197, |
| "logits/rejected": -1.1329045295715332, |
| "logps/chosen": -35.237884521484375, |
| "logps/rejected": -38.39533996582031, |
| "loss": 0.6914, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.014047741889953613, |
| "rewards/margins": 0.0001532137393951416, |
| "rewards/rejected": -0.014200955629348755, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.11864406779661017, |
| "grad_norm": 74.36407794239695, |
| "learning_rate": 7.291666666666667e-08, |
| "logits/chosen": -1.2061525583267212, |
| "logits/rejected": -1.1756294965744019, |
| "logps/chosen": -29.565950393676758, |
| "logps/rejected": -33.2646484375, |
| "loss": 0.7034, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.09937351942062378, |
| "rewards/margins": 0.04770404100418091, |
| "rewards/rejected": 0.05166947841644287, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.13559322033898305, |
| "grad_norm": 72.94427557648902, |
| "learning_rate": 8.333333333333333e-08, |
| "logits/chosen": -1.2344228029251099, |
| "logits/rejected": -1.255335807800293, |
| "logps/chosen": -26.22496223449707, |
| "logps/rejected": -39.68927764892578, |
| "loss": 0.7084, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": 0.005427069962024689, |
| "rewards/margins": -0.014611326158046722, |
| "rewards/rejected": 0.02003839612007141, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.15254237288135594, |
| "grad_norm": 69.72083135093712, |
| "learning_rate": 9.375e-08, |
| "logits/chosen": -1.1401718854904175, |
| "logits/rejected": -1.1010041236877441, |
| "logps/chosen": -33.88338851928711, |
| "logps/rejected": -28.835594177246094, |
| "loss": 0.7, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.037932395935058594, |
| "rewards/margins": -0.005142271518707275, |
| "rewards/rejected": 0.04307466745376587, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.1694915254237288, |
| "grad_norm": 71.74909213913158, |
| "learning_rate": 1.0416666666666667e-07, |
| "logits/chosen": -1.180943250656128, |
| "logits/rejected": -1.090497612953186, |
| "logps/chosen": -37.923099517822266, |
| "logps/rejected": -36.79877471923828, |
| "loss": 0.6975, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.031415216624736786, |
| "rewards/margins": -0.0013954713940620422, |
| "rewards/rejected": 0.03281068801879883, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.1864406779661017, |
| "grad_norm": 75.11980776081754, |
| "learning_rate": 1.1458333333333332e-07, |
| "logits/chosen": -0.9823209047317505, |
| "logits/rejected": -0.9876938462257385, |
| "logps/chosen": -27.41145896911621, |
| "logps/rejected": -45.299991607666016, |
| "loss": 0.7077, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.0018342137336730957, |
| "rewards/margins": 0.0696558952331543, |
| "rewards/rejected": -0.07149010896682739, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.2033898305084746, |
| "grad_norm": 71.48536897103016, |
| "learning_rate": 1.25e-07, |
| "logits/chosen": -0.9993177652359009, |
| "logits/rejected": -0.9222314953804016, |
| "logps/chosen": -23.32973289489746, |
| "logps/rejected": -32.4486083984375, |
| "loss": 0.6838, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.01836332678794861, |
| "rewards/margins": 0.00046828389167785645, |
| "rewards/rejected": -0.018831610679626465, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.22033898305084745, |
| "grad_norm": 69.73428591175114, |
| "learning_rate": 1.3541666666666666e-07, |
| "logits/chosen": -1.4160847663879395, |
| "logits/rejected": -1.2769191265106201, |
| "logps/chosen": -26.515804290771484, |
| "logps/rejected": -36.50257110595703, |
| "loss": 0.7069, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.09053070843219757, |
| "rewards/margins": 0.059635356068611145, |
| "rewards/rejected": 0.030895352363586426, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.23728813559322035, |
| "grad_norm": 74.26548135080421, |
| "learning_rate": 1.4583333333333335e-07, |
| "logits/chosen": -1.2310669422149658, |
| "logits/rejected": -1.1061973571777344, |
| "logps/chosen": -38.83403396606445, |
| "logps/rejected": -57.466835021972656, |
| "loss": 0.7022, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.014897465705871582, |
| "rewards/margins": 0.015999972820281982, |
| "rewards/rejected": -0.030897438526153564, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.2542372881355932, |
| "grad_norm": 70.61107518338639, |
| "learning_rate": 1.5624999999999999e-07, |
| "logits/chosen": -1.245528221130371, |
| "logits/rejected": -1.2389111518859863, |
| "logps/chosen": -24.09255027770996, |
| "logps/rejected": -35.16242218017578, |
| "loss": 0.7081, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.009011238813400269, |
| "rewards/margins": 0.01104736328125, |
| "rewards/rejected": -0.0020361244678497314, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.2711864406779661, |
| "grad_norm": 69.06624100974847, |
| "learning_rate": 1.6666666666666665e-07, |
| "logits/chosen": -1.1713544130325317, |
| "logits/rejected": -1.2104028463363647, |
| "logps/chosen": -27.774211883544922, |
| "logps/rejected": -32.56517791748047, |
| "loss": 0.6959, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.06307366490364075, |
| "rewards/margins": -0.055293723940849304, |
| "rewards/rejected": -0.007779940962791443, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.288135593220339, |
| "grad_norm": 74.45322195801226, |
| "learning_rate": 1.7708333333333334e-07, |
| "logits/chosen": -1.2374873161315918, |
| "logits/rejected": -1.2554068565368652, |
| "logps/chosen": -24.268505096435547, |
| "logps/rejected": -35.4179573059082, |
| "loss": 0.6963, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.0001607835292816162, |
| "rewards/margins": -0.01779848337173462, |
| "rewards/rejected": 0.017637699842453003, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.3050847457627119, |
| "grad_norm": 71.43793235652528, |
| "learning_rate": 1.875e-07, |
| "logits/chosen": -1.1041090488433838, |
| "logits/rejected": -1.0679348707199097, |
| "logps/chosen": -23.000707626342773, |
| "logps/rejected": -29.853412628173828, |
| "loss": 0.705, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.055961236357688904, |
| "rewards/margins": -0.05573050677776337, |
| "rewards/rejected": -0.0002307295799255371, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.3220338983050847, |
| "grad_norm": 71.50823519836709, |
| "learning_rate": 1.9791666666666664e-07, |
| "logits/chosen": -1.0583523511886597, |
| "logits/rejected": -1.0313684940338135, |
| "logps/chosen": -21.926410675048828, |
| "logps/rejected": -35.9990348815918, |
| "loss": 0.7084, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.018382668495178223, |
| "rewards/margins": 0.02424493432044983, |
| "rewards/rejected": -0.04262760281562805, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.3389830508474576, |
| "grad_norm": 65.57787832871358, |
| "learning_rate": 2.0833333333333333e-07, |
| "logits/chosen": -1.2984358072280884, |
| "logits/rejected": -1.2342230081558228, |
| "logps/chosen": -30.60858154296875, |
| "logps/rejected": -38.88047790527344, |
| "loss": 0.7012, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.01342683658003807, |
| "rewards/margins": 0.014368299394845963, |
| "rewards/rejected": -0.027795135974884033, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.3559322033898305, |
| "grad_norm": 68.54840408145243, |
| "learning_rate": 2.1875e-07, |
| "logits/chosen": -1.3012466430664062, |
| "logits/rejected": -1.3128973245620728, |
| "logps/chosen": -26.805089950561523, |
| "logps/rejected": -41.52635192871094, |
| "loss": 0.6853, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.01039344072341919, |
| "rewards/margins": 0.011234819889068604, |
| "rewards/rejected": -0.0008413791656494141, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.3728813559322034, |
| "grad_norm": 75.35032250165732, |
| "learning_rate": 2.2916666666666663e-07, |
| "logits/chosen": -1.1717727184295654, |
| "logits/rejected": -1.1936640739440918, |
| "logps/chosen": -21.9468994140625, |
| "logps/rejected": -27.52823257446289, |
| "loss": 0.6756, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.030159294605255127, |
| "rewards/margins": 0.03967534005641937, |
| "rewards/rejected": -0.0698346346616745, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.3898305084745763, |
| "grad_norm": 69.76965900033692, |
| "learning_rate": 2.3958333333333335e-07, |
| "logits/chosen": -1.227396845817566, |
| "logits/rejected": -1.2514605522155762, |
| "logps/chosen": -30.95319938659668, |
| "logps/rejected": -33.107421875, |
| "loss": 0.6483, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.009671658277511597, |
| "rewards/margins": 0.06378874182701111, |
| "rewards/rejected": -0.05411708354949951, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.4067796610169492, |
| "grad_norm": 71.41404237507972, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.1064453125, |
| "logits/rejected": -1.0061161518096924, |
| "logps/chosen": -32.68113327026367, |
| "logps/rejected": -38.1193962097168, |
| "loss": 0.6682, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.034807443618774414, |
| "rewards/margins": 0.0188644677400589, |
| "rewards/rejected": -0.05367191135883331, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.423728813559322, |
| "grad_norm": 70.49202249008462, |
| "learning_rate": 2.604166666666667e-07, |
| "logits/chosen": -1.1899397373199463, |
| "logits/rejected": -1.1957398653030396, |
| "logps/chosen": -37.75098419189453, |
| "logps/rejected": -34.21184539794922, |
| "loss": 0.6635, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.003971487283706665, |
| "rewards/margins": 0.10494436323642731, |
| "rewards/rejected": -0.10097287595272064, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.4406779661016949, |
| "grad_norm": 67.14613715257252, |
| "learning_rate": 2.708333333333333e-07, |
| "logits/chosen": -1.3827494382858276, |
| "logits/rejected": -1.3526558876037598, |
| "logps/chosen": -27.053857803344727, |
| "logps/rejected": -33.783485412597656, |
| "loss": 0.6528, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.015252411365509033, |
| "rewards/margins": 0.1568407416343689, |
| "rewards/rejected": -0.14158833026885986, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.4576271186440678, |
| "grad_norm": 68.86332311369058, |
| "learning_rate": 2.8125e-07, |
| "logits/chosen": -1.0096489191055298, |
| "logits/rejected": -1.0554242134094238, |
| "logps/chosen": -33.25010681152344, |
| "logps/rejected": -35.958675384521484, |
| "loss": 0.6573, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.04330983757972717, |
| "rewards/margins": 0.12464988231658936, |
| "rewards/rejected": -0.16795971989631653, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.4745762711864407, |
| "grad_norm": 66.94051666468431, |
| "learning_rate": 2.916666666666667e-07, |
| "logits/chosen": -1.3319611549377441, |
| "logits/rejected": -1.2341638803482056, |
| "logps/chosen": -23.90947914123535, |
| "logps/rejected": -31.997909545898438, |
| "loss": 0.6585, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.028478246182203293, |
| "rewards/margins": 0.07672013342380524, |
| "rewards/rejected": -0.10519838333129883, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.4915254237288136, |
| "grad_norm": 62.13361032578897, |
| "learning_rate": 3.020833333333333e-07, |
| "logits/chosen": -1.058496356010437, |
| "logits/rejected": -1.061716914176941, |
| "logps/chosen": -27.815153121948242, |
| "logps/rejected": -35.63865661621094, |
| "loss": 0.6417, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.002513296902179718, |
| "rewards/margins": 0.08875668793916702, |
| "rewards/rejected": -0.0862433910369873, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.5084745762711864, |
| "grad_norm": 69.40663934648452, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -1.148958683013916, |
| "logits/rejected": -1.0955352783203125, |
| "logps/chosen": -29.258386611938477, |
| "logps/rejected": -37.004207611083984, |
| "loss": 0.6333, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.018777184188365936, |
| "rewards/margins": 0.270157128572464, |
| "rewards/rejected": -0.28893429040908813, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.5254237288135594, |
| "grad_norm": 63.75111913966312, |
| "learning_rate": 3.2291666666666666e-07, |
| "logits/chosen": -1.1732263565063477, |
| "logits/rejected": -1.0661126375198364, |
| "logps/chosen": -31.076984405517578, |
| "logps/rejected": -34.91364288330078, |
| "loss": 0.6028, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.04974186420440674, |
| "rewards/margins": 0.24639210104942322, |
| "rewards/rejected": -0.19665023684501648, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.5423728813559322, |
| "grad_norm": 63.46121996218027, |
| "learning_rate": 3.333333333333333e-07, |
| "logits/chosen": -1.2128429412841797, |
| "logits/rejected": -1.2360285520553589, |
| "logps/chosen": -36.98728942871094, |
| "logps/rejected": -38.032920837402344, |
| "loss": 0.6111, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.015447601675987244, |
| "rewards/margins": 0.3491629958152771, |
| "rewards/rejected": -0.33371537923812866, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.559322033898305, |
| "grad_norm": 64.01628740909891, |
| "learning_rate": 3.4375e-07, |
| "logits/chosen": -1.2774078845977783, |
| "logits/rejected": -1.252654790878296, |
| "logps/chosen": -30.110694885253906, |
| "logps/rejected": -39.82551956176758, |
| "loss": 0.5955, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.12499848008155823, |
| "rewards/margins": 0.44781434535980225, |
| "rewards/rejected": -0.5728127956390381, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.576271186440678, |
| "grad_norm": 61.04313764198192, |
| "learning_rate": 3.541666666666667e-07, |
| "logits/chosen": -1.2864689826965332, |
| "logits/rejected": -1.1125664710998535, |
| "logps/chosen": -29.14852523803711, |
| "logps/rejected": -35.5062255859375, |
| "loss": 0.5925, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.07132556289434433, |
| "rewards/margins": 0.29910576343536377, |
| "rewards/rejected": -0.3704313337802887, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.5932203389830508, |
| "grad_norm": 61.863987476747845, |
| "learning_rate": 3.645833333333333e-07, |
| "logits/chosen": -1.1560570001602173, |
| "logits/rejected": -1.1620285511016846, |
| "logps/chosen": -21.810134887695312, |
| "logps/rejected": -45.566375732421875, |
| "loss": 0.5656, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.03650933504104614, |
| "rewards/margins": 0.7581607699394226, |
| "rewards/rejected": -0.7946701049804688, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.6101694915254238, |
| "grad_norm": 59.689844232963395, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": -1.1466398239135742, |
| "logits/rejected": -1.1647982597351074, |
| "logps/chosen": -24.348777770996094, |
| "logps/rejected": -29.572818756103516, |
| "loss": 0.5827, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.0986182689666748, |
| "rewards/margins": 0.30504968762397766, |
| "rewards/rejected": -0.40366795659065247, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.6271186440677966, |
| "grad_norm": 66.8530097795874, |
| "learning_rate": 3.8541666666666665e-07, |
| "logits/chosen": -1.1411319971084595, |
| "logits/rejected": -1.1138074398040771, |
| "logps/chosen": -29.88798713684082, |
| "logps/rejected": -30.512065887451172, |
| "loss": 0.5748, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.1340489685535431, |
| "rewards/margins": 0.17574873566627502, |
| "rewards/rejected": -0.3097977042198181, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.6440677966101694, |
| "grad_norm": 59.54927270244813, |
| "learning_rate": 3.958333333333333e-07, |
| "logits/chosen": -1.2669272422790527, |
| "logits/rejected": -1.2261359691619873, |
| "logps/chosen": -33.28898239135742, |
| "logps/rejected": -52.16841506958008, |
| "loss": 0.5415, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.1103217601776123, |
| "rewards/margins": 0.4893791675567627, |
| "rewards/rejected": -0.599700927734375, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.6610169491525424, |
| "grad_norm": 61.22261904686975, |
| "learning_rate": 4.0625e-07, |
| "logits/chosen": -1.2993597984313965, |
| "logits/rejected": -1.2113492488861084, |
| "logps/chosen": -38.285438537597656, |
| "logps/rejected": -51.55641555786133, |
| "loss": 0.5969, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.14819863438606262, |
| "rewards/margins": 0.8797410726547241, |
| "rewards/rejected": -1.027939796447754, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.6779661016949152, |
| "grad_norm": 60.12706158104359, |
| "learning_rate": 4.1666666666666667e-07, |
| "logits/chosen": -1.1191147565841675, |
| "logits/rejected": -0.9404773712158203, |
| "logps/chosen": -27.851015090942383, |
| "logps/rejected": -40.25725173950195, |
| "loss": 0.5564, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.17417365312576294, |
| "rewards/margins": 0.5528932809829712, |
| "rewards/rejected": -0.7270669341087341, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.6949152542372882, |
| "grad_norm": 57.92726362837684, |
| "learning_rate": 4.270833333333333e-07, |
| "logits/chosen": -1.0899916887283325, |
| "logits/rejected": -0.9453008770942688, |
| "logps/chosen": -26.283432006835938, |
| "logps/rejected": -45.57765579223633, |
| "loss": 0.5461, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.13252140581607819, |
| "rewards/margins": 0.8450538516044617, |
| "rewards/rejected": -0.9775752425193787, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.711864406779661, |
| "grad_norm": 58.169474320406394, |
| "learning_rate": 4.375e-07, |
| "logits/chosen": -1.1068731546401978, |
| "logits/rejected": -1.0945335626602173, |
| "logps/chosen": -24.04115104675293, |
| "logps/rejected": -36.62716293334961, |
| "loss": 0.5406, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.10331939160823822, |
| "rewards/margins": 0.5659677386283875, |
| "rewards/rejected": -0.6692871451377869, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.7288135593220338, |
| "grad_norm": 66.66372536069926, |
| "learning_rate": 4.479166666666667e-07, |
| "logits/chosen": -1.1794297695159912, |
| "logits/rejected": -1.193390965461731, |
| "logps/chosen": -46.91223907470703, |
| "logps/rejected": -41.650333404541016, |
| "loss": 0.5573, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.3094208538532257, |
| "rewards/margins": 0.14515355229377747, |
| "rewards/rejected": -0.4545744061470032, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.7457627118644068, |
| "grad_norm": 56.78835490397593, |
| "learning_rate": 4.5833333333333327e-07, |
| "logits/chosen": -1.0120888948440552, |
| "logits/rejected": -0.8681447505950928, |
| "logps/chosen": -33.3481330871582, |
| "logps/rejected": -48.61557388305664, |
| "loss": 0.5516, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.093438059091568, |
| "rewards/margins": 0.624420166015625, |
| "rewards/rejected": -0.7178582549095154, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.7627118644067796, |
| "grad_norm": 59.752166781716426, |
| "learning_rate": 4.6874999999999996e-07, |
| "logits/chosen": -1.2927764654159546, |
| "logits/rejected": -1.212005615234375, |
| "logps/chosen": -27.116588592529297, |
| "logps/rejected": -48.95513153076172, |
| "loss": 0.5622, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.08109956979751587, |
| "rewards/margins": 1.1761130094528198, |
| "rewards/rejected": -1.2572126388549805, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.7796610169491526, |
| "grad_norm": 56.0712139430381, |
| "learning_rate": 4.791666666666667e-07, |
| "logits/chosen": -1.310462474822998, |
| "logits/rejected": -1.1343291997909546, |
| "logps/chosen": -32.363792419433594, |
| "logps/rejected": -41.77903747558594, |
| "loss": 0.4924, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.1404968947172165, |
| "rewards/margins": 0.6709720492362976, |
| "rewards/rejected": -0.8114689588546753, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.7966101694915254, |
| "grad_norm": 53.687876472368266, |
| "learning_rate": 4.895833333333333e-07, |
| "logits/chosen": -1.0838322639465332, |
| "logits/rejected": -1.1518497467041016, |
| "logps/chosen": -34.07114791870117, |
| "logps/rejected": -38.58074188232422, |
| "loss": 0.475, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.14443516731262207, |
| "rewards/margins": 0.33194631338119507, |
| "rewards/rejected": -0.47638148069381714, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.8135593220338984, |
| "grad_norm": 57.81488552818391, |
| "learning_rate": 5e-07, |
| "logits/chosen": -1.3274493217468262, |
| "logits/rejected": -1.099511981010437, |
| "logps/chosen": -33.7588005065918, |
| "logps/rejected": -51.24869918823242, |
| "loss": 0.46, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.2817200720310211, |
| "rewards/margins": 1.4236443042755127, |
| "rewards/rejected": -1.705364465713501, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.8305084745762712, |
| "grad_norm": 51.9232811376309, |
| "learning_rate": 4.999931375995349e-07, |
| "logits/chosen": -1.2194585800170898, |
| "logits/rejected": -1.091806173324585, |
| "logps/chosen": -28.378108978271484, |
| "logps/rejected": -36.51158905029297, |
| "loss": 0.4926, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.31702089309692383, |
| "rewards/margins": 0.6066832542419434, |
| "rewards/rejected": -0.9237041473388672, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.847457627118644, |
| "grad_norm": 58.79932831253633, |
| "learning_rate": 4.999725507748798e-07, |
| "logits/chosen": -1.4276092052459717, |
| "logits/rejected": -1.2703173160552979, |
| "logps/chosen": -27.375112533569336, |
| "logps/rejected": -45.86325454711914, |
| "loss": 0.5289, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.21876651048660278, |
| "rewards/margins": 1.0810855627059937, |
| "rewards/rejected": -1.2998521327972412, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.864406779661017, |
| "grad_norm": 56.405768925690126, |
| "learning_rate": 4.99938240656235e-07, |
| "logits/chosen": -1.1859700679779053, |
| "logits/rejected": -1.110828161239624, |
| "logps/chosen": -30.097301483154297, |
| "logps/rejected": -56.47822189331055, |
| "loss": 0.4842, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.14388218522071838, |
| "rewards/margins": 0.8802270889282227, |
| "rewards/rejected": -1.0241093635559082, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.8813559322033898, |
| "grad_norm": 54.84466981430871, |
| "learning_rate": 4.998902091271985e-07, |
| "logits/chosen": -1.2907037734985352, |
| "logits/rejected": -1.198894739151001, |
| "logps/chosen": -23.458566665649414, |
| "logps/rejected": -36.68768310546875, |
| "loss": 0.4399, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.09736257791519165, |
| "rewards/margins": 0.8091447353363037, |
| "rewards/rejected": -0.9065073132514954, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.8983050847457628, |
| "grad_norm": 53.59884977562605, |
| "learning_rate": 4.998284588246634e-07, |
| "logits/chosen": -1.2666661739349365, |
| "logits/rejected": -1.1450353860855103, |
| "logps/chosen": -32.4919319152832, |
| "logps/rejected": -36.02308654785156, |
| "loss": 0.4643, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.2568899691104889, |
| "rewards/margins": 0.8810305595397949, |
| "rewards/rejected": -1.137920618057251, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.9152542372881356, |
| "grad_norm": 56.86352488023535, |
| "learning_rate": 4.997529931386719e-07, |
| "logits/chosen": -1.2422281503677368, |
| "logits/rejected": -1.2324570417404175, |
| "logps/chosen": -33.5006217956543, |
| "logps/rejected": -30.282136917114258, |
| "loss": 0.4844, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.3842281401157379, |
| "rewards/margins": 0.29243284463882446, |
| "rewards/rejected": -0.6766610145568848, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.9322033898305084, |
| "grad_norm": 53.15925076232375, |
| "learning_rate": 4.996638162122302e-07, |
| "logits/chosen": -1.2868903875350952, |
| "logits/rejected": -1.1907857656478882, |
| "logps/chosen": -32.84450149536133, |
| "logps/rejected": -39.07723617553711, |
| "loss": 0.4865, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.19609573483467102, |
| "rewards/margins": 1.1344835758209229, |
| "rewards/rejected": -1.3305792808532715, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.9491525423728814, |
| "grad_norm": 50.6107831993107, |
| "learning_rate": 4.995609329410804e-07, |
| "logits/chosen": -1.1869336366653442, |
| "logits/rejected": -1.0896047353744507, |
| "logps/chosen": -25.43665885925293, |
| "logps/rejected": -37.78445816040039, |
| "loss": 0.4331, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.2436077892780304, |
| "rewards/margins": 1.5263582468032837, |
| "rewards/rejected": -1.7699658870697021, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.9661016949152542, |
| "grad_norm": 53.83410279226871, |
| "learning_rate": 4.994443489734322e-07, |
| "logits/chosen": -1.140820860862732, |
| "logits/rejected": -1.0942250490188599, |
| "logps/chosen": -28.2945499420166, |
| "logps/rejected": -44.74162292480469, |
| "loss": 0.4513, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.28495079278945923, |
| "rewards/margins": 1.6222182512283325, |
| "rewards/rejected": -1.9071691036224365, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.9830508474576272, |
| "grad_norm": 57.20949918087681, |
| "learning_rate": 4.993140707096525e-07, |
| "logits/chosen": -1.2853411436080933, |
| "logits/rejected": -1.1984596252441406, |
| "logps/chosen": -35.58999252319336, |
| "logps/rejected": -42.4873161315918, |
| "loss": 0.4307, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.26272836327552795, |
| "rewards/margins": 1.1384968757629395, |
| "rewards/rejected": -1.4012253284454346, |
| "step": 58 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 49.633369993703724, |
| "learning_rate": 4.991701053019145e-07, |
| "logits/chosen": -1.2317020893096924, |
| "logits/rejected": -1.2267297506332397, |
| "logps/chosen": -28.742332458496094, |
| "logps/rejected": -50.28435516357422, |
| "loss": 0.4259, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.22372712194919586, |
| "rewards/margins": 1.6622364521026611, |
| "rewards/rejected": -1.8859635591506958, |
| "step": 59 |
| }, |
| { |
| "epoch": 1.0169491525423728, |
| "grad_norm": 43.19271219759473, |
| "learning_rate": 4.990124606538042e-07, |
| "logits/chosen": -1.27030348777771, |
| "logits/rejected": -1.2970830202102661, |
| "logps/chosen": -22.824378967285156, |
| "logps/rejected": -43.53241729736328, |
| "loss": 0.3445, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.11317068338394165, |
| "rewards/margins": 1.718346118927002, |
| "rewards/rejected": -1.8315167427062988, |
| "step": 60 |
| }, |
| { |
| "epoch": 1.0338983050847457, |
| "grad_norm": 46.56532536625075, |
| "learning_rate": 4.988411454198874e-07, |
| "logits/chosen": -1.2344048023223877, |
| "logits/rejected": -1.376710295677185, |
| "logps/chosen": -30.1800537109375, |
| "logps/rejected": -34.68271255493164, |
| "loss": 0.4151, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.3319806754589081, |
| "rewards/margins": 0.3052244484424591, |
| "rewards/rejected": -0.637205183506012, |
| "step": 61 |
| }, |
| { |
| "epoch": 1.0508474576271187, |
| "grad_norm": 45.932270396743505, |
| "learning_rate": 4.98656169005234e-07, |
| "logits/chosen": -1.300619125366211, |
| "logits/rejected": -1.1745063066482544, |
| "logps/chosen": -32.63179016113281, |
| "logps/rejected": -39.57646942138672, |
| "loss": 0.3826, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.021081820130348206, |
| "rewards/margins": 1.4243545532226562, |
| "rewards/rejected": -1.4454363584518433, |
| "step": 62 |
| }, |
| { |
| "epoch": 1.0677966101694916, |
| "grad_norm": 41.51255029654015, |
| "learning_rate": 4.984575415649018e-07, |
| "logits/chosen": -1.272727370262146, |
| "logits/rejected": -1.1292237043380737, |
| "logps/chosen": -26.08396339416504, |
| "logps/rejected": -47.56403350830078, |
| "loss": 0.3506, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.11639979481697083, |
| "rewards/margins": 2.3725996017456055, |
| "rewards/rejected": -2.488999366760254, |
| "step": 63 |
| }, |
| { |
| "epoch": 1.0847457627118644, |
| "grad_norm": 38.246979074883626, |
| "learning_rate": 4.982452740033792e-07, |
| "logits/chosen": -1.22151517868042, |
| "logits/rejected": -1.082349181175232, |
| "logps/chosen": -27.676637649536133, |
| "logps/rejected": -33.685211181640625, |
| "loss": 0.3068, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.00457368791103363, |
| "rewards/margins": 1.3627952337265015, |
| "rewards/rejected": -1.358221411705017, |
| "step": 64 |
| }, |
| { |
| "epoch": 1.1016949152542372, |
| "grad_norm": 40.134511453074204, |
| "learning_rate": 4.980193779739863e-07, |
| "logits/chosen": -1.1573750972747803, |
| "logits/rejected": -1.0462363958358765, |
| "logps/chosen": -29.544757843017578, |
| "logps/rejected": -45.139732360839844, |
| "loss": 0.3303, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.12963010370731354, |
| "rewards/margins": 1.9951367378234863, |
| "rewards/rejected": -2.1247665882110596, |
| "step": 65 |
| }, |
| { |
| "epoch": 1.11864406779661, |
| "grad_norm": 42.89070654695283, |
| "learning_rate": 4.977798658782351e-07, |
| "logits/chosen": -1.3436741828918457, |
| "logits/rejected": -1.2738423347473145, |
| "logps/chosen": -29.10391616821289, |
| "logps/rejected": -42.66758346557617, |
| "loss": 0.349, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.23598036170005798, |
| "rewards/margins": 1.5764625072479248, |
| "rewards/rejected": -1.8124428987503052, |
| "step": 66 |
| }, |
| { |
| "epoch": 1.1355932203389831, |
| "grad_norm": 41.22280324142371, |
| "learning_rate": 4.975267508651491e-07, |
| "logits/chosen": -1.202622890472412, |
| "logits/rejected": -1.047250509262085, |
| "logps/chosen": -27.090843200683594, |
| "logps/rejected": -30.239742279052734, |
| "loss": 0.3463, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.1256638914346695, |
| "rewards/margins": 1.4247705936431885, |
| "rewards/rejected": -1.5504344701766968, |
| "step": 67 |
| }, |
| { |
| "epoch": 1.152542372881356, |
| "grad_norm": 41.21696455893867, |
| "learning_rate": 4.97260046830541e-07, |
| "logits/chosen": -1.166504144668579, |
| "logits/rejected": -0.8773643970489502, |
| "logps/chosen": -22.584096908569336, |
| "logps/rejected": -42.063133239746094, |
| "loss": 0.3428, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.07130265235900879, |
| "rewards/margins": 1.9287859201431274, |
| "rewards/rejected": -2.000088691711426, |
| "step": 68 |
| }, |
| { |
| "epoch": 1.1694915254237288, |
| "grad_norm": 43.16486538306169, |
| "learning_rate": 4.969797684162497e-07, |
| "logits/chosen": -1.4693577289581299, |
| "logits/rejected": -1.3569204807281494, |
| "logps/chosen": -25.79123878479004, |
| "logps/rejected": -37.476409912109375, |
| "loss": 0.3706, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.10858143866062164, |
| "rewards/margins": 1.385354995727539, |
| "rewards/rejected": -1.4939366579055786, |
| "step": 69 |
| }, |
| { |
| "epoch": 1.1864406779661016, |
| "grad_norm": 42.7942644360943, |
| "learning_rate": 4.966859310093372e-07, |
| "logits/chosen": -1.168703317642212, |
| "logits/rejected": -1.0707366466522217, |
| "logps/chosen": -30.724082946777344, |
| "logps/rejected": -41.230712890625, |
| "loss": 0.354, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.13678063452243805, |
| "rewards/margins": 1.567689061164856, |
| "rewards/rejected": -1.7044697999954224, |
| "step": 70 |
| }, |
| { |
| "epoch": 1.2033898305084745, |
| "grad_norm": 39.25048654439285, |
| "learning_rate": 4.96378550741243e-07, |
| "logits/chosen": -1.3091105222702026, |
| "logits/rejected": -1.2656583786010742, |
| "logps/chosen": -30.91085433959961, |
| "logps/rejected": -41.49336624145508, |
| "loss": 0.307, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.22276180982589722, |
| "rewards/margins": 1.757472276687622, |
| "rewards/rejected": -1.980234146118164, |
| "step": 71 |
| }, |
| { |
| "epoch": 1.2203389830508475, |
| "grad_norm": 40.23010655586107, |
| "learning_rate": 4.960576444868992e-07, |
| "logits/chosen": -1.4617582559585571, |
| "logits/rejected": -1.4617631435394287, |
| "logps/chosen": -28.715673446655273, |
| "logps/rejected": -48.552024841308594, |
| "loss": 0.317, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.21487601101398468, |
| "rewards/margins": 2.4703986644744873, |
| "rewards/rejected": -2.685274362564087, |
| "step": 72 |
| }, |
| { |
| "epoch": 1.2372881355932204, |
| "grad_norm": 45.05758163705817, |
| "learning_rate": 4.957232298638035e-07, |
| "logits/chosen": -1.3103129863739014, |
| "logits/rejected": -1.297128677368164, |
| "logps/chosen": -29.75772476196289, |
| "logps/rejected": -42.82856750488281, |
| "loss": 0.3574, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.16225725412368774, |
| "rewards/margins": 1.7887846231460571, |
| "rewards/rejected": -1.9510418176651, |
| "step": 73 |
| }, |
| { |
| "epoch": 1.2542372881355932, |
| "grad_norm": 40.337686604932905, |
| "learning_rate": 4.953753252310525e-07, |
| "logits/chosen": -1.351475477218628, |
| "logits/rejected": -1.2956455945968628, |
| "logps/chosen": -30.769058227539062, |
| "logps/rejected": -37.68085479736328, |
| "loss": 0.3211, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.22842487692832947, |
| "rewards/margins": 1.4851303100585938, |
| "rewards/rejected": -1.713555097579956, |
| "step": 74 |
| }, |
| { |
| "epoch": 1.271186440677966, |
| "grad_norm": 41.87339924313547, |
| "learning_rate": 4.950139496883334e-07, |
| "logits/chosen": -1.1046594381332397, |
| "logits/rejected": -1.083458662033081, |
| "logps/chosen": -23.34552764892578, |
| "logps/rejected": -33.603294372558594, |
| "loss": 0.2996, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.22441284358501434, |
| "rewards/margins": 1.64926016330719, |
| "rewards/rejected": -1.8736729621887207, |
| "step": 75 |
| }, |
| { |
| "epoch": 1.288135593220339, |
| "grad_norm": 40.59281009638228, |
| "learning_rate": 4.94639123074876e-07, |
| "logits/chosen": -1.5633933544158936, |
| "logits/rejected": -1.3900222778320312, |
| "logps/chosen": -28.575279235839844, |
| "logps/rejected": -40.71543502807617, |
| "loss": 0.3253, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.23141731321811676, |
| "rewards/margins": 1.784515619277954, |
| "rewards/rejected": -2.015933036804199, |
| "step": 76 |
| }, |
| { |
| "epoch": 1.305084745762712, |
| "grad_norm": 40.512541982486866, |
| "learning_rate": 4.942508659683626e-07, |
| "logits/chosen": -1.2337239980697632, |
| "logits/rejected": -1.1204659938812256, |
| "logps/chosen": -37.800514221191406, |
| "logps/rejected": -57.468048095703125, |
| "loss": 0.3173, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.0504547655582428, |
| "rewards/margins": 2.774332046508789, |
| "rewards/rejected": -2.824786901473999, |
| "step": 77 |
| }, |
| { |
| "epoch": 1.3220338983050848, |
| "grad_norm": 44.23393681805476, |
| "learning_rate": 4.938491996837994e-07, |
| "logits/chosen": -1.1162344217300415, |
| "logits/rejected": -1.1677778959274292, |
| "logps/chosen": -21.936918258666992, |
| "logps/rejected": -36.89491653442383, |
| "loss": 0.3389, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.04354112595319748, |
| "rewards/margins": 1.7271283864974976, |
| "rewards/rejected": -1.7706694602966309, |
| "step": 78 |
| }, |
| { |
| "epoch": 1.3389830508474576, |
| "grad_norm": 41.690947911283914, |
| "learning_rate": 4.934341462723454e-07, |
| "logits/chosen": -1.3248709440231323, |
| "logits/rejected": -1.2905504703521729, |
| "logps/chosen": -22.56363868713379, |
| "logps/rejected": -36.43506622314453, |
| "loss": 0.3275, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.11233043670654297, |
| "rewards/margins": 1.878725528717041, |
| "rewards/rejected": -1.991055965423584, |
| "step": 79 |
| }, |
| { |
| "epoch": 1.3559322033898304, |
| "grad_norm": 40.04810360539708, |
| "learning_rate": 4.930057285201027e-07, |
| "logits/chosen": -1.1112794876098633, |
| "logits/rejected": -1.0777575969696045, |
| "logps/chosen": -24.657052993774414, |
| "logps/rejected": -39.67708969116211, |
| "loss": 0.3229, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.13391758501529694, |
| "rewards/margins": 2.0548856258392334, |
| "rewards/rejected": -2.18880295753479, |
| "step": 80 |
| }, |
| { |
| "epoch": 1.3728813559322033, |
| "grad_norm": 34.996138065577895, |
| "learning_rate": 4.925639699468645e-07, |
| "logits/chosen": -1.2954164743423462, |
| "logits/rejected": -1.2493962049484253, |
| "logps/chosen": -22.976837158203125, |
| "logps/rejected": -30.558788299560547, |
| "loss": 0.2742, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.00974225252866745, |
| "rewards/margins": 1.3260502815246582, |
| "rewards/rejected": -1.3357923030853271, |
| "step": 81 |
| }, |
| { |
| "epoch": 1.3898305084745763, |
| "grad_norm": 37.19153270519795, |
| "learning_rate": 4.921088948048246e-07, |
| "logits/chosen": -1.0799801349639893, |
| "logits/rejected": -1.029328465461731, |
| "logps/chosen": -21.34770965576172, |
| "logps/rejected": -27.523284912109375, |
| "loss": 0.3197, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.0038819462060928345, |
| "rewards/margins": 1.294053316116333, |
| "rewards/rejected": -1.2979353666305542, |
| "step": 82 |
| }, |
| { |
| "epoch": 1.4067796610169492, |
| "grad_norm": 38.782824257992516, |
| "learning_rate": 4.916405280772462e-07, |
| "logits/chosen": -1.2136616706848145, |
| "logits/rejected": -1.1575380563735962, |
| "logps/chosen": -34.565452575683594, |
| "logps/rejected": -37.70560836791992, |
| "loss": 0.2773, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.11810773611068726, |
| "rewards/margins": 1.5449273586273193, |
| "rewards/rejected": -1.6630350351333618, |
| "step": 83 |
| }, |
| { |
| "epoch": 1.423728813559322, |
| "grad_norm": 46.057268803202, |
| "learning_rate": 4.911588954770896e-07, |
| "logits/chosen": -1.323722004890442, |
| "logits/rejected": -1.2598729133605957, |
| "logps/chosen": -28.280380249023438, |
| "logps/rejected": -35.962127685546875, |
| "loss": 0.3575, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.18131475150585175, |
| "rewards/margins": 1.3694193363189697, |
| "rewards/rejected": -1.5507341623306274, |
| "step": 84 |
| }, |
| { |
| "epoch": 1.4406779661016949, |
| "grad_norm": 39.078212479916175, |
| "learning_rate": 4.906640234456011e-07, |
| "logits/chosen": -1.254352331161499, |
| "logits/rejected": -1.220250129699707, |
| "logps/chosen": -22.146629333496094, |
| "logps/rejected": -34.24107360839844, |
| "loss": 0.3198, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.16238263249397278, |
| "rewards/margins": 2.182830810546875, |
| "rewards/rejected": -2.3452136516571045, |
| "step": 85 |
| }, |
| { |
| "epoch": 1.457627118644068, |
| "grad_norm": 40.017911222526, |
| "learning_rate": 4.90155939150861e-07, |
| "logits/chosen": -1.2662581205368042, |
| "logits/rejected": -1.1520745754241943, |
| "logps/chosen": -28.800233840942383, |
| "logps/rejected": -45.41657638549805, |
| "loss": 0.2885, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.10078342258930206, |
| "rewards/margins": 2.74984073638916, |
| "rewards/rejected": -2.8506245613098145, |
| "step": 86 |
| }, |
| { |
| "epoch": 1.4745762711864407, |
| "grad_norm": 38.422387432652926, |
| "learning_rate": 4.896346704862927e-07, |
| "logits/chosen": -1.1133819818496704, |
| "logits/rejected": -1.1244077682495117, |
| "logps/chosen": -26.18191909790039, |
| "logps/rejected": -38.19125747680664, |
| "loss": 0.2784, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2561756372451782, |
| "rewards/margins": 2.0794429779052734, |
| "rewards/rejected": -2.335618734359741, |
| "step": 87 |
| }, |
| { |
| "epoch": 1.4915254237288136, |
| "grad_norm": 43.19306665638686, |
| "learning_rate": 4.891002460691305e-07, |
| "logits/chosen": -1.1532127857208252, |
| "logits/rejected": -1.0514928102493286, |
| "logps/chosen": -33.20591735839844, |
| "logps/rejected": -44.78691101074219, |
| "loss": 0.2942, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.31594839692115784, |
| "rewards/margins": 2.774695634841919, |
| "rewards/rejected": -3.090643882751465, |
| "step": 88 |
| }, |
| { |
| "epoch": 1.5084745762711864, |
| "grad_norm": 37.55592044811218, |
| "learning_rate": 4.885526952388497e-07, |
| "logits/chosen": -1.5443884134292603, |
| "logits/rejected": -1.491202473640442, |
| "logps/chosen": -29.413482666015625, |
| "logps/rejected": -44.91068649291992, |
| "loss": 0.2738, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.08295662701129913, |
| "rewards/margins": 2.9935202598571777, |
| "rewards/rejected": -3.076476812362671, |
| "step": 89 |
| }, |
| { |
| "epoch": 1.5254237288135593, |
| "grad_norm": 39.41480347028368, |
| "learning_rate": 4.879920480555549e-07, |
| "logits/chosen": -1.2505245208740234, |
| "logits/rejected": -1.1800575256347656, |
| "logps/chosen": -34.517356872558594, |
| "logps/rejected": -54.461875915527344, |
| "loss": 0.3016, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2518163323402405, |
| "rewards/margins": 2.4639229774475098, |
| "rewards/rejected": -2.7157392501831055, |
| "step": 90 |
| }, |
| { |
| "epoch": 1.542372881355932, |
| "grad_norm": 41.233244462870424, |
| "learning_rate": 4.874183352983297e-07, |
| "logits/chosen": -1.2128483057022095, |
| "logits/rejected": -1.1637994050979614, |
| "logps/chosen": -26.66428565979004, |
| "logps/rejected": -30.274539947509766, |
| "loss": 0.3237, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.016297563910484314, |
| "rewards/margins": 1.5082069635391235, |
| "rewards/rejected": -1.4919092655181885, |
| "step": 91 |
| }, |
| { |
| "epoch": 1.559322033898305, |
| "grad_norm": 33.074028930414165, |
| "learning_rate": 4.868315884635478e-07, |
| "logits/chosen": -1.3729506731033325, |
| "logits/rejected": -1.363387107849121, |
| "logps/chosen": -32.76537322998047, |
| "logps/rejected": -40.88609313964844, |
| "loss": 0.2209, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.49252671003341675, |
| "rewards/margins": 1.6107347011566162, |
| "rewards/rejected": -2.1032614707946777, |
| "step": 92 |
| }, |
| { |
| "epoch": 1.576271186440678, |
| "grad_norm": 44.45767411612402, |
| "learning_rate": 4.862318397631433e-07, |
| "logits/chosen": -1.290561318397522, |
| "logits/rejected": -1.2587978839874268, |
| "logps/chosen": -27.68136215209961, |
| "logps/rejected": -37.53421401977539, |
| "loss": 0.3135, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.045494288206100464, |
| "rewards/margins": 2.3608152866363525, |
| "rewards/rejected": -2.3153209686279297, |
| "step": 93 |
| }, |
| { |
| "epoch": 1.5932203389830508, |
| "grad_norm": 38.75459636795706, |
| "learning_rate": 4.856191221228422e-07, |
| "logits/chosen": -1.4851233959197998, |
| "logits/rejected": -1.3868330717086792, |
| "logps/chosen": -25.41338348388672, |
| "logps/rejected": -50.90999984741211, |
| "loss": 0.3055, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.057565659284591675, |
| "rewards/margins": 3.016381025314331, |
| "rewards/rejected": -2.958815336227417, |
| "step": 94 |
| }, |
| { |
| "epoch": 1.6101694915254239, |
| "grad_norm": 39.42511408680881, |
| "learning_rate": 4.84993469180355e-07, |
| "logits/chosen": -1.5986624956130981, |
| "logits/rejected": -1.5039199590682983, |
| "logps/chosen": -23.876976013183594, |
| "logps/rejected": -37.30643081665039, |
| "loss": 0.2918, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.1558499038219452, |
| "rewards/margins": 2.534642219543457, |
| "rewards/rejected": -2.3787922859191895, |
| "step": 95 |
| }, |
| { |
| "epoch": 1.6271186440677967, |
| "grad_norm": 40.62251102213997, |
| "learning_rate": 4.843549152835302e-07, |
| "logits/chosen": -1.4016410112380981, |
| "logits/rejected": -1.330747127532959, |
| "logps/chosen": -35.39458465576172, |
| "logps/rejected": -41.057533264160156, |
| "loss": 0.2683, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.30797258019447327, |
| "rewards/margins": 1.5779187679290771, |
| "rewards/rejected": -1.8858911991119385, |
| "step": 96 |
| }, |
| { |
| "epoch": 1.6440677966101696, |
| "grad_norm": 37.47292375573573, |
| "learning_rate": 4.837034954884681e-07, |
| "logits/chosen": -1.2742365598678589, |
| "logits/rejected": -1.16330885887146, |
| "logps/chosen": -19.86534881591797, |
| "logps/rejected": -37.20149230957031, |
| "loss": 0.3136, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.16833439469337463, |
| "rewards/margins": 2.6455376148223877, |
| "rewards/rejected": -2.8138718605041504, |
| "step": 97 |
| }, |
| { |
| "epoch": 1.6610169491525424, |
| "grad_norm": 34.360879838757626, |
| "learning_rate": 4.83039245557597e-07, |
| "logits/chosen": -1.4751869440078735, |
| "logits/rejected": -1.3903852701187134, |
| "logps/chosen": -27.246294021606445, |
| "logps/rejected": -37.46033477783203, |
| "loss": 0.2373, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.015960171818733215, |
| "rewards/margins": 1.7550795078277588, |
| "rewards/rejected": -1.739119291305542, |
| "step": 98 |
| }, |
| { |
| "epoch": 1.6779661016949152, |
| "grad_norm": 36.086860100703234, |
| "learning_rate": 4.823622019577088e-07, |
| "logits/chosen": -1.5154139995574951, |
| "logits/rejected": -1.435178279876709, |
| "logps/chosen": -29.518024444580078, |
| "logps/rejected": -34.60513687133789, |
| "loss": 0.2638, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.023779883980751038, |
| "rewards/margins": 1.6297731399536133, |
| "rewards/rejected": -1.6059932708740234, |
| "step": 99 |
| }, |
| { |
| "epoch": 1.694915254237288, |
| "grad_norm": 44.37484563053148, |
| "learning_rate": 4.816724018579583e-07, |
| "logits/chosen": -1.3004710674285889, |
| "logits/rejected": -1.1905782222747803, |
| "logps/chosen": -44.38223648071289, |
| "logps/rejected": -44.088531494140625, |
| "loss": 0.3148, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.6569697856903076, |
| "rewards/margins": 1.929584264755249, |
| "rewards/rejected": -2.5865538120269775, |
| "step": 100 |
| }, |
| { |
| "epoch": 1.711864406779661, |
| "grad_norm": 37.67689426318597, |
| "learning_rate": 4.809698831278217e-07, |
| "logits/chosen": -1.1910172700881958, |
| "logits/rejected": -1.085993766784668, |
| "logps/chosen": -25.45205307006836, |
| "logps/rejected": -38.646697998046875, |
| "loss": 0.2924, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.10635387897491455, |
| "rewards/margins": 2.1210451126098633, |
| "rewards/rejected": -2.2273988723754883, |
| "step": 101 |
| }, |
| { |
| "epoch": 1.7288135593220337, |
| "grad_norm": 44.53554229157193, |
| "learning_rate": 4.802546843350177e-07, |
| "logits/chosen": -1.3502918481826782, |
| "logits/rejected": -1.3793635368347168, |
| "logps/chosen": -31.245201110839844, |
| "logps/rejected": -38.174922943115234, |
| "loss": 0.308, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.04055324196815491, |
| "rewards/margins": 2.123584032058716, |
| "rewards/rejected": -2.164137363433838, |
| "step": 102 |
| }, |
| { |
| "epoch": 1.7457627118644068, |
| "grad_norm": 33.6410465461232, |
| "learning_rate": 4.795268447433906e-07, |
| "logits/chosen": -1.5270867347717285, |
| "logits/rejected": -1.5600392818450928, |
| "logps/chosen": -26.281536102294922, |
| "logps/rejected": -40.25182342529297, |
| "loss": 0.237, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.38495194911956787, |
| "rewards/margins": 2.3371963500976562, |
| "rewards/rejected": -2.7221484184265137, |
| "step": 103 |
| }, |
| { |
| "epoch": 1.7627118644067796, |
| "grad_norm": 37.86791531616575, |
| "learning_rate": 4.787864043107546e-07, |
| "logits/chosen": -1.1703252792358398, |
| "logits/rejected": -1.2337673902511597, |
| "logps/chosen": -28.018447875976562, |
| "logps/rejected": -26.48525619506836, |
| "loss": 0.2904, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.1365218162536621, |
| "rewards/margins": 1.0608235597610474, |
| "rewards/rejected": -1.197345495223999, |
| "step": 104 |
| }, |
| { |
| "epoch": 1.7796610169491527, |
| "grad_norm": 39.89897092814635, |
| "learning_rate": 4.780334036866996e-07, |
| "logits/chosen": -1.4013991355895996, |
| "logits/rejected": -1.3012598752975464, |
| "logps/chosen": -30.404062271118164, |
| "logps/rejected": -50.464717864990234, |
| "loss": 0.2444, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.36456581950187683, |
| "rewards/margins": 2.707120895385742, |
| "rewards/rejected": -3.0716869831085205, |
| "step": 105 |
| }, |
| { |
| "epoch": 1.7966101694915255, |
| "grad_norm": 38.70079485674391, |
| "learning_rate": 4.772678842103605e-07, |
| "logits/chosen": -1.396234393119812, |
| "logits/rejected": -1.2844393253326416, |
| "logps/chosen": -30.791566848754883, |
| "logps/rejected": -41.80755615234375, |
| "loss": 0.2245, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2901880741119385, |
| "rewards/margins": 2.5994958877563477, |
| "rewards/rejected": -2.8896842002868652, |
| "step": 106 |
| }, |
| { |
| "epoch": 1.8135593220338984, |
| "grad_norm": 31.615929528424363, |
| "learning_rate": 4.764898879081467e-07, |
| "logits/chosen": -1.3441611528396606, |
| "logits/rejected": -1.3465229272842407, |
| "logps/chosen": -25.936952590942383, |
| "logps/rejected": -44.00107192993164, |
| "loss": 0.2261, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.0497933030128479, |
| "rewards/margins": 1.7863792181015015, |
| "rewards/rejected": -1.7365858554840088, |
| "step": 107 |
| }, |
| { |
| "epoch": 1.8305084745762712, |
| "grad_norm": 36.20936022186147, |
| "learning_rate": 4.7569945749143586e-07, |
| "logits/chosen": -1.3387551307678223, |
| "logits/rejected": -1.329017996788025, |
| "logps/chosen": -26.50401496887207, |
| "logps/rejected": -48.64241027832031, |
| "loss": 0.2448, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1088000237941742, |
| "rewards/margins": 2.8309779167175293, |
| "rewards/rejected": -2.9397776126861572, |
| "step": 108 |
| }, |
| { |
| "epoch": 1.847457627118644, |
| "grad_norm": 36.766498225229036, |
| "learning_rate": 4.748966363542285e-07, |
| "logits/chosen": -1.2870004177093506, |
| "logits/rejected": -1.2100244760513306, |
| "logps/chosen": -24.399860382080078, |
| "logps/rejected": -37.19389343261719, |
| "loss": 0.2463, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.04130461812019348, |
| "rewards/margins": 2.3760554790496826, |
| "rewards/rejected": -2.3347508907318115, |
| "step": 109 |
| }, |
| { |
| "epoch": 1.8644067796610169, |
| "grad_norm": 37.505379590427104, |
| "learning_rate": 4.7408146857076563e-07, |
| "logits/chosen": -1.2373127937316895, |
| "logits/rejected": -1.1650217771530151, |
| "logps/chosen": -39.90864562988281, |
| "logps/rejected": -40.89439392089844, |
| "loss": 0.2646, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.0051874518394470215, |
| "rewards/margins": 1.6469688415527344, |
| "rewards/rejected": -1.6417814493179321, |
| "step": 110 |
| }, |
| { |
| "epoch": 1.8813559322033897, |
| "grad_norm": 35.644427389294876, |
| "learning_rate": 4.732539988931096e-07, |
| "logits/chosen": -1.3618909120559692, |
| "logits/rejected": -1.4531258344650269, |
| "logps/chosen": -26.969541549682617, |
| "logps/rejected": -42.32440185546875, |
| "loss": 0.2208, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.25816747546195984, |
| "rewards/margins": 2.6419811248779297, |
| "rewards/rejected": -2.900148391723633, |
| "step": 111 |
| }, |
| { |
| "epoch": 1.8983050847457628, |
| "grad_norm": 39.403893355046144, |
| "learning_rate": 4.7241427274868683e-07, |
| "logits/chosen": -1.392714262008667, |
| "logits/rejected": -1.2956047058105469, |
| "logps/chosen": -25.744760513305664, |
| "logps/rejected": -43.44940185546875, |
| "loss": 0.2692, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.19541674852371216, |
| "rewards/margins": 2.4149329662323, |
| "rewards/rejected": -2.610349655151367, |
| "step": 112 |
| }, |
| { |
| "epoch": 1.9152542372881356, |
| "grad_norm": 42.63537438222289, |
| "learning_rate": 4.7156233623779383e-07, |
| "logits/chosen": -1.3376697301864624, |
| "logits/rejected": -1.3315945863723755, |
| "logps/chosen": -31.033275604248047, |
| "logps/rejected": -34.59294128417969, |
| "loss": 0.262, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.01679442822933197, |
| "rewards/margins": 1.9989566802978516, |
| "rewards/rejected": -2.015751361846924, |
| "step": 113 |
| }, |
| { |
| "epoch": 1.9322033898305084, |
| "grad_norm": 37.518445889823454, |
| "learning_rate": 4.7069823613106687e-07, |
| "logits/chosen": -1.3494609594345093, |
| "logits/rejected": -1.4168843030929565, |
| "logps/chosen": -37.80765914916992, |
| "logps/rejected": -50.918888092041016, |
| "loss": 0.2293, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.5224637985229492, |
| "rewards/margins": 3.094416618347168, |
| "rewards/rejected": -3.6168806552886963, |
| "step": 114 |
| }, |
| { |
| "epoch": 1.9491525423728815, |
| "grad_norm": 42.51900093030587, |
| "learning_rate": 4.698220198669136e-07, |
| "logits/chosen": -1.646604299545288, |
| "logits/rejected": -1.517140507698059, |
| "logps/chosen": -27.87681770324707, |
| "logps/rejected": -42.672096252441406, |
| "loss": 0.2965, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.16765162348747253, |
| "rewards/margins": 2.965381145477295, |
| "rewards/rejected": -3.133033275604248, |
| "step": 115 |
| }, |
| { |
| "epoch": 1.9661016949152543, |
| "grad_norm": 33.55277119465526, |
| "learning_rate": 4.6893373554890917e-07, |
| "logits/chosen": -1.5241327285766602, |
| "logits/rejected": -1.2908215522766113, |
| "logps/chosen": -33.22343444824219, |
| "logps/rejected": -44.93342208862305, |
| "loss": 0.226, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.36639106273651123, |
| "rewards/margins": 2.9403634071350098, |
| "rewards/rejected": -3.3067543506622314, |
| "step": 116 |
| }, |
| { |
| "epoch": 1.9830508474576272, |
| "grad_norm": 36.89279198796042, |
| "learning_rate": 4.6803343194315546e-07, |
| "logits/chosen": -1.3500347137451172, |
| "logits/rejected": -1.3750879764556885, |
| "logps/chosen": -33.52631759643555, |
| "logps/rejected": -46.71184539794922, |
| "loss": 0.2323, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.17364656925201416, |
| "rewards/margins": 2.6467905044555664, |
| "rewards/rejected": -2.820436954498291, |
| "step": 117 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 29.673763942339104, |
| "learning_rate": 4.6712115847560353e-07, |
| "logits/chosen": -1.1543025970458984, |
| "logits/rejected": -1.1016186475753784, |
| "logps/chosen": -27.275094985961914, |
| "logps/rejected": -35.34591293334961, |
| "loss": 0.2172, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.010025471448898315, |
| "rewards/margins": 2.681260824203491, |
| "rewards/rejected": -2.6712355613708496, |
| "step": 118 |
| }, |
| { |
| "epoch": 2.016949152542373, |
| "grad_norm": 18.754455801142957, |
| "learning_rate": 4.661969652293402e-07, |
| "logits/chosen": -1.3498600721359253, |
| "logits/rejected": -1.2991336584091187, |
| "logps/chosen": -24.873241424560547, |
| "logps/rejected": -45.2943000793457, |
| "loss": 0.1226, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.13535727560520172, |
| "rewards/margins": 3.1097800731658936, |
| "rewards/rejected": -3.2451369762420654, |
| "step": 119 |
| }, |
| { |
| "epoch": 2.0338983050847457, |
| "grad_norm": 22.75173167375844, |
| "learning_rate": 4.652609029418388e-07, |
| "logits/chosen": -1.283535122871399, |
| "logits/rejected": -1.2337732315063477, |
| "logps/chosen": -26.472030639648438, |
| "logps/rejected": -41.27444076538086, |
| "loss": 0.1627, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.14066343009471893, |
| "rewards/margins": 2.8250813484191895, |
| "rewards/rejected": -2.9657444953918457, |
| "step": 120 |
| }, |
| { |
| "epoch": 2.0508474576271185, |
| "grad_norm": 20.99893381194414, |
| "learning_rate": 4.6431302300217366e-07, |
| "logits/chosen": -1.4252784252166748, |
| "logits/rejected": -1.421356439590454, |
| "logps/chosen": -31.787378311157227, |
| "logps/rejected": -33.3957405090332, |
| "loss": 0.1657, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.40507441759109497, |
| "rewards/margins": 1.8697203397750854, |
| "rewards/rejected": -1.4646459817886353, |
| "step": 121 |
| }, |
| { |
| "epoch": 2.0677966101694913, |
| "grad_norm": 18.786273208958082, |
| "learning_rate": 4.633533774481987e-07, |
| "logits/chosen": -1.2753486633300781, |
| "logits/rejected": -1.1164805889129639, |
| "logps/chosen": -31.65105438232422, |
| "logps/rejected": -46.21112823486328, |
| "loss": 0.1353, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.07824045419692993, |
| "rewards/margins": 3.2129859924316406, |
| "rewards/rejected": -3.291226387023926, |
| "step": 122 |
| }, |
| { |
| "epoch": 2.084745762711864, |
| "grad_norm": 20.917039008362774, |
| "learning_rate": 4.623820189636905e-07, |
| "logits/chosen": -1.3903650045394897, |
| "logits/rejected": -1.3099316358566284, |
| "logps/chosen": -29.19454002380371, |
| "logps/rejected": -46.168941497802734, |
| "loss": 0.1536, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.4621618092060089, |
| "rewards/margins": 2.845735788345337, |
| "rewards/rejected": -2.3835740089416504, |
| "step": 123 |
| }, |
| { |
| "epoch": 2.1016949152542375, |
| "grad_norm": 20.45027445287867, |
| "learning_rate": 4.613990008754565e-07, |
| "logits/chosen": -1.3704748153686523, |
| "logits/rejected": -1.2672369480133057, |
| "logps/chosen": -34.432945251464844, |
| "logps/rejected": -39.66848373413086, |
| "loss": 0.1588, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.1847638189792633, |
| "rewards/margins": 2.653686761856079, |
| "rewards/rejected": -2.4689230918884277, |
| "step": 124 |
| }, |
| { |
| "epoch": 2.1186440677966103, |
| "grad_norm": 20.088146504282733, |
| "learning_rate": 4.60404377150407e-07, |
| "logits/chosen": -1.2935415506362915, |
| "logits/rejected": -1.2802854776382446, |
| "logps/chosen": -26.330522537231445, |
| "logps/rejected": -41.638004302978516, |
| "loss": 0.1543, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.15763472020626068, |
| "rewards/margins": 2.74660587310791, |
| "rewards/rejected": -2.904240131378174, |
| "step": 125 |
| }, |
| { |
| "epoch": 2.135593220338983, |
| "grad_norm": 23.38109010648461, |
| "learning_rate": 4.593982023925925e-07, |
| "logits/chosen": -1.1731081008911133, |
| "logits/rejected": -1.0896246433258057, |
| "logps/chosen": -33.23085021972656, |
| "logps/rejected": -40.83133316040039, |
| "loss": 0.1689, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.19674523174762726, |
| "rewards/margins": 2.6324305534362793, |
| "rewards/rejected": -2.8291759490966797, |
| "step": 126 |
| }, |
| { |
| "epoch": 2.152542372881356, |
| "grad_norm": 19.53451419738541, |
| "learning_rate": 4.58380531840206e-07, |
| "logits/chosen": -1.3832257986068726, |
| "logits/rejected": -1.1368017196655273, |
| "logps/chosen": -31.488880157470703, |
| "logps/rejected": -37.851097106933594, |
| "loss": 0.1448, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.38223376870155334, |
| "rewards/margins": 3.1422624588012695, |
| "rewards/rejected": -2.76002836227417, |
| "step": 127 |
| }, |
| { |
| "epoch": 2.169491525423729, |
| "grad_norm": 19.597219846897513, |
| "learning_rate": 4.5735142136255045e-07, |
| "logits/chosen": -1.3937269449234009, |
| "logits/rejected": -1.3436622619628906, |
| "logps/chosen": -28.44049644470215, |
| "logps/rejected": -49.33514404296875, |
| "loss": 0.1489, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.09995569288730621, |
| "rewards/margins": 3.583566665649414, |
| "rewards/rejected": -3.4836111068725586, |
| "step": 128 |
| }, |
| { |
| "epoch": 2.1864406779661016, |
| "grad_norm": 18.394803732689535, |
| "learning_rate": 4.5631092745697164e-07, |
| "logits/chosen": -1.1625999212265015, |
| "logits/rejected": -1.1585655212402344, |
| "logps/chosen": -29.666309356689453, |
| "logps/rejected": -41.36751174926758, |
| "loss": 0.119, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2516571879386902, |
| "rewards/margins": 2.918815851211548, |
| "rewards/rejected": -2.667158603668213, |
| "step": 129 |
| }, |
| { |
| "epoch": 2.2033898305084745, |
| "grad_norm": 19.452927999271022, |
| "learning_rate": 4.5525910724575645e-07, |
| "logits/chosen": -1.298140048980713, |
| "logits/rejected": -1.238019585609436, |
| "logps/chosen": -30.736804962158203, |
| "logps/rejected": -49.28110885620117, |
| "loss": 0.1411, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.11459851264953613, |
| "rewards/margins": 4.15794563293457, |
| "rewards/rejected": -4.043347358703613, |
| "step": 130 |
| }, |
| { |
| "epoch": 2.2203389830508473, |
| "grad_norm": 16.937231864925646, |
| "learning_rate": 4.54196018472997e-07, |
| "logits/chosen": -1.2767530679702759, |
| "logits/rejected": -1.1214213371276855, |
| "logps/chosen": -27.268341064453125, |
| "logps/rejected": -52.620338439941406, |
| "loss": 0.105, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.010926365852355957, |
| "rewards/margins": 4.242362976074219, |
| "rewards/rejected": -4.231436729431152, |
| "step": 131 |
| }, |
| { |
| "epoch": 2.23728813559322, |
| "grad_norm": 19.91080195782553, |
| "learning_rate": 4.5312171950142033e-07, |
| "logits/chosen": -1.443523645401001, |
| "logits/rejected": -1.3692708015441895, |
| "logps/chosen": -23.356483459472656, |
| "logps/rejected": -39.884552001953125, |
| "loss": 0.137, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.09224121272563934, |
| "rewards/margins": 3.541024923324585, |
| "rewards/rejected": -3.4487838745117188, |
| "step": 132 |
| }, |
| { |
| "epoch": 2.2542372881355934, |
| "grad_norm": 20.224862204629513, |
| "learning_rate": 4.520362693091845e-07, |
| "logits/chosen": -1.3375797271728516, |
| "logits/rejected": -1.2925443649291992, |
| "logps/chosen": -24.31032371520996, |
| "logps/rejected": -31.916282653808594, |
| "loss": 0.1541, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.05450062453746796, |
| "rewards/margins": 1.724971055984497, |
| "rewards/rejected": -1.6704705953598022, |
| "step": 133 |
| }, |
| { |
| "epoch": 2.2711864406779663, |
| "grad_norm": 19.256180369703056, |
| "learning_rate": 4.5093972748664087e-07, |
| "logits/chosen": -1.3231240510940552, |
| "logits/rejected": -1.2512582540512085, |
| "logps/chosen": -35.62217330932617, |
| "logps/rejected": -48.332237243652344, |
| "loss": 0.1134, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.26731669902801514, |
| "rewards/margins": 3.2187671661376953, |
| "rewards/rejected": -3.486083745956421, |
| "step": 134 |
| }, |
| { |
| "epoch": 2.288135593220339, |
| "grad_norm": 17.92618734739177, |
| "learning_rate": 4.498321542330622e-07, |
| "logits/chosen": -1.4986979961395264, |
| "logits/rejected": -1.4225276708602905, |
| "logps/chosen": -26.092084884643555, |
| "logps/rejected": -52.08736038208008, |
| "loss": 0.1094, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.04193298518657684, |
| "rewards/margins": 3.300752639770508, |
| "rewards/rejected": -3.3426852226257324, |
| "step": 135 |
| }, |
| { |
| "epoch": 2.305084745762712, |
| "grad_norm": 19.524384947131523, |
| "learning_rate": 4.4871361035333833e-07, |
| "logits/chosen": -1.302308201789856, |
| "logits/rejected": -1.306333065032959, |
| "logps/chosen": -25.217451095581055, |
| "logps/rejected": -43.95090103149414, |
| "loss": 0.1352, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.24911952018737793, |
| "rewards/margins": 3.123009204864502, |
| "rewards/rejected": -2.873889684677124, |
| "step": 136 |
| }, |
| { |
| "epoch": 2.3220338983050848, |
| "grad_norm": 21.654262830345974, |
| "learning_rate": 4.475841572546374e-07, |
| "logits/chosen": -1.333143711090088, |
| "logits/rejected": -1.1934046745300293, |
| "logps/chosen": -29.572952270507812, |
| "logps/rejected": -42.41865539550781, |
| "loss": 0.1608, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2631007432937622, |
| "rewards/margins": 2.9843344688415527, |
| "rewards/rejected": -3.2474350929260254, |
| "step": 137 |
| }, |
| { |
| "epoch": 2.3389830508474576, |
| "grad_norm": 18.2902071706345, |
| "learning_rate": 4.464438569430353e-07, |
| "logits/chosen": -1.4010558128356934, |
| "logits/rejected": -1.3596662282943726, |
| "logps/chosen": -27.541452407836914, |
| "logps/rejected": -37.1332893371582, |
| "loss": 0.1122, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.17605237662792206, |
| "rewards/margins": 2.4087536334991455, |
| "rewards/rejected": -2.232701301574707, |
| "step": 138 |
| }, |
| { |
| "epoch": 2.3559322033898304, |
| "grad_norm": 20.193211619680806, |
| "learning_rate": 4.452927720201112e-07, |
| "logits/chosen": -1.1419758796691895, |
| "logits/rejected": -1.2467293739318848, |
| "logps/chosen": -24.366790771484375, |
| "logps/rejected": -44.22784423828125, |
| "loss": 0.1357, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2860787510871887, |
| "rewards/margins": 3.2284557819366455, |
| "rewards/rejected": -2.9423768520355225, |
| "step": 139 |
| }, |
| { |
| "epoch": 2.3728813559322033, |
| "grad_norm": 16.846442500740498, |
| "learning_rate": 4.441309656795106e-07, |
| "logits/chosen": -1.3143264055252075, |
| "logits/rejected": -1.1588001251220703, |
| "logps/chosen": -25.230070114135742, |
| "logps/rejected": -50.344722747802734, |
| "loss": 0.1133, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2081325650215149, |
| "rewards/margins": 3.1986870765686035, |
| "rewards/rejected": -2.990554094314575, |
| "step": 140 |
| }, |
| { |
| "epoch": 2.389830508474576, |
| "grad_norm": 19.91817673644514, |
| "learning_rate": 4.429585017034766e-07, |
| "logits/chosen": -1.355256199836731, |
| "logits/rejected": -1.3759305477142334, |
| "logps/chosen": -30.238567352294922, |
| "logps/rejected": -43.92167663574219, |
| "loss": 0.1377, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.17302727699279785, |
| "rewards/margins": 3.385795831680298, |
| "rewards/rejected": -3.558823347091675, |
| "step": 141 |
| }, |
| { |
| "epoch": 2.406779661016949, |
| "grad_norm": 17.91719154232722, |
| "learning_rate": 4.417754444593478e-07, |
| "logits/chosen": -1.452804684638977, |
| "logits/rejected": -1.394730567932129, |
| "logps/chosen": -29.204275131225586, |
| "logps/rejected": -45.112300872802734, |
| "loss": 0.1072, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.046540290117263794, |
| "rewards/margins": 4.2701945304870605, |
| "rewards/rejected": -4.223654270172119, |
| "step": 142 |
| }, |
| { |
| "epoch": 2.423728813559322, |
| "grad_norm": 17.305416636387555, |
| "learning_rate": 4.4058185889602497e-07, |
| "logits/chosen": -1.375995397567749, |
| "logits/rejected": -1.3006415367126465, |
| "logps/chosen": -17.108240127563477, |
| "logps/rejected": -35.922874450683594, |
| "loss": 0.1445, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.28150078654289246, |
| "rewards/margins": 3.295111656188965, |
| "rewards/rejected": -3.01361083984375, |
| "step": 143 |
| }, |
| { |
| "epoch": 2.440677966101695, |
| "grad_norm": 23.303228169178016, |
| "learning_rate": 4.39377810540405e-07, |
| "logits/chosen": -1.4052019119262695, |
| "logits/rejected": -1.4526053667068481, |
| "logps/chosen": -41.58496856689453, |
| "logps/rejected": -40.82295227050781, |
| "loss": 0.1742, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.30576008558273315, |
| "rewards/margins": 2.3252716064453125, |
| "rewards/rejected": -2.6310315132141113, |
| "step": 144 |
| }, |
| { |
| "epoch": 2.457627118644068, |
| "grad_norm": 19.121378802553227, |
| "learning_rate": 4.38163365493784e-07, |
| "logits/chosen": -1.5245730876922607, |
| "logits/rejected": -1.401250958442688, |
| "logps/chosen": -34.83244705200195, |
| "logps/rejected": -64.91217041015625, |
| "loss": 0.1261, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.20400622487068176, |
| "rewards/margins": 4.261959552764893, |
| "rewards/rejected": -4.057952880859375, |
| "step": 145 |
| }, |
| { |
| "epoch": 2.4745762711864407, |
| "grad_norm": 19.034390396216985, |
| "learning_rate": 4.3693859042822774e-07, |
| "logits/chosen": -1.2971210479736328, |
| "logits/rejected": -1.2576966285705566, |
| "logps/chosen": -33.01571273803711, |
| "logps/rejected": -45.87381362915039, |
| "loss": 0.1277, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.19635039567947388, |
| "rewards/margins": 3.7301583290100098, |
| "rewards/rejected": -3.5338077545166016, |
| "step": 146 |
| }, |
| { |
| "epoch": 2.4915254237288136, |
| "grad_norm": 18.357502635593228, |
| "learning_rate": 4.3570355258291223e-07, |
| "logits/chosen": -1.2477927207946777, |
| "logits/rejected": -1.2029328346252441, |
| "logps/chosen": -28.15603256225586, |
| "logps/rejected": -39.89695358276367, |
| "loss": 0.1305, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.18680232763290405, |
| "rewards/margins": 2.8544976711273193, |
| "rewards/rejected": -2.6676955223083496, |
| "step": 147 |
| }, |
| { |
| "epoch": 2.5084745762711864, |
| "grad_norm": 13.757078257538927, |
| "learning_rate": 4.344583197604318e-07, |
| "logits/chosen": -1.280619740486145, |
| "logits/rejected": -1.2003180980682373, |
| "logps/chosen": -24.585142135620117, |
| "logps/rejected": -49.09882354736328, |
| "loss": 0.0795, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.009844973683357239, |
| "rewards/margins": 3.587639331817627, |
| "rewards/rejected": -3.5974843502044678, |
| "step": 148 |
| }, |
| { |
| "epoch": 2.5254237288135593, |
| "grad_norm": 20.493002641349737, |
| "learning_rate": 4.332029603230767e-07, |
| "logits/chosen": -1.2804765701293945, |
| "logits/rejected": -1.2299047708511353, |
| "logps/chosen": -38.74854278564453, |
| "logps/rejected": -40.60469055175781, |
| "loss": 0.1209, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.31216520071029663, |
| "rewards/margins": 3.550785541534424, |
| "rewards/rejected": -3.2386200428009033, |
| "step": 149 |
| }, |
| { |
| "epoch": 2.542372881355932, |
| "grad_norm": 18.873521616647515, |
| "learning_rate": 4.319375431890806e-07, |
| "logits/chosen": -1.5655155181884766, |
| "logits/rejected": -1.5413960218429565, |
| "logps/chosen": -29.58742904663086, |
| "logps/rejected": -38.81654739379883, |
| "loss": 0.1254, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10849936306476593, |
| "rewards/margins": 4.284536361694336, |
| "rewards/rejected": -4.393035888671875, |
| "step": 150 |
| }, |
| { |
| "epoch": 2.559322033898305, |
| "grad_norm": 19.108581899079446, |
| "learning_rate": 4.306621378288364e-07, |
| "logits/chosen": -1.227691650390625, |
| "logits/rejected": -1.206729769706726, |
| "logps/chosen": -29.225234985351562, |
| "logps/rejected": -47.78348159790039, |
| "loss": 0.1189, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.18846926093101501, |
| "rewards/margins": 3.5759382247924805, |
| "rewards/rejected": -3.3874690532684326, |
| "step": 151 |
| }, |
| { |
| "epoch": 2.576271186440678, |
| "grad_norm": 17.54679094674662, |
| "learning_rate": 4.2937681426108275e-07, |
| "logits/chosen": -1.2980151176452637, |
| "logits/rejected": -1.2792203426361084, |
| "logps/chosen": -29.963348388671875, |
| "logps/rejected": -36.31569290161133, |
| "loss": 0.119, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.15102972090244293, |
| "rewards/margins": 2.2603495121002197, |
| "rewards/rejected": -2.411379098892212, |
| "step": 152 |
| }, |
| { |
| "epoch": 2.593220338983051, |
| "grad_norm": 20.293681732169816, |
| "learning_rate": 4.280816430490602e-07, |
| "logits/chosen": -1.6751010417938232, |
| "logits/rejected": -1.579331874847412, |
| "logps/chosen": -27.965721130371094, |
| "logps/rejected": -41.831912994384766, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.03810068964958191, |
| "rewards/margins": 3.356261968612671, |
| "rewards/rejected": -3.3181610107421875, |
| "step": 153 |
| }, |
| { |
| "epoch": 2.610169491525424, |
| "grad_norm": 18.30214205628312, |
| "learning_rate": 4.2677669529663686e-07, |
| "logits/chosen": -1.2230945825576782, |
| "logits/rejected": -1.3257890939712524, |
| "logps/chosen": -22.84085464477539, |
| "logps/rejected": -30.21630859375, |
| "loss": 0.1296, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4078897535800934, |
| "rewards/margins": 2.349480152130127, |
| "rewards/rejected": -1.9415905475616455, |
| "step": 154 |
| }, |
| { |
| "epoch": 2.6271186440677967, |
| "grad_norm": 16.933166318116392, |
| "learning_rate": 4.254620426444053e-07, |
| "logits/chosen": -1.2983089685440063, |
| "logits/rejected": -1.2278701066970825, |
| "logps/chosen": -28.5270938873291, |
| "logps/rejected": -47.79298400878906, |
| "loss": 0.1053, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.39613279700279236, |
| "rewards/margins": 4.555056571960449, |
| "rewards/rejected": -4.158923625946045, |
| "step": 155 |
| }, |
| { |
| "epoch": 2.6440677966101696, |
| "grad_norm": 16.954026895333676, |
| "learning_rate": 4.2413775726574923e-07, |
| "logits/chosen": -1.4136017560958862, |
| "logits/rejected": -1.308146357536316, |
| "logps/chosen": -26.033409118652344, |
| "logps/rejected": -45.159584045410156, |
| "loss": 0.1098, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3105984926223755, |
| "rewards/margins": 3.2515172958374023, |
| "rewards/rejected": -3.5621156692504883, |
| "step": 156 |
| }, |
| { |
| "epoch": 2.6610169491525424, |
| "grad_norm": 23.018543543838756, |
| "learning_rate": 4.228039118628815e-07, |
| "logits/chosen": -1.3158849477767944, |
| "logits/rejected": -1.1991815567016602, |
| "logps/chosen": -25.21765899658203, |
| "logps/rejected": -40.39363479614258, |
| "loss": 0.1629, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.04916132241487503, |
| "rewards/margins": 2.8159801959991455, |
| "rewards/rejected": -2.7668187618255615, |
| "step": 157 |
| }, |
| { |
| "epoch": 2.6779661016949152, |
| "grad_norm": 18.04681645919104, |
| "learning_rate": 4.214605796628526e-07, |
| "logits/chosen": -1.3597509860992432, |
| "logits/rejected": -1.375614881515503, |
| "logps/chosen": -26.287364959716797, |
| "logps/rejected": -41.234405517578125, |
| "loss": 0.1155, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.18970844149589539, |
| "rewards/margins": 3.569676399230957, |
| "rewards/rejected": -3.3799679279327393, |
| "step": 158 |
| }, |
| { |
| "epoch": 2.694915254237288, |
| "grad_norm": 14.068268306629102, |
| "learning_rate": 4.201078344135306e-07, |
| "logits/chosen": -1.497442364692688, |
| "logits/rejected": -1.4510717391967773, |
| "logps/chosen": -27.79704475402832, |
| "logps/rejected": -43.55573272705078, |
| "loss": 0.101, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.037288397550582886, |
| "rewards/margins": 3.6583452224731445, |
| "rewards/rejected": -3.6210567951202393, |
| "step": 159 |
| }, |
| { |
| "epoch": 2.711864406779661, |
| "grad_norm": 20.29404199604278, |
| "learning_rate": 4.187457503795526e-07, |
| "logits/chosen": -1.6163471937179565, |
| "logits/rejected": -1.5419741868972778, |
| "logps/chosen": -26.61087417602539, |
| "logps/rejected": -30.924564361572266, |
| "loss": 0.1336, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.18512818217277527, |
| "rewards/margins": 2.91485857963562, |
| "rewards/rejected": -2.7297308444976807, |
| "step": 160 |
| }, |
| { |
| "epoch": 2.7288135593220337, |
| "grad_norm": 14.433423874260647, |
| "learning_rate": 4.173744023382474e-07, |
| "logits/chosen": -1.6092435121536255, |
| "logits/rejected": -1.4594898223876953, |
| "logps/chosen": -24.235836029052734, |
| "logps/rejected": -42.99752426147461, |
| "loss": 0.0838, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.11106225848197937, |
| "rewards/margins": 3.523545742034912, |
| "rewards/rejected": -3.6346077919006348, |
| "step": 161 |
| }, |
| { |
| "epoch": 2.7457627118644066, |
| "grad_norm": 18.073004370156372, |
| "learning_rate": 4.159938655755306e-07, |
| "logits/chosen": -1.2371647357940674, |
| "logits/rejected": -1.249834656715393, |
| "logps/chosen": -26.12664031982422, |
| "logps/rejected": -41.84228515625, |
| "loss": 0.117, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.07593405246734619, |
| "rewards/margins": 3.008237361907959, |
| "rewards/rejected": -2.9323031902313232, |
| "step": 162 |
| }, |
| { |
| "epoch": 2.7627118644067794, |
| "grad_norm": 17.580682721769048, |
| "learning_rate": 4.1460421588177094e-07, |
| "logits/chosen": -1.3883872032165527, |
| "logits/rejected": -1.284624695777893, |
| "logps/chosen": -24.988061904907227, |
| "logps/rejected": -44.79413986206055, |
| "loss": 0.1081, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.04829689860343933, |
| "rewards/margins": 4.283046722412109, |
| "rewards/rejected": -4.234749794006348, |
| "step": 163 |
| }, |
| { |
| "epoch": 2.7796610169491527, |
| "grad_norm": 14.683936946298637, |
| "learning_rate": 4.1320552954763037e-07, |
| "logits/chosen": -1.3138792514801025, |
| "logits/rejected": -1.3704159259796143, |
| "logps/chosen": -34.95574188232422, |
| "logps/rejected": -41.10405731201172, |
| "loss": 0.0829, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08980777859687805, |
| "rewards/margins": 3.34732723236084, |
| "rewards/rejected": -3.257519483566284, |
| "step": 164 |
| }, |
| { |
| "epoch": 2.7966101694915255, |
| "grad_norm": 19.53577404998919, |
| "learning_rate": 4.117978833598747e-07, |
| "logits/chosen": -1.4069619178771973, |
| "logits/rejected": -1.3315298557281494, |
| "logps/chosen": -38.807586669921875, |
| "logps/rejected": -41.68088912963867, |
| "loss": 0.1398, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.32595694065093994, |
| "rewards/margins": 2.6362242698669434, |
| "rewards/rejected": -2.962181329727173, |
| "step": 165 |
| }, |
| { |
| "epoch": 2.8135593220338984, |
| "grad_norm": 15.630747955128056, |
| "learning_rate": 4.1038135459715885e-07, |
| "logits/chosen": -1.3965390920639038, |
| "logits/rejected": -1.3702296018600464, |
| "logps/chosen": -17.21418571472168, |
| "logps/rejected": -33.933189392089844, |
| "loss": 0.0941, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.29006946086883545, |
| "rewards/margins": 3.5608468055725098, |
| "rewards/rejected": -3.270777463912964, |
| "step": 166 |
| }, |
| { |
| "epoch": 2.830508474576271, |
| "grad_norm": 15.319247967225127, |
| "learning_rate": 4.0895602102578373e-07, |
| "logits/chosen": -1.2587236166000366, |
| "logits/rejected": -1.2784702777862549, |
| "logps/chosen": -33.745662689208984, |
| "logps/rejected": -53.36175537109375, |
| "loss": 0.0943, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5272909998893738, |
| "rewards/margins": 3.7912838459014893, |
| "rewards/rejected": -4.318574905395508, |
| "step": 167 |
| }, |
| { |
| "epoch": 2.847457627118644, |
| "grad_norm": 17.91197797858757, |
| "learning_rate": 4.075219608954278e-07, |
| "logits/chosen": -1.220982313156128, |
| "logits/rejected": -1.119415044784546, |
| "logps/chosen": -25.41081428527832, |
| "logps/rejected": -43.74126434326172, |
| "loss": 0.1181, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.16412924230098724, |
| "rewards/margins": 3.79860258102417, |
| "rewards/rejected": -3.6344728469848633, |
| "step": 168 |
| }, |
| { |
| "epoch": 2.864406779661017, |
| "grad_norm": 19.216931577152895, |
| "learning_rate": 4.0607925293484997e-07, |
| "logits/chosen": -1.3364936113357544, |
| "logits/rejected": -1.3348599672317505, |
| "logps/chosen": -28.252498626708984, |
| "logps/rejected": -36.41025161743164, |
| "loss": 0.1405, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.07597078382968903, |
| "rewards/margins": 2.509685516357422, |
| "rewards/rejected": -2.5856566429138184, |
| "step": 169 |
| }, |
| { |
| "epoch": 2.8813559322033897, |
| "grad_norm": 18.004667351375737, |
| "learning_rate": 4.046279763475687e-07, |
| "logits/chosen": -1.4610190391540527, |
| "logits/rejected": -1.467834234237671, |
| "logps/chosen": -23.43694305419922, |
| "logps/rejected": -40.96953582763672, |
| "loss": 0.12, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.00839678943157196, |
| "rewards/margins": 3.222750186920166, |
| "rewards/rejected": -3.214353561401367, |
| "step": 170 |
| }, |
| { |
| "epoch": 2.898305084745763, |
| "grad_norm": 16.724266176013543, |
| "learning_rate": 4.031682108075128e-07, |
| "logits/chosen": -1.3930025100708008, |
| "logits/rejected": -1.3053603172302246, |
| "logps/chosen": -26.837438583374023, |
| "logps/rejected": -46.042606353759766, |
| "loss": 0.115, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3490511476993561, |
| "rewards/margins": 3.341874122619629, |
| "rewards/rejected": -3.6909255981445312, |
| "step": 171 |
| }, |
| { |
| "epoch": 2.915254237288136, |
| "grad_norm": 17.501626125315028, |
| "learning_rate": 4.0170003645464835e-07, |
| "logits/chosen": -1.4981375932693481, |
| "logits/rejected": -1.4686487913131714, |
| "logps/chosen": -31.091188430786133, |
| "logps/rejected": -41.67449188232422, |
| "loss": 0.1159, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.08511877059936523, |
| "rewards/margins": 3.4050216674804688, |
| "rewards/rejected": -3.490140438079834, |
| "step": 172 |
| }, |
| { |
| "epoch": 2.9322033898305087, |
| "grad_norm": 17.505812920779697, |
| "learning_rate": 4.0022353389057793e-07, |
| "logits/chosen": -1.5352020263671875, |
| "logits/rejected": -1.4121986627578735, |
| "logps/chosen": -29.729156494140625, |
| "logps/rejected": -48.48210525512695, |
| "loss": 0.1104, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.050699926912784576, |
| "rewards/margins": 3.282029390335083, |
| "rewards/rejected": -3.2313296794891357, |
| "step": 173 |
| }, |
| { |
| "epoch": 2.9491525423728815, |
| "grad_norm": 15.718044631960725, |
| "learning_rate": 3.9873878417411685e-07, |
| "logits/chosen": -1.3283764123916626, |
| "logits/rejected": -1.2386645078659058, |
| "logps/chosen": -31.367496490478516, |
| "logps/rejected": -48.3538818359375, |
| "loss": 0.0849, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.02908661961555481, |
| "rewards/margins": 4.318341255187988, |
| "rewards/rejected": -4.3474273681640625, |
| "step": 174 |
| }, |
| { |
| "epoch": 2.9661016949152543, |
| "grad_norm": 19.307114282390735, |
| "learning_rate": 3.97245868816842e-07, |
| "logits/chosen": -1.61316978931427, |
| "logits/rejected": -1.4183673858642578, |
| "logps/chosen": -25.636213302612305, |
| "logps/rejected": -31.219690322875977, |
| "loss": 0.1342, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.19882389903068542, |
| "rewards/margins": 2.8216686248779297, |
| "rewards/rejected": -2.622844696044922, |
| "step": 175 |
| }, |
| { |
| "epoch": 2.983050847457627, |
| "grad_norm": 19.852640661420647, |
| "learning_rate": 3.95744869778618e-07, |
| "logits/chosen": -1.3724339008331299, |
| "logits/rejected": -1.2494310140609741, |
| "logps/chosen": -37.323822021484375, |
| "logps/rejected": -48.69821548461914, |
| "loss": 0.1322, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2126758098602295, |
| "rewards/margins": 3.236673355102539, |
| "rewards/rejected": -3.4493494033813477, |
| "step": 176 |
| }, |
| { |
| "epoch": 3.0, |
| "grad_norm": 17.961834616955002, |
| "learning_rate": 3.942358694630967e-07, |
| "logits/chosen": -1.4469400644302368, |
| "logits/rejected": -1.4965747594833374, |
| "logps/chosen": -27.073068618774414, |
| "logps/rejected": -47.31336212158203, |
| "loss": 0.1379, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2840898931026459, |
| "rewards/margins": 3.4035041332244873, |
| "rewards/rejected": -3.687593936920166, |
| "step": 177 |
| }, |
| { |
| "epoch": 3.016949152542373, |
| "grad_norm": 11.404351341704286, |
| "learning_rate": 3.927189507131938e-07, |
| "logits/chosen": -1.340846300125122, |
| "logits/rejected": -1.3331762552261353, |
| "logps/chosen": -29.038299560546875, |
| "logps/rejected": -44.76627731323242, |
| "loss": 0.0734, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.44353172183036804, |
| "rewards/margins": 3.546970844268799, |
| "rewards/rejected": -3.99050235748291, |
| "step": 178 |
| }, |
| { |
| "epoch": 3.0338983050847457, |
| "grad_norm": 13.04527280968449, |
| "learning_rate": 3.9119419680654083e-07, |
| "logits/chosen": -1.3644622564315796, |
| "logits/rejected": -1.2482867240905762, |
| "logps/chosen": -29.963117599487305, |
| "logps/rejected": -44.21002960205078, |
| "loss": 0.0936, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.05338460952043533, |
| "rewards/margins": 3.796813488006592, |
| "rewards/rejected": -3.7434287071228027, |
| "step": 179 |
| }, |
| { |
| "epoch": 3.0508474576271185, |
| "grad_norm": 10.551592699584061, |
| "learning_rate": 3.896616914509131e-07, |
| "logits/chosen": -1.1854358911514282, |
| "logits/rejected": -1.2167768478393555, |
| "logps/chosen": -28.745718002319336, |
| "logps/rejected": -39.30556869506836, |
| "loss": 0.0617, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.07138124108314514, |
| "rewards/margins": 3.6734557151794434, |
| "rewards/rejected": -3.60207462310791, |
| "step": 180 |
| }, |
| { |
| "epoch": 3.0677966101694913, |
| "grad_norm": 12.613695441723, |
| "learning_rate": 3.881215187796344e-07, |
| "logits/chosen": -1.5086756944656372, |
| "logits/rejected": -1.4750981330871582, |
| "logps/chosen": -24.299089431762695, |
| "logps/rejected": -48.68684005737305, |
| "loss": 0.088, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.2250959873199463, |
| "rewards/margins": 5.23423433303833, |
| "rewards/rejected": -5.0091376304626465, |
| "step": 181 |
| }, |
| { |
| "epoch": 3.084745762711864, |
| "grad_norm": 12.720295333105653, |
| "learning_rate": 3.865737633469579e-07, |
| "logits/chosen": -1.456554889678955, |
| "logits/rejected": -1.4655078649520874, |
| "logps/chosen": -34.70986557006836, |
| "logps/rejected": -45.1373405456543, |
| "loss": 0.0913, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5282381176948547, |
| "rewards/margins": 3.892458438873291, |
| "rewards/rejected": -4.420697212219238, |
| "step": 182 |
| }, |
| { |
| "epoch": 3.1016949152542375, |
| "grad_norm": 11.42456365348978, |
| "learning_rate": 3.8501851012342444e-07, |
| "logits/chosen": -1.429445505142212, |
| "logits/rejected": -1.2520623207092285, |
| "logps/chosen": -33.7725830078125, |
| "logps/rejected": -48.29920196533203, |
| "loss": 0.0673, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.20377352833747864, |
| "rewards/margins": 3.905850887298584, |
| "rewards/rejected": -4.10962438583374, |
| "step": 183 |
| }, |
| { |
| "epoch": 3.1186440677966103, |
| "grad_norm": 11.707964288174598, |
| "learning_rate": 3.834558444911977e-07, |
| "logits/chosen": -1.442047119140625, |
| "logits/rejected": -1.2768933773040771, |
| "logps/chosen": -31.306930541992188, |
| "logps/rejected": -55.08317184448242, |
| "loss": 0.0851, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1050567626953125, |
| "rewards/margins": 4.629131317138672, |
| "rewards/rejected": -4.524074554443359, |
| "step": 184 |
| }, |
| { |
| "epoch": 3.135593220338983, |
| "grad_norm": 12.341349720055733, |
| "learning_rate": 3.818858522393763e-07, |
| "logits/chosen": -1.4654240608215332, |
| "logits/rejected": -1.2980097532272339, |
| "logps/chosen": -25.214473724365234, |
| "logps/rejected": -48.52466583251953, |
| "loss": 0.084, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.06900361180305481, |
| "rewards/margins": 3.743170738220215, |
| "rewards/rejected": -3.8121743202209473, |
| "step": 185 |
| }, |
| { |
| "epoch": 3.152542372881356, |
| "grad_norm": 11.013651552230316, |
| "learning_rate": 3.8030861955928496e-07, |
| "logits/chosen": -1.4930305480957031, |
| "logits/rejected": -1.49806809425354, |
| "logps/chosen": -33.961334228515625, |
| "logps/rejected": -60.0030403137207, |
| "loss": 0.0587, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1520581841468811, |
| "rewards/margins": 4.225584983825684, |
| "rewards/rejected": -4.377642631530762, |
| "step": 186 |
| }, |
| { |
| "epoch": 3.169491525423729, |
| "grad_norm": 11.565802520156074, |
| "learning_rate": 3.787242330397418e-07, |
| "logits/chosen": -1.151625633239746, |
| "logits/rejected": -1.229320764541626, |
| "logps/chosen": -27.614501953125, |
| "logps/rejected": -46.39374542236328, |
| "loss": 0.0744, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1534918248653412, |
| "rewards/margins": 3.757617235183716, |
| "rewards/rejected": -3.6041250228881836, |
| "step": 187 |
| }, |
| { |
| "epoch": 3.1864406779661016, |
| "grad_norm": 11.91938723600508, |
| "learning_rate": 3.7713277966230513e-07, |
| "logits/chosen": -1.3814018964767456, |
| "logits/rejected": -1.3967022895812988, |
| "logps/chosen": -39.28313446044922, |
| "logps/rejected": -51.139671325683594, |
| "loss": 0.0781, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.14791953563690186, |
| "rewards/margins": 3.787137508392334, |
| "rewards/rejected": -3.639218330383301, |
| "step": 188 |
| }, |
| { |
| "epoch": 3.2033898305084745, |
| "grad_norm": 11.7798233218555, |
| "learning_rate": 3.755343467964981e-07, |
| "logits/chosen": -1.4196237325668335, |
| "logits/rejected": -1.281465768814087, |
| "logps/chosen": -30.973888397216797, |
| "logps/rejected": -60.48612976074219, |
| "loss": 0.0728, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.04596884548664093, |
| "rewards/margins": 5.157370567321777, |
| "rewards/rejected": -5.203339099884033, |
| "step": 189 |
| }, |
| { |
| "epoch": 3.2203389830508473, |
| "grad_norm": 9.848668366337494, |
| "learning_rate": 3.739290221950123e-07, |
| "logits/chosen": -1.4920192956924438, |
| "logits/rejected": -1.3025527000427246, |
| "logps/chosen": -20.232595443725586, |
| "logps/rejected": -40.525020599365234, |
| "loss": 0.0603, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6132769584655762, |
| "rewards/margins": 4.133052349090576, |
| "rewards/rejected": -3.519775390625, |
| "step": 190 |
| }, |
| { |
| "epoch": 3.23728813559322, |
| "grad_norm": 11.384419516356248, |
| "learning_rate": 3.723168939888901e-07, |
| "logits/chosen": -1.4284577369689941, |
| "logits/rejected": -1.3038253784179688, |
| "logps/chosen": -36.025821685791016, |
| "logps/rejected": -47.912906646728516, |
| "loss": 0.0729, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4135657548904419, |
| "rewards/margins": 4.515513896942139, |
| "rewards/rejected": -4.101947784423828, |
| "step": 191 |
| }, |
| { |
| "epoch": 3.2542372881355934, |
| "grad_norm": 12.741788906869417, |
| "learning_rate": 3.7069805068268624e-07, |
| "logits/chosen": -1.190822958946228, |
| "logits/rejected": -1.2386127710342407, |
| "logps/chosen": -25.53938865661621, |
| "logps/rejected": -41.572757720947266, |
| "loss": 0.0968, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.21493886411190033, |
| "rewards/margins": 3.464285373687744, |
| "rewards/rejected": -3.6792244911193848, |
| "step": 192 |
| }, |
| { |
| "epoch": 3.2711864406779663, |
| "grad_norm": 11.543121161706487, |
| "learning_rate": 3.6907258114960915e-07, |
| "logits/chosen": -1.44771146774292, |
| "logits/rejected": -1.3371340036392212, |
| "logps/chosen": -22.689008712768555, |
| "logps/rejected": -33.55266571044922, |
| "loss": 0.0759, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.20511241257190704, |
| "rewards/margins": 3.822577953338623, |
| "rewards/rejected": -3.617465019226074, |
| "step": 193 |
| }, |
| { |
| "epoch": 3.288135593220339, |
| "grad_norm": 12.699281503282757, |
| "learning_rate": 3.6744057462664194e-07, |
| "logits/chosen": -1.2974333763122559, |
| "logits/rejected": -1.2633615732192993, |
| "logps/chosen": -35.57712936401367, |
| "logps/rejected": -41.26565170288086, |
| "loss": 0.079, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1042805016040802, |
| "rewards/margins": 3.9843153953552246, |
| "rewards/rejected": -4.088595867156982, |
| "step": 194 |
| }, |
| { |
| "epoch": 3.305084745762712, |
| "grad_norm": 11.346679553734594, |
| "learning_rate": 3.658021207096432e-07, |
| "logits/chosen": -1.250510811805725, |
| "logits/rejected": -1.2325608730316162, |
| "logps/chosen": -27.381729125976562, |
| "logps/rejected": -39.44231414794922, |
| "loss": 0.071, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.14005836844444275, |
| "rewards/margins": 3.225729465484619, |
| "rewards/rejected": -3.0856711864471436, |
| "step": 195 |
| }, |
| { |
| "epoch": 3.3220338983050848, |
| "grad_norm": 11.675211775416328, |
| "learning_rate": 3.6415730934842825e-07, |
| "logits/chosen": -1.4330198764801025, |
| "logits/rejected": -1.2990684509277344, |
| "logps/chosen": -26.091453552246094, |
| "logps/rejected": -37.71536636352539, |
| "loss": 0.0855, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5305294990539551, |
| "rewards/margins": 3.7357966899871826, |
| "rewards/rejected": -3.2052674293518066, |
| "step": 196 |
| }, |
| { |
| "epoch": 3.3389830508474576, |
| "grad_norm": 10.595079308232455, |
| "learning_rate": 3.625062308418311e-07, |
| "logits/chosen": -1.4256861209869385, |
| "logits/rejected": -1.3300725221633911, |
| "logps/chosen": -44.14484786987305, |
| "logps/rejected": -49.18852233886719, |
| "loss": 0.0731, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.23929768800735474, |
| "rewards/margins": 3.831993818283081, |
| "rewards/rejected": -4.071290969848633, |
| "step": 197 |
| }, |
| { |
| "epoch": 3.3559322033898304, |
| "grad_norm": 10.589818972158676, |
| "learning_rate": 3.6084897583274715e-07, |
| "logits/chosen": -1.5123655796051025, |
| "logits/rejected": -1.4504189491271973, |
| "logps/chosen": -21.742530822753906, |
| "logps/rejected": -43.59285354614258, |
| "loss": 0.0587, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08142367005348206, |
| "rewards/margins": 4.206247329711914, |
| "rewards/rejected": -4.124823570251465, |
| "step": 198 |
| }, |
| { |
| "epoch": 3.3728813559322033, |
| "grad_norm": 10.064171955563651, |
| "learning_rate": 3.591856353031566e-07, |
| "logits/chosen": -1.436945915222168, |
| "logits/rejected": -1.475320816040039, |
| "logps/chosen": -22.611244201660156, |
| "logps/rejected": -43.448341369628906, |
| "loss": 0.0701, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2055773138999939, |
| "rewards/margins": 4.247138500213623, |
| "rewards/rejected": -4.041561126708984, |
| "step": 199 |
| }, |
| { |
| "epoch": 3.389830508474576, |
| "grad_norm": 9.40202931235145, |
| "learning_rate": 3.5751630056913013e-07, |
| "logits/chosen": -1.5867615938186646, |
| "logits/rejected": -1.559362769126892, |
| "logps/chosen": -27.019208908081055, |
| "logps/rejected": -38.67339324951172, |
| "loss": 0.059, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.21613261103630066, |
| "rewards/margins": 3.4724230766296387, |
| "rewards/rejected": -3.2562904357910156, |
| "step": 200 |
| }, |
| { |
| "epoch": 3.406779661016949, |
| "grad_norm": 10.1184235462117, |
| "learning_rate": 3.558410632758153e-07, |
| "logits/chosen": -1.4771666526794434, |
| "logits/rejected": -1.4069215059280396, |
| "logps/chosen": -23.256723403930664, |
| "logps/rejected": -42.76215744018555, |
| "loss": 0.0723, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2688101828098297, |
| "rewards/margins": 4.244396686553955, |
| "rewards/rejected": -3.9755868911743164, |
| "step": 201 |
| }, |
| { |
| "epoch": 3.423728813559322, |
| "grad_norm": 13.152928025513987, |
| "learning_rate": 3.5416001539240574e-07, |
| "logits/chosen": -1.5284346342086792, |
| "logits/rejected": -1.4866607189178467, |
| "logps/chosen": -24.796672821044922, |
| "logps/rejected": -51.007835388183594, |
| "loss": 0.098, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.19576720893383026, |
| "rewards/margins": 4.06268835067749, |
| "rewards/rejected": -4.258455753326416, |
| "step": 202 |
| }, |
| { |
| "epoch": 3.440677966101695, |
| "grad_norm": 11.395001548472651, |
| "learning_rate": 3.5247324920709147e-07, |
| "logits/chosen": -1.3313159942626953, |
| "logits/rejected": -1.2498857975006104, |
| "logps/chosen": -30.9143009185791, |
| "logps/rejected": -41.243194580078125, |
| "loss": 0.07, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.04568520188331604, |
| "rewards/margins": 3.3133530616760254, |
| "rewards/rejected": -3.267667770385742, |
| "step": 203 |
| }, |
| { |
| "epoch": 3.457627118644068, |
| "grad_norm": 9.512575018928173, |
| "learning_rate": 3.5078085732199307e-07, |
| "logits/chosen": -1.567063331604004, |
| "logits/rejected": -1.4618712663650513, |
| "logps/chosen": -25.394840240478516, |
| "logps/rejected": -49.12550354003906, |
| "loss": 0.0601, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.02158541977405548, |
| "rewards/margins": 4.889016628265381, |
| "rewards/rejected": -4.867431163787842, |
| "step": 204 |
| }, |
| { |
| "epoch": 3.4745762711864407, |
| "grad_norm": 11.586078932679674, |
| "learning_rate": 3.490829326480773e-07, |
| "logits/chosen": -1.4906607866287231, |
| "logits/rejected": -1.3863712549209595, |
| "logps/chosen": -31.574989318847656, |
| "logps/rejected": -48.25616455078125, |
| "loss": 0.0787, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.011842876672744751, |
| "rewards/margins": 4.549272537231445, |
| "rewards/rejected": -4.561115741729736, |
| "step": 205 |
| }, |
| { |
| "epoch": 3.4915254237288136, |
| "grad_norm": 12.547434003386876, |
| "learning_rate": 3.4737956840005684e-07, |
| "logits/chosen": -1.3435784578323364, |
| "logits/rejected": -1.3757704496383667, |
| "logps/chosen": -25.303531646728516, |
| "logps/rejected": -39.15208053588867, |
| "loss": 0.086, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.02988174557685852, |
| "rewards/margins": 3.5366082191467285, |
| "rewards/rejected": -3.5067262649536133, |
| "step": 206 |
| }, |
| { |
| "epoch": 3.5084745762711864, |
| "grad_norm": 9.788457273992568, |
| "learning_rate": 3.4567085809127245e-07, |
| "logits/chosen": -1.5033990144729614, |
| "logits/rejected": -1.4766664505004883, |
| "logps/chosen": -26.12259864807129, |
| "logps/rejected": -52.18391418457031, |
| "loss": 0.0617, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08881500363349915, |
| "rewards/margins": 4.650575160980225, |
| "rewards/rejected": -4.561759948730469, |
| "step": 207 |
| }, |
| { |
| "epoch": 3.5254237288135593, |
| "grad_norm": 11.249455307964162, |
| "learning_rate": 3.439568955285595e-07, |
| "logits/chosen": -1.600437045097351, |
| "logits/rejected": -1.5185781717300415, |
| "logps/chosen": -20.018888473510742, |
| "logps/rejected": -41.524349212646484, |
| "loss": 0.0704, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.08864006400108337, |
| "rewards/margins": 3.782201051712036, |
| "rewards/rejected": -3.8708412647247314, |
| "step": 208 |
| }, |
| { |
| "epoch": 3.542372881355932, |
| "grad_norm": 9.990126506747924, |
| "learning_rate": 3.4223777480709804e-07, |
| "logits/chosen": -1.3552483320236206, |
| "logits/rejected": -1.2629144191741943, |
| "logps/chosen": -21.040348052978516, |
| "logps/rejected": -41.19926071166992, |
| "loss": 0.0595, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.106040358543396, |
| "rewards/margins": 4.690126419067383, |
| "rewards/rejected": -4.796167373657227, |
| "step": 209 |
| }, |
| { |
| "epoch": 3.559322033898305, |
| "grad_norm": 9.838092383663449, |
| "learning_rate": 3.405135903052465e-07, |
| "logits/chosen": -1.4207416772842407, |
| "logits/rejected": -1.2667282819747925, |
| "logps/chosen": -30.103904724121094, |
| "logps/rejected": -46.88092803955078, |
| "loss": 0.0606, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3432961106300354, |
| "rewards/margins": 4.319201946258545, |
| "rewards/rejected": -4.6624979972839355, |
| "step": 210 |
| }, |
| { |
| "epoch": 3.576271186440678, |
| "grad_norm": 9.67907653448832, |
| "learning_rate": 3.3878443667936136e-07, |
| "logits/chosen": -1.2791118621826172, |
| "logits/rejected": -1.2004616260528564, |
| "logps/chosen": -40.10679244995117, |
| "logps/rejected": -60.650394439697266, |
| "loss": 0.0526, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6684077978134155, |
| "rewards/margins": 4.151851654052734, |
| "rewards/rejected": -4.820259094238281, |
| "step": 211 |
| }, |
| { |
| "epoch": 3.593220338983051, |
| "grad_norm": 9.853143082693608, |
| "learning_rate": 3.3705040885859967e-07, |
| "logits/chosen": -1.5361111164093018, |
| "logits/rejected": -1.4038530588150024, |
| "logps/chosen": -37.90250015258789, |
| "logps/rejected": -45.059024810791016, |
| "loss": 0.0492, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.0030466020107269287, |
| "rewards/margins": 3.964195728302002, |
| "rewards/rejected": -3.9672420024871826, |
| "step": 212 |
| }, |
| { |
| "epoch": 3.610169491525424, |
| "grad_norm": 9.44047901416316, |
| "learning_rate": 3.3531160203970805e-07, |
| "logits/chosen": -1.4581642150878906, |
| "logits/rejected": -1.4039422273635864, |
| "logps/chosen": -30.59682846069336, |
| "logps/rejected": -44.48968505859375, |
| "loss": 0.0694, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1339409351348877, |
| "rewards/margins": 4.1027374267578125, |
| "rewards/rejected": -4.236677646636963, |
| "step": 213 |
| }, |
| { |
| "epoch": 3.6271186440677967, |
| "grad_norm": 11.435287785470077, |
| "learning_rate": 3.3356811168179627e-07, |
| "logits/chosen": -1.2856285572052002, |
| "logits/rejected": -1.177187204360962, |
| "logps/chosen": -28.70745086669922, |
| "logps/rejected": -39.22813415527344, |
| "loss": 0.0779, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.319486528635025, |
| "rewards/margins": 4.963109016418457, |
| "rewards/rejected": -4.643622398376465, |
| "step": 214 |
| }, |
| { |
| "epoch": 3.6440677966101696, |
| "grad_norm": 10.041339419341044, |
| "learning_rate": 3.318200335010967e-07, |
| "logits/chosen": -1.720375418663025, |
| "logits/rejected": -1.5207815170288086, |
| "logps/chosen": -26.1734676361084, |
| "logps/rejected": -41.08108901977539, |
| "loss": 0.0651, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5092735290527344, |
| "rewards/margins": 4.717857360839844, |
| "rewards/rejected": -4.208584308624268, |
| "step": 215 |
| }, |
| { |
| "epoch": 3.6610169491525424, |
| "grad_norm": 10.722521061649259, |
| "learning_rate": 3.3006746346570935e-07, |
| "logits/chosen": -1.5194891691207886, |
| "logits/rejected": -1.5225661993026733, |
| "logps/chosen": -21.88874626159668, |
| "logps/rejected": -31.313539505004883, |
| "loss": 0.0652, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.42684128880500793, |
| "rewards/margins": 3.6699740886688232, |
| "rewards/rejected": -3.2431328296661377, |
| "step": 216 |
| }, |
| { |
| "epoch": 3.6779661016949152, |
| "grad_norm": 11.239607248752012, |
| "learning_rate": 3.2831049779033395e-07, |
| "logits/chosen": -1.585048794746399, |
| "logits/rejected": -1.447473406791687, |
| "logps/chosen": -44.55853271484375, |
| "logps/rejected": -62.56214141845703, |
| "loss": 0.0718, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5582241415977478, |
| "rewards/margins": 5.07534122467041, |
| "rewards/rejected": -5.633565425872803, |
| "step": 217 |
| }, |
| { |
| "epoch": 3.694915254237288, |
| "grad_norm": 11.121200361780803, |
| "learning_rate": 3.2654923293098666e-07, |
| "logits/chosen": -1.4354138374328613, |
| "logits/rejected": -1.431335210800171, |
| "logps/chosen": -29.88983917236328, |
| "logps/rejected": -41.21036911010742, |
| "loss": 0.0722, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.12309768795967102, |
| "rewards/margins": 4.238857746124268, |
| "rewards/rejected": -4.361955642700195, |
| "step": 218 |
| }, |
| { |
| "epoch": 3.711864406779661, |
| "grad_norm": 8.46695075496944, |
| "learning_rate": 3.247837655797061e-07, |
| "logits/chosen": -1.4146300554275513, |
| "logits/rejected": -1.3639909029006958, |
| "logps/chosen": -23.79798698425293, |
| "logps/rejected": -42.03700256347656, |
| "loss": 0.0509, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.14847984910011292, |
| "rewards/margins": 4.2562971115112305, |
| "rewards/rejected": -4.107817649841309, |
| "step": 219 |
| }, |
| { |
| "epoch": 3.7288135593220337, |
| "grad_norm": 10.875773708222669, |
| "learning_rate": 3.2301419265924393e-07, |
| "logits/chosen": -1.3614307641983032, |
| "logits/rejected": -1.263061285018921, |
| "logps/chosen": -26.913053512573242, |
| "logps/rejected": -38.41120910644531, |
| "loss": 0.0775, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.12246422469615936, |
| "rewards/margins": 3.324322462081909, |
| "rewards/rejected": -3.2018585205078125, |
| "step": 220 |
| }, |
| { |
| "epoch": 3.7457627118644066, |
| "grad_norm": 10.149586335689621, |
| "learning_rate": 3.2124061131774443e-07, |
| "logits/chosen": -1.3240911960601807, |
| "logits/rejected": -1.3163329362869263, |
| "logps/chosen": -25.73955535888672, |
| "logps/rejected": -49.833221435546875, |
| "loss": 0.0624, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2507280111312866, |
| "rewards/margins": 4.252786636352539, |
| "rewards/rejected": -4.002058982849121, |
| "step": 221 |
| }, |
| { |
| "epoch": 3.7627118644067794, |
| "grad_norm": 9.140932864263574, |
| "learning_rate": 3.194631189234109e-07, |
| "logits/chosen": -1.6033962965011597, |
| "logits/rejected": -1.4346346855163574, |
| "logps/chosen": -35.82508087158203, |
| "logps/rejected": -44.355918884277344, |
| "loss": 0.0458, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.09858936071395874, |
| "rewards/margins": 4.44521951675415, |
| "rewards/rejected": -4.543808937072754, |
| "step": 222 |
| }, |
| { |
| "epoch": 3.7796610169491527, |
| "grad_norm": 9.775231597153738, |
| "learning_rate": 3.1768181305916063e-07, |
| "logits/chosen": -1.4034669399261475, |
| "logits/rejected": -1.2983992099761963, |
| "logps/chosen": -36.48939514160156, |
| "logps/rejected": -50.544036865234375, |
| "loss": 0.0503, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08162027597427368, |
| "rewards/margins": 4.144961357116699, |
| "rewards/rejected": -4.06334114074707, |
| "step": 223 |
| }, |
| { |
| "epoch": 3.7966101694915255, |
| "grad_norm": 11.927845816995063, |
| "learning_rate": 3.158967915172669e-07, |
| "logits/chosen": -1.4804027080535889, |
| "logits/rejected": -1.349886178970337, |
| "logps/chosen": -27.929018020629883, |
| "logps/rejected": -37.72224426269531, |
| "loss": 0.0819, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.21573437750339508, |
| "rewards/margins": 3.54447078704834, |
| "rewards/rejected": -3.760205030441284, |
| "step": 224 |
| }, |
| { |
| "epoch": 3.8135593220338984, |
| "grad_norm": 11.052999347550179, |
| "learning_rate": 3.141081522939911e-07, |
| "logits/chosen": -1.4779460430145264, |
| "logits/rejected": -1.349549412727356, |
| "logps/chosen": -37.28151321411133, |
| "logps/rejected": -47.440765380859375, |
| "loss": 0.0651, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3217180073261261, |
| "rewards/margins": 4.7478485107421875, |
| "rewards/rejected": -5.06956672668457, |
| "step": 225 |
| }, |
| { |
| "epoch": 3.830508474576271, |
| "grad_norm": 10.970513938481727, |
| "learning_rate": 3.1231599358420233e-07, |
| "logits/chosen": -1.3360522985458374, |
| "logits/rejected": -1.2304054498672485, |
| "logps/chosen": -25.628524780273438, |
| "logps/rejected": -38.20055389404297, |
| "loss": 0.0619, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2233581840991974, |
| "rewards/margins": 4.349393844604492, |
| "rewards/rejected": -4.126035690307617, |
| "step": 226 |
| }, |
| { |
| "epoch": 3.847457627118644, |
| "grad_norm": 9.433851363193908, |
| "learning_rate": 3.105204137759867e-07, |
| "logits/chosen": -1.2719545364379883, |
| "logits/rejected": -1.330519676208496, |
| "logps/chosen": -34.64011001586914, |
| "logps/rejected": -52.32704162597656, |
| "loss": 0.0631, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2245815396308899, |
| "rewards/margins": 4.7216315269470215, |
| "rewards/rejected": -4.9462127685546875, |
| "step": 227 |
| }, |
| { |
| "epoch": 3.864406779661017, |
| "grad_norm": 10.439526673557062, |
| "learning_rate": 3.0872151144524594e-07, |
| "logits/chosen": -1.6759734153747559, |
| "logits/rejected": -1.579871654510498, |
| "logps/chosen": -27.196908950805664, |
| "logps/rejected": -50.7559700012207, |
| "loss": 0.069, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.22928810119628906, |
| "rewards/margins": 4.84774923324585, |
| "rewards/rejected": -5.077037811279297, |
| "step": 228 |
| }, |
| { |
| "epoch": 3.8813559322033897, |
| "grad_norm": 11.392292474810198, |
| "learning_rate": 3.069193853502855e-07, |
| "logits/chosen": -1.5869011878967285, |
| "logits/rejected": -1.5820071697235107, |
| "logps/chosen": -25.63404655456543, |
| "logps/rejected": -38.69296646118164, |
| "loss": 0.0774, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3039775788784027, |
| "rewards/margins": 3.5710806846618652, |
| "rewards/rejected": -3.875058174133301, |
| "step": 229 |
| }, |
| { |
| "epoch": 3.898305084745763, |
| "grad_norm": 10.225877059440036, |
| "learning_rate": 3.0511413442639297e-07, |
| "logits/chosen": -1.434234619140625, |
| "logits/rejected": -1.3351249694824219, |
| "logps/chosen": -24.643152236938477, |
| "logps/rejected": -60.13444519042969, |
| "loss": 0.0577, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10867035388946533, |
| "rewards/margins": 6.075805187225342, |
| "rewards/rejected": -6.184475898742676, |
| "step": 230 |
| }, |
| { |
| "epoch": 3.915254237288136, |
| "grad_norm": 9.295415437658166, |
| "learning_rate": 3.0330585778040675e-07, |
| "logits/chosen": -1.3108859062194824, |
| "logits/rejected": -1.2643885612487793, |
| "logps/chosen": -24.4619140625, |
| "logps/rejected": -34.88125228881836, |
| "loss": 0.0527, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.13806024193763733, |
| "rewards/margins": 3.5883846282958984, |
| "rewards/rejected": -3.726444721221924, |
| "step": 231 |
| }, |
| { |
| "epoch": 3.9322033898305087, |
| "grad_norm": 9.365373350685292, |
| "learning_rate": 3.0149465468527457e-07, |
| "logits/chosen": -1.4852244853973389, |
| "logits/rejected": -1.5285433530807495, |
| "logps/chosen": -25.74740982055664, |
| "logps/rejected": -39.37394332885742, |
| "loss": 0.0535, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.005698531866073608, |
| "rewards/margins": 4.059448719024658, |
| "rewards/rejected": -4.0651469230651855, |
| "step": 232 |
| }, |
| { |
| "epoch": 3.9491525423728815, |
| "grad_norm": 8.974250718981809, |
| "learning_rate": 2.9968062457460437e-07, |
| "logits/chosen": -1.5756818056106567, |
| "logits/rejected": -1.4879854917526245, |
| "logps/chosen": -24.878393173217773, |
| "logps/rejected": -43.14397048950195, |
| "loss": 0.0495, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1284530758857727, |
| "rewards/margins": 4.223859786987305, |
| "rewards/rejected": -4.3523125648498535, |
| "step": 233 |
| }, |
| { |
| "epoch": 3.9661016949152543, |
| "grad_norm": 11.054495120717768, |
| "learning_rate": 2.978638670372047e-07, |
| "logits/chosen": -1.4321879148483276, |
| "logits/rejected": -1.3549392223358154, |
| "logps/chosen": -35.11801528930664, |
| "logps/rejected": -50.857975006103516, |
| "loss": 0.0675, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5975885391235352, |
| "rewards/margins": 4.9589338302612305, |
| "rewards/rejected": -5.556522846221924, |
| "step": 234 |
| }, |
| { |
| "epoch": 3.983050847457627, |
| "grad_norm": 10.55567875619197, |
| "learning_rate": 2.9604448181161755e-07, |
| "logits/chosen": -1.4181180000305176, |
| "logits/rejected": -1.247128963470459, |
| "logps/chosen": -24.004146575927734, |
| "logps/rejected": -40.258331298828125, |
| "loss": 0.0771, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.09078972041606903, |
| "rewards/margins": 3.5850982666015625, |
| "rewards/rejected": -3.6758880615234375, |
| "step": 235 |
| }, |
| { |
| "epoch": 4.0, |
| "grad_norm": 8.789282244689215, |
| "learning_rate": 2.9422256878064324e-07, |
| "logits/chosen": -1.3495515584945679, |
| "logits/rejected": -1.3198853731155396, |
| "logps/chosen": -35.33921432495117, |
| "logps/rejected": -48.3183708190918, |
| "loss": 0.0458, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6972587704658508, |
| "rewards/margins": 4.445403575897217, |
| "rewards/rejected": -5.142662525177002, |
| "step": 236 |
| }, |
| { |
| "epoch": 4.016949152542373, |
| "grad_norm": 6.75184146677303, |
| "learning_rate": 2.923982279658564e-07, |
| "logits/chosen": -1.633570909500122, |
| "logits/rejected": -1.5905429124832153, |
| "logps/chosen": -38.822261810302734, |
| "logps/rejected": -48.57697296142578, |
| "loss": 0.0414, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.40955615043640137, |
| "rewards/margins": 4.950255393981934, |
| "rewards/rejected": -5.359812259674072, |
| "step": 237 |
| }, |
| { |
| "epoch": 4.033898305084746, |
| "grad_norm": 9.349730549756508, |
| "learning_rate": 2.90571559522115e-07, |
| "logits/chosen": -1.0709235668182373, |
| "logits/rejected": -1.1218361854553223, |
| "logps/chosen": -29.26075553894043, |
| "logps/rejected": -34.48569869995117, |
| "loss": 0.061, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08125007152557373, |
| "rewards/margins": 3.5675737857818604, |
| "rewards/rejected": -3.486323833465576, |
| "step": 238 |
| }, |
| { |
| "epoch": 4.0508474576271185, |
| "grad_norm": 8.738621249147451, |
| "learning_rate": 2.8874266373206215e-07, |
| "logits/chosen": -1.5969672203063965, |
| "logits/rejected": -1.5486068725585938, |
| "logps/chosen": -32.66074752807617, |
| "logps/rejected": -43.64617919921875, |
| "loss": 0.0515, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.13383600115776062, |
| "rewards/margins": 4.7373738288879395, |
| "rewards/rejected": -4.603538513183594, |
| "step": 239 |
| }, |
| { |
| "epoch": 4.067796610169491, |
| "grad_norm": 8.414244576812601, |
| "learning_rate": 2.8691164100062034e-07, |
| "logits/chosen": -1.4231804609298706, |
| "logits/rejected": -1.2792776823043823, |
| "logps/chosen": -31.69137954711914, |
| "logps/rejected": -54.63775634765625, |
| "loss": 0.0499, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08693331480026245, |
| "rewards/margins": 5.980134963989258, |
| "rewards/rejected": -5.89320182800293, |
| "step": 240 |
| }, |
| { |
| "epoch": 4.084745762711864, |
| "grad_norm": 8.9348895906443, |
| "learning_rate": 2.8507859184947953e-07, |
| "logits/chosen": -1.4366453886032104, |
| "logits/rejected": -1.3272255659103394, |
| "logps/chosen": -26.352733612060547, |
| "logps/rejected": -46.89934158325195, |
| "loss": 0.0619, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2997795641422272, |
| "rewards/margins": 3.753899574279785, |
| "rewards/rejected": -4.053679466247559, |
| "step": 241 |
| }, |
| { |
| "epoch": 4.101694915254237, |
| "grad_norm": 8.250352811668128, |
| "learning_rate": 2.8324361691157853e-07, |
| "logits/chosen": -1.374919056892395, |
| "logits/rejected": -1.421012043952942, |
| "logps/chosen": -31.536663055419922, |
| "logps/rejected": -55.928749084472656, |
| "loss": 0.0425, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2735545039176941, |
| "rewards/margins": 4.6181135177612305, |
| "rewards/rejected": -4.891667366027832, |
| "step": 242 |
| }, |
| { |
| "epoch": 4.11864406779661, |
| "grad_norm": 9.40465609028747, |
| "learning_rate": 2.8140681692558034e-07, |
| "logits/chosen": -1.7210100889205933, |
| "logits/rejected": -1.6137436628341675, |
| "logps/chosen": -29.081104278564453, |
| "logps/rejected": -42.040245056152344, |
| "loss": 0.0608, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.13167661428451538, |
| "rewards/margins": 4.6320929527282715, |
| "rewards/rejected": -4.500416278839111, |
| "step": 243 |
| }, |
| { |
| "epoch": 4.135593220338983, |
| "grad_norm": 7.575529614099465, |
| "learning_rate": 2.7956829273034146e-07, |
| "logits/chosen": -1.3278542757034302, |
| "logits/rejected": -1.2801333665847778, |
| "logps/chosen": -28.36415672302246, |
| "logps/rejected": -45.0773811340332, |
| "loss": 0.0521, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.0763159990310669, |
| "rewards/margins": 4.670356750488281, |
| "rewards/rejected": -4.594040393829346, |
| "step": 244 |
| }, |
| { |
| "epoch": 4.1525423728813555, |
| "grad_norm": 7.841549437235225, |
| "learning_rate": 2.7772814525937634e-07, |
| "logits/chosen": -1.521530032157898, |
| "logits/rejected": -1.3261444568634033, |
| "logps/chosen": -25.693857192993164, |
| "logps/rejected": -44.95489501953125, |
| "loss": 0.0461, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.31940776109695435, |
| "rewards/margins": 4.865126132965088, |
| "rewards/rejected": -4.545718193054199, |
| "step": 245 |
| }, |
| { |
| "epoch": 4.169491525423728, |
| "grad_norm": 7.437246530691066, |
| "learning_rate": 2.7588647553531576e-07, |
| "logits/chosen": -1.3834595680236816, |
| "logits/rejected": -1.366625428199768, |
| "logps/chosen": -27.27410125732422, |
| "logps/rejected": -47.83835220336914, |
| "loss": 0.0479, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1564972996711731, |
| "rewards/margins": 4.690642833709717, |
| "rewards/rejected": -4.534145355224609, |
| "step": 246 |
| }, |
| { |
| "epoch": 4.186440677966102, |
| "grad_norm": 8.374701794746429, |
| "learning_rate": 2.7404338466436116e-07, |
| "logits/chosen": -1.4731414318084717, |
| "logits/rejected": -1.4213758707046509, |
| "logps/chosen": -32.91023635864258, |
| "logps/rejected": -48.37405014038086, |
| "loss": 0.0475, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.12676534056663513, |
| "rewards/margins": 5.037107944488525, |
| "rewards/rejected": -4.910342216491699, |
| "step": 247 |
| }, |
| { |
| "epoch": 4.203389830508475, |
| "grad_norm": 8.684915932739882, |
| "learning_rate": 2.721989738307337e-07, |
| "logits/chosen": -1.5974880456924438, |
| "logits/rejected": -1.5572988986968994, |
| "logps/chosen": -29.11380386352539, |
| "logps/rejected": -43.61833953857422, |
| "loss": 0.0524, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.09620954096317291, |
| "rewards/margins": 3.6240577697753906, |
| "rewards/rejected": -3.527848482131958, |
| "step": 248 |
| }, |
| { |
| "epoch": 4.220338983050848, |
| "grad_norm": 8.422615969153167, |
| "learning_rate": 2.7035334429111955e-07, |
| "logits/chosen": -1.4860804080963135, |
| "logits/rejected": -1.4129899740219116, |
| "logps/chosen": -38.593204498291016, |
| "logps/rejected": -55.535247802734375, |
| "loss": 0.0493, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.007733196020126343, |
| "rewards/margins": 4.608980178833008, |
| "rewards/rejected": -4.616713047027588, |
| "step": 249 |
| }, |
| { |
| "epoch": 4.237288135593221, |
| "grad_norm": 8.150205311828831, |
| "learning_rate": 2.685065973691107e-07, |
| "logits/chosen": -1.509846806526184, |
| "logits/rejected": -1.440779447555542, |
| "logps/chosen": -32.75022506713867, |
| "logps/rejected": -53.621826171875, |
| "loss": 0.0429, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10550498962402344, |
| "rewards/margins": 5.389019966125488, |
| "rewards/rejected": -5.49452543258667, |
| "step": 250 |
| }, |
| { |
| "epoch": 4.254237288135593, |
| "grad_norm": 8.158516596968319, |
| "learning_rate": 2.6665883444964277e-07, |
| "logits/chosen": -1.2622435092926025, |
| "logits/rejected": -1.2126017808914185, |
| "logps/chosen": -22.526697158813477, |
| "logps/rejected": -49.31510543823242, |
| "loss": 0.0487, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2732163071632385, |
| "rewards/margins": 5.386449337005615, |
| "rewards/rejected": -5.659666061401367, |
| "step": 251 |
| }, |
| { |
| "epoch": 4.271186440677966, |
| "grad_norm": 8.708471545136758, |
| "learning_rate": 2.6481015697342856e-07, |
| "logits/chosen": -1.3532803058624268, |
| "logits/rejected": -1.263253092765808, |
| "logps/chosen": -18.96270751953125, |
| "logps/rejected": -40.195404052734375, |
| "loss": 0.0488, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.013826161623001099, |
| "rewards/margins": 4.144740581512451, |
| "rewards/rejected": -4.130914688110352, |
| "step": 252 |
| }, |
| { |
| "epoch": 4.288135593220339, |
| "grad_norm": 7.692204259130146, |
| "learning_rate": 2.629606664313896e-07, |
| "logits/chosen": -1.4633291959762573, |
| "logits/rejected": -1.2908146381378174, |
| "logps/chosen": -26.60018539428711, |
| "logps/rejected": -42.72929763793945, |
| "loss": 0.0435, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.05975392460823059, |
| "rewards/margins": 3.8384809494018555, |
| "rewards/rejected": -3.8982346057891846, |
| "step": 253 |
| }, |
| { |
| "epoch": 4.305084745762712, |
| "grad_norm": 7.840483273382048, |
| "learning_rate": 2.611104643590838e-07, |
| "logits/chosen": -1.3723235130310059, |
| "logits/rejected": -1.3505971431732178, |
| "logps/chosen": -22.22472381591797, |
| "logps/rejected": -48.225982666015625, |
| "loss": 0.0488, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1287948191165924, |
| "rewards/margins": 4.58843994140625, |
| "rewards/rejected": -4.4596452713012695, |
| "step": 254 |
| }, |
| { |
| "epoch": 4.322033898305085, |
| "grad_norm": 7.007309716178494, |
| "learning_rate": 2.592596523311317e-07, |
| "logits/chosen": -1.6482080221176147, |
| "logits/rejected": -1.5536173582077026, |
| "logps/chosen": -34.07274627685547, |
| "logps/rejected": -39.57206344604492, |
| "loss": 0.0379, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.029447555541992188, |
| "rewards/margins": 4.474746227264404, |
| "rewards/rejected": -4.445298194885254, |
| "step": 255 |
| }, |
| { |
| "epoch": 4.338983050847458, |
| "grad_norm": 8.655754602681508, |
| "learning_rate": 2.5740833195563994e-07, |
| "logits/chosen": -1.4583147764205933, |
| "logits/rejected": -1.4426932334899902, |
| "logps/chosen": -28.750776290893555, |
| "logps/rejected": -42.55610656738281, |
| "loss": 0.0613, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.13989472389221191, |
| "rewards/margins": 4.060596942901611, |
| "rewards/rejected": -4.200491905212402, |
| "step": 256 |
| }, |
| { |
| "epoch": 4.3559322033898304, |
| "grad_norm": 7.666532517465646, |
| "learning_rate": 2.5555660486862293e-07, |
| "logits/chosen": -1.3961790800094604, |
| "logits/rejected": -1.3545511960983276, |
| "logps/chosen": -30.55471420288086, |
| "logps/rejected": -46.441307067871094, |
| "loss": 0.0527, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.01744025945663452, |
| "rewards/margins": 4.828250885009766, |
| "rewards/rejected": -4.810810565948486, |
| "step": 257 |
| }, |
| { |
| "epoch": 4.372881355932203, |
| "grad_norm": 7.38904178241979, |
| "learning_rate": 2.5370457272842315e-07, |
| "logits/chosen": -1.2144039869308472, |
| "logits/rejected": -1.1987119913101196, |
| "logps/chosen": -31.387115478515625, |
| "logps/rejected": -44.41786193847656, |
| "loss": 0.0499, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.37475213408470154, |
| "rewards/margins": 4.554241180419922, |
| "rewards/rejected": -4.179489612579346, |
| "step": 258 |
| }, |
| { |
| "epoch": 4.389830508474576, |
| "grad_norm": 7.357227804203978, |
| "learning_rate": 2.5185233721013053e-07, |
| "logits/chosen": -1.564170479774475, |
| "logits/rejected": -1.5079408884048462, |
| "logps/chosen": -24.936302185058594, |
| "logps/rejected": -44.029632568359375, |
| "loss": 0.0424, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1059635728597641, |
| "rewards/margins": 4.297895908355713, |
| "rewards/rejected": -4.403859615325928, |
| "step": 259 |
| }, |
| { |
| "epoch": 4.406779661016949, |
| "grad_norm": 9.22648891874881, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.3310877084732056, |
| "logits/rejected": -1.3538789749145508, |
| "logps/chosen": -24.936809539794922, |
| "logps/rejected": -49.063575744628906, |
| "loss": 0.0569, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.07093064486980438, |
| "rewards/margins": 4.918918132781982, |
| "rewards/rejected": -4.989849090576172, |
| "step": 260 |
| }, |
| { |
| "epoch": 4.423728813559322, |
| "grad_norm": 6.836226567179652, |
| "learning_rate": 2.4814766278986944e-07, |
| "logits/chosen": -1.6707507371902466, |
| "logits/rejected": -1.5290323495864868, |
| "logps/chosen": -29.512426376342773, |
| "logps/rejected": -54.653114318847656, |
| "loss": 0.0401, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.0224665105342865, |
| "rewards/margins": 5.460002899169922, |
| "rewards/rejected": -5.437536239624023, |
| "step": 261 |
| }, |
| { |
| "epoch": 4.440677966101695, |
| "grad_norm": 8.63432296696573, |
| "learning_rate": 2.462954272715768e-07, |
| "logits/chosen": -1.5188168287277222, |
| "logits/rejected": -1.4816163778305054, |
| "logps/chosen": -31.95643424987793, |
| "logps/rejected": -42.647403717041016, |
| "loss": 0.0511, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3785313367843628, |
| "rewards/margins": 4.157118320465088, |
| "rewards/rejected": -4.535649299621582, |
| "step": 262 |
| }, |
| { |
| "epoch": 4.4576271186440675, |
| "grad_norm": 8.141729762323788, |
| "learning_rate": 2.4444339513137716e-07, |
| "logits/chosen": -1.590928316116333, |
| "logits/rejected": -1.5425117015838623, |
| "logps/chosen": -30.981969833374023, |
| "logps/rejected": -55.22663116455078, |
| "loss": 0.0517, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1410723626613617, |
| "rewards/margins": 5.534295082092285, |
| "rewards/rejected": -5.393222808837891, |
| "step": 263 |
| }, |
| { |
| "epoch": 4.47457627118644, |
| "grad_norm": 6.553586136504455, |
| "learning_rate": 2.4259166804436003e-07, |
| "logits/chosen": -1.6111406087875366, |
| "logits/rejected": -1.5660130977630615, |
| "logps/chosen": -30.944381713867188, |
| "logps/rejected": -48.914039611816406, |
| "loss": 0.0361, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1980370730161667, |
| "rewards/margins": 5.023958683013916, |
| "rewards/rejected": -5.2219953536987305, |
| "step": 264 |
| }, |
| { |
| "epoch": 4.491525423728813, |
| "grad_norm": 7.7695916088798045, |
| "learning_rate": 2.4074034766886826e-07, |
| "logits/chosen": -1.4081194400787354, |
| "logits/rejected": -1.4115426540374756, |
| "logps/chosen": -26.77755355834961, |
| "logps/rejected": -44.787628173828125, |
| "loss": 0.0479, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2828730642795563, |
| "rewards/margins": 5.210730075836182, |
| "rewards/rejected": -5.493603229522705, |
| "step": 265 |
| }, |
| { |
| "epoch": 4.508474576271187, |
| "grad_norm": 9.138228801008614, |
| "learning_rate": 2.3888953564091616e-07, |
| "logits/chosen": -1.5158981084823608, |
| "logits/rejected": -1.5783250331878662, |
| "logps/chosen": -34.667503356933594, |
| "logps/rejected": -53.5840950012207, |
| "loss": 0.056, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.20706957578659058, |
| "rewards/margins": 5.503992080688477, |
| "rewards/rejected": -5.711061477661133, |
| "step": 266 |
| }, |
| { |
| "epoch": 4.52542372881356, |
| "grad_norm": 6.507174183795552, |
| "learning_rate": 2.3703933356861044e-07, |
| "logits/chosen": -1.399531602859497, |
| "logits/rejected": -1.419585108757019, |
| "logps/chosen": -33.15267562866211, |
| "logps/rejected": -45.78288269042969, |
| "loss": 0.0436, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5228755474090576, |
| "rewards/margins": 4.130328178405762, |
| "rewards/rejected": -4.653203964233398, |
| "step": 267 |
| }, |
| { |
| "epoch": 4.5423728813559325, |
| "grad_norm": 7.507653099421495, |
| "learning_rate": 2.3518984302657144e-07, |
| "logits/chosen": -1.5033714771270752, |
| "logits/rejected": -1.427080750465393, |
| "logps/chosen": -24.752490997314453, |
| "logps/rejected": -57.89167022705078, |
| "loss": 0.0403, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4027895927429199, |
| "rewards/margins": 5.957125186920166, |
| "rewards/rejected": -6.359914779663086, |
| "step": 268 |
| }, |
| { |
| "epoch": 4.559322033898305, |
| "grad_norm": 7.12313349874887, |
| "learning_rate": 2.333411655503572e-07, |
| "logits/chosen": -1.4686946868896484, |
| "logits/rejected": -1.3002254962921143, |
| "logps/chosen": -29.05103302001953, |
| "logps/rejected": -53.72389221191406, |
| "loss": 0.0382, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.0036399289965629578, |
| "rewards/margins": 5.409298419952393, |
| "rewards/rejected": -5.412938117980957, |
| "step": 269 |
| }, |
| { |
| "epoch": 4.576271186440678, |
| "grad_norm": 7.93618380947219, |
| "learning_rate": 2.3149340263088927e-07, |
| "logits/chosen": -1.7106562852859497, |
| "logits/rejected": -1.5941462516784668, |
| "logps/chosen": -23.763492584228516, |
| "logps/rejected": -47.54367446899414, |
| "loss": 0.0517, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.3024110496044159, |
| "rewards/margins": 5.066253185272217, |
| "rewards/rejected": -4.7638421058654785, |
| "step": 270 |
| }, |
| { |
| "epoch": 4.593220338983051, |
| "grad_norm": 7.441146999396367, |
| "learning_rate": 2.296466557088805e-07, |
| "logits/chosen": -1.432921051979065, |
| "logits/rejected": -1.415549635887146, |
| "logps/chosen": -27.215051651000977, |
| "logps/rejected": -50.206668853759766, |
| "loss": 0.0465, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3797184228897095, |
| "rewards/margins": 5.635693550109863, |
| "rewards/rejected": -6.015412330627441, |
| "step": 271 |
| }, |
| { |
| "epoch": 4.610169491525424, |
| "grad_norm": 7.203519404793784, |
| "learning_rate": 2.278010261692663e-07, |
| "logits/chosen": -1.6040518283843994, |
| "logits/rejected": -1.5052812099456787, |
| "logps/chosen": -27.241743087768555, |
| "logps/rejected": -47.83567810058594, |
| "loss": 0.0384, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10971636325120926, |
| "rewards/margins": 5.601438999176025, |
| "rewards/rejected": -5.711155891418457, |
| "step": 272 |
| }, |
| { |
| "epoch": 4.627118644067797, |
| "grad_norm": 7.1515886159837505, |
| "learning_rate": 2.2595661533563887e-07, |
| "logits/chosen": -1.5058000087738037, |
| "logits/rejected": -1.4968637228012085, |
| "logps/chosen": -31.407421112060547, |
| "logps/rejected": -50.27019500732422, |
| "loss": 0.0469, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.42635661363601685, |
| "rewards/margins": 4.159647464752197, |
| "rewards/rejected": -4.58600378036499, |
| "step": 273 |
| }, |
| { |
| "epoch": 4.6440677966101696, |
| "grad_norm": 7.65650118033261, |
| "learning_rate": 2.2411352446468424e-07, |
| "logits/chosen": -1.3374431133270264, |
| "logits/rejected": -1.3260188102722168, |
| "logps/chosen": -21.7060489654541, |
| "logps/rejected": -47.063289642333984, |
| "loss": 0.0408, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.14809060096740723, |
| "rewards/margins": 4.805155277252197, |
| "rewards/rejected": -4.657064437866211, |
| "step": 274 |
| }, |
| { |
| "epoch": 4.661016949152542, |
| "grad_norm": 8.277053566352693, |
| "learning_rate": 2.2227185474062374e-07, |
| "logits/chosen": -1.49006986618042, |
| "logits/rejected": -1.3998165130615234, |
| "logps/chosen": -24.71628761291504, |
| "logps/rejected": -47.67393493652344, |
| "loss": 0.051, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.02221532166004181, |
| "rewards/margins": 4.3083624839782715, |
| "rewards/rejected": -4.286147117614746, |
| "step": 275 |
| }, |
| { |
| "epoch": 4.677966101694915, |
| "grad_norm": 7.8454820880953715, |
| "learning_rate": 2.2043170726965857e-07, |
| "logits/chosen": -1.4486889839172363, |
| "logits/rejected": -1.4862079620361328, |
| "logps/chosen": -27.138608932495117, |
| "logps/rejected": -44.235687255859375, |
| "loss": 0.0498, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.02510838210582733, |
| "rewards/margins": 4.74564266204834, |
| "rewards/rejected": -4.720534324645996, |
| "step": 276 |
| }, |
| { |
| "epoch": 4.694915254237288, |
| "grad_norm": 7.405600784416248, |
| "learning_rate": 2.1859318307441966e-07, |
| "logits/chosen": -1.444726586341858, |
| "logits/rejected": -1.4401050806045532, |
| "logps/chosen": -32.68350601196289, |
| "logps/rejected": -45.99779510498047, |
| "loss": 0.0469, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.024221569299697876, |
| "rewards/margins": 5.507167816162109, |
| "rewards/rejected": -5.482946872711182, |
| "step": 277 |
| }, |
| { |
| "epoch": 4.711864406779661, |
| "grad_norm": 8.104155066599384, |
| "learning_rate": 2.1675638308842142e-07, |
| "logits/chosen": -1.2842341661453247, |
| "logits/rejected": -1.330397605895996, |
| "logps/chosen": -27.432506561279297, |
| "logps/rejected": -40.2528076171875, |
| "loss": 0.0472, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.19671624898910522, |
| "rewards/margins": 4.634299278259277, |
| "rewards/rejected": -4.4375834465026855, |
| "step": 278 |
| }, |
| { |
| "epoch": 4.728813559322034, |
| "grad_norm": 5.864508514550939, |
| "learning_rate": 2.149214081505205e-07, |
| "logits/chosen": -1.5249521732330322, |
| "logits/rejected": -1.404898762702942, |
| "logps/chosen": -32.8936653137207, |
| "logps/rejected": -37.96503829956055, |
| "loss": 0.0318, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.00440371036529541, |
| "rewards/margins": 4.577935218811035, |
| "rewards/rejected": -4.573531627655029, |
| "step": 279 |
| }, |
| { |
| "epoch": 4.745762711864407, |
| "grad_norm": 7.425009165674246, |
| "learning_rate": 2.1308835899937972e-07, |
| "logits/chosen": -1.463742733001709, |
| "logits/rejected": -1.4091339111328125, |
| "logps/chosen": -28.56261444091797, |
| "logps/rejected": -44.104896545410156, |
| "loss": 0.0411, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.04231783747673035, |
| "rewards/margins": 4.795005798339844, |
| "rewards/rejected": -4.837324142456055, |
| "step": 280 |
| }, |
| { |
| "epoch": 4.762711864406779, |
| "grad_norm": 8.25860440969276, |
| "learning_rate": 2.112573362679379e-07, |
| "logits/chosen": -1.4964463710784912, |
| "logits/rejected": -1.355745553970337, |
| "logps/chosen": -37.27642059326172, |
| "logps/rejected": -61.14379119873047, |
| "loss": 0.0524, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.36257362365722656, |
| "rewards/margins": 5.591864109039307, |
| "rewards/rejected": -5.22929048538208, |
| "step": 281 |
| }, |
| { |
| "epoch": 4.779661016949152, |
| "grad_norm": 8.825180205529128, |
| "learning_rate": 2.09428440477885e-07, |
| "logits/chosen": -1.5772864818572998, |
| "logits/rejected": -1.3255836963653564, |
| "logps/chosen": -27.248455047607422, |
| "logps/rejected": -47.95212173461914, |
| "loss": 0.0507, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1756860911846161, |
| "rewards/margins": 6.540701866149902, |
| "rewards/rejected": -6.71638822555542, |
| "step": 282 |
| }, |
| { |
| "epoch": 4.796610169491525, |
| "grad_norm": 6.459544380114144, |
| "learning_rate": 2.0760177203414366e-07, |
| "logits/chosen": -1.4836134910583496, |
| "logits/rejected": -1.5253633260726929, |
| "logps/chosen": -28.740909576416016, |
| "logps/rejected": -38.53902816772461, |
| "loss": 0.0349, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.034703388810157776, |
| "rewards/margins": 4.7387285232543945, |
| "rewards/rejected": -4.773431777954102, |
| "step": 283 |
| }, |
| { |
| "epoch": 4.813559322033898, |
| "grad_norm": 8.201593648267453, |
| "learning_rate": 2.0577743121935682e-07, |
| "logits/chosen": -1.5238087177276611, |
| "logits/rejected": -1.479065179824829, |
| "logps/chosen": -23.523155212402344, |
| "logps/rejected": -49.87913131713867, |
| "loss": 0.0576, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.09851184487342834, |
| "rewards/margins": 4.452397346496582, |
| "rewards/rejected": -4.550909519195557, |
| "step": 284 |
| }, |
| { |
| "epoch": 4.830508474576272, |
| "grad_norm": 8.037104047323464, |
| "learning_rate": 2.0395551818838243e-07, |
| "logits/chosen": -1.5587154626846313, |
| "logits/rejected": -1.473785638809204, |
| "logps/chosen": -38.94770812988281, |
| "logps/rejected": -56.7118034362793, |
| "loss": 0.0483, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5131532549858093, |
| "rewards/margins": 5.551197052001953, |
| "rewards/rejected": -6.06434965133667, |
| "step": 285 |
| }, |
| { |
| "epoch": 4.847457627118644, |
| "grad_norm": 6.619262748940799, |
| "learning_rate": 2.021361329627953e-07, |
| "logits/chosen": -1.5830734968185425, |
| "logits/rejected": -1.4701610803604126, |
| "logps/chosen": -25.216768264770508, |
| "logps/rejected": -52.16200637817383, |
| "loss": 0.0422, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.12712815403938293, |
| "rewards/margins": 5.188778877258301, |
| "rewards/rejected": -5.3159074783325195, |
| "step": 286 |
| }, |
| { |
| "epoch": 4.864406779661017, |
| "grad_norm": 7.108190229695481, |
| "learning_rate": 2.003193754253957e-07, |
| "logits/chosen": -1.499477744102478, |
| "logits/rejected": -1.442081093788147, |
| "logps/chosen": -29.464977264404297, |
| "logps/rejected": -44.91366958618164, |
| "loss": 0.0467, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.14971515536308289, |
| "rewards/margins": 4.699173927307129, |
| "rewards/rejected": -4.848889350891113, |
| "step": 287 |
| }, |
| { |
| "epoch": 4.88135593220339, |
| "grad_norm": 6.501653298714645, |
| "learning_rate": 1.9850534531472544e-07, |
| "logits/chosen": -1.384242057800293, |
| "logits/rejected": -1.2570266723632812, |
| "logps/chosen": -27.370962142944336, |
| "logps/rejected": -41.63351058959961, |
| "loss": 0.0416, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.09189403057098389, |
| "rewards/margins": 5.2285356521606445, |
| "rewards/rejected": -5.136641502380371, |
| "step": 288 |
| }, |
| { |
| "epoch": 4.898305084745763, |
| "grad_norm": 6.999053945822559, |
| "learning_rate": 1.966941422195933e-07, |
| "logits/chosen": -1.4301297664642334, |
| "logits/rejected": -1.3571842908859253, |
| "logps/chosen": -30.701709747314453, |
| "logps/rejected": -47.53770065307617, |
| "loss": 0.0442, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2589109241962433, |
| "rewards/margins": 4.9735002517700195, |
| "rewards/rejected": -5.232410907745361, |
| "step": 289 |
| }, |
| { |
| "epoch": 4.915254237288136, |
| "grad_norm": 7.2234737195409275, |
| "learning_rate": 1.94885865573607e-07, |
| "logits/chosen": -1.5849264860153198, |
| "logits/rejected": -1.5440596342086792, |
| "logps/chosen": -22.58307647705078, |
| "logps/rejected": -42.317161560058594, |
| "loss": 0.0549, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.05918128788471222, |
| "rewards/margins": 4.245813846588135, |
| "rewards/rejected": -4.304995536804199, |
| "step": 290 |
| }, |
| { |
| "epoch": 4.932203389830509, |
| "grad_norm": 7.881613271557854, |
| "learning_rate": 1.930806146497146e-07, |
| "logits/chosen": -1.5185688734054565, |
| "logits/rejected": -1.498981237411499, |
| "logps/chosen": -27.832983016967773, |
| "logps/rejected": -44.11700439453125, |
| "loss": 0.0452, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.012274429202079773, |
| "rewards/margins": 4.80919885635376, |
| "rewards/rejected": -4.796924591064453, |
| "step": 291 |
| }, |
| { |
| "epoch": 4.9491525423728815, |
| "grad_norm": 7.929413571170103, |
| "learning_rate": 1.912784885547541e-07, |
| "logits/chosen": -1.4354244470596313, |
| "logits/rejected": -1.2839109897613525, |
| "logps/chosen": -28.58751678466797, |
| "logps/rejected": -52.627559661865234, |
| "loss": 0.05, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.09678968787193298, |
| "rewards/margins": 3.94634747505188, |
| "rewards/rejected": -4.043137550354004, |
| "step": 292 |
| }, |
| { |
| "epoch": 4.966101694915254, |
| "grad_norm": 8.10839834515343, |
| "learning_rate": 1.8947958622401328e-07, |
| "logits/chosen": -1.2912473678588867, |
| "logits/rejected": -1.3136945962905884, |
| "logps/chosen": -28.488567352294922, |
| "logps/rejected": -44.88406753540039, |
| "loss": 0.0492, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5085054636001587, |
| "rewards/margins": 4.020839691162109, |
| "rewards/rejected": -4.529345512390137, |
| "step": 293 |
| }, |
| { |
| "epoch": 4.983050847457627, |
| "grad_norm": 6.68942425307024, |
| "learning_rate": 1.876840064157976e-07, |
| "logits/chosen": -1.4644904136657715, |
| "logits/rejected": -1.4102816581726074, |
| "logps/chosen": -26.936355590820312, |
| "logps/rejected": -47.06075668334961, |
| "loss": 0.0416, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.07672211527824402, |
| "rewards/margins": 4.622737884521484, |
| "rewards/rejected": -4.699460029602051, |
| "step": 294 |
| }, |
| { |
| "epoch": 5.0, |
| "grad_norm": 8.672333086068912, |
| "learning_rate": 1.858918477060089e-07, |
| "logits/chosen": -1.4006508588790894, |
| "logits/rejected": -1.4321238994598389, |
| "logps/chosen": -25.137971878051758, |
| "logps/rejected": -42.432716369628906, |
| "loss": 0.0478, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.02563352882862091, |
| "rewards/margins": 4.927007675170898, |
| "rewards/rejected": -4.952641010284424, |
| "step": 295 |
| }, |
| { |
| "epoch": 5.016949152542373, |
| "grad_norm": 6.442205099947888, |
| "learning_rate": 1.8410320848273313e-07, |
| "logits/chosen": -1.4923574924468994, |
| "logits/rejected": -1.4842016696929932, |
| "logps/chosen": -26.232227325439453, |
| "logps/rejected": -48.02200698852539, |
| "loss": 0.0339, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.0712697505950928, |
| "rewards/margins": 5.136242866516113, |
| "rewards/rejected": -6.207512855529785, |
| "step": 296 |
| }, |
| { |
| "epoch": 5.033898305084746, |
| "grad_norm": 6.132421236608343, |
| "learning_rate": 1.8231818694083938e-07, |
| "logits/chosen": -1.4439387321472168, |
| "logits/rejected": -1.4443602561950684, |
| "logps/chosen": -39.26667404174805, |
| "logps/rejected": -56.92470169067383, |
| "loss": 0.0313, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1077936589717865, |
| "rewards/margins": 6.438662528991699, |
| "rewards/rejected": -6.546456336975098, |
| "step": 297 |
| }, |
| { |
| "epoch": 5.0508474576271185, |
| "grad_norm": 5.496092091450637, |
| "learning_rate": 1.8053688107658905e-07, |
| "logits/chosen": -1.2399489879608154, |
| "logits/rejected": -1.1487674713134766, |
| "logps/chosen": -24.482892990112305, |
| "logps/rejected": -38.75669860839844, |
| "loss": 0.0391, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1850048452615738, |
| "rewards/margins": 4.235044479370117, |
| "rewards/rejected": -4.050039291381836, |
| "step": 298 |
| }, |
| { |
| "epoch": 5.067796610169491, |
| "grad_norm": 6.371224857913382, |
| "learning_rate": 1.787593886822556e-07, |
| "logits/chosen": -1.5012649297714233, |
| "logits/rejected": -1.5702931880950928, |
| "logps/chosen": -26.034366607666016, |
| "logps/rejected": -54.3376579284668, |
| "loss": 0.0339, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.206559419631958, |
| "rewards/margins": 5.6937360763549805, |
| "rewards/rejected": -5.900295257568359, |
| "step": 299 |
| }, |
| { |
| "epoch": 5.084745762711864, |
| "grad_norm": 6.572183005650043, |
| "learning_rate": 1.7698580734075607e-07, |
| "logits/chosen": -1.48176908493042, |
| "logits/rejected": -1.4655022621154785, |
| "logps/chosen": -27.89720344543457, |
| "logps/rejected": -45.596282958984375, |
| "loss": 0.0356, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.14924107491970062, |
| "rewards/margins": 4.559385299682617, |
| "rewards/rejected": -4.7086262702941895, |
| "step": 300 |
| }, |
| { |
| "epoch": 5.101694915254237, |
| "grad_norm": 6.034869634249788, |
| "learning_rate": 1.7521623442029388e-07, |
| "logits/chosen": -1.5313202142715454, |
| "logits/rejected": -1.564162015914917, |
| "logps/chosen": -22.828670501708984, |
| "logps/rejected": -48.36279296875, |
| "loss": 0.0363, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2727990448474884, |
| "rewards/margins": 4.790719985961914, |
| "rewards/rejected": -4.51792049407959, |
| "step": 301 |
| }, |
| { |
| "epoch": 5.11864406779661, |
| "grad_norm": 6.764929177752538, |
| "learning_rate": 1.7345076706901326e-07, |
| "logits/chosen": -1.4571880102157593, |
| "logits/rejected": -1.4540932178497314, |
| "logps/chosen": -30.096073150634766, |
| "logps/rejected": -55.99176788330078, |
| "loss": 0.0389, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1132214367389679, |
| "rewards/margins": 5.543183326721191, |
| "rewards/rejected": -5.656404972076416, |
| "step": 302 |
| }, |
| { |
| "epoch": 5.135593220338983, |
| "grad_norm": 7.398683409083888, |
| "learning_rate": 1.7168950220966614e-07, |
| "logits/chosen": -1.430177927017212, |
| "logits/rejected": -1.482927918434143, |
| "logps/chosen": -30.467327117919922, |
| "logps/rejected": -45.27473831176758, |
| "loss": 0.0516, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.06274604797363281, |
| "rewards/margins": 4.2044267654418945, |
| "rewards/rejected": -4.267173767089844, |
| "step": 303 |
| }, |
| { |
| "epoch": 5.1525423728813555, |
| "grad_norm": 5.9915990011633555, |
| "learning_rate": 1.6993253653429062e-07, |
| "logits/chosen": -1.4992166757583618, |
| "logits/rejected": -1.4459744691848755, |
| "logps/chosen": -33.046180725097656, |
| "logps/rejected": -52.499366760253906, |
| "loss": 0.0365, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5871363878250122, |
| "rewards/margins": 5.686320781707764, |
| "rewards/rejected": -6.2734575271606445, |
| "step": 304 |
| }, |
| { |
| "epoch": 5.169491525423728, |
| "grad_norm": 6.387235286093505, |
| "learning_rate": 1.681799664989033e-07, |
| "logits/chosen": -1.4004420042037964, |
| "logits/rejected": -1.3709527254104614, |
| "logps/chosen": -26.409391403198242, |
| "logps/rejected": -37.188838958740234, |
| "loss": 0.0425, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.3655552864074707, |
| "rewards/margins": 4.699047565460205, |
| "rewards/rejected": -4.333492279052734, |
| "step": 305 |
| }, |
| { |
| "epoch": 5.186440677966102, |
| "grad_norm": 5.71477855668128, |
| "learning_rate": 1.6643188831820374e-07, |
| "logits/chosen": -1.4717719554901123, |
| "logits/rejected": -1.5952577590942383, |
| "logps/chosen": -27.931270599365234, |
| "logps/rejected": -50.50065231323242, |
| "loss": 0.0397, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6083439588546753, |
| "rewards/margins": 5.1790900230407715, |
| "rewards/rejected": -5.787433624267578, |
| "step": 306 |
| }, |
| { |
| "epoch": 5.203389830508475, |
| "grad_norm": 5.369095807459635, |
| "learning_rate": 1.6468839796029198e-07, |
| "logits/chosen": -1.5481698513031006, |
| "logits/rejected": -1.4016451835632324, |
| "logps/chosen": -33.04383850097656, |
| "logps/rejected": -58.85445022583008, |
| "loss": 0.0321, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.07381519675254822, |
| "rewards/margins": 5.094147205352783, |
| "rewards/rejected": -5.167962074279785, |
| "step": 307 |
| }, |
| { |
| "epoch": 5.220338983050848, |
| "grad_norm": 6.478677176582888, |
| "learning_rate": 1.6294959114140033e-07, |
| "logits/chosen": -1.6205213069915771, |
| "logits/rejected": -1.5254000425338745, |
| "logps/chosen": -29.354225158691406, |
| "logps/rejected": -43.66874694824219, |
| "loss": 0.0357, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.08936242759227753, |
| "rewards/margins": 4.020651817321777, |
| "rewards/rejected": -4.110013961791992, |
| "step": 308 |
| }, |
| { |
| "epoch": 5.237288135593221, |
| "grad_norm": 6.290350105724975, |
| "learning_rate": 1.6121556332063861e-07, |
| "logits/chosen": -1.3170948028564453, |
| "logits/rejected": -1.281385898590088, |
| "logps/chosen": -37.28167724609375, |
| "logps/rejected": -45.006961822509766, |
| "loss": 0.0352, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.06686612963676453, |
| "rewards/margins": 4.521815776824951, |
| "rewards/rejected": -4.588682174682617, |
| "step": 309 |
| }, |
| { |
| "epoch": 5.254237288135593, |
| "grad_norm": 7.295722008430091, |
| "learning_rate": 1.5948640969475345e-07, |
| "logits/chosen": -1.5421946048736572, |
| "logits/rejected": -1.406693935394287, |
| "logps/chosen": -26.546001434326172, |
| "logps/rejected": -40.070228576660156, |
| "loss": 0.0464, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.11074566841125488, |
| "rewards/margins": 4.666855812072754, |
| "rewards/rejected": -4.77760124206543, |
| "step": 310 |
| }, |
| { |
| "epoch": 5.271186440677966, |
| "grad_norm": 6.299381642433967, |
| "learning_rate": 1.5776222519290204e-07, |
| "logits/chosen": -1.5537474155426025, |
| "logits/rejected": -1.501166820526123, |
| "logps/chosen": -26.153173446655273, |
| "logps/rejected": -50.52534103393555, |
| "loss": 0.0353, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.541397213935852, |
| "rewards/margins": 5.92680025100708, |
| "rewards/rejected": -6.468197822570801, |
| "step": 311 |
| }, |
| { |
| "epoch": 5.288135593220339, |
| "grad_norm": 6.901736790648853, |
| "learning_rate": 1.560431044714405e-07, |
| "logits/chosen": -1.547554850578308, |
| "logits/rejected": -1.4092559814453125, |
| "logps/chosen": -32.46575927734375, |
| "logps/rejected": -52.17520523071289, |
| "loss": 0.042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.18227432668209076, |
| "rewards/margins": 5.195803642272949, |
| "rewards/rejected": -5.378078460693359, |
| "step": 312 |
| }, |
| { |
| "epoch": 5.305084745762712, |
| "grad_norm": 6.537170506793814, |
| "learning_rate": 1.5432914190872756e-07, |
| "logits/chosen": -1.394778847694397, |
| "logits/rejected": -1.3562414646148682, |
| "logps/chosen": -26.12644386291504, |
| "logps/rejected": -38.51447677612305, |
| "loss": 0.036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.43324902653694153, |
| "rewards/margins": 4.649719715118408, |
| "rewards/rejected": -4.216470718383789, |
| "step": 313 |
| }, |
| { |
| "epoch": 5.322033898305085, |
| "grad_norm": 6.822269638404017, |
| "learning_rate": 1.5262043159994314e-07, |
| "logits/chosen": -1.603257179260254, |
| "logits/rejected": -1.41001558303833, |
| "logps/chosen": -27.069252014160156, |
| "logps/rejected": -54.39444351196289, |
| "loss": 0.037, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2375459223985672, |
| "rewards/margins": 5.968645095825195, |
| "rewards/rejected": -5.731099605560303, |
| "step": 314 |
| }, |
| { |
| "epoch": 5.338983050847458, |
| "grad_norm": 6.852429555090263, |
| "learning_rate": 1.5091706735192266e-07, |
| "logits/chosen": -1.5043295621871948, |
| "logits/rejected": -1.4766517877578735, |
| "logps/chosen": -21.581401824951172, |
| "logps/rejected": -50.377071380615234, |
| "loss": 0.0566, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.24750420451164246, |
| "rewards/margins": 5.493175506591797, |
| "rewards/rejected": -5.24567174911499, |
| "step": 315 |
| }, |
| { |
| "epoch": 5.3559322033898304, |
| "grad_norm": 6.421398270201425, |
| "learning_rate": 1.4921914267800699e-07, |
| "logits/chosen": -1.4200242757797241, |
| "logits/rejected": -1.3987623453140259, |
| "logps/chosen": -19.360551834106445, |
| "logps/rejected": -35.37394714355469, |
| "loss": 0.0374, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.14433270692825317, |
| "rewards/margins": 3.7558863162994385, |
| "rewards/rejected": -3.611553430557251, |
| "step": 316 |
| }, |
| { |
| "epoch": 5.372881355932203, |
| "grad_norm": 6.030116263122066, |
| "learning_rate": 1.4752675079290848e-07, |
| "logits/chosen": -1.3269891738891602, |
| "logits/rejected": -1.2499594688415527, |
| "logps/chosen": -29.73949432373047, |
| "logps/rejected": -37.97323989868164, |
| "loss": 0.0307, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.44939878582954407, |
| "rewards/margins": 3.9090001583099365, |
| "rewards/rejected": -4.358399391174316, |
| "step": 317 |
| }, |
| { |
| "epoch": 5.389830508474576, |
| "grad_norm": 6.16190997278292, |
| "learning_rate": 1.458399846075942e-07, |
| "logits/chosen": -1.5378894805908203, |
| "logits/rejected": -1.5542147159576416, |
| "logps/chosen": -33.65290832519531, |
| "logps/rejected": -57.606727600097656, |
| "loss": 0.0361, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.47632092237472534, |
| "rewards/margins": 5.512063980102539, |
| "rewards/rejected": -5.98838472366333, |
| "step": 318 |
| }, |
| { |
| "epoch": 5.406779661016949, |
| "grad_norm": 6.084242732250159, |
| "learning_rate": 1.441589367241846e-07, |
| "logits/chosen": -1.3787221908569336, |
| "logits/rejected": -1.2671631574630737, |
| "logps/chosen": -25.762453079223633, |
| "logps/rejected": -41.90483093261719, |
| "loss": 0.0351, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.07579733431339264, |
| "rewards/margins": 4.506519317626953, |
| "rewards/rejected": -4.430721759796143, |
| "step": 319 |
| }, |
| { |
| "epoch": 5.423728813559322, |
| "grad_norm": 6.400287756731778, |
| "learning_rate": 1.4248369943086995e-07, |
| "logits/chosen": -1.609413504600525, |
| "logits/rejected": -1.440029501914978, |
| "logps/chosen": -29.613908767700195, |
| "logps/rejected": -45.63286590576172, |
| "loss": 0.0406, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1749686449766159, |
| "rewards/margins": 4.805515766143799, |
| "rewards/rejected": -4.980484485626221, |
| "step": 320 |
| }, |
| { |
| "epoch": 5.440677966101695, |
| "grad_norm": 5.58673826227153, |
| "learning_rate": 1.4081436469684337e-07, |
| "logits/chosen": -1.4598129987716675, |
| "logits/rejected": -1.3000261783599854, |
| "logps/chosen": -26.955453872680664, |
| "logps/rejected": -49.03952407836914, |
| "loss": 0.0351, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2353724241256714, |
| "rewards/margins": 5.155057907104492, |
| "rewards/rejected": -5.390429973602295, |
| "step": 321 |
| }, |
| { |
| "epoch": 5.4576271186440675, |
| "grad_norm": 5.7806024491841885, |
| "learning_rate": 1.3915102416725286e-07, |
| "logits/chosen": -1.5266464948654175, |
| "logits/rejected": -1.406123161315918, |
| "logps/chosen": -21.365182876586914, |
| "logps/rejected": -44.53782653808594, |
| "loss": 0.0369, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.19776439666748047, |
| "rewards/margins": 4.6383209228515625, |
| "rewards/rejected": -4.440556526184082, |
| "step": 322 |
| }, |
| { |
| "epoch": 5.47457627118644, |
| "grad_norm": 6.663910066076515, |
| "learning_rate": 1.3749376915816885e-07, |
| "logits/chosen": -1.5701993703842163, |
| "logits/rejected": -1.552007794380188, |
| "logps/chosen": -36.67780685424805, |
| "logps/rejected": -47.98915481567383, |
| "loss": 0.0398, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.8618345856666565, |
| "rewards/margins": 4.562122821807861, |
| "rewards/rejected": -5.423957347869873, |
| "step": 323 |
| }, |
| { |
| "epoch": 5.491525423728813, |
| "grad_norm": 6.220022143578053, |
| "learning_rate": 1.3584269065157172e-07, |
| "logits/chosen": -1.3734331130981445, |
| "logits/rejected": -1.3631477355957031, |
| "logps/chosen": -36.97685623168945, |
| "logps/rejected": -53.4107780456543, |
| "loss": 0.0326, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10500967502593994, |
| "rewards/margins": 4.521078109741211, |
| "rewards/rejected": -4.6260881423950195, |
| "step": 324 |
| }, |
| { |
| "epoch": 5.508474576271187, |
| "grad_norm": 6.094670518205716, |
| "learning_rate": 1.341978792903568e-07, |
| "logits/chosen": -1.4123265743255615, |
| "logits/rejected": -1.3930362462997437, |
| "logps/chosen": -24.757308959960938, |
| "logps/rejected": -50.27605438232422, |
| "loss": 0.0323, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1657731980085373, |
| "rewards/margins": 6.393916606903076, |
| "rewards/rejected": -6.22814416885376, |
| "step": 325 |
| }, |
| { |
| "epoch": 5.52542372881356, |
| "grad_norm": 7.344149893164419, |
| "learning_rate": 1.3255942537335804e-07, |
| "logits/chosen": -1.4898028373718262, |
| "logits/rejected": -1.529905915260315, |
| "logps/chosen": -30.63724136352539, |
| "logps/rejected": -47.23683547973633, |
| "loss": 0.0557, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2261982262134552, |
| "rewards/margins": 4.666712760925293, |
| "rewards/rejected": -4.892910957336426, |
| "step": 326 |
| }, |
| { |
| "epoch": 5.5423728813559325, |
| "grad_norm": 5.4257600584522185, |
| "learning_rate": 1.3092741885039085e-07, |
| "logits/chosen": -1.430129051208496, |
| "logits/rejected": -1.2972102165222168, |
| "logps/chosen": -29.192171096801758, |
| "logps/rejected": -61.473453521728516, |
| "loss": 0.0452, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.42023560404777527, |
| "rewards/margins": 5.649378299713135, |
| "rewards/rejected": -6.069613933563232, |
| "step": 327 |
| }, |
| { |
| "epoch": 5.559322033898305, |
| "grad_norm": 6.750041211234325, |
| "learning_rate": 1.2930194931731382e-07, |
| "logits/chosen": -1.5282580852508545, |
| "logits/rejected": -1.464250087738037, |
| "logps/chosen": -21.08121109008789, |
| "logps/rejected": -36.48762512207031, |
| "loss": 0.0373, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.21070019900798798, |
| "rewards/margins": 4.739938735961914, |
| "rewards/rejected": -4.950639247894287, |
| "step": 328 |
| }, |
| { |
| "epoch": 5.576271186440678, |
| "grad_norm": 6.346382606047745, |
| "learning_rate": 1.2768310601110993e-07, |
| "logits/chosen": -1.544804334640503, |
| "logits/rejected": -1.5036790370941162, |
| "logps/chosen": -25.425878524780273, |
| "logps/rejected": -57.694950103759766, |
| "loss": 0.0329, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.15332584083080292, |
| "rewards/margins": 6.2687883377075195, |
| "rewards/rejected": -6.115462303161621, |
| "step": 329 |
| }, |
| { |
| "epoch": 5.593220338983051, |
| "grad_norm": 5.891747930904324, |
| "learning_rate": 1.260709778049877e-07, |
| "logits/chosen": -1.4886832237243652, |
| "logits/rejected": -1.5345224142074585, |
| "logps/chosen": -24.217357635498047, |
| "logps/rejected": -38.936588287353516, |
| "loss": 0.031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.14570964872837067, |
| "rewards/margins": 3.9201903343200684, |
| "rewards/rejected": -3.7744803428649902, |
| "step": 330 |
| }, |
| { |
| "epoch": 5.610169491525424, |
| "grad_norm": 6.479009923461722, |
| "learning_rate": 1.2446565320350182e-07, |
| "logits/chosen": -1.6119954586029053, |
| "logits/rejected": -1.5358697175979614, |
| "logps/chosen": -23.517757415771484, |
| "logps/rejected": -43.86408233642578, |
| "loss": 0.0457, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.01591832935810089, |
| "rewards/margins": 5.240238189697266, |
| "rewards/rejected": -5.2243194580078125, |
| "step": 331 |
| }, |
| { |
| "epoch": 5.627118644067797, |
| "grad_norm": 5.354023576639168, |
| "learning_rate": 1.2286722033769492e-07, |
| "logits/chosen": -1.630919337272644, |
| "logits/rejected": -1.538849949836731, |
| "logps/chosen": -28.93423080444336, |
| "logps/rejected": -53.66632080078125, |
| "loss": 0.0289, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.17018574476242065, |
| "rewards/margins": 5.786593437194824, |
| "rewards/rejected": -5.9567790031433105, |
| "step": 332 |
| }, |
| { |
| "epoch": 5.6440677966101696, |
| "grad_norm": 5.571846024722403, |
| "learning_rate": 1.2127576696025826e-07, |
| "logits/chosen": -1.3449130058288574, |
| "logits/rejected": -1.3209434747695923, |
| "logps/chosen": -28.3823299407959, |
| "logps/rejected": -57.359275817871094, |
| "loss": 0.0353, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.19171355664730072, |
| "rewards/margins": 6.2648420333862305, |
| "rewards/rejected": -6.4565558433532715, |
| "step": 333 |
| }, |
| { |
| "epoch": 5.661016949152542, |
| "grad_norm": 6.138956177743218, |
| "learning_rate": 1.19691380440715e-07, |
| "logits/chosen": -1.358786702156067, |
| "logits/rejected": -1.3909467458724976, |
| "logps/chosen": -30.639657974243164, |
| "logps/rejected": -48.24367141723633, |
| "loss": 0.0411, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4908216595649719, |
| "rewards/margins": 4.40742301940918, |
| "rewards/rejected": -4.898244857788086, |
| "step": 334 |
| }, |
| { |
| "epoch": 5.677966101694915, |
| "grad_norm": 4.507338496018484, |
| "learning_rate": 1.1811414776062365e-07, |
| "logits/chosen": -1.2691717147827148, |
| "logits/rejected": -1.2181487083435059, |
| "logps/chosen": -32.79166793823242, |
| "logps/rejected": -39.962955474853516, |
| "loss": 0.0227, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.39959853887557983, |
| "rewards/margins": 4.66073751449585, |
| "rewards/rejected": -4.261138916015625, |
| "step": 335 |
| }, |
| { |
| "epoch": 5.694915254237288, |
| "grad_norm": 6.700687388710383, |
| "learning_rate": 1.1654415550880242e-07, |
| "logits/chosen": -1.4832508563995361, |
| "logits/rejected": -1.4977812767028809, |
| "logps/chosen": -25.554107666015625, |
| "logps/rejected": -42.260276794433594, |
| "loss": 0.0325, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.07877922058105469, |
| "rewards/margins": 4.69774055480957, |
| "rewards/rejected": -4.776519775390625, |
| "step": 336 |
| }, |
| { |
| "epoch": 5.711864406779661, |
| "grad_norm": 6.852779788605717, |
| "learning_rate": 1.1498148987657549e-07, |
| "logits/chosen": -1.2651898860931396, |
| "logits/rejected": -1.2874916791915894, |
| "logps/chosen": -31.26464080810547, |
| "logps/rejected": -56.77597427368164, |
| "loss": 0.0453, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4548006057739258, |
| "rewards/margins": 6.219876289367676, |
| "rewards/rejected": -6.67467737197876, |
| "step": 337 |
| }, |
| { |
| "epoch": 5.728813559322034, |
| "grad_norm": 5.857910583133337, |
| "learning_rate": 1.1342623665304207e-07, |
| "logits/chosen": -1.5274604558944702, |
| "logits/rejected": -1.5151126384735107, |
| "logps/chosen": -26.850894927978516, |
| "logps/rejected": -47.350067138671875, |
| "loss": 0.0367, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5475198030471802, |
| "rewards/margins": 4.643810272216797, |
| "rewards/rejected": -5.1913299560546875, |
| "step": 338 |
| }, |
| { |
| "epoch": 5.745762711864407, |
| "grad_norm": 7.203472778094057, |
| "learning_rate": 1.1187848122036562e-07, |
| "logits/chosen": -1.6401658058166504, |
| "logits/rejected": -1.6699111461639404, |
| "logps/chosen": -28.927499771118164, |
| "logps/rejected": -36.704795837402344, |
| "loss": 0.0422, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.15964704751968384, |
| "rewards/margins": 4.4911370277404785, |
| "rewards/rejected": -4.3314900398254395, |
| "step": 339 |
| }, |
| { |
| "epoch": 5.762711864406779, |
| "grad_norm": 7.2779221460952, |
| "learning_rate": 1.1033830854908691e-07, |
| "logits/chosen": -1.502543330192566, |
| "logits/rejected": -1.4199045896530151, |
| "logps/chosen": -23.966299057006836, |
| "logps/rejected": -46.33234786987305, |
| "loss": 0.0506, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10161617398262024, |
| "rewards/margins": 5.133113861083984, |
| "rewards/rejected": -5.2347307205200195, |
| "step": 340 |
| }, |
| { |
| "epoch": 5.779661016949152, |
| "grad_norm": 7.302109736865858, |
| "learning_rate": 1.0880580319345919e-07, |
| "logits/chosen": -1.4564778804779053, |
| "logits/rejected": -1.5227388143539429, |
| "logps/chosen": -33.69976806640625, |
| "logps/rejected": -42.49582290649414, |
| "loss": 0.0396, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1525229811668396, |
| "rewards/margins": 4.674422264099121, |
| "rewards/rejected": -4.8269453048706055, |
| "step": 341 |
| }, |
| { |
| "epoch": 5.796610169491525, |
| "grad_norm": 7.067583835233995, |
| "learning_rate": 1.0728104928680623e-07, |
| "logits/chosen": -1.5719774961471558, |
| "logits/rejected": -1.502136468887329, |
| "logps/chosen": -21.33773422241211, |
| "logps/rejected": -42.372676849365234, |
| "loss": 0.0433, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.12551027536392212, |
| "rewards/margins": 5.258235454559326, |
| "rewards/rejected": -5.3837456703186035, |
| "step": 342 |
| }, |
| { |
| "epoch": 5.813559322033898, |
| "grad_norm": 5.957807006157361, |
| "learning_rate": 1.0576413053690326e-07, |
| "logits/chosen": -1.4468128681182861, |
| "logits/rejected": -1.3548585176467896, |
| "logps/chosen": -24.791799545288086, |
| "logps/rejected": -44.49803161621094, |
| "loss": 0.039, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.11925530433654785, |
| "rewards/margins": 5.184385299682617, |
| "rewards/rejected": -5.06512975692749, |
| "step": 343 |
| }, |
| { |
| "epoch": 5.830508474576272, |
| "grad_norm": 6.401362630388005, |
| "learning_rate": 1.0425513022138202e-07, |
| "logits/chosen": -1.491408348083496, |
| "logits/rejected": -1.3371365070343018, |
| "logps/chosen": -29.460378646850586, |
| "logps/rejected": -53.63321304321289, |
| "loss": 0.0354, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3325144648551941, |
| "rewards/margins": 5.398627758026123, |
| "rewards/rejected": -5.731142520904541, |
| "step": 344 |
| }, |
| { |
| "epoch": 5.847457627118644, |
| "grad_norm": 5.5542840449584405, |
| "learning_rate": 1.0275413118315798e-07, |
| "logits/chosen": -1.3796604871749878, |
| "logits/rejected": -1.377408504486084, |
| "logps/chosen": -26.488426208496094, |
| "logps/rejected": -46.708282470703125, |
| "loss": 0.0331, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1414252519607544, |
| "rewards/margins": 5.361431121826172, |
| "rewards/rejected": -5.220005989074707, |
| "step": 345 |
| }, |
| { |
| "epoch": 5.864406779661017, |
| "grad_norm": 6.360073336005713, |
| "learning_rate": 1.0126121582588315e-07, |
| "logits/chosen": -1.5218093395233154, |
| "logits/rejected": -1.3046934604644775, |
| "logps/chosen": -41.216285705566406, |
| "logps/rejected": -43.98678970336914, |
| "loss": 0.0402, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5423004627227783, |
| "rewards/margins": 4.604074001312256, |
| "rewards/rejected": -5.146374702453613, |
| "step": 346 |
| }, |
| { |
| "epoch": 5.88135593220339, |
| "grad_norm": 5.539790259394075, |
| "learning_rate": 9.977646610942201e-08, |
| "logits/chosen": -1.4420253038406372, |
| "logits/rejected": -1.4948465824127197, |
| "logps/chosen": -37.67629623413086, |
| "logps/rejected": -53.65730667114258, |
| "loss": 0.0283, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.8092373609542847, |
| "rewards/margins": 5.268472671508789, |
| "rewards/rejected": -6.077709674835205, |
| "step": 347 |
| }, |
| { |
| "epoch": 5.898305084745763, |
| "grad_norm": 6.554303392876224, |
| "learning_rate": 9.829996354535172e-08, |
| "logits/chosen": -1.5454871654510498, |
| "logits/rejected": -1.5333365201950073, |
| "logps/chosen": -21.350933074951172, |
| "logps/rejected": -49.00358200073242, |
| "loss": 0.0408, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.24133503437042236, |
| "rewards/margins": 4.911167144775391, |
| "rewards/rejected": -5.152502059936523, |
| "step": 348 |
| }, |
| { |
| "epoch": 5.915254237288136, |
| "grad_norm": 6.546704494284942, |
| "learning_rate": 9.68317891924871e-08, |
| "logits/chosen": -1.5749708414077759, |
| "logits/rejected": -1.433280110359192, |
| "logps/chosen": -34.201988220214844, |
| "logps/rejected": -51.28511428833008, |
| "loss": 0.0356, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3138273358345032, |
| "rewards/margins": 4.917507648468018, |
| "rewards/rejected": -5.231335163116455, |
| "step": 349 |
| }, |
| { |
| "epoch": 5.932203389830509, |
| "grad_norm": 6.294270520592479, |
| "learning_rate": 9.53720236524313e-08, |
| "logits/chosen": -1.5306050777435303, |
| "logits/rejected": -1.442819356918335, |
| "logps/chosen": -39.66340255737305, |
| "logps/rejected": -47.36227798461914, |
| "loss": 0.034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3249303102493286, |
| "rewards/margins": 4.536349296569824, |
| "rewards/rejected": -4.861279487609863, |
| "step": 350 |
| }, |
| { |
| "epoch": 5.9491525423728815, |
| "grad_norm": 6.24196436709209, |
| "learning_rate": 9.392074706515002e-08, |
| "logits/chosen": -1.3876663446426392, |
| "logits/rejected": -1.3757574558258057, |
| "logps/chosen": -29.42030143737793, |
| "logps/rejected": -51.045021057128906, |
| "loss": 0.0347, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.15947160124778748, |
| "rewards/margins": 4.901638031005859, |
| "rewards/rejected": -5.06110954284668, |
| "step": 351 |
| }, |
| { |
| "epoch": 5.966101694915254, |
| "grad_norm": 7.215999575793012, |
| "learning_rate": 9.247803910457225e-08, |
| "logits/chosen": -1.5945814847946167, |
| "logits/rejected": -1.5649042129516602, |
| "logps/chosen": -23.595874786376953, |
| "logps/rejected": -45.547088623046875, |
| "loss": 0.0492, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.08822253346443176, |
| "rewards/margins": 4.996213912963867, |
| "rewards/rejected": -5.084437370300293, |
| "step": 352 |
| }, |
| { |
| "epoch": 5.983050847457627, |
| "grad_norm": 6.19260019151604, |
| "learning_rate": 9.104397897421623e-08, |
| "logits/chosen": -1.4307911396026611, |
| "logits/rejected": -1.3201934099197388, |
| "logps/chosen": -26.220760345458984, |
| "logps/rejected": -51.94208526611328, |
| "loss": 0.0311, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.05383840203285217, |
| "rewards/margins": 5.268675804138184, |
| "rewards/rejected": -5.322514057159424, |
| "step": 353 |
| }, |
| { |
| "epoch": 6.0, |
| "grad_norm": 5.881521162946629, |
| "learning_rate": 8.961864540284119e-08, |
| "logits/chosen": -1.46354079246521, |
| "logits/rejected": -1.369706392288208, |
| "logps/chosen": -21.57988739013672, |
| "logps/rejected": -39.70952606201172, |
| "loss": 0.0307, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.02779284119606018, |
| "rewards/margins": 4.516615867614746, |
| "rewards/rejected": -4.488822937011719, |
| "step": 354 |
| }, |
| { |
| "epoch": 6.016949152542373, |
| "grad_norm": 6.0145355879659625, |
| "learning_rate": 8.82021166401253e-08, |
| "logits/chosen": -1.124272108078003, |
| "logits/rejected": -1.1505348682403564, |
| "logps/chosen": -46.4174919128418, |
| "logps/rejected": -55.31433868408203, |
| "loss": 0.0382, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.7474560737609863, |
| "rewards/margins": 5.104282379150391, |
| "rewards/rejected": -5.851738452911377, |
| "step": 355 |
| }, |
| { |
| "epoch": 6.033898305084746, |
| "grad_norm": 5.799187572293291, |
| "learning_rate": 8.679447045236962e-08, |
| "logits/chosen": -1.4048717021942139, |
| "logits/rejected": -1.479861855506897, |
| "logps/chosen": -21.200166702270508, |
| "logps/rejected": -38.02268600463867, |
| "loss": 0.0352, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.14473703503608704, |
| "rewards/margins": 5.024458885192871, |
| "rewards/rejected": -5.169196128845215, |
| "step": 356 |
| }, |
| { |
| "epoch": 6.0508474576271185, |
| "grad_norm": 7.041713270895366, |
| "learning_rate": 8.539578411822901e-08, |
| "logits/chosen": -1.3607594966888428, |
| "logits/rejected": -1.3202852010726929, |
| "logps/chosen": -27.951833724975586, |
| "logps/rejected": -46.871917724609375, |
| "loss": 0.0439, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.13125565648078918, |
| "rewards/margins": 4.497035026550293, |
| "rewards/rejected": -4.365779399871826, |
| "step": 357 |
| }, |
| { |
| "epoch": 6.067796610169491, |
| "grad_norm": 4.947363488557164, |
| "learning_rate": 8.400613442446947e-08, |
| "logits/chosen": -1.571787714958191, |
| "logits/rejected": -1.4330024719238281, |
| "logps/chosen": -28.946853637695312, |
| "logps/rejected": -45.973785400390625, |
| "loss": 0.0311, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.589256763458252, |
| "rewards/margins": 5.187601566314697, |
| "rewards/rejected": -5.776858329772949, |
| "step": 358 |
| }, |
| { |
| "epoch": 6.084745762711864, |
| "grad_norm": 5.199584429039793, |
| "learning_rate": 8.262559766175253e-08, |
| "logits/chosen": -1.4506189823150635, |
| "logits/rejected": -1.3975802659988403, |
| "logps/chosen": -24.713668823242188, |
| "logps/rejected": -51.15769577026367, |
| "loss": 0.0267, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.11659543216228485, |
| "rewards/margins": 6.322598457336426, |
| "rewards/rejected": -6.439194202423096, |
| "step": 359 |
| }, |
| { |
| "epoch": 6.101694915254237, |
| "grad_norm": 5.228661804484342, |
| "learning_rate": 8.125424962044741e-08, |
| "logits/chosen": -1.679395079612732, |
| "logits/rejected": -1.5920395851135254, |
| "logps/chosen": -32.77759552001953, |
| "logps/rejected": -51.247283935546875, |
| "loss": 0.0256, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6732353568077087, |
| "rewards/margins": 5.932892322540283, |
| "rewards/rejected": -6.6061272621154785, |
| "step": 360 |
| }, |
| { |
| "epoch": 6.11864406779661, |
| "grad_norm": 4.99263433146697, |
| "learning_rate": 7.989216558646941e-08, |
| "logits/chosen": -1.5995450019836426, |
| "logits/rejected": -1.5377026796340942, |
| "logps/chosen": -30.13187026977539, |
| "logps/rejected": -47.109947204589844, |
| "loss": 0.0269, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2468140721321106, |
| "rewards/margins": 5.169532299041748, |
| "rewards/rejected": -5.416346073150635, |
| "step": 361 |
| }, |
| { |
| "epoch": 6.135593220338983, |
| "grad_norm": 5.703896702738386, |
| "learning_rate": 7.853942033714736e-08, |
| "logits/chosen": -1.5143556594848633, |
| "logits/rejected": -1.3968244791030884, |
| "logps/chosen": -37.41975402832031, |
| "logps/rejected": -56.349769592285156, |
| "loss": 0.0326, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.31496256589889526, |
| "rewards/margins": 5.113517761230469, |
| "rewards/rejected": -5.428481101989746, |
| "step": 362 |
| }, |
| { |
| "epoch": 6.1525423728813555, |
| "grad_norm": 5.916237621370921, |
| "learning_rate": 7.719608813711847e-08, |
| "logits/chosen": -1.5250320434570312, |
| "logits/rejected": -1.475509762763977, |
| "logps/chosen": -26.13006591796875, |
| "logps/rejected": -35.852359771728516, |
| "loss": 0.0313, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.13443481922149658, |
| "rewards/margins": 4.1385908126831055, |
| "rewards/rejected": -4.00415563583374, |
| "step": 363 |
| }, |
| { |
| "epoch": 6.169491525423728, |
| "grad_norm": 6.510656924645236, |
| "learning_rate": 7.586224273425081e-08, |
| "logits/chosen": -1.3499259948730469, |
| "logits/rejected": -1.244497299194336, |
| "logps/chosen": -34.813541412353516, |
| "logps/rejected": -47.603271484375, |
| "loss": 0.0405, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.11887124180793762, |
| "rewards/margins": 4.864650726318359, |
| "rewards/rejected": -4.745779514312744, |
| "step": 364 |
| }, |
| { |
| "epoch": 6.186440677966102, |
| "grad_norm": 5.630103461727817, |
| "learning_rate": 7.45379573555947e-08, |
| "logits/chosen": -1.4300577640533447, |
| "logits/rejected": -1.4099435806274414, |
| "logps/chosen": -34.18559646606445, |
| "logps/rejected": -45.739933013916016, |
| "loss": 0.0314, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.07826349139213562, |
| "rewards/margins": 4.771849155426025, |
| "rewards/rejected": -4.8501129150390625, |
| "step": 365 |
| }, |
| { |
| "epoch": 6.203389830508475, |
| "grad_norm": 5.238611793677947, |
| "learning_rate": 7.322330470336313e-08, |
| "logits/chosen": -1.5490187406539917, |
| "logits/rejected": -1.6131647825241089, |
| "logps/chosen": -29.01973533630371, |
| "logps/rejected": -48.46769332885742, |
| "loss": 0.0323, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.043659090995788574, |
| "rewards/margins": 4.948011875152588, |
| "rewards/rejected": -4.991670608520508, |
| "step": 366 |
| }, |
| { |
| "epoch": 6.220338983050848, |
| "grad_norm": 7.178172383273588, |
| "learning_rate": 7.19183569509398e-08, |
| "logits/chosen": -1.4348180294036865, |
| "logits/rejected": -1.4342725276947021, |
| "logps/chosen": -26.032867431640625, |
| "logps/rejected": -35.465248107910156, |
| "loss": 0.0433, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.11982511729001999, |
| "rewards/margins": 4.798130035400391, |
| "rewards/rejected": -4.678304672241211, |
| "step": 367 |
| }, |
| { |
| "epoch": 6.237288135593221, |
| "grad_norm": 5.80114325638975, |
| "learning_rate": 7.062318573891715e-08, |
| "logits/chosen": -1.3932957649230957, |
| "logits/rejected": -1.3466596603393555, |
| "logps/chosen": -26.193689346313477, |
| "logps/rejected": -43.341983795166016, |
| "loss": 0.035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.26808837056159973, |
| "rewards/margins": 5.052790641784668, |
| "rewards/rejected": -4.784702301025391, |
| "step": 368 |
| }, |
| { |
| "epoch": 6.254237288135593, |
| "grad_norm": 5.8991450258769875, |
| "learning_rate": 6.933786217116364e-08, |
| "logits/chosen": -1.5599933862686157, |
| "logits/rejected": -1.4719116687774658, |
| "logps/chosen": -26.387096405029297, |
| "logps/rejected": -41.33942794799805, |
| "loss": 0.0386, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.40677589178085327, |
| "rewards/margins": 4.713696479797363, |
| "rewards/rejected": -4.306921005249023, |
| "step": 369 |
| }, |
| { |
| "epoch": 6.271186440677966, |
| "grad_norm": 5.827867650343381, |
| "learning_rate": 6.806245681091944e-08, |
| "logits/chosen": -1.4101459980010986, |
| "logits/rejected": -1.3021355867385864, |
| "logps/chosen": -27.591569900512695, |
| "logps/rejected": -47.71153259277344, |
| "loss": 0.0338, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1557801067829132, |
| "rewards/margins": 5.925251483917236, |
| "rewards/rejected": -5.769471168518066, |
| "step": 370 |
| }, |
| { |
| "epoch": 6.288135593220339, |
| "grad_norm": 5.390338841256569, |
| "learning_rate": 6.679703967692321e-08, |
| "logits/chosen": -1.584707498550415, |
| "logits/rejected": -1.4573631286621094, |
| "logps/chosen": -21.927671432495117, |
| "logps/rejected": -49.91468048095703, |
| "loss": 0.0317, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.393250048160553, |
| "rewards/margins": 5.488755226135254, |
| "rewards/rejected": -5.095505714416504, |
| "step": 371 |
| }, |
| { |
| "epoch": 6.305084745762712, |
| "grad_norm": 4.571609396535961, |
| "learning_rate": 6.554168023956816e-08, |
| "logits/chosen": -1.4562060832977295, |
| "logits/rejected": -1.3005712032318115, |
| "logps/chosen": -25.927963256835938, |
| "logps/rejected": -44.458621978759766, |
| "loss": 0.0284, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2479822039604187, |
| "rewards/margins": 4.508713722229004, |
| "rewards/rejected": -4.756695747375488, |
| "step": 372 |
| }, |
| { |
| "epoch": 6.322033898305085, |
| "grad_norm": 6.286732270647367, |
| "learning_rate": 6.429644741708779e-08, |
| "logits/chosen": -1.4326915740966797, |
| "logits/rejected": -1.3545727729797363, |
| "logps/chosen": -25.131105422973633, |
| "logps/rejected": -40.29548263549805, |
| "loss": 0.0324, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.04497765004634857, |
| "rewards/margins": 4.998070240020752, |
| "rewards/rejected": -4.953092575073242, |
| "step": 373 |
| }, |
| { |
| "epoch": 6.338983050847458, |
| "grad_norm": 6.303722387593113, |
| "learning_rate": 6.306140957177225e-08, |
| "logits/chosen": -1.452634334564209, |
| "logits/rejected": -1.427751898765564, |
| "logps/chosen": -26.491840362548828, |
| "logps/rejected": -47.47591781616211, |
| "loss": 0.0331, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.015405148267745972, |
| "rewards/margins": 5.423872947692871, |
| "rewards/rejected": -5.439278602600098, |
| "step": 374 |
| }, |
| { |
| "epoch": 6.3559322033898304, |
| "grad_norm": 5.766682311294022, |
| "learning_rate": 6.183663450621607e-08, |
| "logits/chosen": -1.506758213043213, |
| "logits/rejected": -1.4310308694839478, |
| "logps/chosen": -38.03307342529297, |
| "logps/rejected": -46.39859390258789, |
| "loss": 0.0361, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.16017396748065948, |
| "rewards/margins": 4.915417194366455, |
| "rewards/rejected": -5.075592041015625, |
| "step": 375 |
| }, |
| { |
| "epoch": 6.372881355932203, |
| "grad_norm": 4.912000082420141, |
| "learning_rate": 6.062218945959496e-08, |
| "logits/chosen": -1.4634639024734497, |
| "logits/rejected": -1.5328123569488525, |
| "logps/chosen": -34.974857330322266, |
| "logps/rejected": -43.447689056396484, |
| "loss": 0.0265, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.06525677442550659, |
| "rewards/margins": 5.0226850509643555, |
| "rewards/rejected": -5.087942123413086, |
| "step": 376 |
| }, |
| { |
| "epoch": 6.389830508474576, |
| "grad_norm": 4.923323155963838, |
| "learning_rate": 5.9418141103975026e-08, |
| "logits/chosen": -1.4239155054092407, |
| "logits/rejected": -1.4835269451141357, |
| "logps/chosen": -30.768068313598633, |
| "logps/rejected": -60.89370346069336, |
| "loss": 0.0235, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3536137342453003, |
| "rewards/margins": 6.844842910766602, |
| "rewards/rejected": -7.198456764221191, |
| "step": 377 |
| }, |
| { |
| "epoch": 6.406779661016949, |
| "grad_norm": 6.31358841065967, |
| "learning_rate": 5.822455554065217e-08, |
| "logits/chosen": -1.3272855281829834, |
| "logits/rejected": -1.2906978130340576, |
| "logps/chosen": -25.13174819946289, |
| "logps/rejected": -41.51945877075195, |
| "loss": 0.0377, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2586425840854645, |
| "rewards/margins": 5.224736213684082, |
| "rewards/rejected": -4.966093063354492, |
| "step": 378 |
| }, |
| { |
| "epoch": 6.423728813559322, |
| "grad_norm": 5.9217345486120365, |
| "learning_rate": 5.704149829652341e-08, |
| "logits/chosen": -1.5300796031951904, |
| "logits/rejected": -1.4160737991333008, |
| "logps/chosen": -29.946929931640625, |
| "logps/rejected": -49.227298736572266, |
| "loss": 0.0318, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1305888444185257, |
| "rewards/margins": 5.067564487457275, |
| "rewards/rejected": -5.198153495788574, |
| "step": 379 |
| }, |
| { |
| "epoch": 6.440677966101695, |
| "grad_norm": 5.394022162647855, |
| "learning_rate": 5.586903432048942e-08, |
| "logits/chosen": -1.5190213918685913, |
| "logits/rejected": -1.4037725925445557, |
| "logps/chosen": -31.318918228149414, |
| "logps/rejected": -49.82917404174805, |
| "loss": 0.0302, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.9247037768363953, |
| "rewards/margins": 5.2036919593811035, |
| "rewards/rejected": -6.128396034240723, |
| "step": 380 |
| }, |
| { |
| "epoch": 6.4576271186440675, |
| "grad_norm": 5.307172264281174, |
| "learning_rate": 5.470722797988883e-08, |
| "logits/chosen": -1.3760260343551636, |
| "logits/rejected": -1.3698216676712036, |
| "logps/chosen": -24.693878173828125, |
| "logps/rejected": -37.211387634277344, |
| "loss": 0.0353, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.05154569447040558, |
| "rewards/margins": 4.664517879486084, |
| "rewards/rejected": -4.612972736358643, |
| "step": 381 |
| }, |
| { |
| "epoch": 6.47457627118644, |
| "grad_norm": 5.039925613354933, |
| "learning_rate": 5.355614305696468e-08, |
| "logits/chosen": -1.56493079662323, |
| "logits/rejected": -1.5143284797668457, |
| "logps/chosen": -27.657060623168945, |
| "logps/rejected": -43.6398811340332, |
| "loss": 0.0262, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.09710407257080078, |
| "rewards/margins": 4.8609700202941895, |
| "rewards/rejected": -4.958074569702148, |
| "step": 382 |
| }, |
| { |
| "epoch": 6.491525423728813, |
| "grad_norm": 6.304781751843407, |
| "learning_rate": 5.241584274536259e-08, |
| "logits/chosen": -1.3160762786865234, |
| "logits/rejected": -1.426667332649231, |
| "logps/chosen": -31.549245834350586, |
| "logps/rejected": -49.771148681640625, |
| "loss": 0.0322, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3802323043346405, |
| "rewards/margins": 5.496129035949707, |
| "rewards/rejected": -5.876360893249512, |
| "step": 383 |
| }, |
| { |
| "epoch": 6.508474576271187, |
| "grad_norm": 5.063738993506685, |
| "learning_rate": 5.1286389646661654e-08, |
| "logits/chosen": -1.396496057510376, |
| "logits/rejected": -1.3829492330551147, |
| "logps/chosen": -28.830703735351562, |
| "logps/rejected": -47.824317932128906, |
| "loss": 0.0246, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2612743079662323, |
| "rewards/margins": 4.95106315612793, |
| "rewards/rejected": -5.212337017059326, |
| "step": 384 |
| }, |
| { |
| "epoch": 6.52542372881356, |
| "grad_norm": 5.637690264352778, |
| "learning_rate": 5.0167845766937806e-08, |
| "logits/chosen": -1.4855574369430542, |
| "logits/rejected": -1.3984534740447998, |
| "logps/chosen": -27.114734649658203, |
| "logps/rejected": -43.69512939453125, |
| "loss": 0.0414, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.21004855632781982, |
| "rewards/margins": 4.845144748687744, |
| "rewards/rejected": -5.055192947387695, |
| "step": 385 |
| }, |
| { |
| "epoch": 6.5423728813559325, |
| "grad_norm": 6.025392312759281, |
| "learning_rate": 4.906027251335917e-08, |
| "logits/chosen": -1.4550882577896118, |
| "logits/rejected": -1.4542597532272339, |
| "logps/chosen": -23.857240676879883, |
| "logps/rejected": -47.817237854003906, |
| "loss": 0.0352, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.31768307089805603, |
| "rewards/margins": 4.948706150054932, |
| "rewards/rejected": -5.266389846801758, |
| "step": 386 |
| }, |
| { |
| "epoch": 6.559322033898305, |
| "grad_norm": 5.587911152077424, |
| "learning_rate": 4.7963730690815467e-08, |
| "logits/chosen": -1.4407036304473877, |
| "logits/rejected": -1.3747568130493164, |
| "logps/chosen": -19.481971740722656, |
| "logps/rejected": -42.736568450927734, |
| "loss": 0.0309, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.19361047446727753, |
| "rewards/margins": 5.198715686798096, |
| "rewards/rejected": -5.392325401306152, |
| "step": 387 |
| }, |
| { |
| "epoch": 6.576271186440678, |
| "grad_norm": 6.586187305125031, |
| "learning_rate": 4.687828049857967e-08, |
| "logits/chosen": -1.3945553302764893, |
| "logits/rejected": -1.322791337966919, |
| "logps/chosen": -29.22484016418457, |
| "logps/rejected": -40.85685348510742, |
| "loss": 0.0546, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.17647796869277954, |
| "rewards/margins": 4.56741189956665, |
| "rewards/rejected": -4.390933990478516, |
| "step": 388 |
| }, |
| { |
| "epoch": 6.593220338983051, |
| "grad_norm": 6.218480481228633, |
| "learning_rate": 4.580398152700304e-08, |
| "logits/chosen": -1.6018644571304321, |
| "logits/rejected": -1.4887254238128662, |
| "logps/chosen": -25.33257484436035, |
| "logps/rejected": -45.91154479980469, |
| "loss": 0.0352, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.20426103472709656, |
| "rewards/margins": 5.0216498374938965, |
| "rewards/rejected": -5.225910186767578, |
| "step": 389 |
| }, |
| { |
| "epoch": 6.610169491525424, |
| "grad_norm": 6.569813291973876, |
| "learning_rate": 4.47408927542435e-08, |
| "logits/chosen": -1.3770880699157715, |
| "logits/rejected": -1.286376714706421, |
| "logps/chosen": -23.488061904907227, |
| "logps/rejected": -38.757720947265625, |
| "loss": 0.0388, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2643284499645233, |
| "rewards/margins": 3.8822245597839355, |
| "rewards/rejected": -4.146553039550781, |
| "step": 390 |
| }, |
| { |
| "epoch": 6.627118644067797, |
| "grad_norm": 5.39352903238084, |
| "learning_rate": 4.368907254302837e-08, |
| "logits/chosen": -1.5808112621307373, |
| "logits/rejected": -1.5275750160217285, |
| "logps/chosen": -20.95781707763672, |
| "logps/rejected": -38.6187744140625, |
| "loss": 0.0366, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.05846305191516876, |
| "rewards/margins": 4.244011878967285, |
| "rewards/rejected": -4.302474498748779, |
| "step": 391 |
| }, |
| { |
| "epoch": 6.6440677966101696, |
| "grad_norm": 5.323944978561868, |
| "learning_rate": 4.264857863744956e-08, |
| "logits/chosen": -1.5523847341537476, |
| "logits/rejected": -1.39801824092865, |
| "logps/chosen": -22.804271697998047, |
| "logps/rejected": -38.31208419799805, |
| "loss": 0.0275, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4400111436843872, |
| "rewards/margins": 5.238194465637207, |
| "rewards/rejected": -4.798183917999268, |
| "step": 392 |
| }, |
| { |
| "epoch": 6.661016949152542, |
| "grad_norm": 7.086981471555236, |
| "learning_rate": 4.161946815979403e-08, |
| "logits/chosen": -1.4906023740768433, |
| "logits/rejected": -1.4377923011779785, |
| "logps/chosen": -37.56535720825195, |
| "logps/rejected": -52.383487701416016, |
| "loss": 0.0451, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3086543679237366, |
| "rewards/margins": 5.0754618644714355, |
| "rewards/rejected": -5.384116172790527, |
| "step": 393 |
| }, |
| { |
| "epoch": 6.677966101694915, |
| "grad_norm": 5.2671704490811075, |
| "learning_rate": 4.0601797607407505e-08, |
| "logits/chosen": -1.4550049304962158, |
| "logits/rejected": -1.3966783285140991, |
| "logps/chosen": -22.132400512695312, |
| "logps/rejected": -41.57554626464844, |
| "loss": 0.0324, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2699892520904541, |
| "rewards/margins": 4.588739395141602, |
| "rewards/rejected": -4.858728408813477, |
| "step": 394 |
| }, |
| { |
| "epoch": 6.694915254237288, |
| "grad_norm": 5.32453491166638, |
| "learning_rate": 3.9595622849593e-08, |
| "logits/chosen": -1.512937307357788, |
| "logits/rejected": -1.3621115684509277, |
| "logps/chosen": -28.885116577148438, |
| "logps/rejected": -50.2530517578125, |
| "loss": 0.0366, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5175858736038208, |
| "rewards/margins": 5.390725135803223, |
| "rewards/rejected": -5.908310890197754, |
| "step": 395 |
| }, |
| { |
| "epoch": 6.711864406779661, |
| "grad_norm": 5.792264114569628, |
| "learning_rate": 3.8600999124543455e-08, |
| "logits/chosen": -1.5662391185760498, |
| "logits/rejected": -1.42247474193573, |
| "logps/chosen": -25.263622283935547, |
| "logps/rejected": -45.11002731323242, |
| "loss": 0.0315, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.27481862902641296, |
| "rewards/margins": 5.28562593460083, |
| "rewards/rejected": -5.010807037353516, |
| "step": 396 |
| }, |
| { |
| "epoch": 6.728813559322034, |
| "grad_norm": 5.997208149184034, |
| "learning_rate": 3.7617981036309533e-08, |
| "logits/chosen": -1.5715502500534058, |
| "logits/rejected": -1.6154096126556396, |
| "logps/chosen": -23.581790924072266, |
| "logps/rejected": -45.5838623046875, |
| "loss": 0.0377, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.18038499355316162, |
| "rewards/margins": 4.884888172149658, |
| "rewards/rejected": -5.065273284912109, |
| "step": 397 |
| }, |
| { |
| "epoch": 6.745762711864407, |
| "grad_norm": 5.245999983483119, |
| "learning_rate": 3.664662255180134e-08, |
| "logits/chosen": -1.4324719905853271, |
| "logits/rejected": -1.329777717590332, |
| "logps/chosen": -26.615516662597656, |
| "logps/rejected": -41.86264419555664, |
| "loss": 0.0394, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.05792872607707977, |
| "rewards/margins": 4.163052082061768, |
| "rewards/rejected": -4.105123519897461, |
| "step": 398 |
| }, |
| { |
| "epoch": 6.762711864406779, |
| "grad_norm": 4.518676670270046, |
| "learning_rate": 3.5686976997826245e-08, |
| "logits/chosen": -1.6349799633026123, |
| "logits/rejected": -1.6590909957885742, |
| "logps/chosen": -41.426727294921875, |
| "logps/rejected": -52.470558166503906, |
| "loss": 0.0236, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4126446843147278, |
| "rewards/margins": 5.763060569763184, |
| "rewards/rejected": -6.175705432891846, |
| "step": 399 |
| }, |
| { |
| "epoch": 6.779661016949152, |
| "grad_norm": 5.721240252343364, |
| "learning_rate": 3.473909705816111e-08, |
| "logits/chosen": -1.4431383609771729, |
| "logits/rejected": -1.3899328708648682, |
| "logps/chosen": -36.3721923828125, |
| "logps/rejected": -52.30086898803711, |
| "loss": 0.0351, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.106335163116455, |
| "rewards/margins": 5.6671295166015625, |
| "rewards/rejected": -6.773464679718018, |
| "step": 400 |
| }, |
| { |
| "epoch": 6.796610169491525, |
| "grad_norm": 5.164348258065598, |
| "learning_rate": 3.3803034770659824e-08, |
| "logits/chosen": -1.6824212074279785, |
| "logits/rejected": -1.5971221923828125, |
| "logps/chosen": -37.007022857666016, |
| "logps/rejected": -74.48016357421875, |
| "loss": 0.0286, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6765580177307129, |
| "rewards/margins": 7.421613693237305, |
| "rewards/rejected": -8.09817123413086, |
| "step": 401 |
| }, |
| { |
| "epoch": 6.813559322033898, |
| "grad_norm": 4.808650804090459, |
| "learning_rate": 3.287884152439646e-08, |
| "logits/chosen": -1.3810476064682007, |
| "logits/rejected": -1.4028950929641724, |
| "logps/chosen": -29.925891876220703, |
| "logps/rejected": -49.08871841430664, |
| "loss": 0.0311, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.16381314396858215, |
| "rewards/margins": 6.113283157348633, |
| "rewards/rejected": -5.949470520019531, |
| "step": 402 |
| }, |
| { |
| "epoch": 6.830508474576272, |
| "grad_norm": 6.236701718893695, |
| "learning_rate": 3.19665680568445e-08, |
| "logits/chosen": -1.5909409523010254, |
| "logits/rejected": -1.5455948114395142, |
| "logps/chosen": -33.62629699707031, |
| "logps/rejected": -39.20241165161133, |
| "loss": 0.0374, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.20260879397392273, |
| "rewards/margins": 4.192948341369629, |
| "rewards/rejected": -4.395556926727295, |
| "step": 403 |
| }, |
| { |
| "epoch": 6.847457627118644, |
| "grad_norm": 4.563195691909389, |
| "learning_rate": 3.106626445109081e-08, |
| "logits/chosen": -1.4258350133895874, |
| "logits/rejected": -1.471103549003601, |
| "logps/chosen": -32.27821731567383, |
| "logps/rejected": -55.883445739746094, |
| "loss": 0.0274, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.37013059854507446, |
| "rewards/margins": 5.775850296020508, |
| "rewards/rejected": -6.1459808349609375, |
| "step": 404 |
| }, |
| { |
| "epoch": 6.864406779661017, |
| "grad_norm": 6.319828827539619, |
| "learning_rate": 3.017798013308645e-08, |
| "logits/chosen": -1.6110063791275024, |
| "logits/rejected": -1.5970886945724487, |
| "logps/chosen": -33.942317962646484, |
| "logps/rejected": -42.78126525878906, |
| "loss": 0.0394, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08474293351173401, |
| "rewards/margins": 4.63005256652832, |
| "rewards/rejected": -4.545310020446777, |
| "step": 405 |
| }, |
| { |
| "epoch": 6.88135593220339, |
| "grad_norm": 6.956985946783924, |
| "learning_rate": 2.9301763868933153e-08, |
| "logits/chosen": -1.4388988018035889, |
| "logits/rejected": -1.4342849254608154, |
| "logps/chosen": -23.345102310180664, |
| "logps/rejected": -40.68294906616211, |
| "loss": 0.0458, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.11982996761798859, |
| "rewards/margins": 4.858503818511963, |
| "rewards/rejected": -4.978332996368408, |
| "step": 406 |
| }, |
| { |
| "epoch": 6.898305084745763, |
| "grad_norm": 5.4291102026527716, |
| "learning_rate": 2.843766376220616e-08, |
| "logits/chosen": -1.7093875408172607, |
| "logits/rejected": -1.6210181713104248, |
| "logps/chosen": -29.290470123291016, |
| "logps/rejected": -50.43269729614258, |
| "loss": 0.0278, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.7307620048522949, |
| "rewards/margins": 5.429152488708496, |
| "rewards/rejected": -6.159914016723633, |
| "step": 407 |
| }, |
| { |
| "epoch": 6.915254237288136, |
| "grad_norm": 5.393300173055395, |
| "learning_rate": 2.7585727251313195e-08, |
| "logits/chosen": -1.3670405149459839, |
| "logits/rejected": -1.1494945287704468, |
| "logps/chosen": -38.649288177490234, |
| "logps/rejected": -55.98222351074219, |
| "loss": 0.0388, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4593397378921509, |
| "rewards/margins": 5.185694694519043, |
| "rewards/rejected": -5.645034313201904, |
| "step": 408 |
| }, |
| { |
| "epoch": 6.932203389830509, |
| "grad_norm": 4.837513841498393, |
| "learning_rate": 2.6746001106890377e-08, |
| "logits/chosen": -1.656294584274292, |
| "logits/rejected": -1.5977482795715332, |
| "logps/chosen": -29.096288681030273, |
| "logps/rejected": -46.76082229614258, |
| "loss": 0.0252, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.21669504046440125, |
| "rewards/margins": 5.154686450958252, |
| "rewards/rejected": -5.371381759643555, |
| "step": 409 |
| }, |
| { |
| "epoch": 6.9491525423728815, |
| "grad_norm": 5.248560918966433, |
| "learning_rate": 2.5918531429234364e-08, |
| "logits/chosen": -1.592848300933838, |
| "logits/rejected": -1.540168285369873, |
| "logps/chosen": -31.085020065307617, |
| "logps/rejected": -53.143741607666016, |
| "loss": 0.0298, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6363227963447571, |
| "rewards/margins": 5.588076591491699, |
| "rewards/rejected": -6.224399089813232, |
| "step": 410 |
| }, |
| { |
| "epoch": 6.966101694915254, |
| "grad_norm": 5.2195974344109715, |
| "learning_rate": 2.5103363645771536e-08, |
| "logits/chosen": -1.65935218334198, |
| "logits/rejected": -1.6805386543273926, |
| "logps/chosen": -34.833702087402344, |
| "logps/rejected": -48.904422760009766, |
| "loss": 0.0352, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1815856397151947, |
| "rewards/margins": 5.786221981048584, |
| "rewards/rejected": -5.967808246612549, |
| "step": 411 |
| }, |
| { |
| "epoch": 6.983050847457627, |
| "grad_norm": 5.470555151396108, |
| "learning_rate": 2.4300542508564114e-08, |
| "logits/chosen": -1.4322288036346436, |
| "logits/rejected": -1.3301326036453247, |
| "logps/chosen": -28.12655258178711, |
| "logps/rejected": -46.921104431152344, |
| "loss": 0.0395, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.31704282760620117, |
| "rewards/margins": 4.567241191864014, |
| "rewards/rejected": -4.884284019470215, |
| "step": 412 |
| }, |
| { |
| "epoch": 7.0, |
| "grad_norm": 5.578953824776314, |
| "learning_rate": 2.3510112091853357e-08, |
| "logits/chosen": -1.276556134223938, |
| "logits/rejected": -1.2297403812408447, |
| "logps/chosen": -21.443805694580078, |
| "logps/rejected": -48.807159423828125, |
| "loss": 0.0301, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.017134033143520355, |
| "rewards/margins": 5.347229957580566, |
| "rewards/rejected": -5.330096244812012, |
| "step": 413 |
| }, |
| { |
| "epoch": 7.016949152542373, |
| "grad_norm": 6.999197685418954, |
| "learning_rate": 2.27321157896396e-08, |
| "logits/chosen": -1.5116349458694458, |
| "logits/rejected": -1.3849687576293945, |
| "logps/chosen": -27.718652725219727, |
| "logps/rejected": -47.36982345581055, |
| "loss": 0.0412, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1311918944120407, |
| "rewards/margins": 5.171109676361084, |
| "rewards/rejected": -5.039917469024658, |
| "step": 414 |
| }, |
| { |
| "epoch": 7.033898305084746, |
| "grad_norm": 4.883886921689459, |
| "learning_rate": 2.1966596313300362e-08, |
| "logits/chosen": -1.6032100915908813, |
| "logits/rejected": -1.518362045288086, |
| "logps/chosen": -29.143009185791016, |
| "logps/rejected": -40.435001373291016, |
| "loss": 0.0341, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.24970372021198273, |
| "rewards/margins": 4.1580705642700195, |
| "rewards/rejected": -4.407774448394775, |
| "step": 415 |
| }, |
| { |
| "epoch": 7.0508474576271185, |
| "grad_norm": 5.855476388920514, |
| "learning_rate": 2.1213595689245384e-08, |
| "logits/chosen": -1.350822925567627, |
| "logits/rejected": -1.3723689317703247, |
| "logps/chosen": -23.788740158081055, |
| "logps/rejected": -40.06031799316406, |
| "loss": 0.0317, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.003228999674320221, |
| "rewards/margins": 4.06948184967041, |
| "rewards/rejected": -4.0727105140686035, |
| "step": 416 |
| }, |
| { |
| "epoch": 7.067796610169491, |
| "grad_norm": 5.942075006867736, |
| "learning_rate": 2.0473155256609363e-08, |
| "logits/chosen": -1.6869771480560303, |
| "logits/rejected": -1.6048226356506348, |
| "logps/chosen": -27.607175827026367, |
| "logps/rejected": -46.171112060546875, |
| "loss": 0.0478, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.15510573983192444, |
| "rewards/margins": 4.710604667663574, |
| "rewards/rejected": -4.865710258483887, |
| "step": 417 |
| }, |
| { |
| "epoch": 7.084745762711864, |
| "grad_norm": 5.9322015599453595, |
| "learning_rate": 1.9745315664982277e-08, |
| "logits/chosen": -1.5960508584976196, |
| "logits/rejected": -1.4311569929122925, |
| "logps/chosen": -20.32171630859375, |
| "logps/rejected": -41.58867645263672, |
| "loss": 0.0372, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.00745101273059845, |
| "rewards/margins": 5.94549560546875, |
| "rewards/rejected": -5.938044548034668, |
| "step": 418 |
| }, |
| { |
| "epoch": 7.101694915254237, |
| "grad_norm": 5.918275937210799, |
| "learning_rate": 1.9030116872178314e-08, |
| "logits/chosen": -1.699689269065857, |
| "logits/rejected": -1.5470737218856812, |
| "logps/chosen": -28.80027198791504, |
| "logps/rejected": -46.22833251953125, |
| "loss": 0.0378, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4729538857936859, |
| "rewards/margins": 4.527037143707275, |
| "rewards/rejected": -4.999991416931152, |
| "step": 419 |
| }, |
| { |
| "epoch": 7.11864406779661, |
| "grad_norm": 6.2740518918594015, |
| "learning_rate": 1.8327598142041656e-08, |
| "logits/chosen": -1.4966247081756592, |
| "logits/rejected": -1.5006301403045654, |
| "logps/chosen": -39.597660064697266, |
| "logps/rejected": -59.670684814453125, |
| "loss": 0.0464, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.10089424252510071, |
| "rewards/margins": 5.958191871643066, |
| "rewards/rejected": -5.857297420501709, |
| "step": 420 |
| }, |
| { |
| "epoch": 7.135593220338983, |
| "grad_norm": 5.848230208495223, |
| "learning_rate": 1.7637798042291125e-08, |
| "logits/chosen": -1.4466344118118286, |
| "logits/rejected": -1.4158880710601807, |
| "logps/chosen": -34.30101013183594, |
| "logps/rejected": -41.198646545410156, |
| "loss": 0.0301, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5382071733474731, |
| "rewards/margins": 4.64283561706543, |
| "rewards/rejected": -5.1810431480407715, |
| "step": 421 |
| }, |
| { |
| "epoch": 7.1525423728813555, |
| "grad_norm": 4.83673555496754, |
| "learning_rate": 1.696075444240305e-08, |
| "logits/chosen": -1.481490135192871, |
| "logits/rejected": -1.3944220542907715, |
| "logps/chosen": -23.669601440429688, |
| "logps/rejected": -44.325191497802734, |
| "loss": 0.0259, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.12855800986289978, |
| "rewards/margins": 4.879583835601807, |
| "rewards/rejected": -5.008142471313477, |
| "step": 422 |
| }, |
| { |
| "epoch": 7.169491525423728, |
| "grad_norm": 5.610847549034329, |
| "learning_rate": 1.6296504511531834e-08, |
| "logits/chosen": -1.418038249015808, |
| "logits/rejected": -1.3723573684692383, |
| "logps/chosen": -27.353422164916992, |
| "logps/rejected": -49.684974670410156, |
| "loss": 0.0335, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.7425976395606995, |
| "rewards/margins": 4.4584550857543945, |
| "rewards/rejected": -5.201053142547607, |
| "step": 423 |
| }, |
| { |
| "epoch": 7.186440677966102, |
| "grad_norm": 5.7865923023012975, |
| "learning_rate": 1.5645084716469776e-08, |
| "logits/chosen": -1.6516090631484985, |
| "logits/rejected": -1.5412788391113281, |
| "logps/chosen": -33.868499755859375, |
| "logps/rejected": -49.333709716796875, |
| "loss": 0.0308, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2233295440673828, |
| "rewards/margins": 5.964905261993408, |
| "rewards/rejected": -6.188235282897949, |
| "step": 424 |
| }, |
| { |
| "epoch": 7.203389830508475, |
| "grad_norm": 5.701505367137342, |
| "learning_rate": 1.5006530819644923e-08, |
| "logits/chosen": -1.416776418685913, |
| "logits/rejected": -1.5832912921905518, |
| "logps/chosen": -32.08591842651367, |
| "logps/rejected": -47.3320198059082, |
| "loss": 0.0324, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.342715859413147, |
| "rewards/margins": 5.240063667297363, |
| "rewards/rejected": -5.582779884338379, |
| "step": 425 |
| }, |
| { |
| "epoch": 7.220338983050848, |
| "grad_norm": 4.437542789195646, |
| "learning_rate": 1.4380877877157832e-08, |
| "logits/chosen": -1.4847077131271362, |
| "logits/rejected": -1.4457132816314697, |
| "logps/chosen": -32.51253128051758, |
| "logps/rejected": -53.59151077270508, |
| "loss": 0.0218, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6102309226989746, |
| "rewards/margins": 5.707812786102295, |
| "rewards/rejected": -6.3180437088012695, |
| "step": 426 |
| }, |
| { |
| "epoch": 7.237288135593221, |
| "grad_norm": 5.065631167693018, |
| "learning_rate": 1.3768160236856674e-08, |
| "logits/chosen": -1.4158098697662354, |
| "logits/rejected": -1.4317882061004639, |
| "logps/chosen": -29.830453872680664, |
| "logps/rejected": -54.25461196899414, |
| "loss": 0.024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.038821518421173096, |
| "rewards/margins": 5.01283597946167, |
| "rewards/rejected": -5.051657676696777, |
| "step": 427 |
| }, |
| { |
| "epoch": 7.254237288135593, |
| "grad_norm": 5.220392921602965, |
| "learning_rate": 1.316841153645215e-08, |
| "logits/chosen": -1.560931921005249, |
| "logits/rejected": -1.5064420700073242, |
| "logps/chosen": -30.01360321044922, |
| "logps/rejected": -48.26612854003906, |
| "loss": 0.0276, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3863828778266907, |
| "rewards/margins": 5.301400184631348, |
| "rewards/rejected": -5.687783241271973, |
| "step": 428 |
| }, |
| { |
| "epoch": 7.271186440677966, |
| "grad_norm": 5.688311376504444, |
| "learning_rate": 1.2581664701670296e-08, |
| "logits/chosen": -1.52787446975708, |
| "logits/rejected": -1.4021316766738892, |
| "logps/chosen": -28.248119354248047, |
| "logps/rejected": -41.31006622314453, |
| "loss": 0.0392, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3207528591156006, |
| "rewards/margins": 5.154188632965088, |
| "rewards/rejected": -5.474941253662109, |
| "step": 429 |
| }, |
| { |
| "epoch": 7.288135593220339, |
| "grad_norm": 5.800049450685864, |
| "learning_rate": 1.2007951944445121e-08, |
| "logits/chosen": -1.4032049179077148, |
| "logits/rejected": -1.3456979990005493, |
| "logps/chosen": -24.016460418701172, |
| "logps/rejected": -41.04709243774414, |
| "loss": 0.0278, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.04568904638290405, |
| "rewards/margins": 3.968392848968506, |
| "rewards/rejected": -4.0140814781188965, |
| "step": 430 |
| }, |
| { |
| "epoch": 7.305084745762712, |
| "grad_norm": 6.591249552173131, |
| "learning_rate": 1.144730476115019e-08, |
| "logits/chosen": -1.4663312435150146, |
| "logits/rejected": -1.437060832977295, |
| "logps/chosen": -25.586219787597656, |
| "logps/rejected": -59.44462585449219, |
| "loss": 0.0416, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.418854296207428, |
| "rewards/margins": 6.551999092102051, |
| "rewards/rejected": -6.970853328704834, |
| "step": 431 |
| }, |
| { |
| "epoch": 7.322033898305085, |
| "grad_norm": 5.1436479741963765, |
| "learning_rate": 1.0899753930869394e-08, |
| "logits/chosen": -1.479379415512085, |
| "logits/rejected": -1.4856846332550049, |
| "logps/chosen": -24.105491638183594, |
| "logps/rejected": -42.0874137878418, |
| "loss": 0.0283, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.016550958156585693, |
| "rewards/margins": 4.623503684997559, |
| "rewards/rejected": -4.640054702758789, |
| "step": 432 |
| }, |
| { |
| "epoch": 7.338983050847458, |
| "grad_norm": 6.13135090791532, |
| "learning_rate": 1.036532951370736e-08, |
| "logits/chosen": -1.5114121437072754, |
| "logits/rejected": -1.487284541130066, |
| "logps/chosen": -30.636520385742188, |
| "logps/rejected": -55.582489013671875, |
| "loss": 0.0421, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.0013150721788406372, |
| "rewards/margins": 6.1965837478637695, |
| "rewards/rejected": -6.1952691078186035, |
| "step": 433 |
| }, |
| { |
| "epoch": 7.3559322033898304, |
| "grad_norm": 5.332642447483903, |
| "learning_rate": 9.844060849138997e-09, |
| "logits/chosen": -1.599272608757019, |
| "logits/rejected": -1.610948085784912, |
| "logps/chosen": -24.916534423828125, |
| "logps/rejected": -39.19096755981445, |
| "loss": 0.0336, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.01982739567756653, |
| "rewards/margins": 4.651766777038574, |
| "rewards/rejected": -4.671594142913818, |
| "step": 434 |
| }, |
| { |
| "epoch": 7.372881355932203, |
| "grad_norm": 6.134889313671512, |
| "learning_rate": 9.335976554398912e-09, |
| "logits/chosen": -1.540908932685852, |
| "logits/rejected": -1.5109443664550781, |
| "logps/chosen": -32.279666900634766, |
| "logps/rejected": -38.13484191894531, |
| "loss": 0.0326, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6547858715057373, |
| "rewards/margins": 3.962502956390381, |
| "rewards/rejected": -4.617289066314697, |
| "step": 435 |
| }, |
| { |
| "epoch": 7.389830508474576, |
| "grad_norm": 5.165828974962686, |
| "learning_rate": 8.841104522910342e-09, |
| "logits/chosen": -1.5303874015808105, |
| "logits/rejected": -1.4300156831741333, |
| "logps/chosen": -33.03150939941406, |
| "logps/rejected": -49.22145080566406, |
| "loss": 0.0242, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.05898144841194153, |
| "rewards/margins": 5.506203651428223, |
| "rewards/rejected": -5.565184593200684, |
| "step": 436 |
| }, |
| { |
| "epoch": 7.406779661016949, |
| "grad_norm": 5.59136345566115, |
| "learning_rate": 8.359471922753714e-09, |
| "logits/chosen": -1.5242373943328857, |
| "logits/rejected": -1.3456315994262695, |
| "logps/chosen": -29.918407440185547, |
| "logps/rejected": -53.36083984375, |
| "loss": 0.0359, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.07045301795005798, |
| "rewards/margins": 5.991230487823486, |
| "rewards/rejected": -5.920777320861816, |
| "step": 437 |
| }, |
| { |
| "epoch": 7.423728813559322, |
| "grad_norm": 5.700922848308937, |
| "learning_rate": 7.891105195175356e-09, |
| "logits/chosen": -1.421931505203247, |
| "logits/rejected": -1.4313148260116577, |
| "logps/chosen": -30.72463607788086, |
| "logps/rejected": -39.33094787597656, |
| "loss": 0.0314, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.35566258430480957, |
| "rewards/margins": 4.320624351501465, |
| "rewards/rejected": -4.676286697387695, |
| "step": 438 |
| }, |
| { |
| "epoch": 7.440677966101695, |
| "grad_norm": 4.895447924779327, |
| "learning_rate": 7.4360300531355894e-09, |
| "logits/chosen": -1.2597088813781738, |
| "logits/rejected": -1.1974666118621826, |
| "logps/chosen": -34.747318267822266, |
| "logps/rejected": -60.813480377197266, |
| "loss": 0.028, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4029719829559326, |
| "rewards/margins": 5.782626628875732, |
| "rewards/rejected": -6.185598373413086, |
| "step": 439 |
| }, |
| { |
| "epoch": 7.4576271186440675, |
| "grad_norm": 4.753436790014126, |
| "learning_rate": 6.994271479897313e-09, |
| "logits/chosen": -1.2228598594665527, |
| "logits/rejected": -1.2148244380950928, |
| "logps/chosen": -23.159412384033203, |
| "logps/rejected": -39.971435546875, |
| "loss": 0.0222, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.20260876417160034, |
| "rewards/margins": 4.695430278778076, |
| "rewards/rejected": -4.49282169342041, |
| "step": 440 |
| }, |
| { |
| "epoch": 7.47457627118644, |
| "grad_norm": 5.465778826325062, |
| "learning_rate": 6.565853727654502e-09, |
| "logits/chosen": -1.6256568431854248, |
| "logits/rejected": -1.6748077869415283, |
| "logps/chosen": -35.98029327392578, |
| "logps/rejected": -51.9576530456543, |
| "loss": 0.0343, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.7835733890533447, |
| "rewards/margins": 5.101205825805664, |
| "rewards/rejected": -5.884779453277588, |
| "step": 441 |
| }, |
| { |
| "epoch": 7.491525423728813, |
| "grad_norm": 5.19068823671553, |
| "learning_rate": 6.150800316200605e-09, |
| "logits/chosen": -1.5622632503509521, |
| "logits/rejected": -1.6051169633865356, |
| "logps/chosen": -26.639972686767578, |
| "logps/rejected": -39.97083282470703, |
| "loss": 0.0279, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.18523569405078888, |
| "rewards/margins": 5.357626914978027, |
| "rewards/rejected": -5.172390937805176, |
| "step": 442 |
| }, |
| { |
| "epoch": 7.508474576271187, |
| "grad_norm": 5.290094026909458, |
| "learning_rate": 5.7491340316373485e-09, |
| "logits/chosen": -1.3767602443695068, |
| "logits/rejected": -1.2797472476959229, |
| "logps/chosen": -26.007776260375977, |
| "logps/rejected": -53.1204719543457, |
| "loss": 0.031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.03208550810813904, |
| "rewards/margins": 6.148884296417236, |
| "rewards/rejected": -6.18096923828125, |
| "step": 443 |
| }, |
| { |
| "epoch": 7.52542372881356, |
| "grad_norm": 5.384685543036387, |
| "learning_rate": 5.360876925123992e-09, |
| "logits/chosen": -1.660988450050354, |
| "logits/rejected": -1.495483160018921, |
| "logps/chosen": -38.71052551269531, |
| "logps/rejected": -63.9692497253418, |
| "loss": 0.026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.767485499382019, |
| "rewards/margins": 6.337566375732422, |
| "rewards/rejected": -7.105052947998047, |
| "step": 444 |
| }, |
| { |
| "epoch": 7.5423728813559325, |
| "grad_norm": 6.204818072148143, |
| "learning_rate": 4.9860503116665176e-09, |
| "logits/chosen": -1.7063543796539307, |
| "logits/rejected": -1.692810297012329, |
| "logps/chosen": -25.562942504882812, |
| "logps/rejected": -48.51639175415039, |
| "loss": 0.041, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.01574818789958954, |
| "rewards/margins": 4.9537529945373535, |
| "rewards/rejected": -4.969500541687012, |
| "step": 445 |
| }, |
| { |
| "epoch": 7.559322033898305, |
| "grad_norm": 5.655663554463141, |
| "learning_rate": 4.624674768947484e-09, |
| "logits/chosen": -1.598649024963379, |
| "logits/rejected": -1.4650629758834839, |
| "logps/chosen": -27.018341064453125, |
| "logps/rejected": -46.08781051635742, |
| "loss": 0.0369, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.0027409642934799194, |
| "rewards/margins": 4.6786322593688965, |
| "rewards/rejected": -4.681373596191406, |
| "step": 446 |
| }, |
| { |
| "epoch": 7.576271186440678, |
| "grad_norm": 5.701136400159739, |
| "learning_rate": 4.2767701361964835e-09, |
| "logits/chosen": -1.3610193729400635, |
| "logits/rejected": -1.3563426733016968, |
| "logps/chosen": -35.585716247558594, |
| "logps/rejected": -50.32651901245117, |
| "loss": 0.0389, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6664023995399475, |
| "rewards/margins": 4.827932834625244, |
| "rewards/rejected": -5.494334697723389, |
| "step": 447 |
| }, |
| { |
| "epoch": 7.593220338983051, |
| "grad_norm": 6.209794879281624, |
| "learning_rate": 3.942355513100792e-09, |
| "logits/chosen": -1.4733314514160156, |
| "logits/rejected": -1.4402073621749878, |
| "logps/chosen": -28.033966064453125, |
| "logps/rejected": -54.08881378173828, |
| "loss": 0.0345, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.519957423210144, |
| "rewards/margins": 5.857639312744141, |
| "rewards/rejected": -6.377596855163574, |
| "step": 448 |
| }, |
| { |
| "epoch": 7.610169491525424, |
| "grad_norm": 4.997367172232428, |
| "learning_rate": 3.6214492587569313e-09, |
| "logits/chosen": -1.5576658248901367, |
| "logits/rejected": -1.6168861389160156, |
| "logps/chosen": -32.894317626953125, |
| "logps/rejected": -41.69158172607422, |
| "loss": 0.0284, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.22915619611740112, |
| "rewards/margins": 4.693375110626221, |
| "rewards/rejected": -4.922531604766846, |
| "step": 449 |
| }, |
| { |
| "epoch": 7.627118644067797, |
| "grad_norm": 5.4083679319206555, |
| "learning_rate": 3.314068990662805e-09, |
| "logits/chosen": -1.619524359703064, |
| "logits/rejected": -1.469055414199829, |
| "logps/chosen": -25.54304313659668, |
| "logps/rejected": -38.69428634643555, |
| "loss": 0.0327, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.059964776039123535, |
| "rewards/margins": 5.082298278808594, |
| "rewards/rejected": -5.02233362197876, |
| "step": 450 |
| }, |
| { |
| "epoch": 7.6440677966101696, |
| "grad_norm": 5.217030218721459, |
| "learning_rate": 3.0202315837502545e-09, |
| "logits/chosen": -1.4242594242095947, |
| "logits/rejected": -1.439193844795227, |
| "logps/chosen": -31.73979949951172, |
| "logps/rejected": -39.235633850097656, |
| "loss": 0.0321, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5601508617401123, |
| "rewards/margins": 4.23047399520874, |
| "rewards/rejected": -4.790624618530273, |
| "step": 451 |
| }, |
| { |
| "epoch": 7.661016949152542, |
| "grad_norm": 5.102186940922818, |
| "learning_rate": 2.7399531694589917e-09, |
| "logits/chosen": -1.5301525592803955, |
| "logits/rejected": -1.5297596454620361, |
| "logps/chosen": -27.26993179321289, |
| "logps/rejected": -46.745147705078125, |
| "loss": 0.0316, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.15520310401916504, |
| "rewards/margins": 5.617114543914795, |
| "rewards/rejected": -5.772317409515381, |
| "step": 452 |
| }, |
| { |
| "epoch": 7.677966101694915, |
| "grad_norm": 5.403786442321042, |
| "learning_rate": 2.473249134850808e-09, |
| "logits/chosen": -1.3538662195205688, |
| "logits/rejected": -1.3246500492095947, |
| "logps/chosen": -22.38913345336914, |
| "logps/rejected": -45.14683532714844, |
| "loss": 0.0325, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.044996634125709534, |
| "rewards/margins": 4.89539098739624, |
| "rewards/rejected": -4.850394248962402, |
| "step": 453 |
| }, |
| { |
| "epoch": 7.694915254237288, |
| "grad_norm": 6.421766143852716, |
| "learning_rate": 2.220134121764833e-09, |
| "logits/chosen": -1.4915080070495605, |
| "logits/rejected": -1.452484130859375, |
| "logps/chosen": -16.578571319580078, |
| "logps/rejected": -37.22169876098633, |
| "loss": 0.0441, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8915931582450867, |
| "rewards/margins": 5.497744560241699, |
| "rewards/rejected": -4.606151580810547, |
| "step": 454 |
| }, |
| { |
| "epoch": 7.711864406779661, |
| "grad_norm": 5.924650241904462, |
| "learning_rate": 1.9806220260137065e-09, |
| "logits/chosen": -1.5789787769317627, |
| "logits/rejected": -1.4192255735397339, |
| "logps/chosen": -30.962202072143555, |
| "logps/rejected": -46.20201873779297, |
| "loss": 0.0511, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.06508506834506989, |
| "rewards/margins": 5.4069318771362305, |
| "rewards/rejected": -5.341846466064453, |
| "step": 455 |
| }, |
| { |
| "epoch": 7.728813559322034, |
| "grad_norm": 5.712693057718049, |
| "learning_rate": 1.7547259966207705e-09, |
| "logits/chosen": -1.5617210865020752, |
| "logits/rejected": -1.489423394203186, |
| "logps/chosen": -30.215980529785156, |
| "logps/rejected": -46.79343795776367, |
| "loss": 0.0302, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2140485942363739, |
| "rewards/margins": 6.268076419830322, |
| "rewards/rejected": -6.482125282287598, |
| "step": 456 |
| }, |
| { |
| "epoch": 7.745762711864407, |
| "grad_norm": 4.765004207763158, |
| "learning_rate": 1.5424584350981485e-09, |
| "logits/chosen": -1.5075204372406006, |
| "logits/rejected": -1.446047067642212, |
| "logps/chosen": -25.82567596435547, |
| "logps/rejected": -43.98649597167969, |
| "loss": 0.0296, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2662041187286377, |
| "rewards/margins": 5.002760887145996, |
| "rewards/rejected": -5.268965244293213, |
| "step": 457 |
| }, |
| { |
| "epoch": 7.762711864406779, |
| "grad_norm": 4.613094215982073, |
| "learning_rate": 1.343830994765982e-09, |
| "logits/chosen": -1.5371060371398926, |
| "logits/rejected": -1.4451144933700562, |
| "logps/chosen": -26.201602935791016, |
| "logps/rejected": -57.06352996826172, |
| "loss": 0.0367, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1458522379398346, |
| "rewards/margins": 6.390477657318115, |
| "rewards/rejected": -6.53632926940918, |
| "step": 458 |
| }, |
| { |
| "epoch": 7.779661016949152, |
| "grad_norm": 4.4872892882814535, |
| "learning_rate": 1.1588545801125837e-09, |
| "logits/chosen": -1.7496167421340942, |
| "logits/rejected": -1.6176550388336182, |
| "logps/chosen": -36.218074798583984, |
| "logps/rejected": -55.342689514160156, |
| "loss": 0.0286, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4042947292327881, |
| "rewards/margins": 5.6281538009643555, |
| "rewards/rejected": -6.032447814941406, |
| "step": 459 |
| }, |
| { |
| "epoch": 7.796610169491525, |
| "grad_norm": 5.575240209393775, |
| "learning_rate": 9.87539346195776e-10, |
| "logits/chosen": -1.4099677801132202, |
| "logits/rejected": -1.2582119703292847, |
| "logps/chosen": -29.023414611816406, |
| "logps/rejected": -42.5074577331543, |
| "loss": 0.0318, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.27348968386650085, |
| "rewards/margins": 4.847697734832764, |
| "rewards/rejected": -5.121187210083008, |
| "step": 460 |
| }, |
| { |
| "epoch": 7.813559322033898, |
| "grad_norm": 5.199396752692828, |
| "learning_rate": 8.298946980855315e-10, |
| "logits/chosen": -1.472285509109497, |
| "logits/rejected": -1.330193281173706, |
| "logps/chosen": -29.86899185180664, |
| "logps/rejected": -41.80485153198242, |
| "loss": 0.0294, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.15588414669036865, |
| "rewards/margins": 5.636477470397949, |
| "rewards/rejected": -5.792361259460449, |
| "step": 461 |
| }, |
| { |
| "epoch": 7.830508474576272, |
| "grad_norm": 4.521518393613341, |
| "learning_rate": 6.8592929034747e-10, |
| "logits/chosen": -1.4151493310928345, |
| "logits/rejected": -1.4073522090911865, |
| "logps/chosen": -27.720748901367188, |
| "logps/rejected": -53.330909729003906, |
| "loss": 0.0224, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1728513091802597, |
| "rewards/margins": 5.166114807128906, |
| "rewards/rejected": -5.338965892791748, |
| "step": 462 |
| }, |
| { |
| "epoch": 7.847457627118644, |
| "grad_norm": 4.567347653583189, |
| "learning_rate": 5.556510265678771e-10, |
| "logits/chosen": -1.570703387260437, |
| "logits/rejected": -1.5427438020706177, |
| "logps/chosen": -24.184301376342773, |
| "logps/rejected": -44.638912200927734, |
| "loss": 0.0259, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.29552435874938965, |
| "rewards/margins": 5.448472499847412, |
| "rewards/rejected": -5.743997573852539, |
| "step": 463 |
| }, |
| { |
| "epoch": 7.864406779661017, |
| "grad_norm": 5.544082314499855, |
| "learning_rate": 4.390670589196621e-10, |
| "logits/chosen": -1.6661534309387207, |
| "logits/rejected": -1.5269910097122192, |
| "logps/chosen": -24.99061393737793, |
| "logps/rejected": -45.773014068603516, |
| "loss": 0.0279, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4490078091621399, |
| "rewards/margins": 5.556340217590332, |
| "rewards/rejected": -6.005348205566406, |
| "step": 464 |
| }, |
| { |
| "epoch": 7.88135593220339, |
| "grad_norm": 4.867190786041254, |
| "learning_rate": 3.3618378776981147e-10, |
| "logits/chosen": -1.6171151399612427, |
| "logits/rejected": -1.5128624439239502, |
| "logps/chosen": -28.165693283081055, |
| "logps/rejected": -41.43475341796875, |
| "loss": 0.0235, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2984722852706909, |
| "rewards/margins": 4.441769599914551, |
| "rewards/rejected": -4.14329719543457, |
| "step": 465 |
| }, |
| { |
| "epoch": 7.898305084745763, |
| "grad_norm": 5.660749716669039, |
| "learning_rate": 2.4700686132803075e-10, |
| "logits/chosen": -1.5283398628234863, |
| "logits/rejected": -1.484012484550476, |
| "logps/chosen": -29.490188598632812, |
| "logps/rejected": -45.15831756591797, |
| "loss": 0.035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.029849261045455933, |
| "rewards/margins": 5.0221357345581055, |
| "rewards/rejected": -4.992286682128906, |
| "step": 466 |
| }, |
| { |
| "epoch": 7.915254237288136, |
| "grad_norm": 6.365487263229808, |
| "learning_rate": 1.715411753365481e-10, |
| "logits/chosen": -1.649171233177185, |
| "logits/rejected": -1.701267957687378, |
| "logps/chosen": -25.891307830810547, |
| "logps/rejected": -47.70183181762695, |
| "loss": 0.0396, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.6622889041900635, |
| "rewards/margins": 5.317337989807129, |
| "rewards/rejected": -5.9796271324157715, |
| "step": 467 |
| }, |
| { |
| "epoch": 7.932203389830509, |
| "grad_norm": 4.885763801093592, |
| "learning_rate": 1.0979087280141297e-10, |
| "logits/chosen": -1.2668333053588867, |
| "logits/rejected": -1.3214877843856812, |
| "logps/chosen": -21.41905403137207, |
| "logps/rejected": -38.722625732421875, |
| "loss": 0.0272, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.182024747133255, |
| "rewards/margins": 4.632743835449219, |
| "rewards/rejected": -4.814768314361572, |
| "step": 468 |
| }, |
| { |
| "epoch": 7.9491525423728815, |
| "grad_norm": 5.3854848332900085, |
| "learning_rate": 6.175934376509429e-11, |
| "logits/chosen": -1.5046411752700806, |
| "logits/rejected": -1.5680880546569824, |
| "logps/chosen": -28.198741912841797, |
| "logps/rejected": -59.644596099853516, |
| "loss": 0.0301, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.0053573548793792725, |
| "rewards/margins": 6.178599834442139, |
| "rewards/rejected": -6.173242568969727, |
| "step": 469 |
| }, |
| { |
| "epoch": 7.966101694915254, |
| "grad_norm": 4.75232754040335, |
| "learning_rate": 2.7449225120268482e-11, |
| "logits/chosen": -1.4780237674713135, |
| "logits/rejected": -1.427555799484253, |
| "logps/chosen": -27.112102508544922, |
| "logps/rejected": -52.26079177856445, |
| "loss": 0.0277, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.19119581580162048, |
| "rewards/margins": 6.167375087738037, |
| "rewards/rejected": -6.3585710525512695, |
| "step": 470 |
| }, |
| { |
| "epoch": 7.983050847457627, |
| "grad_norm": 5.233750895631027, |
| "learning_rate": 6.862400465157403e-12, |
| "logits/chosen": -1.5693848133087158, |
| "logits/rejected": -1.5205610990524292, |
| "logps/chosen": -33.29484558105469, |
| "logps/rejected": -41.09092712402344, |
| "loss": 0.0276, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.30308201909065247, |
| "rewards/margins": 4.717187881469727, |
| "rewards/rejected": -5.020270347595215, |
| "step": 471 |
| }, |
| { |
| "epoch": 8.0, |
| "grad_norm": 6.397324083693935, |
| "learning_rate": 0.0, |
| "logits/chosen": -1.639676570892334, |
| "logits/rejected": -1.6202343702316284, |
| "logps/chosen": -33.1722526550293, |
| "logps/rejected": -40.651100158691406, |
| "loss": 0.0363, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.057152003049850464, |
| "rewards/margins": 4.614887237548828, |
| "rewards/rejected": -4.672039031982422, |
| "step": 472 |
| }, |
| { |
| "epoch": 8.0, |
| "step": 472, |
| "total_flos": 0.0, |
| "train_loss": 0.15622181608736263, |
| "train_runtime": 4721.8854, |
| "train_samples_per_second": 12.788, |
| "train_steps_per_second": 0.1 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 472, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 8, |
| "save_steps": 400, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
