{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.999711843242724,
  "eval_steps": 500,
  "global_step": 5204,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completion_length": 220.77500915527344,
      "epoch": 0.00019210450485063874,
      "grad_norm": 2.5577025413513184,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.038,
      "reward": 0.37062498927116394,
      "reward_std": 0.34713491797447205,
      "rewards/code_format_reward": 0.26875001192092896,
      "rewards/code_reward": 0.11812499910593033,
      "step": 1,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.01640424354829722,
      "clip_ratio/high_mean": 0.003707133045989192,
      "clip_ratio/low_mean": 0.0004983297904901621,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004205462749167863,
      "completion_length": 164.34375381469727,
      "epoch": 0.0019210450485063874,
      "grad_norm": 2.2875964641571045,
      "kl": 0.13929970601263145,
      "learning_rate": 9.999947520846931e-07,
      "loss": 0.0575,
      "reward": 0.655464380979538,
      "reward_std": 0.6216425597667694,
      "rewards/code_format_reward": 0.5078125074505806,
      "rewards/code_reward": 0.20077905245125294,
      "step": 10,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio/high_max": 0.04116484243422747,
      "clip_ratio/high_mean": 0.007335515914019197,
      "clip_ratio/low_mean": 0.00010183055419474841,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007437346538063138,
      "completion_length": 100.84750213623047,
      "epoch": 0.003842090097012775,
      "grad_norm": 2.409867286682129,
      "kl": 1.1695969879627228,
      "learning_rate": 9.999734326385416e-07,
      "loss": -0.0111,
      "reward": 0.9829235672950745,
      "reward_std": 0.5127422153949738,
      "rewards/code_format_reward": 0.84375,
      "rewards/code_reward": 0.2805242508649826,
      "step": 20,
      "zero_std_ratio": 0.075
    },
    {
      "clip_ratio/high_max": 0.039504543878138065,
      "clip_ratio/high_mean": 0.0051267803879454735,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0051267803879454735,
      "completion_length": 97.57750091552734,
      "epoch": 0.005763135145519163,
      "grad_norm": 4.608696460723877,
      "kl": 2.0701700329780577,
      "learning_rate": 9.99935714443203e-07,
      "loss": -0.019,
      "reward": 1.1568554759025573,
      "reward_std": 0.6407819569110871,
      "rewards/code_format_reward": 0.8674999952316285,
      "rewards/code_reward": 0.3615527212619781,
      "step": 30,
      "zero_std_ratio": 0.025
    },
    {
      "clip_ratio/high_max": 0.005034898268058896,
      "clip_ratio/high_mean": 0.0006811309984186664,
      "clip_ratio/low_mean": 0.00013664715661434456,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008177781579433941,
      "completion_length": 83.10500030517578,
      "epoch": 0.00768418019402555,
      "grad_norm": 4.833131313323975,
      "kl": 2.2019619703292848,
      "learning_rate": 9.99881598873272e-07,
      "loss": -0.02,
      "reward": 1.1795239448547363,
      "reward_std": 0.7194581270217896,
      "rewards/code_format_reward": 0.8987499952316285,
      "rewards/code_reward": 0.36507447361946105,
      "step": 40,
      "zero_std_ratio": 0.05
    },
    {
      "clip_ratio/high_max": 0.00872214906848967,
      "clip_ratio/high_mean": 0.0010902686335612088,
      "clip_ratio/low_mean": 0.00040422612219117584,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0014944947557523846,
      "completion_length": 88.22500152587891,
      "epoch": 0.009605225242531937,
      "grad_norm": 2.778585433959961,
      "kl": 2.4498987793922424,
      "learning_rate": 9.998110879009265e-07,
      "loss": -0.0035,
      "reward": 1.2663686752319336,
      "reward_std": 0.6244019389152526,
      "rewards/code_format_reward": 0.918750011920929,
      "rewards/code_reward": 0.40349680185317993,
      "step": 50,
      "zero_std_ratio": 0.075
    },
    {
      "clip_ratio/high_max": 0.016367838624864815,
      "clip_ratio/high_mean": 0.002741052128840238,
      "clip_ratio/low_mean": 0.0008432979579083621,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003584350028540939,
      "completion_length": 91.3375,
      "epoch": 0.011526270291038325,
      "grad_norm": 2.5201120376586914,
      "kl": 2.7947509050369264,
      "learning_rate": 9.997241840958557e-07,
      "loss": 0.005,
      "reward": 1.0697558522224426,
      "reward_std": 0.49940577149391174,
      "rewards/code_format_reward": 0.9200000047683716,
      "rewards/code_reward": 0.30487790107727053,
      "step": 60,
      "zero_std_ratio": 0.025
    },
    {
      "clip_ratio/high_max": 0.031629907339811324,
      "clip_ratio/high_mean": 0.005140213097911328,
      "clip_ratio/low_mean": 0.003656612744089216,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008796826144680381,
      "completion_length": 84.79750213623046,
      "epoch": 0.013447315339544713,
      "grad_norm": 7.281564712524414,
      "kl": 1.7218781247735024,
      "learning_rate": 9.99620890625166e-07,
      "loss": -0.0261,
      "reward": 1.1421246886253358,
      "reward_std": 0.5977877795696258,
      "rewards/code_format_reward": 0.9275000095367432,
      "rewards/code_reward": 0.33918734490871427,
      "step": 70,
      "zero_std_ratio": 0.05
    },
    {
      "clip_ratio/high_max": 0.10261552361771464,
      "clip_ratio/high_mean": 0.014289343578275293,
      "clip_ratio/low_mean": 0.0031720689148642123,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017461412807460875,
      "completion_length": 75.67250061035156,
      "epoch": 0.0153683603880511,
      "grad_norm": 3.359511137008667,
      "kl": 0.3687619216740131,
      "learning_rate": 9.995012112532654e-07,
      "loss": -0.0037,
      "reward": 1.2640612244606018,
      "reward_std": 0.5189764618873596,
      "rewards/code_format_reward": 0.9087499976158142,
      "rewards/code_reward": 0.40484309792518614,
      "step": 80,
      "zero_std_ratio": 0.075
    },
    {
      "clip_ratio/high_max": 0.053048994287382814,
      "clip_ratio/high_mean": 0.0092369354548282,
      "clip_ratio/low_mean": 0.00010360952001065016,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009340544970473274,
      "completion_length": 84.56500091552735,
      "epoch": 0.01728940543655749,
      "grad_norm": 2.177191734313965,
      "kl": 0.5693678379058837,
      "learning_rate": 9.993651503417269e-07,
      "loss": -0.008,
      "reward": 1.1986377000808717,
      "reward_std": 0.49277395009994507,
      "rewards/code_format_reward": 0.9112500071525573,
      "rewards/code_reward": 0.3715063512325287,
      "step": 90,
      "zero_std_ratio": 0.1
    },
    {
      "clip_ratio/high_max": 0.04136249013245106,
      "clip_ratio/high_mean": 0.00551328391302377,
      "clip_ratio/low_mean": 0.001408070686738938,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006921354681253433,
      "completion_length": 80.8550033569336,
      "epoch": 0.019210450485063875,
      "grad_norm": 2.0033416748046875,
      "kl": 0.8493028253316879,
      "learning_rate": 9.992127128491296e-07,
      "loss": 0.0027,
      "reward": 1.1780336141586303,
      "reward_std": 0.4479735493659973,
      "rewards/code_format_reward": 0.9275000095367432,
      "rewards/code_reward": 0.3571417987346649,
      "step": 100,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio/high_max": 0.0585523322224617,
      "clip_ratio/high_mean": 0.008032404945697635,
      "clip_ratio/low_mean": 0.006125411042012275,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014157815964426845,
      "completion_length": 74.19000091552735,
      "epoch": 0.02113149553357026,
      "grad_norm": 2.267624855041504,
      "kl": 1.1039492040872574,
      "learning_rate": 9.990439043308776e-07,
      "loss": -0.0238,
      "reward": 1.2784739494323731,
      "reward_std": 0.49057124853134154,
      "rewards/code_format_reward": 0.9475000023841857,
      "rewards/code_reward": 0.40236196517944334,
      "step": 110,
      "zero_std_ratio": 0.175
    },
    {
      "clip_ratio/high_max": 0.07124514738097787,
      "clip_ratio/high_mean": 0.01569047374650836,
      "clip_ratio/low_mean": 0.0004420768018462695,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01613254987169057,
      "completion_length": 68.35750045776368,
      "epoch": 0.02305254058207665,
      "grad_norm": 4.1564249992370605,
      "kl": 1.4338344126939773,
      "learning_rate": 9.988587309389975e-07,
      "loss": -0.0026,
      "reward": 1.1606964468955994,
      "reward_std": 0.46601226925849915,
      "rewards/code_format_reward": 0.9475000023841857,
      "rewards/code_reward": 0.34347322285175325,
      "step": 120,
      "zero_std_ratio": 0.175
    },
    {
      "clip_ratio/high_max": 0.07592196827754379,
      "clip_ratio/high_mean": 0.014454811741597951,
      "clip_ratio/low_mean": 0.0013599038298707455,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015814715722808615,
      "completion_length": 72.15750122070312,
      "epoch": 0.024973585630583037,
      "grad_norm": 3.9662575721740723,
      "kl": 1.5312897458672523,
      "learning_rate": 9.98657199421914e-07,
      "loss": -0.0024,
      "reward": 1.1610160946846009,
      "reward_std": 0.3773229032754898,
      "rewards/code_format_reward": 0.9587499976158143,
      "rewards/code_reward": 0.3408205330371857,
      "step": 130,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.07943324451334774,
      "clip_ratio/high_mean": 0.014111382194096222,
      "clip_ratio/low_mean": 0.0036704083904623985,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017781790602020918,
      "completion_length": 83.8675033569336,
      "epoch": 0.026894630679089426,
      "grad_norm": 9.37182331085205,
      "kl": 0.5293755233287811,
      "learning_rate": 9.984393171242054e-07,
      "loss": -0.0045,
      "reward": 1.3634901762008667,
      "reward_std": 0.5678210258483887,
      "rewards/code_format_reward": 0.9512500047683716,
      "rewards/code_reward": 0.4439325869083405,
      "step": 140,
      "zero_std_ratio": 0.175
    },
    {
      "clip_ratio/high_max": 0.13982175141572953,
      "clip_ratio/high_mean": 0.018845621962100267,
      "clip_ratio/low_mean": 0.0009358229042845778,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019781444873660802,
      "completion_length": 79.41999969482421,
      "epoch": 0.028815675727595812,
      "grad_norm": 3.3437957763671875,
      "kl": 1.0034890450537204,
      "learning_rate": 9.982050919863332e-07,
      "loss": -0.0003,
      "reward": 1.332119607925415,
      "reward_std": 0.4401752531528473,
      "rewards/code_format_reward": 0.9674999952316284,
      "rewards/code_reward": 0.4241847813129425,
      "step": 150,
      "zero_std_ratio": 0.2
    },
    {
      "clip_ratio/high_max": 0.08667803611606359,
      "clip_ratio/high_mean": 0.01204416286200285,
      "clip_ratio/low_mean": 0.0012475995084969328,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013291762379230932,
      "completion_length": 80.43250122070313,
      "epoch": 0.0307367207761022,
      "grad_norm": 3.763737678527832,
      "kl": 0.9024959966540337,
      "learning_rate": 9.979545325443564e-07,
      "loss": -0.0043,
      "reward": 1.3518987059593202,
      "reward_std": 0.46767728328704833,
      "rewards/code_format_reward": 0.9450000047683715,
      "rewards/code_reward": 0.4396993488073349,
      "step": 160,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.08449154160916805,
      "clip_ratio/high_mean": 0.012011481402441859,
      "clip_ratio/low_mean": 0.00172541297506541,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013736894307658076,
      "completion_length": 78.61250152587891,
      "epoch": 0.03265776582460859,
      "grad_norm": 7.203779220581055,
      "kl": 0.9198675453662872,
      "learning_rate": 9.976876479296167e-07,
      "loss": -0.0013,
      "reward": 1.3803849458694457,
      "reward_std": 0.4038102596998215,
      "rewards/code_format_reward": 0.9587499976158143,
      "rewards/code_reward": 0.4505049705505371,
      "step": 170,
      "zero_std_ratio": 0.2
    },
    {
      "clip_ratio/high_max": 0.07188423536717892,
      "clip_ratio/high_mean": 0.013125935778953135,
      "clip_ratio/low_mean": 0.0036661239922977985,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016792060085572304,
      "completion_length": 77.1875,
      "epoch": 0.03457881087311498,
      "grad_norm": 4.487454414367676,
      "kl": 1.7507148087024689,
      "learning_rate": 9.974044478684084e-07,
      "loss": 0.0129,
      "reward": 1.3845421075820923,
      "reward_std": 0.5211645245552063,
      "rewards/code_format_reward": 0.9325000047683716,
      "rewards/code_reward": 0.4591460168361664,
      "step": 180,
      "zero_std_ratio": 0.175
    },
    {
      "clip_ratio/high_max": 0.03638382372446358,
      "clip_ratio/high_mean": 0.005234482995001599,
      "clip_ratio/low_mean": 0.0020637288223952057,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007298211794113741,
      "completion_length": 72.73499984741211,
      "epoch": 0.03649985592162136,
      "grad_norm": 1.9644516706466675,
      "kl": 1.5947209149599075,
      "learning_rate": 9.97104942681622e-07,
      "loss": -0.0015,
      "reward": 1.5394827842712402,
      "reward_std": 0.42243914008140565,
      "rewards/code_format_reward": 0.9625,
      "rewards/code_reward": 0.5291163563728333,
      "step": 190,
      "zero_std_ratio": 0.225
    },
    {
      "clip_ratio/high_max": 0.2028519107028842,
      "clip_ratio/high_mean": 0.031164265819825232,
      "clip_ratio/low_mean": 0.00270410452503711,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03386837020516396,
      "completion_length": 69.81000061035157,
      "epoch": 0.03842090097012775,
      "grad_norm": 3.170088052749634,
      "kl": 1.0117133632302284,
      "learning_rate": 9.9678914328437e-07,
      "loss": 0.0113,
      "reward": 1.4108091354370118,
      "reward_std": 0.43394198417663576,
      "rewards/code_format_reward": 0.9675000071525574,
      "rewards/code_reward": 0.46352959871292115,
      "step": 200,
      "zero_std_ratio": 0.225
    },
    {
      "clip_ratio/high_max": 0.053923821565695106,
      "clip_ratio/high_mean": 0.009883182743215002,
      "clip_ratio/low_mean": 0.0038779765891376883,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013761159335263073,
      "completion_length": 69.04750213623046,
      "epoch": 0.04034194601863414,
      "grad_norm": 2.5729293823242188,
      "kl": 1.1861489608883857,
      "learning_rate": 9.964570611855874e-07,
      "loss": -0.007,
      "reward": 1.4398113250732423,
      "reward_std": 0.39351261258125303,
      "rewards/code_format_reward": 0.9650000095367431,
      "rewards/code_reward": 0.47865564227104185,
      "step": 210,
      "zero_std_ratio": 0.3
    },
    {
      "clip_ratio/high_max": 0.1575187448877841,
      "clip_ratio/high_mean": 0.020484526228392495,
      "clip_ratio/low_mean": 0.011988240911159664,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03247276756446808,
      "completion_length": 61.67000122070313,
      "epoch": 0.04226299106714052,
      "grad_norm": 9.919574737548828,
      "kl": 3.983895111083984,
      "learning_rate": 9.961087084876135e-07,
      "loss": 0.0076,
      "reward": 1.2202381372451783,
      "reward_std": 0.26475468575954436,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.36793155074119566,
      "step": 220,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.141914052516222,
      "clip_ratio/high_mean": 0.023408634401857854,
      "clip_ratio/low_mean": 0.004240041392040439,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.027648675863747484,
      "completion_length": 67.37250213623047,
      "epoch": 0.04418403611564691,
      "grad_norm": 106.46134185791016,
      "kl": 2.121386268734932,
      "learning_rate": 9.957440978857498e-07,
      "loss": -0.0021,
      "reward": 1.3681801557540894,
      "reward_std": 0.37111111879348757,
      "rewards/code_format_reward": 0.9675000071525574,
      "rewards/code_reward": 0.4422150731086731,
      "step": 230,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.07315623210743069,
      "clip_ratio/high_mean": 0.01155225959373638,
      "clip_ratio/low_mean": 0.005734288269013632,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01728654802427627,
      "completion_length": 72.65750198364258,
      "epoch": 0.0461050811641533,
      "grad_norm": 3.1017534732818604,
      "kl": 0.882834991812706,
      "learning_rate": 9.953632426677983e-07,
      "loss": -0.0093,
      "reward": 1.484795618057251,
      "reward_std": 0.4526777356863022,
      "rewards/code_format_reward": 0.9662500023841858,
      "rewards/code_reward": 0.5008352994918823,
      "step": 240,
      "zero_std_ratio": 0.2
    },
    {
      "clip_ratio/high_max": 0.06419091664720326,
      "clip_ratio/high_mean": 0.008793376400717534,
      "clip_ratio/low_mean": 0.0021902987034991385,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010983675080933609,
      "completion_length": 88.21500091552734,
      "epoch": 0.048026126212659684,
      "grad_norm": 5.3243279457092285,
      "kl": 2.7226425796747207,
      "learning_rate": 9.94966156713577e-07,
      "loss": -0.0127,
      "reward": 1.455380654335022,
      "reward_std": 0.4675000965595245,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.4842528164386749,
      "step": 250,
      "zero_std_ratio": 0.2
    },
    {
      "clip_ratio/high_max": 0.0741606397787109,
      "clip_ratio/high_mean": 0.012071207936969586,
      "clip_ratio/low_mean": 0.003062122967094183,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015133331064134836,
      "completion_length": 88.98250122070313,
      "epoch": 0.04994717126116607,
      "grad_norm": 2.7804369926452637,
      "kl": 0.6169008180499077,
      "learning_rate": 9.94552854494413e-07,
      "loss": 0.0033,
      "reward": 1.427869987487793,
      "reward_std": 0.4851543098688126,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.4717474699020386,
      "step": 260,
      "zero_std_ratio": 0.1
    },
    {
      "clip_ratio/high_max": 0.03937563952058554,
      "clip_ratio/high_mean": 0.0065028761862777175,
      "clip_ratio/low_mean": 0.004409579199273139,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01091245551360771,
      "completion_length": 87.42750091552735,
      "epoch": 0.05186821630967246,
      "grad_norm": 6.559643745422363,
      "kl": 0.4433484449982643,
      "learning_rate": 9.941233510726168e-07,
      "loss": -0.0018,
      "reward": 1.4182387351989747,
      "reward_std": 0.4612067699432373,
      "rewards/code_format_reward": 0.9412499904632569,
      "rewards/code_reward": 0.4738068819046021,
      "step": 270,
      "zero_std_ratio": 0.175
    },
    {
      "clip_ratio/high_max": 0.057588514033705,
      "clip_ratio/high_mean": 0.008462971181143076,
      "clip_ratio/low_mean": 0.007865038787713274,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016328010114375503,
      "completion_length": 79.69500122070312,
      "epoch": 0.05378926135817885,
      "grad_norm": 6.077131271362305,
      "kl": 0.6961165189743042,
      "learning_rate": 9.936776621009322e-07,
      "loss": 0.0038,
      "reward": 1.5715951919555664,
      "reward_std": 0.4179812580347061,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5420475661754608,
      "step": 280,
      "zero_std_ratio": 0.2
    },
    {
      "clip_ratio/high_max": 0.025062982086092235,
      "clip_ratio/high_mean": 0.004761367203900591,
      "clip_ratio/low_mean": 0.0028623046877328307,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007623671973124147,
      "completion_length": 83.21750183105469,
      "epoch": 0.055710306406685235,
      "grad_norm": 6.066061019897461,
      "kl": 0.7484225794672966,
      "learning_rate": 9.932158038219662e-07,
      "loss": -0.0052,
      "reward": 1.1587857127189636,
      "reward_std": 0.39943512678146365,
      "rewards/code_format_reward": 0.9637500047683716,
      "rewards/code_reward": 0.3384553253650665,
      "step": 290,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.10117955654859542,
      "clip_ratio/high_mean": 0.013649052195250987,
      "clip_ratio/low_mean": 0.0008885912131518126,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014537643361836671,
      "completion_length": 84.00750122070312,
      "epoch": 0.057631351455191625,
      "grad_norm": 3.23887038230896,
      "kl": 0.8064253896474838,
      "learning_rate": 9.92737793067597e-07,
      "loss": -0.0034,
      "reward": 1.3393104553222657,
      "reward_std": 0.4101540923118591,
      "rewards/code_format_reward": 0.9549999952316284,
      "rewards/code_reward": 0.43090522289276123,
      "step": 300,
      "zero_std_ratio": 0.15
    },
    {
      "clip_ratio/high_max": 0.04703736044466496,
      "clip_ratio/high_mean": 0.007716302154585719,
      "clip_ratio/low_mean": 0.0006432932626921683,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008359595513320528,
      "completion_length": 77.70500030517579,
      "epoch": 0.059552396503698014,
      "grad_norm": 3.357680320739746,
      "kl": 0.6727996915578842,
      "learning_rate": 9.922436472583614e-07,
      "loss": 0.0013,
      "reward": 1.6670202493667603,
      "reward_std": 0.4320096135139465,
      "rewards/code_format_reward": 0.9712500095367431,
      "rewards/code_reward": 0.5906976163387299,
      "step": 310,
      "zero_std_ratio": 0.3
    },
    {
      "clip_ratio/high_max": 0.16380154211074113,
      "clip_ratio/high_mean": 0.03262772373855114,
      "clip_ratio/low_mean": 0.0011754593724617735,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03380318162962794,
      "completion_length": 72.78750152587891,
      "epoch": 0.0614734415522044,
      "grad_norm": 3.652451992034912,
      "kl": 1.8953835844993592,
      "learning_rate": 9.91733384402818e-07,
      "loss": -0.005,
      "reward": 1.4837595462799071,
      "reward_std": 0.45500350296497344,
      "rewards/code_format_reward": 0.9662500023841858,
      "rewards/code_reward": 0.5003172576427459,
      "step": 320,
      "zero_std_ratio": 0.225
    },
    {
      "clip_ratio/high_max": 0.034284231485798955,
      "clip_ratio/high_mean": 0.005935872689587995,
      "clip_ratio/low_mean": 0.000911827472737059,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0068477002554573115,
      "completion_length": 74.17000274658203,
      "epoch": 0.06339448660071079,
      "grad_norm": 1.5065704584121704,
      "kl": 0.40838020071387293,
      "learning_rate": 9.912070230968928e-07,
      "loss": -0.0054,
      "reward": 1.3848075151443482,
      "reward_std": 0.3038723856210709,
      "rewards/code_format_reward": 0.9612499952316285,
      "rewards/code_reward": 0.45209125280380247,
      "step": 330,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.05724322898313403,
      "clip_ratio/high_mean": 0.009350239217747002,
      "clip_ratio/low_mean": 0.0077414238592609765,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0170916625414975,
      "completion_length": 80.06500091552735,
      "epoch": 0.06531553164921718,
      "grad_norm": 3.77842116355896,
      "kl": 0.8782595857977867,
      "learning_rate": 9.906645825232008e-07,
      "loss": -0.0023,
      "reward": 1.294193172454834,
      "reward_std": 0.3676457226276398,
      "rewards/code_format_reward": 0.9549999952316284,
      "rewards/code_reward": 0.4083465874195099,
      "step": 340,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.10199148450046777,
      "clip_ratio/high_mean": 0.018657304299995302,
      "clip_ratio/low_mean": 0.004165191331412643,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022822496155276893,
      "completion_length": 86.5000015258789,
      "epoch": 0.06723657669772357,
      "grad_norm": 3.2845616340637207,
      "kl": 0.9463568836450577,
      "learning_rate": 9.901060824503463e-07,
      "loss": -0.0115,
      "reward": 1.485135293006897,
      "reward_std": 0.48840407729148866,
      "rewards/code_format_reward": 0.9487499833106995,
      "rewards/code_reward": 0.5053801357746124,
      "step": 350,
      "zero_std_ratio": 0.225
    },
    {
      "clip_ratio/high_max": 0.07233364712446928,
      "clip_ratio/high_mean": 0.009769158461131156,
      "clip_ratio/low_mean": 0.019356250233249737,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029125408595427872,
      "completion_length": 80.54000091552734,
      "epoch": 0.06915762174622996,
      "grad_norm": 19.32016944885254,
      "kl": 1.1565445899963378,
      "learning_rate": 9.89531543232204e-07,
      "loss": 0.0045,
      "reward": 1.3412477493286132,
      "reward_std": 0.49785757064819336,
      "rewards/code_format_reward": 0.9599999904632568,
      "rewards/code_reward": 0.43062385320663454,
      "step": 360,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.11471173651516438,
      "clip_ratio/high_mean": 0.02246011425741017,
      "clip_ratio/low_mean": 0.00892345790634863,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.031383572798222306,
      "completion_length": 74.02000274658204,
      "epoch": 0.07107866679473633,
      "grad_norm": 2.2520923614501953,
      "kl": 1.074078917503357,
      "learning_rate": 9.889409858071753e-07,
      "loss": -0.0059,
      "reward": 1.5273491621017456,
      "reward_std": 0.414175683259964,
      "rewards/code_format_reward": 0.9775000095367432,
      "rewards/code_reward": 0.519299578666687,
      "step": 370,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.06336253914050757,
      "clip_ratio/high_mean": 0.01199121386744082,
      "clip_ratio/low_mean": 0.009130357182584703,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021121570840477943,
      "completion_length": 86.4000015258789,
      "epoch": 0.07299971184324272,
      "grad_norm": 4.1052961349487305,
      "kl": 1.3110491752624511,
      "learning_rate": 9.883344316974266e-07,
      "loss": -0.0079,
      "reward": 1.5908024072647096,
      "reward_std": 0.47413656711578367,
      "rewards/code_format_reward": 0.9600000023841858,
      "rewards/code_reward": 0.555401211977005,
      "step": 380,
      "zero_std_ratio": 0.2
    },
    {
      "clip_ratio/high_max": 0.04433182019274682,
      "clip_ratio/high_mean": 0.008989717412623577,
      "clip_ratio/low_mean": 0.006074265367351473,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015063982826541178,
      "completion_length": 86.175,
      "epoch": 0.07492075689174911,
      "grad_norm": 4.5202155113220215,
      "kl": 0.830048742890358,
      "learning_rate": 9.877119030081048e-07,
      "loss": -0.0051,
      "reward": 1.492829155921936,
      "reward_std": 0.3874175697565079,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.5007895469665528,
      "step": 390,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.1522485612425953,
      "clip_ratio/high_mean": 0.0220908185117878,
      "clip_ratio/low_mean": 0.012701757764443756,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03479257607832551,
      "completion_length": 78.91000137329101,
      "epoch": 0.0768418019402555,
      "grad_norm": 2.6146676540374756,
      "kl": 0.8627120085060597,
      "learning_rate": 9.870734224265308e-07,
      "loss": -0.0059,
      "reward": 1.5748756647109985,
      "reward_std": 0.3048340857028961,
      "rewards/code_format_reward": 0.987500011920929,
      "rewards/code_reward": 0.5405627965927124,
      "step": 400,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.16312104668468236,
      "clip_ratio/high_mean": 0.025311203207820654,
      "clip_ratio/low_mean": 0.008227485651150345,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03353868862614036,
      "completion_length": 77.27750091552734,
      "epoch": 0.07876284698876189,
      "grad_norm": 1.7234841585159302,
      "kl": 0.8750749856233597,
      "learning_rate": 9.864190132213742e-07,
      "loss": -0.0062,
      "reward": 1.6338460445404053,
      "reward_std": 0.3537067860364914,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.570673018693924,
      "step": 410,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.0936438184697181,
      "clip_ratio/high_mean": 0.014423616812564433,
      "clip_ratio/low_mean": 0.010347768076462672,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024771385360509157,
      "completion_length": 75.69749908447265,
      "epoch": 0.08068389203726828,
      "grad_norm": 2.0902154445648193,
      "kl": 1.264050543308258,
      "learning_rate": 9.857486992418036e-07,
      "loss": 0.0048,
      "reward": 1.644848608970642,
      "reward_std": 0.277804034948349,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.5774242997169494,
      "step": 420,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.05532362968660891,
      "clip_ratio/high_mean": 0.00992250678827986,
      "clip_ratio/low_mean": 0.004125738283619285,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014048245223239064,
      "completion_length": 69.60749969482421,
      "epoch": 0.08260493708577465,
      "grad_norm": 3.702075481414795,
      "kl": 1.7400359451770782,
      "learning_rate": 9.850625049166189e-07,
      "loss": -0.0008,
      "reward": 1.5316168069839478,
      "reward_std": 0.275749945640564,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.5223708748817444,
      "step": 430,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.15841160174459218,
      "clip_ratio/high_mean": 0.02351265251636505,
      "clip_ratio/low_mean": 0.010281538363778963,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.033794190967455506,
      "completion_length": 74.51000061035157,
      "epoch": 0.08452598213428104,
      "grad_norm": 3.4808361530303955,
      "kl": 1.2856003642082214,
      "learning_rate": 9.8436045525336e-07,
      "loss": -0.0035,
      "reward": 1.5067368984222411,
      "reward_std": 0.28293364942073823,
      "rewards/code_format_reward": 0.9737499833106995,
      "rewards/code_reward": 0.5099309325218201,
      "step": 440,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.06043836465105414,
      "clip_ratio/high_mean": 0.009103650611359626,
      "clip_ratio/low_mean": 0.002932069695089012,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012035720515996218,
      "completion_length": 76.08250122070312,
      "epoch": 0.08644702718278743,
      "grad_norm": 3.665134906768799,
      "kl": 1.0338351279497147,
      "learning_rate": 9.836425758373958e-07,
      "loss": 0.0011,
      "reward": 1.4822889804840087,
      "reward_std": 0.18996141627430915,
      "rewards/code_format_reward": 0.9674999952316284,
      "rewards/code_reward": 0.49926944375038146,
      "step": 450,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.21641009524464608,
      "clip_ratio/high_mean": 0.03260216782800853,
      "clip_ratio/low_mean": 0.007402116784942336,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.040004284400492904,
      "completion_length": 73.13500213623047,
      "epoch": 0.08836807223129382,
      "grad_norm": 3.1982343196868896,
      "kl": 0.6477661892771721,
      "learning_rate": 9.829088928309923e-07,
      "loss": -0.0043,
      "reward": 1.7202057361602783,
      "reward_std": 0.25773381292819975,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.6163528442382813,
      "step": 460,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.09453173456713557,
      "clip_ratio/high_mean": 0.015337946941144764,
      "clip_ratio/low_mean": 0.005975433619460091,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02131338034523651,
      "completion_length": 81.9000015258789,
      "epoch": 0.09028911727980021,
      "grad_norm": 1.441091775894165,
      "kl": 0.6155861958861351,
      "learning_rate": 9.82159432972358e-07,
      "loss": -0.0063,
      "reward": 1.4617766380310058,
      "reward_std": 0.24772228300571442,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.48651331663131714,
      "step": 470,
      "zero_std_ratio": 0.3
    },
    {
      "clip_ratio/high_max": 0.16705528497695923,
      "clip_ratio/high_mean": 0.026639112271368504,
      "clip_ratio/low_mean": 0.0035399875399889425,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03017909936606884,
      "completion_length": 77.79500274658203,
      "epoch": 0.0922101623283066,
      "grad_norm": 47.74139404296875,
      "kl": 1.360982394218445,
      "learning_rate": 9.813942235746705e-07,
      "loss": 0.0034,
      "reward": 1.5168325901031494,
      "reward_std": 0.3997103154659271,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5149787843227387,
      "step": 480,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.26956315375864504,
      "clip_ratio/high_mean": 0.04211876043118536,
      "clip_ratio/low_mean": 0.002336682367604226,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04445544336922467,
      "completion_length": 86.21500091552734,
      "epoch": 0.09413120737681299,
      "grad_norm": 3.7244272232055664,
      "kl": 2.59437358379364,
      "learning_rate": 9.80613292525081e-07,
      "loss": 0.0038,
      "reward": 1.6131777048110962,
      "reward_std": 0.32231712639331817,
      "rewards/code_format_reward": 0.9799999833106995,
      "rewards/code_reward": 0.5615888297557831,
      "step": 490,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.22240130547434092,
      "clip_ratio/high_mean": 0.044074146053753795,
      "clip_ratio/low_mean": 0.012573283386882395,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0566474299877882,
      "completion_length": 72.23500061035156,
      "epoch": 0.09605225242531937,
      "grad_norm": 2.852999687194824,
      "kl": 1.615745335817337,
      "learning_rate": 9.79816668283697e-07,
      "loss": 0.0017,
      "reward": 1.5203128576278686,
      "reward_std": 0.3012717217206955,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.517031443119049,
      "step": 500,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.15330625362694264,
      "clip_ratio/high_mean": 0.02403738833963871,
      "clip_ratio/low_mean": 0.004583830677438528,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028621218353509902,
      "completion_length": 74.30000076293945,
      "epoch": 0.09797329747382576,
      "grad_norm": 2.484840154647827,
      "kl": 2.1540999174118043,
      "learning_rate": 9.790043798825458e-07,
      "loss": 0.0073,
      "reward": 1.5013367414474488,
      "reward_std": 0.24206546545028687,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.508168363571167,
      "step": 510,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.15383050357922912,
      "clip_ratio/high_mean": 0.027125787048134953,
      "clip_ratio/low_mean": 0.002593657124089077,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029719442850910126,
      "completion_length": 65.0400016784668,
      "epoch": 0.09989434252233215,
      "grad_norm": 7.2150959968566895,
      "kl": 1.1968895211815833,
      "learning_rate": 9.781764569245178e-07,
      "loss": -0.006,
      "reward": 1.510750651359558,
      "reward_std": 0.41533524394035337,
      "rewards/code_format_reward": 0.9712500095367431,
      "rewards/code_reward": 0.5125628054141999,
      "step": 520,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.109321213606745,
      "clip_ratio/high_mean": 0.018354640086181463,
      "clip_ratio/low_mean": 0.011131488461978733,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029486127989366652,
      "completion_length": 74.52250213623047,
      "epoch": 0.10181538757083854,
      "grad_norm": 1.8456060886383057,
      "kl": 0.7155197218060494,
      "learning_rate": 9.773329295822844e-07,
      "loss": 0.0073,
      "reward": 1.5899319171905517,
      "reward_std": 0.3179755389690399,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5512159705162049,
      "step": 530,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.04905872759409249,
      "clip_ratio/high_mean": 0.008021075790748,
      "clip_ratio/low_mean": 0.004390958754811436,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012412034533917904,
      "completion_length": 67.07500076293945,
      "epoch": 0.10373643261934493,
      "grad_norm": 4.641266345977783,
      "kl": 0.7290919035673141,
      "learning_rate": 9.764738285972015e-07,
      "loss": 0.0008,
      "reward": 1.300760817527771,
      "reward_std": 0.3361863404512405,
      "rewards/code_format_reward": 0.9537500143051147,
      "rewards/code_reward": 0.4119428813457489,
      "step": 540,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.1825170351192355,
      "clip_ratio/high_mean": 0.027253909036517143,
      "clip_ratio/low_mean": 0.0015975978298229166,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028851506439968942,
      "completion_length": 73.99250030517578,
      "epoch": 0.10565747766785132,
      "grad_norm": 1.1770566701889038,
      "kl": 1.328820213675499,
      "learning_rate": 9.755991852781876e-07,
      "loss": -0.0023,
      "reward": 1.5671115159988402,
      "reward_std": 0.34309983551502227,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5401182293891906,
      "step": 550,
      "zero_std_ratio": 0.3
    },
    {
      "clip_ratio/high_max": 0.12550847120583059,
      "clip_ratio/high_mean": 0.025771993771195413,
      "clip_ratio/low_mean": 0.0035689805867150427,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029340974800288678,
      "completion_length": 71.76750030517579,
      "epoch": 0.1075785227163577,
      "grad_norm": 0.3435879647731781,
      "kl": 2.12383970618248,
      "learning_rate": 9.747090315005836e-07,
      "loss": 0.0024,
      "reward": 1.5273173809051515,
      "reward_std": 0.2889336168766022,
      "rewards/code_format_reward": 0.9649999976158142,
      "rewards/code_reward": 0.5224087119102478,
      "step": 560,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.0834595168940723,
      "clip_ratio/high_mean": 0.015262311231344939,
      "clip_ratio/low_mean": 0.021645180485211312,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03690749178640544,
      "completion_length": 79.53250122070312,
      "epoch": 0.10949956776486408,
      "grad_norm": 1.7026695013046265,
      "kl": 1.6705755025148392,
      "learning_rate": 9.738033997049902e-07,
      "loss": 0.1708,
      "reward": 1.5908133745193482,
      "reward_std": 0.3691225051879883,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.5475941836833954,
      "step": 570,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.18124623028561473,
      "clip_ratio/high_mean": 0.02496154889231548,
      "clip_ratio/low_mean": 0.020611650816863402,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04557319916784763,
      "completion_length": 85.51750183105469,
      "epoch": 0.11142061281337047,
      "grad_norm": 18.138025283813477,
      "kl": 4.237766814231873,
      "learning_rate": 9.728823228960862e-07,
      "loss": -0.0051,
      "reward": 1.5469601631164551,
      "reward_std": 0.37420718297362326,
      "rewards/code_format_reward": 0.975000011920929,
      "rewards/code_reward": 0.5297300696372986,
      "step": 580,
      "zero_std_ratio": 0.3
    },
    {
      "clip_ratio/high_max": 0.014268473512493074,
      "clip_ratio/high_mean": 0.0028658110386459157,
      "clip_ratio/low_mean": 0.0056301898322999476,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008496000757440924,
      "completion_length": 80.10750274658203,
      "epoch": 0.11334165786187686,
      "grad_norm": 5.16138219833374,
      "kl": 0.6609396353363991,
      "learning_rate": 9.71945834641426e-07,
      "loss": -0.004,
      "reward": 1.4476024627685546,
      "reward_std": 0.3472218900918961,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.4813012361526489,
      "step": 590,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.17693078136071563,
      "clip_ratio/high_mean": 0.02441923434380442,
      "clip_ratio/low_mean": 0.012987980741309002,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.037407214660197495,
      "completion_length": 83.96500091552734,
      "epoch": 0.11526270291038325,
      "grad_norm": 1.7465465068817139,
      "kl": 1.0383819937705994,
      "learning_rate": 9.709939690702158e-07,
      "loss": -0.0078,
      "reward": 1.4550770282745362,
      "reward_std": 0.3056318134069443,
      "rewards/code_format_reward": 0.9587500095367432,
      "rewards/code_reward": 0.48785099387168884,
      "step": 600,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.1888352295383811,
      "clip_ratio/high_mean": 0.026437551854178308,
      "clip_ratio/low_mean": 0.0054486555512994524,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.031886206939816475,
      "completion_length": 79.63500213623047,
      "epoch": 0.11718374795888964,
      "grad_norm": 5.674210548400879,
      "kl": 1.2073093384504319,
      "learning_rate": 9.700267608720692e-07,
      "loss": -0.0021,
      "reward": 1.4424492359161376,
      "reward_std": 0.3397494524717331,
      "rewards/code_format_reward": 0.9725000143051148,
      "rewards/code_reward": 0.4780996203422546,
      "step": 610,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.09671425293199717,
      "clip_ratio/high_mean": 0.020163473271531986,
      "clip_ratio/low_mean": 0.006395513273309917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02655898590455763,
      "completion_length": 75.22750091552734,
      "epoch": 0.11910479300739603,
      "grad_norm": 5.531320571899414,
      "kl": 2.2407817423343657,
      "learning_rate": 9.690442452957448e-07,
      "loss": -0.0021,
      "reward": 1.5595922470092773,
      "reward_std": 0.28165863305330274,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.5351086378097534,
      "step": 620,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.11810005996376276,
      "clip_ratio/high_mean": 0.02500568316318095,
      "clip_ratio/low_mean": 0.00357620443101041,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028581888042390348,
      "completion_length": 80.09000091552734,
      "epoch": 0.1210258380559024,
      "grad_norm": 2.165558338165283,
      "kl": 1.546025463938713,
      "learning_rate": 9.680464581478594e-07,
      "loss": -0.0037,
      "reward": 1.51439368724823,
      "reward_std": 0.3320598304271698,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.5140718221664429,
      "step": 630,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.10313799739815295,
      "clip_ratio/high_mean": 0.017414161982014776,
      "clip_ratio/low_mean": 0.009596780824358575,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.027010941854678096,
      "completion_length": 76.15749969482422,
      "epoch": 0.1229468831044088,
      "grad_norm": 5.05511999130249,
      "kl": 1.6615911841392517,
      "learning_rate": 9.670334357915852e-07,
      "loss": 0.0033,
      "reward": 1.5930729150772094,
      "reward_std": 0.3864523351192474,
      "rewards/code_format_reward": 0.9662500023841858,
      "rewards/code_reward": 0.554973942041397,
      "step": 640,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.1653188370168209,
      "clip_ratio/high_mean": 0.027094300370663404,
      "clip_ratio/low_mean": 0.0033949258620850744,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03048922661691904,
      "completion_length": 74.23250274658203,
      "epoch": 0.12486792815291518,
      "grad_norm": 1.1590094566345215,
      "kl": 0.39487394616007804,
      "learning_rate": 9.660052151453228e-07,
      "loss": -0.006,
      "reward": 1.7198987245559691,
      "reward_std": 0.3215783953666687,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.613699346780777,
      "step": 650,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.2655821519903839,
      "clip_ratio/high_mean": 0.03813204998150468,
      "clip_ratio/low_mean": 0.017123100493336096,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.05525515023618936,
      "completion_length": 79.31999969482422,
      "epoch": 0.12678897320142157,
      "grad_norm": 2.8189809322357178,
      "kl": 0.9924295842647552,
      "learning_rate": 9.649618336813565e-07,
      "loss": -0.0022,
      "reward": 1.710445189476013,
      "reward_std": 0.2906018912792206,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.6111600875854493,
      "step": 660,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.10447313897311687,
      "clip_ratio/high_mean": 0.017084641277324408,
      "clip_ratio/low_mean": 0.018559307692339645,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03564394909190014,
      "completion_length": 73.31750183105468,
      "epoch": 0.12871001824992795,
      "grad_norm": 7.561813831329346,
      "kl": 1.0190230280160903,
      "learning_rate": 9.639033294244894e-07,
      "loss": -0.0059,
      "reward": 1.4508479833602905,
      "reward_std": 0.2639226779341698,
      "rewards/code_format_reward": 0.9724999904632569,
      "rewards/code_reward": 0.4822989523410797,
      "step": 670,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.17416613902896644,
      "clip_ratio/high_mean": 0.02931727101095021,
      "clip_ratio/low_mean": 0.013709834642941131,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04302710462361574,
      "completion_length": 75.30500183105468,
      "epoch": 0.13063106329843435,
      "grad_norm": 4.0138373374938965,
      "kl": 1.8731355726718903,
      "learning_rate": 9.628297409506558e-07,
      "loss": 0.0038,
      "reward": 1.5990655183792115,
      "reward_std": 0.38845544308423996,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.5554702281951904,
      "step": 680,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.14133978222962468,
      "clip_ratio/high_mean": 0.025468734742025843,
      "clip_ratio/low_mean": 0.0034107466402929277,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028879481457988732,
      "completion_length": 71.69250183105468,
      "epoch": 0.13255210834694073,
      "grad_norm": 2.7108314037323,
      "kl": 1.0770379617810248,
      "learning_rate": 9.61741107385517e-07,
      "loss": 0.0015,
      "reward": 1.357295000553131,
      "reward_std": 0.16353759765625,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.43333501517772677,
      "step": 690,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.2215075224637985,
      "clip_ratio/high_mean": 0.03973329542204738,
      "clip_ratio/low_mean": 0.021483630378497764,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.06121692657470703,
      "completion_length": 77.00250244140625,
      "epoch": 0.13447315339544713,
      "grad_norm": 3.874828338623047,
      "kl": 1.798163938522339,
      "learning_rate": 9.606374684030354e-07,
      "loss": -0.0002,
      "reward": 1.4897700071334838,
      "reward_std": 0.3036611869931221,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.5023849844932556,
      "step": 700,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.26057110670953987,
      "clip_ratio/high_mean": 0.04422192363999784,
      "clip_ratio/low_mean": 0.012507367390207946,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.05672929054126143,
      "completion_length": 68.01749954223632,
      "epoch": 0.1363941984439535,
      "grad_norm": 1.9008493423461914,
      "kl": 1.1601522982120513,
      "learning_rate": 9.595188642240268e-07,
      "loss": -0.006,
      "reward": 1.5408167839050293,
      "reward_std": 0.23992418646812438,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5244708836078644,
      "step": 710,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.11190514008048921,
      "clip_ratio/high_mean": 0.022988432584679686,
      "clip_ratio/low_mean": 0.003842631517909467,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02683106428885367,
      "completion_length": 70.91749954223633,
      "epoch": 0.1383152434924599,
      "grad_norm": 2.230220317840576,
      "kl": 0.6176944851875306,
      "learning_rate": 9.58385335614697e-07,
      "loss": -0.0038,
      "reward": 1.474353313446045,
      "reward_std": 0.22789922058582307,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.49092662930488584,
      "step": 720,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.22790296860039233,
      "clip_ratio/high_mean": 0.043722260277718306,
      "clip_ratio/low_mean": 0.005503303511068225,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0492255637422204,
      "completion_length": 70.33000183105469,
      "epoch": 0.1402362885409663,
      "grad_norm": 3.880234956741333,
      "kl": 1.7978762328624724,
      "learning_rate": 9.572369238851546e-07,
      "loss": -0.01,
      "reward": 1.7555195808410644,
      "reward_std": 0.30654080510139464,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6311972856521606,
      "step": 730,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.13005290240980685,
      "clip_ratio/high_mean": 0.02253831790876575,
      "clip_ratio/low_mean": 0.0076317260100040585,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030170044326223434,
      "completion_length": 67.4625015258789,
      "epoch": 0.14215733358947266,
      "grad_norm": 31014.41015625,
      "kl": 2.5802926242351534,
      "learning_rate": 9.560736708879055e-07,
      "loss": 4.1316,
      "reward": 1.391554856300354,
      "reward_std": 0.3107602626085281,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.4501524269580841,
      "step": 740,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.21672796942293643,
      "clip_ratio/high_mean": 0.03920850001741201,
      "clip_ratio/low_mean": 0.0084746521897614,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.047683153115212915,
      "completion_length": 71.03750076293946,
      "epoch": 0.14407837863797907,
      "grad_norm": 1.3094109296798706,
      "kl": 4.56303431391716,
      "learning_rate": 9.54895619016329e-07,
      "loss": 0.0111,
      "reward": 1.5939582109451294,
      "reward_std": 0.2379148319363594,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.5547916054725647,
      "step": 750,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.08126737037673593,
      "clip_ratio/high_mean": 0.01269659586250782,
      "clip_ratio/low_mean": 0.006480468995869159,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019177064718678593,
      "completion_length": 74.09750213623047,
      "epoch": 0.14599942368648544,
      "grad_norm": 3.0267083644866943,
      "kl": 1.5844107165932655,
      "learning_rate": 9.53702811203131e-07,
      "loss": 0.0048,
      "reward": 1.4744285106658936,
      "reward_std": 0.2754403457045555,
      "rewards/code_format_reward": 0.9900000095367432,
      "rewards/code_reward": 0.489714241027832,
      "step": 760,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.22151243952102959,
      "clip_ratio/high_mean": 0.038386100489879026,
      "clip_ratio/low_mean": 0.001766498590586707,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04015259912703186,
      "completion_length": 73.72750244140624,
      "epoch": 0.14792046873499184,
      "grad_norm": 3596482.75,
      "kl": 0.6901701986789703,
      "learning_rate": 9.524952909187801e-07,
      "loss": 83.9443,
      "reward": 1.4019340753555298,
      "reward_std": 0.24908357337117196,
      "rewards/code_format_reward": 0.9749999880790711,
      "rewards/code_reward": 0.45721703171730044,
      "step": 770,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.07684345319867134,
      "clip_ratio/high_mean": 0.014277776470407844,
      "clip_ratio/low_mean": 0.016169815976172685,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0304475924000144,
      "completion_length": 79.24250183105468,
      "epoch": 0.14984151378349822,
      "grad_norm": 3.468223810195923,
      "kl": 0.45489892959594724,
      "learning_rate": 9.512731021699245e-07,
      "loss": -0.0056,
      "reward": 1.580666732788086,
      "reward_std": 0.41472728848457335,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5459583520889282,
      "step": 780,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.10067678079940379,
      "clip_ratio/high_mean": 0.013439147116150707,
      "clip_ratio/low_mean": 0.023053765966324136,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03649291144683957,
      "completion_length": 72.04750137329101,
      "epoch": 0.15176255883200462,
      "grad_norm": 13.193933486938477,
      "kl": 1.6161374658346177,
      "learning_rate": 9.500362894977864e-07,
      "loss": 0.0007,
      "reward": 1.6252036333084106,
      "reward_std": 0.3433967262506485,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5666643261909485,
      "step": 790,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.11107501722872257,
      "clip_ratio/high_mean": 0.01587685807608068,
      "clip_ratio/low_mean": 0.001843169682251755,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017720027733594178,
      "completion_length": 77.34250183105469,
      "epoch": 0.153683603880511,
      "grad_norm": 3.4086289405822754,
      "kl": 0.735039034485817,
      "learning_rate": 9.487848979765399e-07,
      "loss": -0.0033,
      "reward": 1.7214166164398192,
      "reward_std": 0.3059865742921829,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6125832796096802,
      "step": 800,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.06381021924316883,
      "clip_ratio/high_mean": 0.012221441417932511,
      "clip_ratio/low_mean": 0.002595777277019806,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014817218482494354,
      "completion_length": 78.94500045776367,
      "epoch": 0.15560464892901738,
      "grad_norm": 2.894174098968506,
      "kl": 0.9337424471974373,
      "learning_rate": 9.475189732116677e-07,
      "loss": -0.0074,
      "reward": 1.5309076070785523,
      "reward_std": 0.36832110285758973,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.5201413094997406,
      "step": 810,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.0614451477304101,
      "clip_ratio/high_mean": 0.011137601570226252,
      "clip_ratio/low_mean": 0.015545779425883666,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02668338119983673,
      "completion_length": 80.46750030517578,
      "epoch": 0.15752569397752378,
      "grad_norm": 1.5945316553115845,
      "kl": 1.666656306385994,
      "learning_rate": 9.462385613382997e-07,
      "loss": -0.0138,
      "reward": 1.4196115970611571,
      "reward_std": 0.3273743912577629,
      "rewards/code_format_reward": 0.9625,
      "rewards/code_reward": 0.4691807866096497,
      "step": 820,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.0729204102884978,
      "clip_ratio/high_mean": 0.011435226618777961,
      "clip_ratio/low_mean": 0.0035716916667297483,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015006918273866177,
      "completion_length": 83.92250061035156,
      "epoch": 0.15944673902603015,
      "grad_norm": 3.7898244857788086,
      "kl": 3.157607713341713,
      "learning_rate": 9.449437090195312e-07,
      "loss": 0.6488,
      "reward": 1.5506922006607056,
      "reward_std": 0.3165741294622421,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.532533586025238,
      "step": 830,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.22014709915965797,
      "clip_ratio/high_mean": 0.030960237560793757,
      "clip_ratio/low_mean": 0.008386016800068318,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03934625396504998,
      "completion_length": 79.59750213623047,
      "epoch": 0.16136778407453656,
      "grad_norm": 3.164461851119995,
      "kl": 0.48004563301801684,
      "learning_rate": 9.436344634447226e-07,
      "loss": 0.0002,
      "reward": 1.4315959692001343,
      "reward_std": 0.2676436066627502,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.4714229583740234,
      "step": 840,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.20247683776542544,
      "clip_ratio/high_mean": 0.040387283614836636,
      "clip_ratio/low_mean": 0.0031327656004577877,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04352005030959845,
      "completion_length": 81.20750274658204,
      "epoch": 0.16328882912304293,
      "grad_norm": 3.2722160816192627,
      "kl": 0.8405016213655472,
      "learning_rate": 9.42310872327779e-07,
      "loss": -0.0002,
      "reward": 1.550826621055603,
      "reward_std": 0.4091781198978424,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.5322882652282714,
      "step": 850,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.061589781753718854,
      "clip_ratio/high_mean": 0.011824411456473172,
      "clip_ratio/low_mean": 0.011703617853345349,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023528029827866705,
      "completion_length": 62.61500244140625,
      "epoch": 0.1652098741715493,
      "grad_norm": 0.2732953727245331,
      "kl": 1.4307941138744353,
      "learning_rate": 9.409729839054123e-07,
      "loss": 0.0075,
      "reward": 1.5864750623703003,
      "reward_std": 0.2073097825050354,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5473000288009644,
      "step": 860,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.1379000276327133,
      "clip_ratio/high_mean": 0.02470994950272143,
      "clip_ratio/low_mean": 0.004926441749557853,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029636391997337343,
      "completion_length": 77.2400032043457,
      "epoch": 0.1671309192200557,
      "grad_norm": 3.488050699234009,
      "kl": 0.9351878672838211,
      "learning_rate": 9.396208469353826e-07,
      "loss": -0.0059,
      "reward": 1.5735363721847535,
      "reward_std": 0.3392061233520508,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.5436432063579559,
      "step": 870,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.07835716316476464,
      "clip_ratio/high_mean": 0.014919109572656453,
      "clip_ratio/low_mean": 0.006504692946327851,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021423802757635713,
      "completion_length": 74.78000183105469,
      "epoch": 0.1690519642685621,
      "grad_norm": 5.493437767028809,
      "kl": 1.060418888926506,
      "learning_rate": 9.382545106947214e-07,
      "loss": -0.0036,
      "reward": 1.745260238647461,
      "reward_std": 0.297343048453331,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.6254426181316376,
      "step": 880,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.12418304020538926,
      "clip_ratio/high_mean": 0.022332211420871318,
      "clip_ratio/low_mean": 0.022319327194418294,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04465153906494379,
      "completion_length": 84.53250122070312,
      "epoch": 0.1709730093170685,
      "grad_norm": 5.462327480316162,
      "kl": 1.5445073664188385,
      "learning_rate": 9.368740249779358e-07,
      "loss": 0.0049,
      "reward": 1.473905611038208,
      "reward_std": 0.33463606536388396,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.49351527690887453,
      "step": 890,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio/high_max": 0.08285986992996186,
      "clip_ratio/high_mean": 0.015584854045300744,
      "clip_ratio/low_mean": 0.002020698119304143,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017605551629094406,
      "completion_length": 85.78250122070312,
      "epoch": 0.17289405436557487,
      "grad_norm": 3.7394657135009766,
      "kl": 1.2308152213692665,
      "learning_rate": 9.354794400951942e-07,
      "loss": 0.0006,
      "reward": 1.3064285874366761,
      "reward_std": 0.3360040634870529,
      "rewards/code_format_reward": 0.9787500023841857,
      "rewards/code_reward": 0.40852679312229156,
      "step": 900,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.06636467641219497,
      "clip_ratio/high_mean": 0.01088127460097894,
      "clip_ratio/low_mean": 0.005357642179296818,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01623891657218337,
      "completion_length": 86.17000122070313,
      "epoch": 0.17481509941408127,
      "grad_norm": 3.883023977279663,
      "kl": 0.5634948700666428,
      "learning_rate": 9.340708068704917e-07,
      "loss": -0.0132,
      "reward": 1.6946633338928223,
      "reward_std": 0.2633577108383179,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6004566550254822,
      "step": 910,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.12004003385081888,
      "clip_ratio/high_mean": 0.01987670698435977,
      "clip_ratio/low_mean": 0.00857236894662492,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028449075785465537,
      "completion_length": 83.18000030517578,
      "epoch": 0.17673614446258765,
      "grad_norm": 5.860812187194824,
      "kl": 1.0160879641771317,
      "learning_rate": 9.326481766397991e-07,
      "loss": -0.0011,
      "reward": 1.5558514595031738,
      "reward_std": 0.28839708790183066,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5344882309436798,
      "step": 920,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.06109805963933468,
      "clip_ratio/high_mean": 0.00847023066598922,
      "clip_ratio/low_mean": 0.004858631710521877,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01332886223681271,
      "completion_length": 85.00750122070312,
      "epoch": 0.17865718951109402,
      "grad_norm": 2.287473440170288,
      "kl": 0.629003182053566,
      "learning_rate": 9.312116012491916e-07,
      "loss": -0.0155,
      "reward": 1.3984088182449341,
      "reward_std": 0.38690108954906466,
      "rewards/code_format_reward": 0.9787500023841857,
      "rewards/code_reward": 0.45451690554618834,
      "step": 930,
      "zero_std_ratio": 0.275
    },
    {
      "clip_ratio/high_max": 0.11440350348129869,
      "clip_ratio/high_mean": 0.021249773760791867,
      "clip_ratio/low_mean": 0.010212704542209395,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03146247826516628,
      "completion_length": 85.56500244140625,
      "epoch": 0.18057823455960043,
      "grad_norm": 2.5915870666503906,
      "kl": 0.6908730089664459,
      "learning_rate": 9.297611330529588e-07,
      "loss": -0.0019,
      "reward": 1.5472615003585815,
      "reward_std": 0.34995803236961365,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.529568213224411,
      "step": 940,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.11480946252122522,
      "clip_ratio/high_mean": 0.021491143060848115,
      "clip_ratio/low_mean": 0.007519157652859576,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029010300803929568,
      "completion_length": 72.10000152587891,
      "epoch": 0.1824992796081068,
      "grad_norm": 1.5689059495925903,
      "kl": 0.7929495573043823,
      "learning_rate": 9.282968249116975e-07,
      "loss": -0.0054,
      "reward": 1.8428637742996217,
      "reward_std": 0.2614489495754242,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6745568513870239,
      "step": 950,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.3971266824752092,
      "clip_ratio/high_mean": 0.05282264268025756,
      "clip_ratio/low_mean": 0.004530514683574438,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.05735315615311265,
      "completion_length": 70.86750030517578,
      "epoch": 0.1844203246566132,
      "grad_norm": 3.4463512897491455,
      "kl": 0.8312035664916039,
      "learning_rate": 9.268187301903852e-07,
      "loss": 0.0003,
      "reward": 1.6929683208465576,
      "reward_std": 0.2562918782234192,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.600546681880951,
      "step": 960,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.1673737466800958,
      "clip_ratio/high_mean": 0.03157579629332759,
      "clip_ratio/low_mean": 0.012752554472535848,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.044328349828720096,
      "completion_length": 76.92749938964843,
      "epoch": 0.18634136970511958,
      "grad_norm": 3.0548853874206543,
      "kl": 0.6291002959012986,
      "learning_rate": 9.253269027564339e-07,
      "loss": -0.005,
      "reward": 1.4119353413581848,
      "reward_std": 0.33177118599414823,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.46065517961978913,
      "step": 970,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.15495819319039583,
      "clip_ratio/high_mean": 0.022329012653790413,
      "clip_ratio/low_mean": 0.006486268152366392,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02881528080906719,
      "completion_length": 68.58250122070312,
      "epoch": 0.18826241475362598,
      "grad_norm": 7.065835952758789,
      "kl": 1.0375685960054397,
      "learning_rate": 9.238213969777292e-07,
      "loss": -0.0046,
      "reward": 1.6331373691558837,
      "reward_std": 0.2626490265130997,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.5703186750411987,
      "step": 980,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.10045178183354438,
      "clip_ratio/high_mean": 0.020599483215482904,
      "clip_ratio/low_mean": 0.007835417747264728,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02843490142840892,
      "completion_length": 71.47500076293946,
      "epoch": 0.19018345980213236,
      "grad_norm": 4.533353328704834,
      "kl": 2.011890631914139,
      "learning_rate": 9.223022677206474e-07,
      "loss": -0.0001,
      "reward": 1.7676753044128417,
      "reward_std": 0.25886805951595304,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6382126212120056,
      "step": 990,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.12670890614390373,
      "clip_ratio/high_mean": 0.022856980562210083,
      "clip_ratio/low_mean": 0.016935013599868397,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.039791994355618955,
      "completion_length": 70.40500106811524,
      "epoch": 0.19210450485063874,
      "grad_norm": 9.587749481201172,
      "kl": 1.1125446915626527,
      "learning_rate": 9.207695703480562e-07,
      "loss": -0.0049,
      "reward": 1.5464402914047242,
      "reward_std": 0.30552313327789304,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5257201135158539,
      "step": 1000,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.13173274043947458,
      "clip_ratio/high_mean": 0.021644592471420764,
      "clip_ratio/low_mean": 0.01016495683870744,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03180954959243536,
      "completion_length": 81.64500122070312,
      "epoch": 0.19402554989914514,
      "grad_norm": 61.59896469116211,
      "kl": 1.3899411320686341,
      "learning_rate": 9.192233607172973e-07,
      "loss": 0.0117,
      "reward": 1.5586263418197632,
      "reward_std": 0.32884465754032133,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5327506422996521,
      "step": 1010,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.38223748579621314,
      "clip_ratio/high_mean": 0.05293128285557032,
      "clip_ratio/low_mean": 0.008536407171050087,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.06146768992766738,
      "completion_length": 75.7425033569336,
      "epoch": 0.19594659494765151,
      "grad_norm": 0.8699261546134949,
      "kl": 2.267198386788368,
      "learning_rate": 9.17663695178151e-07,
      "loss": 0.0007,
      "reward": 1.4393709778785706,
      "reward_std": 0.19248414039611816,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.4724979490041733,
      "step": 1020,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.05449964143335819,
      "clip_ratio/high_mean": 0.008484689320903271,
      "clip_ratio/low_mean": 0.0017167545520351268,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010201443906407804,
      "completion_length": 74.80750045776367,
      "epoch": 0.19786763999615792,
      "grad_norm": 3.8721530437469482,
      "kl": 1.034875027090311,
      "learning_rate": 9.160906305707814e-07,
      "loss": -0.0065,
      "reward": 1.6229804277420044,
      "reward_std": 0.21886643767356873,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5624276876449585,
      "step": 1030,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.057783479290083054,
      "clip_ratio/high_mean": 0.008794186974409968,
      "clip_ratio/low_mean": 0.01261859169753734,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02141277852933854,
      "completion_length": 80.94500122070312,
      "epoch": 0.1997886850446643,
      "grad_norm": 2.0369646549224854,
      "kl": 0.47016064152121545,
      "learning_rate": 9.145042242236667e-07,
      "loss": -0.0016,
      "reward": 1.5200274467468262,
      "reward_std": 0.2379522889852524,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.5147012054920197,
      "step": 1040,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.05080003601033241,
      "clip_ratio/high_mean": 0.0081847107532667,
      "clip_ratio/low_mean": 0.003685746184783056,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011870456766337157,
      "completion_length": 86.39750213623047,
      "epoch": 0.2017097300931707,
      "grad_norm": 1.86152184009552,
      "kl": 0.9119557231664658,
      "learning_rate": 9.129045339515085e-07,
      "loss": -0.0025,
      "reward": 1.338998556137085,
      "reward_std": 0.29172809422016144,
      "rewards/code_format_reward": 0.9787500023841857,
      "rewards/code_reward": 0.42481178045272827,
      "step": 1050,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.11181259918957949,
      "clip_ratio/high_mean": 0.01702371232677251,
      "clip_ratio/low_mean": 0.003983464353950694,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021007176581770183,
      "completion_length": 89.0250015258789,
      "epoch": 0.20363077514167707,
      "grad_norm": 1.664932370185852,
      "kl": 1.7415984645485878,
      "learning_rate": 9.112916180531254e-07,
      "loss": -0.0009,
      "reward": 1.6867451906204223,
      "reward_std": 0.26216842532157897,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.5971225798130035,
      "step": 1060,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.1619036693125963,
      "clip_ratio/high_mean": 0.02605230761691928,
      "clip_ratio/low_mean": 0.011786457896232606,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03783876644447446,
      "completion_length": 80.52750091552734,
      "epoch": 0.20555182019018345,
      "grad_norm": 3.1480722427368164,
      "kl": 2.3309426337480543,
      "learning_rate": 9.096655353093286e-07,
      "loss": -0.0108,
      "reward": 1.7797099113464356,
      "reward_std": 0.3243818938732147,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6429799437522888,
      "step": 1070,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.145719656907022,
      "clip_ratio/high_mean": 0.02472380215767771,
      "clip_ratio/low_mean": 0.01881317695369944,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.043536979146301745,
      "completion_length": 75.46750183105469,
      "epoch": 0.20747286523868985,
      "grad_norm": 4.7426347732543945,
      "kl": 0.7767296731472015,
      "learning_rate": 9.080263449807788e-07,
      "loss": 0.0042,
      "reward": 1.5128322124481202,
      "reward_std": 0.26058112680912016,
      "rewards/code_format_reward": 0.9662500023841858,
      "rewards/code_reward": 0.514853572845459,
      "step": 1080,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.04860758520662785,
      "clip_ratio/high_mean": 0.00921072952914983,
      "clip_ratio/low_mean": 0.013458288778201677,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022669017571024595,
      "completion_length": 77.22750244140624,
      "epoch": 0.20939391028719623,
      "grad_norm": 2.2836835384368896,
      "kl": 0.6794285923242569,
      "learning_rate": 9.063741068058278e-07,
      "loss": -0.0028,
      "reward": 1.5665315628051757,
      "reward_std": 0.23656646013259888,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5398283064365387,
      "step": 1090,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.2074673067778349,
      "clip_ratio/high_mean": 0.036228268034756185,
      "clip_ratio/low_mean": 0.003734398238157155,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.039962667226791385,
      "completion_length": 91.18000030517578,
      "epoch": 0.21131495533570263,
      "grad_norm": 7.916996002197266,
      "kl": 1.0919141083955766,
      "learning_rate": 9.0470888099834e-07,
      "loss": 0.1666,
      "reward": 1.68690767288208,
      "reward_std": 0.32907233834266664,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.5984537959098816,
      "step": 1100,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.16652454435825348,
      "clip_ratio/high_mean": 0.027045656740665436,
      "clip_ratio/low_mean": 0.006342244842380751,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03338790265843272,
      "completion_length": 80.02000122070312,
      "epoch": 0.213236000384209,
      "grad_norm": 24.34583282470703,
      "kl": 1.00138920545578,
      "learning_rate": 9.030307282454995e-07,
      "loss": -0.0023,
      "reward": 1.6111816883087158,
      "reward_std": 0.24880893230438234,
      "rewards/code_format_reward": 0.9724999904632569,
      "rewards/code_reward": 0.5624658226966858,
      "step": 1110,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.18198216175660492,
      "clip_ratio/high_mean": 0.02493738690391183,
      "clip_ratio/low_mean": 0.004894328210502863,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029831714881584048,
      "completion_length": 71.95,
      "epoch": 0.2151570454327154,
      "grad_norm": 2.7608304023742676,
      "kl": 0.971074515581131,
      "learning_rate": 9.013397097055971e-07,
      "loss": -0.0022,
      "reward": 1.6884326457977294,
      "reward_std": 0.3369467526674271,
      "rewards/code_format_reward": 0.9712499856948853,
      "rewards/code_reward": 0.6014038324356079,
      "step": 1120,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.16543546952307225,
      "clip_ratio/high_mean": 0.02493141880258918,
      "clip_ratio/low_mean": 0.007064808573340997,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03199622761458158,
      "completion_length": 72.50500030517578,
      "epoch": 0.21707809048122179,
      "grad_norm": 7.147952556610107,
      "kl": 6.163409499824047,
      "learning_rate": 8.996358870058017e-07,
      "loss": 0.0081,
      "reward": 1.5753276348114014,
      "reward_std": 0.2175431028008461,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.5395387947559357,
      "step": 1130,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.05989155264105648,
      "clip_ratio/high_mean": 0.009021314003621227,
      "clip_ratio/low_mean": 0.014251881884410978,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023273196443915366,
      "completion_length": 74.57750091552734,
      "epoch": 0.21899913552972816,
      "grad_norm": 17.58907699584961,
      "kl": 0.9839092344045639,
      "learning_rate": 8.979193222399154e-07,
      "loss": -0.0006,
      "reward": 1.570918822288513,
      "reward_std": 0.27486068904399874,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5420219123363494,
      "step": 1140,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.23600016683340072,
      "clip_ratio/high_mean": 0.04525289600715041,
      "clip_ratio/low_mean": 0.00799154011765495,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.05324443739373237,
      "completion_length": 71.89750061035156,
      "epoch": 0.22092018057823457,
      "grad_norm": 8.010896682739258,
      "kl": 1.0768774889409543,
      "learning_rate": 8.961900779661095e-07,
      "loss": 0.0139,
      "reward": 1.5848765134811402,
      "reward_std": 0.21965934410691262,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5458757638931274,
      "step": 1150,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.10782922431826591,
      "clip_ratio/high_mean": 0.014393238560296595,
      "clip_ratio/low_mean": 0.0046036563231609765,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018996895058080554,
      "completion_length": 78.2925018310547,
      "epoch": 0.22284122562674094,
      "grad_norm": 3.7750465869903564,
      "kl": 0.5210637584328651,
      "learning_rate": 8.944482172046448e-07,
      "loss": -0.0065,
      "reward": 1.6227028608322143,
      "reward_std": 0.2484603613615036,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.5660388946533204,
      "step": 1160,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.13120641289278864,
      "clip_ratio/high_mean": 0.019719564472325146,
      "clip_ratio/low_mean": 0.00696407729992643,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026683641644194723,
      "completion_length": 81.64000091552734,
      "epoch": 0.22476227067524734,
      "grad_norm": 1.1691230535507202,
      "kl": 0.5908193171024323,
      "learning_rate": 8.926938034355751e-07,
      "loss": -0.0008,
      "reward": 1.6598936080932618,
      "reward_std": 0.3073273479938507,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.5830717980861664,
      "step": 1170,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.26425148695707323,
      "clip_ratio/high_mean": 0.03642228813841939,
      "clip_ratio/low_mean": 0.0025068818649742752,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03892916943877935,
      "completion_length": 83.06500244140625,
      "epoch": 0.22668331572375372,
      "grad_norm": 5.047176361083984,
      "kl": 0.8601905956864357,
      "learning_rate": 8.90926900596434e-07,
      "loss": 0.019,
      "reward": 1.6030859470367431,
      "reward_std": 0.18358819633722306,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.5549804508686066,
      "step": 1180,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.20188620835542678,
      "clip_ratio/high_mean": 0.03365288833156228,
      "clip_ratio/low_mean": 0.012162915989756584,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04581580460071564,
      "completion_length": 80.93500061035157,
      "epoch": 0.2286043607722601,
      "grad_norm": 3.431043863296509,
      "kl": 3.284740853309631,
      "learning_rate": 8.891475730799039e-07,
      "loss": -0.0024,
      "reward": 1.719798493385315,
      "reward_std": 0.2678588882088661,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.6127117216587067,
      "step": 1190,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.13805483505129815,
      "clip_ratio/high_mean": 0.02111883880570531,
      "clip_ratio/low_mean": 0.002508872369071469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023627711273729802,
      "completion_length": 87.33250274658204,
      "epoch": 0.2305254058207665,
      "grad_norm": 4.731442928314209,
      "kl": 1.1696231275796891,
      "learning_rate": 8.873558857314706e-07,
      "loss": -0.0053,
      "reward": 1.7580220222473144,
      "reward_std": 0.28411929309368134,
      "rewards/code_format_reward": 0.9900000095367432,
      "rewards/code_reward": 0.6315110087394714,
      "step": 1200,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.07043634681031108,
      "clip_ratio/high_mean": 0.009235845855437219,
      "clip_ratio/low_mean": 0.017453379271319135,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02668922524899244,
      "completion_length": 86.74250030517578,
      "epoch": 0.23244645086927287,
      "grad_norm": 23.686250686645508,
      "kl": 1.7613270074129104,
      "learning_rate": 8.855519038470587e-07,
      "loss": 0.91,
      "reward": 1.8096629619598388,
      "reward_std": 0.2700611263513565,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6592064738273621,
      "step": 1210,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.1193816315382719,
      "clip_ratio/high_mean": 0.01799508691765368,
      "clip_ratio/low_mean": 0.0052341839407745285,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023229270869342143,
      "completion_length": 91.73750152587891,
      "epoch": 0.23436749591777928,
      "grad_norm": 5.015241622924805,
      "kl": 87723751.16166303,
      "learning_rate": 8.83735693170653e-07,
      "loss": 178666.875,
      "reward": 1.5409840583801269,
      "reward_std": 0.3586106300354004,
      "rewards/code_format_reward": 0.9687500119209289,
      "rewards/code_reward": 0.5283045113086701,
      "step": 1220,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.15788686936721205,
      "clip_ratio/high_mean": 0.02180835944600403,
      "clip_ratio/low_mean": 0.004957044991897419,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026765404315665365,
      "completion_length": 83.70250091552734,
      "epoch": 0.23628854096628565,
      "grad_norm": 2.7140953540802,
      "kl": 0.755669391900301,
      "learning_rate": 8.81907319891902e-07,
      "loss": -0.0099,
      "reward": 1.8449480056762695,
      "reward_std": 0.28006095588207247,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6755990028381348,
      "step": 1230,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.12416752465069295,
      "clip_ratio/high_mean": 0.01972346901893616,
      "clip_ratio/low_mean": 0.01847981174942106,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0382032809779048,
      "completion_length": 91.47000274658203,
      "epoch": 0.23820958601479206,
      "grad_norm": 10.781957626342773,
      "kl": 1.0129390999674797,
      "learning_rate": 8.800668506437059e-07,
      "loss": 0.0011,
      "reward": 1.6923505306243896,
      "reward_std": 0.3265227422118187,
      "rewards/code_format_reward": 0.9787500023841857,
      "rewards/code_reward": 0.6014877319335937,
      "step": 1240,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.12042212830856443,
      "clip_ratio/high_mean": 0.017916655144654216,
      "clip_ratio/low_mean": 0.007017276567057707,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02493393179029226,
      "completion_length": 76.6675018310547,
      "epoch": 0.24013063106329843,
      "grad_norm": 47.773136138916016,
      "kl": 1.4071896970272064,
      "learning_rate": 8.782143524997882e-07,
      "loss": 0.0018,
      "reward": 1.6722928285598755,
      "reward_std": 0.25374017357826234,
      "rewards/code_format_reward": 0.9824999809265137,
      "rewards/code_reward": 0.5905213832855225,
      "step": 1250,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.08169625541195273,
      "clip_ratio/high_mean": 0.013112110400106758,
      "clip_ratio/low_mean": 0.003914138658728916,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01702624891186133,
      "completion_length": 78.23750152587891,
      "epoch": 0.2420516761118048,
      "grad_norm": 2688.99462890625,
      "kl": 9.395949372649193,
      "learning_rate": 8.76349892972251e-07,
      "loss": 0.1943,
      "reward": 1.5601455688476562,
      "reward_std": 0.3348282665014267,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5372602701187134,
      "step": 1260,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.13095853393897414,
      "clip_ratio/high_mean": 0.018921413994394242,
      "clip_ratio/low_mean": 0.018763081403449178,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03768449563067407,
      "completion_length": 76.1500015258789,
      "epoch": 0.2439727211603112,
      "grad_norm": 3.0777931213378906,
      "kl": 1.7352074533700943,
      "learning_rate": 8.744735400091154e-07,
      "loss": 0.0055,
      "reward": 1.633968448638916,
      "reward_std": 0.23277063071727752,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.5713592231273651,
      "step": 1270,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.15694143967702984,
      "clip_ratio/high_mean": 0.026766782545018943,
      "clip_ratio/low_mean": 0.010570818380801938,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03733760174363852,
      "completion_length": 76.13500213623047,
      "epoch": 0.2458937662088176,
      "grad_norm": 2.8748385906219482,
      "kl": 3.007472372055054,
      "learning_rate": 8.725853619918444e-07,
      "loss": 0.0249,
      "reward": 1.4643328666687012,
      "reward_std": 0.2899716466665268,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.48716638684272767,
      "step": 1280,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.07734788609668612,
      "clip_ratio/high_mean": 0.013521577988285571,
      "clip_ratio/low_mean": 0.002974278874171432,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01649585694540292,
      "completion_length": 77.6050033569336,
      "epoch": 0.247814811257324,
      "grad_norm": 4.51137638092041,
      "kl": 0.6521440967917442,
      "learning_rate": 8.706854277328507e-07,
      "loss": -0.0065,
      "reward": 1.663088607788086,
      "reward_std": 0.29463320076465604,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.5843567848205566,
      "step": 1290,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.12926983460783958,
      "clip_ratio/high_mean": 0.016393666993826626,
      "clip_ratio/low_mean": 0.024948839796707034,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04134250609204173,
      "completion_length": 74.63750305175782,
      "epoch": 0.24973585630583037,
      "grad_norm": 7.019649982452393,
      "kl": 0.6837658904492855,
      "learning_rate": 8.687738064729902e-07,
      "loss": -0.0022,
      "reward": 1.6927862167358398,
      "reward_std": 0.14656674191355706,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5973306179046631,
      "step": 1300,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.15073961750604212,
      "clip_ratio/high_mean": 0.024888798157917336,
      "clip_ratio/low_mean": 0.004707413475262001,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02959621128393337,
      "completion_length": 79.17500152587891,
      "epoch": 0.25165690135433677,
      "grad_norm": 3.9428677558898926,
      "kl": 1.0088127315044404,
      "learning_rate": 8.668505678790368e-07,
      "loss": 0.7445,
      "reward": 1.5962260961532593,
      "reward_std": 0.22741070687770842,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.5528005361557007,
      "step": 1310,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.0862931152805686,
      "clip_ratio/high_mean": 0.016994312894530593,
      "clip_ratio/low_mean": 0.0031913593309582213,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020185671979561448,
      "completion_length": 79.30500183105468,
      "epoch": 0.25357794640284315,
      "grad_norm": 2.810743808746338,
      "kl": 2.0237294919788837,
      "learning_rate": 8.649157820411451e-07,
      "loss": -0.0028,
      "reward": 1.6300202369689942,
      "reward_std": 0.2859074264764786,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5712601006031036,
      "step": 1320,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.14902311654295772,
      "clip_ratio/high_mean": 0.02855427504691761,
      "clip_ratio/low_mean": 0.012185945303644984,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.040740220062434676,
      "completion_length": 70.88000030517578,
      "epoch": 0.2554989914513495,
      "grad_norm": 4.68557071685791,
      "kl": 1.2288852274417876,
      "learning_rate": 8.629695194702949e-07,
      "loss": -0.0057,
      "reward": 1.4114359855651855,
      "reward_std": 0.2626632884144783,
      "rewards/code_format_reward": 0.9625,
      "rewards/code_reward": 0.46509301066398623,
      "step": 1330,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.11323303133249282,
      "clip_ratio/high_mean": 0.016216285666450857,
      "clip_ratio/low_mean": 0.0045135776337701826,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02072986289858818,
      "completion_length": 71.99250030517578,
      "epoch": 0.2574200364998559,
      "grad_norm": 43.944698333740234,
      "kl": 1.446278090775013,
      "learning_rate": 8.610118510957221e-07,
      "loss": 0.0112,
      "reward": 1.5807109117507934,
      "reward_std": 0.23466840982437134,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5469179153442383,
      "step": 1340,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.20504833161830902,
      "clip_ratio/high_mean": 0.029384778672829272,
      "clip_ratio/low_mean": 0.006734570109983906,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03611934892833233,
      "completion_length": 69.60750198364258,
      "epoch": 0.25934108154836233,
      "grad_norm": 3.4515652656555176,
      "kl": 1.288391387462616,
      "learning_rate": 8.59042848262334e-07,
      "loss": 0.0022,
      "reward": 1.7648874998092652,
      "reward_std": 0.29008678793907167,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.6340062260627747,
      "step": 1350,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.18540791552513838,
      "clip_ratio/high_mean": 0.030647353292442857,
      "clip_ratio/low_mean": 0.0048290589373209515,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03547641267068684,
      "completion_length": 73.8150016784668,
      "epoch": 0.2612621265968687,
      "grad_norm": 24.974191665649414,
      "kl": 1.361786951869726,
      "learning_rate": 8.570625827281077e-07,
      "loss": -0.0015,
      "reward": 1.6352276086807251,
      "reward_std": 0.20483867302536965,
      "rewards/code_format_reward": 0.9712500095367431,
      "rewards/code_reward": 0.5748012781143188,
      "step": 1360,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.25138766765594484,
      "clip_ratio/high_mean": 0.043486443860456345,
      "clip_ratio/low_mean": 0.006613140180706978,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.05009958455339074,
      "completion_length": 85.41999969482421,
      "epoch": 0.2631831716453751,
      "grad_norm": 0.2826422452926636,
      "kl": 1.1484392315149308,
      "learning_rate": 8.550711266614774e-07,
      "loss": -0.0015,
      "reward": 1.5049166679382324,
      "reward_std": 0.17118329852819442,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.5090208292007447,
      "step": 1370,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.10418513733893633,
      "clip_ratio/high_mean": 0.017387184244580568,
      "clip_ratio/low_mean": 0.006483422458404675,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023870606115087865,
      "completion_length": 78.00750274658203,
      "epoch": 0.26510421669388146,
      "grad_norm": 0.43826720118522644,
      "kl": 0.5077251173555851,
      "learning_rate": 8.530685526387023e-07,
      "loss": 0.0071,
      "reward": 1.5417476654052735,
      "reward_std": 0.2806018695235252,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5271238267421723,
      "step": 1380,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.12633447310654447,
      "clip_ratio/high_mean": 0.01944113611098146,
      "clip_ratio/low_mean": 0.02114583211950958,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04058696813735878,
      "completion_length": 69.89499969482422,
      "epoch": 0.26702526174238783,
      "grad_norm": 3.222648859024048,
      "kl": 0.8532382689416409,
      "learning_rate": 8.510549336412227e-07,
      "loss": 0.2832,
      "reward": 1.4325429320335388,
      "reward_std": 0.23379142954945564,
      "rewards/code_format_reward": 0.95625,
      "rewards/code_reward": 0.47720896899700166,
      "step": 1390,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.10978957340121269,
      "clip_ratio/high_mean": 0.015348212420940399,
      "clip_ratio/low_mean": 0.00886362442979589,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024211836606264116,
      "completion_length": 74.32000198364258,
      "epoch": 0.26894630679089426,
      "grad_norm": 511.98333740234375,
      "kl": 6.762348529696465,
      "learning_rate": 8.490303430529996e-07,
      "loss": 0.0097,
      "reward": 1.5433219909667968,
      "reward_std": 0.3002948135137558,
      "rewards/code_format_reward": 0.9787500023841857,
      "rewards/code_reward": 0.5269734919071197,
      "step": 1400,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.021737607452087103,
      "clip_ratio/high_mean": 0.004128801400656812,
      "clip_ratio/low_mean": 0.008135353482794016,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012264154804870486,
      "completion_length": 70.88250122070312,
      "epoch": 0.27086735183940064,
      "grad_norm": 4.558300018310547,
      "kl": 1.0645984336733818,
      "learning_rate": 8.469948546578406e-07,
      "loss": -0.002,
      "reward": 1.711915636062622,
      "reward_std": 0.23479849100112915,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6090827941894531,
      "step": 1410,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.31115832179784775,
      "clip_ratio/high_mean": 0.04542893636971712,
      "clip_ratio/low_mean": 0.004167796808178537,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04959673266857863,
      "completion_length": 82.51750335693359,
      "epoch": 0.272788396887907,
      "grad_norm": 26.85635757446289,
      "kl": 0.6633755072951317,
      "learning_rate": 8.449485426367113e-07,
      "loss": -0.0044,
      "reward": 1.8086278200149537,
      "reward_std": 0.25109012275934217,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6577514052391052,
      "step": 1420,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.21433292645961047,
      "clip_ratio/high_mean": 0.027504962938837706,
      "clip_ratio/low_mean": 0.007746222103014589,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03525118476245552,
      "completion_length": 69.12750015258788,
      "epoch": 0.2747094419364134,
      "grad_norm": 39.272727966308594,
      "kl": 2.1152508199214934,
      "learning_rate": 8.428914815650318e-07,
      "loss": 56.6465,
      "reward": 1.5950207233428955,
      "reward_std": 0.25626782774925233,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.5518853664398193,
      "step": 1430,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.14325670124962925,
      "clip_ratio/high_mean": 0.02268084152601659,
      "clip_ratio/low_mean": 0.006528474338119849,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029209316370543092,
      "completion_length": 67.30000076293945,
      "epoch": 0.2766304869849198,
      "grad_norm": 4.287910461425781,
      "kl": 1.2686308354139328,
      "learning_rate": 8.408237464099576e-07,
      "loss": 9.8201,
      "reward": 1.6364605188369752,
      "reward_std": 0.22813104093074799,
      "rewards/code_format_reward": 0.9749999880790711,
      "rewards/code_reward": 0.5744802415370941,
      "step": 1440,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.18851536950096487,
      "clip_ratio/high_mean": 0.024719347018981354,
      "clip_ratio/low_mean": 0.013444452191470191,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.038163799053290856,
      "completion_length": 82.13250274658203,
      "epoch": 0.2785515320334262,
      "grad_norm": 0.4786536991596222,
      "kl": 8.468844538927078,
      "learning_rate": 8.387454125276494e-07,
      "loss": 0.0456,
      "reward": 1.7758944988250733,
      "reward_std": 0.1511917643249035,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6410722196102142,
      "step": 1450,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.15984937213361264,
      "clip_ratio/high_mean": 0.025054804515093565,
      "clip_ratio/low_mean": 0.01257994698244147,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03763475017622113,
      "completion_length": 79.66000213623047,
      "epoch": 0.2804725770819326,
      "grad_norm": 3.223284959793091,
      "kl": 1.7015444114804268,
      "learning_rate": 8.366565556605258e-07,
      "loss": 0.0276,
      "reward": 1.5976650953292846,
      "reward_std": 0.341750779747963,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.5566450238227845,
      "step": 1460,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.27182711616624144,
      "clip_ratio/high_mean": 0.040798351392732,
      "clip_ratio/low_mean": 0.002200227712455671,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04299857785226777,
      "completion_length": 79.22250213623047,
      "epoch": 0.28239362213043895,
      "grad_norm": 1.4845157861709595,
      "kl": 1.693036738038063,
      "learning_rate": 8.345572519345031e-07,
      "loss": -0.0017,
      "reward": 1.7161717653274535,
      "reward_std": 0.2422049015760422,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.612460857629776,
      "step": 1470,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.17126517184078693,
      "clip_ratio/high_mean": 0.025960111571475864,
      "clip_ratio/low_mean": 0.00444280517695006,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030402917158789934,
      "completion_length": 83.31750183105468,
      "epoch": 0.2843146671789453,
      "grad_norm": 5.96829080581665,
      "kl": 0.574289733916521,
      "learning_rate": 8.324475778562209e-07,
      "loss": -0.0061,
      "reward": 1.7776363611221313,
      "reward_std": 0.2358689785003662,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6428806602954864,
      "step": 1480,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.2015096817165613,
      "clip_ratio/high_mean": 0.03324723746627569,
      "clip_ratio/low_mean": 0.00480144299363019,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.038048680778592824,
      "completion_length": 73.28000106811524,
      "epoch": 0.28623571222745176,
      "grad_norm": 6.496949672698975,
      "kl": 0.6653359919786453,
      "learning_rate": 8.30327610310254e-07,
      "loss": 0.0021,
      "reward": 1.6191941976547242,
      "reward_std": 0.31718442738056185,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.5639720797538758,
      "step": 1490,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.15795932533219456,
      "clip_ratio/high_mean": 0.02212390162749216,
      "clip_ratio/low_mean": 0.00480329486890696,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026927196700125933,
      "completion_length": 73.78000106811524,
      "epoch": 0.28815675727595813,
      "grad_norm": 5.75892972946167,
      "kl": 0.46196936070919037,
      "learning_rate": 8.281974265563108e-07,
      "loss": -0.0045,
      "reward": 1.7829506158828736,
      "reward_std": 0.17953601479530334,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.642725282907486,
      "step": 1500,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.24582542856223882,
      "clip_ratio/high_mean": 0.030850262753665446,
      "clip_ratio/low_mean": 0.005616182333324104,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03646644388791174,
      "completion_length": 77.69500198364258,
      "epoch": 0.2900778023244645,
      "grad_norm": 326340576.0,
      "kl": 0.605505321919918,
      "learning_rate": 8.260571042264166e-07,
      "loss": 8518.9961,
      "reward": 1.7113344192504882,
      "reward_std": 0.18693218380212784,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6087921977043151,
      "step": 1510,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.22759323129430414,
      "clip_ratio/high_mean": 0.03405714362161234,
      "clip_ratio/low_mean": 0.0032101303557283247,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03726727362954989,
      "completion_length": 75.53250122070312,
      "epoch": 0.2919988473729709,
      "grad_norm": 2.2893807888031006,
      "kl": 0.5214515089988708,
      "learning_rate": 8.23906721322086e-07,
      "loss": 0.0027,
      "reward": 1.6311777591705323,
      "reward_std": 0.17696685791015626,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5690263509750366,
      "step": 1520,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.06725322343409061,
      "clip_ratio/high_mean": 0.010706762981135398,
      "clip_ratio/low_mean": 0.0018884234530560206,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012595186498947442,
      "completion_length": 78.90999908447266,
      "epoch": 0.29391989242147726,
      "grad_norm": 2.6211440563201904,
      "kl": 0.5930808052420616,
      "learning_rate": 8.217463562114786e-07,
      "loss": -0.0035,
      "reward": 1.7637510299682617,
      "reward_std": 0.209340962767601,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.6365630030632019,
      "step": 1530,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.06629493543878198,
      "clip_ratio/high_mean": 0.012000571249518543,
      "clip_ratio/low_mean": 0.010053297760896385,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022053868882358073,
      "completion_length": 77.66250152587891,
      "epoch": 0.2958409374699837,
      "grad_norm": 0.5937472581863403,
      "kl": 0.6556157968938351,
      "learning_rate": 8.195760876265438e-07,
      "loss": 0.0023,
      "reward": 1.4144308805465697,
      "reward_std": 0.12647379338741302,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.461590439081192,
      "step": 1540,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.2611659773625433,
      "clip_ratio/high_mean": 0.05069012229796499,
      "clip_ratio/low_mean": 0.009917778367525897,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.06060790033079684,
      "completion_length": 80.60250091552734,
      "epoch": 0.29776198251849006,
      "grad_norm": 7.297484874725342,
      "kl": 2.139972834289074,
      "learning_rate": 8.173959946601519e-07,
      "loss": 0.0662,
      "reward": 1.6416264057159424,
      "reward_std": 0.3118141442537308,
      "rewards/code_format_reward": 0.9749999880790711,
      "rewards/code_reward": 0.5770631790161133,
      "step": 1550,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.14441705606877803,
      "clip_ratio/high_mean": 0.023728324193507434,
      "clip_ratio/low_mean": 0.005098688977886923,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028827012795954943,
      "completion_length": 77.17750244140625,
      "epoch": 0.29968302756699644,
      "grad_norm": 5.614815711975098,
      "kl": 0.5137595549225807,
      "learning_rate": 8.152061567632108e-07,
      "loss": -0.0057,
      "reward": 1.5097593545913697,
      "reward_std": 0.29559260606765747,
      "rewards/code_format_reward": 0.9575000047683716,
      "rewards/code_reward": 0.5155046641826629,
      "step": 1560,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.03715956890955567,
      "clip_ratio/high_mean": 0.006024846772197634,
      "clip_ratio/low_mean": 0.009319488028995692,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015344334836117923,
      "completion_length": 76.34000091552734,
      "epoch": 0.3016040726155028,
      "grad_norm": 5.059381008148193,
      "kl": 0.8711868159472942,
      "learning_rate": 8.130066537417707e-07,
      "loss": -0.0003,
      "reward": 1.4149085521697997,
      "reward_std": 0.19155050422996284,
      "rewards/code_format_reward": 0.9749999880790711,
      "rewards/code_reward": 0.463704252243042,
      "step": 1570,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.0935845285654068,
      "clip_ratio/high_mean": 0.013573423656634987,
      "clip_ratio/low_mean": 0.00990565216197865,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023479075590148567,
      "completion_length": 83.97500152587891,
      "epoch": 0.30352511766400925,
      "grad_norm": 2.025956869125366,
      "kl": 0.9980318561196327,
      "learning_rate": 8.10797565754116e-07,
      "loss": -0.0041,
      "reward": 1.5444376945495606,
      "reward_std": 0.19510383605957032,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.525031316280365,
      "step": 1580,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.11659459788352251,
      "clip_ratio/high_mean": 0.016526972700376064,
      "clip_ratio/low_mean": 0.0030368489184184,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019563821679912507,
      "completion_length": 90.33000335693359,
      "epoch": 0.3054461627125156,
      "grad_norm": 4.901747703552246,
      "kl": 0.6650052145123482,
      "learning_rate": 8.085789733078439e-07,
      "loss": 0.9063,
      "reward": 1.6000897407531738,
      "reward_std": 0.20618843138217927,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5556698679924011,
      "step": 1590,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.1246914654970169,
      "clip_ratio/high_mean": 0.018419789243489505,
      "clip_ratio/low_mean": 0.0033823222620412707,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021802110970020293,
      "completion_length": 82.78250122070312,
      "epoch": 0.307367207761022,
      "grad_norm": 16365.4453125,
      "kl": 83.84930176734925,
      "learning_rate": 8.063509572569303e-07,
      "loss": 0.4123,
      "reward": 1.8164207458496093,
      "reward_std": 0.25260339230298995,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6613353252410888,
      "step": 1600,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.18187000900506972,
      "clip_ratio/high_mean": 0.026620355295017363,
      "clip_ratio/low_mean": 0.011157544914749452,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03777789976447821,
      "completion_length": 72.65250244140626,
      "epoch": 0.3092882528095284,
      "grad_norm": 2.8136842250823975,
      "kl": 0.9565572030842304,
      "learning_rate": 8.041135987987831e-07,
      "loss": 0.0037,
      "reward": 1.7599462985992431,
      "reward_std": 0.26825075447559354,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6324730753898621,
      "step": 1610,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.03404317735694349,
      "clip_ratio/high_mean": 0.006068735342705622,
      "clip_ratio/low_mean": 0.010824382931605214,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016893118200823665,
      "completion_length": 78.07500305175782,
      "epoch": 0.31120929785803475,
      "grad_norm": 31.179058074951172,
      "kl": 0.560398967564106,
      "learning_rate": 8.018669794712835e-07,
      "loss": -0.0011,
      "reward": 1.5130140781402588,
      "reward_std": 0.2716240629553795,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.5115070700645447,
      "step": 1620,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.06218870538286865,
      "clip_ratio/high_mean": 0.008549430634593591,
      "clip_ratio/low_mean": 0.007052442076383158,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01560187318827957,
      "completion_length": 83.56500091552735,
      "epoch": 0.3131303429065412,
      "grad_norm": 0.6899747252464294,
      "kl": 0.7204694971442223,
      "learning_rate": 7.996111811498138e-07,
      "loss": 0.0031,
      "reward": 1.687961721420288,
      "reward_std": 0.19512347355484963,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.5958558440208435,
      "step": 1630,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.17274489336414262,
      "clip_ratio/high_mean": 0.021967002666497138,
      "clip_ratio/low_mean": 0.009596503502689303,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03156350784411188,
      "completion_length": 80.9175033569336,
      "epoch": 0.31505138795504756,
      "grad_norm": 2.105334758758545,
      "kl": 0.8054538488388061,
      "learning_rate": 7.97346286044274e-07,
      "loss": -0.0058,
      "reward": 1.3176400899887084,
      "reward_std": 0.20478213280439378,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.41350752413272857,
      "step": 1640,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.16534482885617763,
      "clip_ratio/high_mean": 0.02735080250131432,
      "clip_ratio/low_mean": 0.0035748321075516286,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030925634037703275,
      "completion_length": 74.01250228881835,
      "epoch": 0.31697243300355393,
      "grad_norm": 184916.921875,
      "kl": 28.671802641451357,
      "learning_rate": 7.950723766960857e-07,
      "loss": 5.579,
      "reward": 1.6360910892486573,
      "reward_std": 0.2874180316925049,
      "rewards/code_format_reward": 0.9687500119209289,
      "rewards/code_reward": 0.5758580267429352,
      "step": 1650,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.10983106552157551,
      "clip_ratio/high_mean": 0.016536441215430388,
      "clip_ratio/low_mean": 0.011150279239518567,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.027686719762277788,
      "completion_length": 84.17750244140625,
      "epoch": 0.3188934780520603,
      "grad_norm": 219305424.0,
      "kl": 106.82060827612877,
      "learning_rate": 7.927895359751835e-07,
      "loss": 5248.6121,
      "reward": 1.5329812049865723,
      "reward_std": 0.22349740117788314,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5221156060695649,
      "step": 1660,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.13622083119116724,
      "clip_ratio/high_mean": 0.01933064509066753,
      "clip_ratio/low_mean": 0.005038347843219526,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024368993006646633,
      "completion_length": 80.39500274658204,
      "epoch": 0.3208145231005667,
      "grad_norm": 9.519110679626465,
      "kl": 0.7214748501777649,
      "learning_rate": 7.904978470769959e-07,
      "loss": -0.0025,
      "reward": 1.6617871284484864,
      "reward_std": 0.27498180270195005,
      "rewards/code_format_reward": 0.95625,
      "rewards/code_reward": 0.5918310403823852,
      "step": 1670,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.09761472065001726,
      "clip_ratio/high_mean": 0.01911984165199101,
      "clip_ratio/low_mean": 0.010301339952275158,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02942118220962584,
      "completion_length": 74.54750213623046,
      "epoch": 0.3227355681490731,
      "grad_norm": 6.143461227416992,
      "kl": 0.7205829441547393,
      "learning_rate": 7.881973935194124e-07,
      "loss": 0.0015,
      "reward": 1.4262179613113404,
      "reward_std": 0.26740061640739443,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.4696714758872986,
      "step": 1680,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.07396706650033594,
      "clip_ratio/high_mean": 0.011737752065528184,
      "clip_ratio/low_mean": 0.005250315659213811,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016988067945931107,
      "completion_length": 75.27500228881836,
      "epoch": 0.3246566131975795,
      "grad_norm": 2.337491989135742,
      "kl": 68.4789316162467,
      "learning_rate": 7.858882591397403e-07,
      "loss": 0.3045,
      "reward": 1.527750849723816,
      "reward_std": 0.26877219378948214,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5163754165172577,
      "step": 1690,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.28693441725336016,
      "clip_ratio/high_mean": 0.04205623795860447,
      "clip_ratio/low_mean": 0.009473194915335626,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.051529432012466715,
      "completion_length": 84.14500274658204,
      "epoch": 0.32657765824608587,
      "grad_norm": 20.964569091796875,
      "kl": 0.5620399042963982,
      "learning_rate": 7.835705280916488e-07,
      "loss": -0.0051,
      "reward": 1.615627408027649,
      "reward_std": 0.2002291887998581,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.5590636849403381,
      "step": 1700,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.2242162274196744,
      "clip_ratio/high_mean": 0.036464582500047985,
      "clip_ratio/low_mean": 0.010222097241785378,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.046686679660342636,
      "completion_length": 78.56000061035157,
      "epoch": 0.32849870329459224,
      "grad_norm": 3.2044875621795654,
      "kl": 0.7747909784317016,
      "learning_rate": 7.812442848421032e-07,
      "loss": -0.0006,
      "reward": 1.6169416427612304,
      "reward_std": 0.24999960064888,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.5612833142280579,
      "step": 1710,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.10125078996643425,
      "clip_ratio/high_mean": 0.019883562461473048,
      "clip_ratio/low_mean": 0.014126901775307487,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.034010464209131896,
      "completion_length": 73.05000152587891,
      "epoch": 0.3304197483430986,
      "grad_norm": 735.9865112304688,
      "kl": 2.3181345582008364,
      "learning_rate": 7.789096141682851e-07,
      "loss": 0.1213,
      "reward": 1.371981406211853,
      "reward_std": 0.17790164202451705,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.44317818284034727,
      "step": 1720,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.15698121464811265,
      "clip_ratio/high_mean": 0.026607585436431692,
      "clip_ratio/low_mean": 0.004372719774255529,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030980303999967873,
      "completion_length": 78.5425033569336,
      "epoch": 0.33234079339160505,
      "grad_norm": 2.3281009197235107,
      "kl": 1.7815167903900146,
      "learning_rate": 7.765666011545045e-07,
      "loss": 0.4359,
      "reward": 1.669968068599701,
      "reward_std": 0.18121034651994705,
      "rewards/code_format_reward": 0.9737499833106995,
      "rewards/code_reward": 0.5915465235710144,
      "step": 1730,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.1189429596066475,
      "clip_ratio/high_mean": 0.021151045989245176,
      "clip_ratio/low_mean": 0.002452358941081911,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023603405337780714,
      "completion_length": 69.71000289916992,
      "epoch": 0.3342618384401114,
      "grad_norm": 1720.8326416015625,
      "kl": 0.7967777937650681,
      "learning_rate": 7.742153311890971e-07,
      "loss": 0.0982,
      "reward": 1.5440645456314086,
      "reward_std": 0.18595425188541412,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5292197823524475,
      "step": 1740,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.08902034647762776,
      "clip_ratio/high_mean": 0.012681722827255725,
      "clip_ratio/low_mean": 0.00311334275174886,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015795065369457007,
      "completion_length": 74.49249954223633,
      "epoch": 0.3361828834886178,
      "grad_norm": 0.09847641736268997,
      "kl": 0.8014414094388485,
      "learning_rate": 7.718558899613143e-07,
      "loss": 0.0099,
      "reward": 1.5567015647888183,
      "reward_std": 0.14754890371114016,
      "rewards/code_format_reward": 0.9649999976158142,
      "rewards/code_reward": 0.5371007978916168,
      "step": 1750,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.15779653917998077,
      "clip_ratio/high_mean": 0.030520046106539668,
      "clip_ratio/low_mean": 0.009007267560809851,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03952731299214065,
      "completion_length": 77.64000091552734,
      "epoch": 0.3381039285371242,
      "grad_norm": 16.5263729095459,
      "kl": 0.7359155111014843,
      "learning_rate": 7.69488363458199e-07,
      "loss": -0.0085,
      "reward": 1.477712869644165,
      "reward_std": 0.26145162880420686,
      "rewards/code_format_reward": 0.993749988079071,
      "rewards/code_reward": 0.49041891694068906,
      "step": 1760,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.17542534926906228,
      "clip_ratio/high_mean": 0.025472976046148687,
      "clip_ratio/low_mean": 0.005083448148798198,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030556425044778734,
      "completion_length": 78.76000061035157,
      "epoch": 0.3400249735856306,
      "grad_norm": 2.440377950668335,
      "kl": 1.2570879265666008,
      "learning_rate": 7.671128379614524e-07,
      "loss": -0.0029,
      "reward": 1.697490382194519,
      "reward_std": 0.21552397906780243,
      "rewards/code_format_reward": 0.9887499809265137,
      "rewards/code_reward": 0.6015576839447021,
      "step": 1770,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.03777246242389083,
      "clip_ratio/high_mean": 0.005805602658074349,
      "clip_ratio/low_mean": 0.006219673785381019,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012025276734493672,
      "completion_length": 78.01500091552734,
      "epoch": 0.341946018634137,
      "grad_norm": 3.58803129196167,
      "kl": 1.3505164757370949,
      "learning_rate": 7.647294000442899e-07,
      "loss": -0.0008,
      "reward": 1.3937680006027222,
      "reward_std": 0.1832626909017563,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.44907149076461794,
      "step": 1780,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.08561003021895885,
      "clip_ratio/high_mean": 0.011109948102966883,
      "clip_ratio/low_mean": 0.0035756964149186387,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014685644480050542,
      "completion_length": 76.20749969482422,
      "epoch": 0.34386706368264336,
      "grad_norm": 10.503286361694336,
      "kl": 0.552098847925663,
      "learning_rate": 7.623381365682855e-07,
      "loss": -0.0015,
      "reward": 1.6644479036331177,
      "reward_std": 0.22849067896604539,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5847239375114441,
      "step": 1790,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.06243175007402897,
      "clip_ratio/high_mean": 0.009089326043613255,
      "clip_ratio/low_mean": 0.005161185140605084,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014250511419959366,
      "completion_length": 69.70000076293945,
      "epoch": 0.34578810873114973,
      "grad_norm": 4.685351371765137,
      "kl": 0.3103115826845169,
      "learning_rate": 7.599391346802063e-07,
      "loss": -0.0003,
      "reward": 1.8390909910202027,
      "reward_std": 0.20207120031118392,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6726704835891724,
      "step": 1800,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.046840774989686904,
      "clip_ratio/high_mean": 0.007519985581166111,
      "clip_ratio/low_mean": 0.004676173024927266,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01219615869631525,
      "completion_length": 80.15500183105469,
      "epoch": 0.3477091537796561,
      "grad_norm": 21886460.0,
      "kl": 0.48781016543507577,
      "learning_rate": 7.575324818088367e-07,
      "loss": 517.7405,
      "reward": 1.6558839797973632,
      "reward_std": 0.2796541228890419,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5845044732093811,
      "step": 1810,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.18512438922189176,
      "clip_ratio/high_mean": 0.0357341198658105,
      "clip_ratio/low_mean": 0.0033004880184307694,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03903460723813623,
      "completion_length": 78.84000091552734,
      "epoch": 0.34963019882816254,
      "grad_norm": 9.198795318603516,
      "kl": 4.244446061551571,
      "learning_rate": 7.551182656617924e-07,
      "loss": 0.0031,
      "reward": 1.5848650455474853,
      "reward_std": 0.17606763169169426,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.5458700299263001,
      "step": 1820,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.07551750033162534,
      "clip_ratio/high_mean": 0.013169253122759983,
      "clip_ratio/low_mean": 0.001537335959437769,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014706589409615844,
      "completion_length": 82.8800033569336,
      "epoch": 0.3515512438766689,
      "grad_norm": 0.724766731262207,
      "kl": 0.9274087265133858,
      "learning_rate": 7.526965742223234e-07,
      "loss": 0.0013,
      "reward": 1.5606717586517334,
      "reward_std": 0.2877893716096878,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5343983888626098,
      "step": 1830,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.1388332260772586,
      "clip_ratio/high_mean": 0.021653852658346295,
      "clip_ratio/low_mean": 0.008576209528837354,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03023006208240986,
      "completion_length": 74.26250076293945,
      "epoch": 0.3534722889251753,
      "grad_norm": 5.426670074462891,
      "kl": 0.7045004338026046,
      "learning_rate": 7.502674957461079e-07,
      "loss": -0.007,
      "reward": 1.5688656568527222,
      "reward_std": 0.30554552264511586,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5384953856468201,
      "step": 1840,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.07899991576559842,
      "clip_ratio/high_mean": 0.013301478006178513,
      "clip_ratio/low_mean": 0.01124582380289212,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024547301628626884,
      "completion_length": 74.45500106811524,
      "epoch": 0.35539333397368167,
      "grad_norm": 2.5104761123657227,
      "kl": 0.6198086604475975,
      "learning_rate": 7.478311187580363e-07,
      "loss": -0.0071,
      "reward": 1.5550098896026612,
      "reward_std": 0.21109988391399384,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.52937992811203,
      "step": 1850,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.063601403683424,
      "clip_ratio/high_mean": 0.010639100335538387,
      "clip_ratio/low_mean": 0.00778028266504407,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018419382628053427,
      "completion_length": 71.92500152587891,
      "epoch": 0.35731437902218804,
      "grad_norm": 3.805928945541382,
      "kl": 1.6179959252476692,
      "learning_rate": 7.453875320489842e-07,
      "loss": 0.3,
      "reward": 1.4410953760147094,
      "reward_std": 0.19501519501209258,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.47523519992828367,
      "step": 1860,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.10860318611375988,
      "clip_ratio/high_mean": 0.018746975070098416,
      "clip_ratio/low_mean": 0.008747255423804745,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02749423016794026,
      "completion_length": 69.9375015258789,
      "epoch": 0.3592354240706945,
      "grad_norm": 2.388782501220703,
      "kl": 0.5952992506325245,
      "learning_rate": 7.429368246725772e-07,
      "loss": 0.0443,
      "reward": 1.6972971916198731,
      "reward_std": 0.17401356399059295,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.6008361041545868,
      "step": 1870,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.08630747124552726,
      "clip_ratio/high_mean": 0.012746809562668205,
      "clip_ratio/low_mean": 0.010304910433478653,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02305172006599605,
      "completion_length": 70.83000183105469,
      "epoch": 0.36115646911920085,
      "grad_norm": 16.255178451538086,
      "kl": 0.8730347856879235,
      "learning_rate": 7.40479085941945e-07,
      "loss": 0.0036,
      "reward": 1.467816424369812,
      "reward_std": 0.17535984218120576,
      "rewards/code_format_reward": 0.9925000071525574,
      "rewards/code_reward": 0.48578319549560545,
      "step": 1880,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.26251301234588026,
      "clip_ratio/high_mean": 0.03827818045392632,
      "clip_ratio/low_mean": 0.005526873719645664,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04380505495937541,
      "completion_length": 64.74750213623047,
      "epoch": 0.3630775141677072,
      "grad_norm": 4.061140060424805,
      "kl": 0.8530658036470413,
      "learning_rate": 7.380144054264669e-07,
      "loss": 0.0197,
      "reward": 1.498781108856201,
      "reward_std": 0.17463037073612214,
      "rewards/code_format_reward": 0.9600000023841858,
      "rewards/code_reward": 0.509390527009964,
      "step": 1890,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.24850870491936802,
      "clip_ratio/high_mean": 0.04144583061570302,
      "clip_ratio/low_mean": 0.00702623330289498,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04847206423291937,
      "completion_length": 75.8375015258789,
      "epoch": 0.3649985592162136,
      "grad_norm": 3.4472062587738037,
      "kl": 1.6324397973716258,
      "learning_rate": 7.355428729485071e-07,
      "loss": -0.001,
      "reward": 1.6619214057922362,
      "reward_std": 0.18103656098246573,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.5840856909751893,
      "step": 1900,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.09173821061849594,
      "clip_ratio/high_mean": 0.014921509474515916,
      "clip_ratio/low_mean": 0.002157307107700035,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017078816797584294,
      "completion_length": 62.185000610351565,
      "epoch": 0.36691960426472003,
      "grad_norm": 2.0225422382354736,
      "kl": 184.02759787738324,
      "learning_rate": 7.330645785801417e-07,
      "loss": 2.9496,
      "reward": 1.7410502433776855,
      "reward_std": 0.10668236091732979,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.6217751204967499,
      "step": 1910,
      "zero_std_ratio": 0.75
    },
    {
      "clip_ratio/high_max": 0.16933906488120556,
      "clip_ratio/high_mean": 0.02619449864141643,
      "clip_ratio/low_mean": 0.014137339405715465,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04033183753490448,
      "completion_length": 79.30000152587891,
      "epoch": 0.3688406493132264,
      "grad_norm": 2.6208443641662598,
      "kl": 1.235317513346672,
      "learning_rate": 7.305796126398758e-07,
      "loss": -0.0012,
      "reward": 1.5036948204040528,
      "reward_std": 0.20645264089107512,
      "rewards/code_format_reward": 0.9762499928474426,
      "rewards/code_reward": 0.5077848553657531,
      "step": 1920,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.2661599090555683,
      "clip_ratio/high_mean": 0.03600101897318382,
      "clip_ratio/low_mean": 0.009155643907433841,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.045156662538647654,
      "completion_length": 78.10000152587891,
      "epoch": 0.3707616943617328,
      "grad_norm": 8.953734397888184,
      "kl": 0.6204134523868561,
      "learning_rate": 7.280880656893518e-07,
      "loss": 0.0025,
      "reward": 1.4915935516357421,
      "reward_std": 0.2376121073961258,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.501109266281128,
      "step": 1930,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.15203024838119744,
      "clip_ratio/high_mean": 0.023713350854814054,
      "clip_ratio/low_mean": 0.004282052081543952,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02799540273845196,
      "completion_length": 74.42500076293945,
      "epoch": 0.37268273941023916,
      "grad_norm": 11.845942497253418,
      "kl": 0.5031724810600281,
      "learning_rate": 7.255900285300496e-07,
      "loss": 0.5255,
      "reward": 1.6400779724121093,
      "reward_std": 0.22267285138368606,
      "rewards/code_format_reward": 0.9649999856948852,
      "rewards/code_reward": 0.5787889719009399,
      "step": 1940,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.09135808227583767,
      "clip_ratio/high_mean": 0.012801296508405358,
      "clip_ratio/low_mean": 0.01690869364247192,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02970999013632536,
      "completion_length": 69.52000198364257,
      "epoch": 0.37460378445874554,
      "grad_norm": 6.7441229820251465,
      "kl": 1.2024895504117012,
      "learning_rate": 7.230855921999769e-07,
      "loss": 44.3651,
      "reward": 1.6912511348724366,
      "reward_std": 0.17418113350868225,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5981255412101746,
      "step": 1950,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.07453169417567551,
      "clip_ratio/high_mean": 0.009913802641676739,
      "clip_ratio/low_mean": 0.003736039294744842,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013649841863662004,
      "completion_length": 74.01250228881835,
      "epoch": 0.37652482950725197,
      "grad_norm": 4.616723537445068,
      "kl": 0.6156632959842682,
      "learning_rate": 7.205748479703515e-07,
      "loss": -0.0005,
      "reward": 1.846400761604309,
      "reward_std": 0.17167636156082153,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6757004141807557,
      "step": 1960,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.09189570704475045,
      "clip_ratio/high_mean": 0.013587052945513278,
      "clip_ratio/low_mean": 0.004667519498616457,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0182545724324882,
      "completion_length": 64.46750030517578,
      "epoch": 0.37844587455575834,
      "grad_norm": 0.17748567461967468,
      "kl": 0.4286219261586666,
      "learning_rate": 7.180578873422757e-07,
      "loss": -0.0046,
      "reward": 1.612094521522522,
      "reward_std": 0.10822201184928418,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.5576097548007966,
      "step": 1970,
      "zero_std_ratio": 0.725
    },
    {
      "clip_ratio/high_max": 0.2088342323899269,
      "clip_ratio/high_mean": 0.028434151923283933,
      "clip_ratio/low_mean": 0.005974846053868532,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.034408997558057305,
      "completion_length": 69.26750106811524,
      "epoch": 0.3803669196042647,
      "grad_norm": 6.238914966583252,
      "kl": 0.7256933867931366,
      "learning_rate": 7.155348020434001e-07,
      "loss": -0.0046,
      "reward": 1.469704508781433,
      "reward_std": 0.24035734832286834,
      "rewards/code_format_reward": 0.9799999833106995,
      "rewards/code_reward": 0.4898522675037384,
      "step": 1980,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.05789460870437324,
      "clip_ratio/high_mean": 0.007717460609273985,
      "clip_ratio/low_mean": 0.003460834617726505,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011178295104764402,
      "completion_length": 70.19000244140625,
      "epoch": 0.3822879646527711,
      "grad_norm": 8.066108703613281,
      "kl": 1.1788517452776432,
      "learning_rate": 7.130056840245824e-07,
      "loss": -0.0005,
      "reward": 1.5026792764663697,
      "reward_std": 0.2312860034406185,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5022771418094635,
      "step": 1990,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.07602796133141965,
      "clip_ratio/high_mean": 0.012856367122731171,
      "clip_ratio/low_mean": 0.0035519548939191735,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016408322553616017,
      "completion_length": 66.4625015258789,
      "epoch": 0.38420900970127747,
      "grad_norm": 3.559206962585449,
      "kl": 1.225260878354311,
      "learning_rate": 7.104706254565358e-07,
      "loss": -0.003,
      "reward": 1.742388916015625,
      "reward_std": 0.12480423972010612,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.623069453239441,
      "step": 2000,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.11271043051965535,
      "clip_ratio/high_mean": 0.017727556044701488,
      "clip_ratio/low_mean": 0.005613272835034877,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02334082857705653,
      "completion_length": 77.1650001525879,
      "epoch": 0.3861300547497839,
      "grad_norm": 3.4077274799346924,
      "kl": 0.8489379599690438,
      "learning_rate": 7.07929718726469e-07,
      "loss": 0.0403,
      "reward": 1.5602745056152343,
      "reward_std": 0.2609230324625969,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.5338872492313385,
      "step": 2010,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.2993799396790564,
      "clip_ratio/high_mean": 0.041865267558023334,
      "clip_ratio/low_mean": 0.006948894041124731,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04881416228599846,
      "completion_length": 74.0150016784668,
      "epoch": 0.3880510997982903,
      "grad_norm": 3.2043685913085938,
      "kl": 6.086115422844887,
      "learning_rate": 7.053830564347206e-07,
      "loss": 2.2989,
      "reward": 1.5310536623001099,
      "reward_std": 0.19302123934030532,
      "rewards/code_format_reward": 0.9837500095367432,
      "rewards/code_reward": 0.5195893287658692,
      "step": 2020,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.06591402762569487,
      "clip_ratio/high_mean": 0.009311116795288399,
      "clip_ratio/low_mean": 0.0017412514251191169,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01105236830189824,
      "completion_length": 73.44250106811523,
      "epoch": 0.38997214484679665,
      "grad_norm": 2.137256622314453,
      "kl": 3.9139866441488267,
      "learning_rate": 7.028307313913838e-07,
      "loss": 0.0061,
      "reward": 1.8796703815460205,
      "reward_std": 0.12868851274251938,
      "rewards/code_format_reward": 0.9974999904632569,
      "rewards/code_reward": 0.6904601573944091,
      "step": 2030,
      "zero_std_ratio": 0.775
    },
    {
      "clip_ratio/high_max": 0.24738994101062417,
      "clip_ratio/high_mean": 0.03705689987400547,
      "clip_ratio/low_mean": 0.007423648721305654,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04448054819367826,
      "completion_length": 67.09500198364258,
      "epoch": 0.39189318989530303,
      "grad_norm": 5.504507541656494,
      "kl": 1.4878595262765884,
      "learning_rate": 7.002728366129242e-07,
      "loss": 0.0166,
      "reward": 1.8640715599060058,
      "reward_std": 0.22610510736703873,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.6870357990264893,
      "step": 2040,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.09283696161583066,
      "clip_ratio/high_mean": 0.014592013147193938,
      "clip_ratio/low_mean": 0.0040809189551509915,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01867293259128928,
      "completion_length": 72.88500137329102,
      "epoch": 0.3938142349438094,
      "grad_norm": 1.877032995223999,
      "kl": 2.3534633785486223,
      "learning_rate": 6.977094653187891e-07,
      "loss": 0.3364,
      "reward": 1.5182712078094482,
      "reward_std": 0.19934598058462144,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5163230776786805,
      "step": 2050,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.047755516692996026,
      "clip_ratio/high_mean": 0.007312651420943439,
      "clip_ratio/low_mean": 0.0007527987050707452,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008065450168214739,
      "completion_length": 67.76500091552734,
      "epoch": 0.39573527999231584,
      "grad_norm": 1.7954281568527222,
      "kl": 2.4329017847776413,
      "learning_rate": 6.95140710928012e-07,
      "loss": 206.5648,
      "reward": 1.3761554956436157,
      "reward_std": 0.21033956706523896,
      "rewards/code_format_reward": 0.9762499928474426,
      "rewards/code_reward": 0.44401525855064394,
      "step": 2060,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.07075442476198077,
      "clip_ratio/high_mean": 0.009443573304452002,
      "clip_ratio/low_mean": 0.003901358728762716,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013344932324253022,
      "completion_length": 68.6150016784668,
      "epoch": 0.3976563250408222,
      "grad_norm": 1.3921815156936646,
      "kl": 0.6283935949206352,
      "learning_rate": 6.925666670558062e-07,
      "loss": 1.5274,
      "reward": 1.4756604433059692,
      "reward_std": 0.2542987480759621,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.49189271330833434,
      "step": 2070,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.09334485791623592,
      "clip_ratio/high_mean": 0.015712386509403587,
      "clip_ratio/low_mean": 0.005205962993204594,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02091834945604205,
      "completion_length": 75.06750183105468,
      "epoch": 0.3995773700893286,
      "grad_norm": 1.3997697830200195,
      "kl": 0.5330163806676864,
      "learning_rate": 6.899874275101538e-07,
      "loss": -0.0031,
      "reward": 1.7522424459457397,
      "reward_std": 0.1803124487400055,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.6286212205886841,
      "step": 2080,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.08916364926844836,
      "clip_ratio/high_mean": 0.014017748599871992,
      "clip_ratio/low_mean": 0.003948131998186,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01796588017605245,
      "completion_length": 78.19750213623047,
      "epoch": 0.40149841513783496,
      "grad_norm": 2296.336669921875,
      "kl": 1.0256180852651595,
      "learning_rate": 6.874030862883879e-07,
      "loss": 0.0318,
      "reward": 1.2450440883636475,
      "reward_std": 0.22890471369028093,
      "rewards/code_format_reward": 0.9775000095367432,
      "rewards/code_reward": 0.3781470343470573,
      "step": 2090,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.22327043637633323,
      "clip_ratio/high_mean": 0.04789549903944135,
      "clip_ratio/low_mean": 0.00559167112223804,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.053487171232700345,
      "completion_length": 70.61250152587891,
      "epoch": 0.4034194601863414,
      "grad_norm": 3.2615253925323486,
      "kl": 8.218332803249359,
      "learning_rate": 6.848137375737652e-07,
      "loss": 0.0058,
      "reward": 1.6430699110031128,
      "reward_std": 0.21420457661151887,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.5793474376201629,
      "step": 2100,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.1533806946128607,
      "clip_ratio/high_mean": 0.02256658235564828,
      "clip_ratio/low_mean": 0.002787484592408873,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025354066491127016,
      "completion_length": 74.33999938964844,
      "epoch": 0.40534050523484777,
      "grad_norm": 4.315516471862793,
      "kl": 1.0426696628332137,
      "learning_rate": 6.822194757320354e-07,
      "loss": 0.0019,
      "reward": 1.6090970516204834,
      "reward_std": 0.1758709292858839,
      "rewards/code_format_reward": 0.993749988079071,
      "rewards/code_reward": 0.5561110019683838,
      "step": 2110,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.1336930485442281,
      "clip_ratio/high_mean": 0.021989132883027195,
      "clip_ratio/low_mean": 0.0070218192064203325,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0290109527297318,
      "completion_length": 73.0250015258789,
      "epoch": 0.40726155028335415,
      "grad_norm": 18.143117904663086,
      "kl": 0.4288759011775255,
      "learning_rate": 6.796203953080007e-07,
      "loss": 0.0005,
      "reward": 1.72017080783844,
      "reward_std": 0.22243313789367675,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6144603788852692,
      "step": 2120,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.08061643750406802,
      "clip_ratio/high_mean": 0.011467291257577016,
      "clip_ratio/low_mean": 0.011395246715983376,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022862538695335388,
      "completion_length": 68.66250152587891,
      "epoch": 0.4091825953318605,
      "grad_norm": 1.0005404949188232,
      "kl": 0.47304695919156076,
      "learning_rate": 6.770165910220709e-07,
      "loss": 0.0006,
      "reward": 1.4831626653671264,
      "reward_std": 0.1916220799088478,
      "rewards/code_format_reward": 0.9837499856948853,
      "rewards/code_reward": 0.4956438183784485,
      "step": 2130,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.06393485199660062,
      "clip_ratio/high_mean": 0.011905963439494372,
      "clip_ratio/low_mean": 0.0023792986408807336,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01428526220843196,
      "completion_length": 74.32250137329102,
      "epoch": 0.4111036403803669,
      "grad_norm": 2.491830825805664,
      "kl": 2.213325946778059,
      "learning_rate": 6.744081577668115e-07,
      "loss": 0.1532,
      "reward": 1.7680244207382203,
      "reward_std": 0.18317916095256806,
      "rewards/code_format_reward": 0.9687499880790711,
      "rewards/code_reward": 0.6418246865272522,
      "step": 2140,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.03965856842696667,
      "clip_ratio/high_mean": 0.00730013819411397,
      "clip_ratio/low_mean": 0.0031650666729547083,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010465204739011824,
      "completion_length": 73.1050018310547,
      "epoch": 0.41302468542887333,
      "grad_norm": 0.353427916765213,
      "kl": 0.2898652456700802,
      "learning_rate": 6.717951906034856e-07,
      "loss": -0.0015,
      "reward": 1.6113624095916748,
      "reward_std": 0.09930019937455654,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5591186702251434,
      "step": 2150,
      "zero_std_ratio": 0.725
    },
    {
      "clip_ratio/high_max": 0.03382167350500822,
      "clip_ratio/high_mean": 0.005409902473911643,
      "clip_ratio/low_mean": 0.0024156818573828785,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007825584337115287,
      "completion_length": 68.12750091552735,
      "epoch": 0.4149457304773797,
      "grad_norm": 3.9950575828552246,
      "kl": 0.789361334592104,
      "learning_rate": 6.691777847585883e-07,
      "loss": 0.048,
      "reward": 1.5698497295379639,
      "reward_std": 0.1552659712731838,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.5417998552322387,
      "step": 2160,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.026565033989027143,
      "clip_ratio/high_mean": 0.004212364956038073,
      "clip_ratio/low_mean": 0.0013839059392921627,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005596270889509469,
      "completion_length": 70.80999984741212,
      "epoch": 0.4168667755258861,
      "grad_norm": 1.3910998106002808,
      "kl": 1.4257395297288895,
      "learning_rate": 6.665560356203784e-07,
      "loss": 0.8731,
      "reward": 1.4512264728546143,
      "reward_std": 0.14117379933595658,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.47748821377754214,
      "step": 2170,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.09546168451197445,
      "clip_ratio/high_mean": 0.01459201174438931,
      "clip_ratio/low_mean": 0.006060798710677773,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020652810629690065,
      "completion_length": 67.89000091552734,
      "epoch": 0.41878782057439246,
      "grad_norm": 0.6732813715934753,
      "kl": 1.1321026906371117,
      "learning_rate": 6.639300387353999e-07,
      "loss": -0.0002,
      "reward": 1.3501636981964111,
      "reward_std": 0.21670444533228875,
      "rewards/code_format_reward": 0.9924999833106994,
      "rewards/code_reward": 0.42695685029029845,
      "step": 2180,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.11407874876167626,
      "clip_ratio/high_mean": 0.01725804756570142,
      "clip_ratio/low_mean": 0.0015681478500482627,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018826195126166567,
      "completion_length": 68.7525016784668,
      "epoch": 0.42070886562289883,
      "grad_norm": 1.5759879350662231,
      "kl": 0.4211964398622513,
      "learning_rate": 6.612998898050014e-07,
      "loss": -0.0021,
      "reward": 1.7485667228698731,
      "reward_std": 0.16526954025030136,
      "rewards/code_format_reward": 0.9612500071525574,
      "rewards/code_reward": 0.6339708626270294,
      "step": 2190,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.10751554854214192,
      "clip_ratio/high_mean": 0.013745604571886361,
      "clip_ratio/low_mean": 0.010064921525190585,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023810526612214743,
      "completion_length": 62.88500137329102,
      "epoch": 0.42262991067140526,
      "grad_norm": 2.4066975116729736,
      "kl": 0.7549011036753654,
      "learning_rate": 6.586656846818477e-07,
      "loss": 0.2999,
      "reward": 1.6932018756866456,
      "reward_std": 0.1608109436929226,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5991009473800659,
      "step": 2200,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.019488381547853352,
      "clip_ratio/high_mean": 0.003436583065195009,
      "clip_ratio/low_mean": 0.002801175639615394,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0062377589056268334,
      "completion_length": 72.55250244140625,
      "epoch": 0.42455095571991164,
      "grad_norm": 2.0696611404418945,
      "kl": 5.306586292386055,
      "learning_rate": 6.56027519366427e-07,
      "loss": 0.011,
      "reward": 1.611876368522644,
      "reward_std": 0.1603232156485319,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.5596881568431854,
      "step": 2210,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.05246827639639377,
      "clip_ratio/high_mean": 0.00732308179140091,
      "clip_ratio/low_mean": 0.0034836977836675944,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010806779406266287,
      "completion_length": 64.31750183105468,
      "epoch": 0.426472000768418,
      "grad_norm": 0.12577353417873383,
      "kl": 0.5850224502384662,
      "learning_rate": 6.533854900035516e-07,
      "loss": -0.0015,
      "reward": 1.7735862731933594,
      "reward_std": 0.13040905613452197,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6399181246757507,
      "step": 2220,
      "zero_std_ratio": 0.7
    },
    {
      "clip_ratio/high_max": 0.24315445288084447,
      "clip_ratio/high_mean": 0.031706276966724546,
      "clip_ratio/low_mean": 0.011593326600268484,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04329960328759626,
      "completion_length": 72.80000152587891,
      "epoch": 0.4283930458169244,
      "grad_norm": 4.765016078948975,
      "kl": 1.5887107208371163,
      "learning_rate": 6.507396928788548e-07,
      "loss": 0.0023,
      "reward": 1.6477301597595215,
      "reward_std": 0.12887158915400504,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5801151037216187,
      "step": 2230,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.044854282308369874,
      "clip_ratio/high_mean": 0.007485381804872304,
      "clip_ratio/low_mean": 0.0028356918206554837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01032107327482663,
      "completion_length": 66.63000183105468,
      "epoch": 0.4303140908654308,
      "grad_norm": 1.5923104286193848,
      "kl": 0.9431760296225548,
      "learning_rate": 6.480902244152813e-07,
      "loss": -0.0021,
      "reward": 1.4723083972930908,
      "reward_std": 0.13776133116334677,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.48865418434143065,
      "step": 2240,
      "zero_std_ratio": 0.7
    },
    {
      "clip_ratio/high_max": 0.08558401605114341,
      "clip_ratio/high_mean": 0.01418596402509138,
      "clip_ratio/low_mean": 0.005716345021210145,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019902308681048454,
      "completion_length": 67.75250015258788,
      "epoch": 0.4322351359139372,
      "grad_norm": 4.213563442230225,
      "kl": 0.7182839468121529,
      "learning_rate": 6.454371811695732e-07,
      "loss": -0.0032,
      "reward": 1.5263491868972778,
      "reward_std": 0.215225650370121,
      "rewards/code_format_reward": 0.975000011920929,
      "rewards/code_reward": 0.51942458152771,
      "step": 2250,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.17924602022394537,
      "clip_ratio/high_mean": 0.02314122476382181,
      "clip_ratio/low_mean": 0.006780697987414897,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029921922995708884,
      "completion_length": 67.31500091552735,
      "epoch": 0.43415618096244357,
      "grad_norm": 2.018653392791748,
      "kl": 0.644180704653263,
      "learning_rate": 6.427806598287522e-07,
      "loss": -0.0031,
      "reward": 1.8284268617630004,
      "reward_std": 0.1590463936328888,
      "rewards/code_format_reward": 0.993749988079071,
      "rewards/code_reward": 0.6657759308815002,
      "step": 2260,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.26562999933958054,
      "clip_ratio/high_mean": 0.04080731603316963,
      "clip_ratio/low_mean": 0.002605196795775555,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.043412512401118875,
      "completion_length": 64.91250076293946,
      "epoch": 0.43607722601094995,
      "grad_norm": 2.8014633655548096,
      "kl": 1.4193657219409943,
      "learning_rate": 6.401207572065942e-07,
      "loss": 0.0075,
      "reward": 1.6795406818389893,
      "reward_std": 0.1340640414506197,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.5913328170776367,
      "step": 2270,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.15035496577620505,
      "clip_ratio/high_mean": 0.021555275144055485,
      "clip_ratio/low_mean": 0.007308500797080342,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028863775311037898,
      "completion_length": 83.20750122070312,
      "epoch": 0.4379982710594563,
      "grad_norm": 5.3116655349731445,
      "kl": 1.7165004715323449,
      "learning_rate": 6.374575702401019e-07,
      "loss": -0.0031,
      "reward": 1.694450354576111,
      "reward_std": 0.2935485541820526,
      "rewards/code_format_reward": 0.9650000095367431,
      "rewards/code_reward": 0.6059751749038697,
      "step": 2280,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.05178634703624994,
      "clip_ratio/high_mean": 0.007199086344917305,
      "clip_ratio/low_mean": 0.004959188599605114,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012158275028923526,
      "completion_length": 68.35500106811523,
      "epoch": 0.43991931610796275,
      "grad_norm": 11.67419719696045,
      "kl": 0.8460408747196198,
      "learning_rate": 6.347911959859725e-07,
      "loss": -0.0013,
      "reward": 1.6080287456512452,
      "reward_std": 0.2270718976855278,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.5615143775939941,
      "step": 2290,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.07622572851832957,
      "clip_ratio/high_mean": 0.011604995708330535,
      "clip_ratio/low_mean": 0.0013341609621420503,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012939156647189521,
      "completion_length": 68.30750274658203,
      "epoch": 0.44184036115646913,
      "grad_norm": 332.7762451171875,
      "kl": 0.7540152728557586,
      "learning_rate": 6.321217316170599e-07,
      "loss": 0.1015,
      "reward": 1.4850183725357056,
      "reward_std": 0.1393202841281891,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.49469670057296755,
      "step": 2300,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.16746083926409483,
      "clip_ratio/high_mean": 0.02103413282893598,
      "clip_ratio/low_mean": 0.0068577720652683635,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.027891904639545828,
      "completion_length": 64.55000152587891,
      "epoch": 0.4437614062049755,
      "grad_norm": 0.36056017875671387,
      "kl": 0.4329931303858757,
      "learning_rate": 6.294492744188335e-07,
      "loss": 0.0002,
      "reward": 1.4963040232658387,
      "reward_std": 0.07247132882475853,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.502214539051056,
      "step": 2310,
      "zero_std_ratio": 0.725
    },
    {
      "clip_ratio/high_max": 0.05429213300812989,
      "clip_ratio/high_mean": 0.007803994990536012,
      "clip_ratio/low_mean": 0.008226435555843636,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01603043078503106,
      "completion_length": 69.78750228881836,
      "epoch": 0.4456824512534819,
      "grad_norm": 0.1676941215991974,
      "kl": 0.276796979829669,
      "learning_rate": 6.267739217858329e-07,
      "loss": -0.0028,
      "reward": 1.7269956827163697,
      "reward_std": 0.1742506742477417,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.6156853199005127,
      "step": 2320,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.03961263382807374,
      "clip_ratio/high_mean": 0.00831791803939268,
      "clip_ratio/low_mean": 0.008615480939624831,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016933399019762874,
      "completion_length": 70.70000228881835,
      "epoch": 0.44760349630198826,
      "grad_norm": 6.724217891693115,
      "kl": 0.544577070325613,
      "learning_rate": 6.240957712181186e-07,
      "loss": -0.0041,
      "reward": 1.3949034690856934,
      "reward_std": 0.21950918734073638,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.45307670831680297,
      "step": 2330,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.2168802363506984,
      "clip_ratio/high_mean": 0.03705684195374488,
      "clip_ratio/low_mean": 0.0028890643618069587,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03994590537622571,
      "completion_length": 74.0625,
      "epoch": 0.4495245413504947,
      "grad_norm": 3.073554277420044,
      "kl": 0.6307030320167542,
      "learning_rate": 6.214149203177182e-07,
      "loss": -0.0002,
      "reward": 1.679004979133606,
      "reward_std": 0.1860196329653263,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.5916899800300598,
      "step": 2340,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.0932310588657856,
      "clip_ratio/high_mean": 0.014429462677799165,
      "clip_ratio/low_mean": 0.0065534046734683216,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020982867432758213,
      "completion_length": 67.18000183105468,
      "epoch": 0.45144558639900106,
      "grad_norm": 3595.65576171875,
      "kl": 1.140541896224022,
      "learning_rate": 6.187314667850697e-07,
      "loss": 0.1447,
      "reward": 1.4676954984664916,
      "reward_std": 0.20568167939782142,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.4875977456569672,
      "step": 2350,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.036702672578394414,
      "clip_ratio/high_mean": 0.006752843782305717,
      "clip_ratio/low_mean": 0.008269340678816661,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015022184286499396,
      "completion_length": 80.57000122070312,
      "epoch": 0.45336663144750744,
      "grad_norm": 2.759171724319458,
      "kl": 10.568821829557418,
      "learning_rate": 6.160455084154613e-07,
      "loss": 1.8532,
      "reward": 1.4545687198638917,
      "reward_std": 0.23069845288991928,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.4813468337059021,
      "step": 2360,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.08144733654335141,
      "clip_ratio/high_mean": 0.014263840962667019,
      "clip_ratio/low_mean": 0.0019872021744959056,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01625104369595647,
      "completion_length": 71.55750045776367,
      "epoch": 0.4552876764960138,
      "grad_norm": 1.9088038206100464,
      "kl": 1.3571255028247833,
      "learning_rate": 6.133571430954667e-07,
      "loss": 0.0026,
      "reward": 1.5344175338745116,
      "reward_std": 0.16607576459646226,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.5237712502479553,
      "step": 2370,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.10026950519531966,
      "clip_ratio/high_mean": 0.01315019663888961,
      "clip_ratio/low_mean": 0.00221524270309601,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015365439187735318,
      "completion_length": 72.40750122070312,
      "epoch": 0.4572087215445202,
      "grad_norm": 4.301158428192139,
      "kl": 0.6290791854262352,
      "learning_rate": 6.106664687993782e-07,
      "loss": -0.0032,
      "reward": 1.5749263525009156,
      "reward_std": 0.16429235637187958,
      "rewards/code_format_reward": 0.9724999785423278,
      "rewards/code_reward": 0.5443381488323211,
      "step": 2380,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.10308512919582427,
      "clip_ratio/high_mean": 0.016338009486207738,
      "clip_ratio/low_mean": 0.0017032683303114028,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018041277857264504,
      "completion_length": 76.43750228881837,
      "epoch": 0.4591297665930266,
      "grad_norm": 6.198258876800537,
      "kl": 408884378.2116049,
      "learning_rate": 6.079735835856362e-07,
      "loss": 1157747.0,
      "reward": 1.5280384778976441,
      "reward_std": 0.19424125757068395,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.517456728219986,
      "step": 2390,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.24319633510895072,
      "clip_ratio/high_mean": 0.037530579004669565,
      "clip_ratio/low_mean": 0.004886501970031531,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04241708111949265,
      "completion_length": 74.5425018310547,
      "epoch": 0.461050811641533,
      "grad_norm": 5.885474681854248,
      "kl": 1.4351533338427545,
      "learning_rate": 6.052785855932548e-07,
      "loss": 0.123,
      "reward": 1.4949720859527589,
      "reward_std": 0.20392217636108398,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.5002985119819641,
      "step": 2400,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.23103972100652753,
      "clip_ratio/high_mean": 0.0305588347138837,
      "clip_ratio/low_mean": 0.002339675696566701,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03289851036388427,
      "completion_length": 70.01750106811524,
      "epoch": 0.4629718566900394,
      "grad_norm": 0.8806352615356445,
      "kl": 1.6503019407391548,
      "learning_rate": 6.025815730382463e-07,
      "loss": 0.8832,
      "reward": 1.6588483333587647,
      "reward_std": 0.19124363958835602,
      "rewards/code_format_reward": 0.9725000143051148,
      "rewards/code_reward": 0.5862991452217102,
      "step": 2410,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.02757756725186482,
      "clip_ratio/high_mean": 0.005332520017691422,
      "clip_ratio/low_mean": 0.019763218611478804,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025095738274103496,
      "completion_length": 71.59250183105469,
      "epoch": 0.46489290173854575,
      "grad_norm": 1.2440141439437866,
      "kl": 2.751401698589325,
      "learning_rate": 5.998826442100412e-07,
      "loss": 362174.725,
      "reward": 1.5159764885902405,
      "reward_std": 0.1902527991682291,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.5129882216453552,
      "step": 2420,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.2530499072512612,
      "clip_ratio/high_mean": 0.03376921496528666,
      "clip_ratio/low_mean": 0.0062858725665137175,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04005508716509212,
      "completion_length": 76.0425018310547,
      "epoch": 0.4668139467870522,
      "grad_norm": 66.4449234008789,
      "kl": 2164149.3861157326,
      "learning_rate": 5.971818974679065e-07,
      "loss": 2449736.0,
      "reward": 1.6650853157043457,
      "reward_std": 0.24712301939725875,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.585355132818222,
      "step": 2430,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.2950001623481512,
      "clip_ratio/high_mean": 0.042542998865246776,
      "clip_ratio/low_mean": 0.0068845050991512835,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.049427504313644025,
      "completion_length": 75.27000198364257,
      "epoch": 0.46873499183555856,
      "grad_norm": 2.206911563873291,
      "kl": 11.237105096876622,
      "learning_rate": 5.944794312373607e-07,
      "loss": 0.0298,
      "reward": 1.7914002895355225,
      "reward_std": 0.22826257348060608,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.649450159072876,
      "step": 2440,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.07424517879262567,
      "clip_ratio/high_mean": 0.010772422759328038,
      "clip_ratio/low_mean": 0.010833968574297614,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02160639046342112,
      "completion_length": 71.40500183105469,
      "epoch": 0.47065603688406493,
      "grad_norm": 76503500980224.0,
      "kl": 393.06428125053645,
      "learning_rate": 5.917753440065869e-07,
      "loss": 909725593.6,
      "reward": 1.4975883960723877,
      "reward_std": 0.28928079828619957,
      "rewards/code_format_reward": 0.9612499833106994,
      "rewards/code_reward": 0.5084816813468933,
      "step": 2450,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.14294563261792065,
      "clip_ratio/high_mean": 0.019953654275741427,
      "clip_ratio/low_mean": 0.004493102640844881,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024446757195983083,
      "completion_length": 79.72250061035156,
      "epoch": 0.4725770819325713,
      "grad_norm": 0.778223991394043,
      "kl": 2.2069298341870307,
      "learning_rate": 5.89069734322844e-07,
      "loss": -0.0085,
      "reward": 1.5203648328781127,
      "reward_std": 0.1896197520196438,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5173698782920837,
      "step": 2460,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.0473501511849463,
      "clip_ratio/high_mean": 0.006591684772865846,
      "clip_ratio/low_mean": 0.0004718510695965961,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007063536025816575,
      "completion_length": 76.70500183105469,
      "epoch": 0.4744981269810777,
      "grad_norm": 0.5978448390960693,
      "kl": 0.6427325546741486,
      "learning_rate": 5.863627007888745e-07,
      "loss": 0.0007,
      "reward": 1.7259918212890626,
      "reward_std": 0.1515914086252451,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.618620878458023,
      "step": 2470,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.048674007039517166,
      "clip_ratio/high_mean": 0.010258768184576184,
      "clip_ratio/low_mean": 0.012727768435433972,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022986536961980164,
      "completion_length": 78.06500244140625,
      "epoch": 0.4764191720295841,
      "grad_norm": 4.168500900268555,
      "kl": 0.5699560895562172,
      "learning_rate": 5.836543420593119e-07,
      "loss": -0.0011,
      "reward": 1.6060274362564086,
      "reward_std": 0.2864475339651108,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.557388699054718,
      "step": 2480,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.014699839102104307,
      "clip_ratio/high_mean": 0.0019118846452329309,
      "clip_ratio/low_mean": 0.0005179177765967325,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0024298023723531514,
      "completion_length": 85.17000122070313,
      "epoch": 0.4783402170780905,
      "grad_norm": 4.149423599243164,
      "kl": 1.3347756370902062,
      "learning_rate": 5.809447568370843e-07,
      "loss": 0.0102,
      "reward": 1.621114158630371,
      "reward_std": 0.21484595835208892,
      "rewards/code_format_reward": 0.9774999856948853,
      "rewards/code_reward": 0.5661820948123932,
      "step": 2490,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.029943293944234027,
      "clip_ratio/high_mean": 0.006927184848609613,
      "clip_ratio/low_mean": 0.0035072380007477475,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010434422721300508,
      "completion_length": 83.86250228881836,
      "epoch": 0.48026126212659687,
      "grad_norm": 5.97049617767334,
      "kl": 4.178053397685289,
      "learning_rate": 5.782340438698185e-07,
      "loss": -0.0063,
      "reward": 1.6789068222045898,
      "reward_std": 0.25779220163822175,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5903908908367157,
      "step": 2500,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.08102847400587052,
      "clip_ratio/high_mean": 0.01392527524731122,
      "clip_ratio/low_mean": 0.0045509199095249645,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018476195022230968,
      "completion_length": 83.11250152587891,
      "epoch": 0.48218230717510324,
      "grad_norm": 5.283038139343262,
      "kl": 1.111867392808199,
      "learning_rate": 5.755223019462401e-07,
      "loss": 17.941,
      "reward": 1.577300524711609,
      "reward_std": 0.22725088596343995,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5442752420902253,
      "step": 2510,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.06463829884305597,
      "clip_ratio/high_mean": 0.008858744835015387,
      "clip_ratio/low_mean": 0.0054666692391037944,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01432541401591152,
      "completion_length": 85.7625015258789,
      "epoch": 0.4841033522236096,
      "grad_norm": 8.200135231018066,
      "kl": 0.4475974731147289,
      "learning_rate": 5.728096298925745e-07,
      "loss": -0.0057,
      "reward": 1.5549763917922974,
      "reward_std": 0.23400793820619584,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5331131994724274,
      "step": 2520,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.031378739466890695,
      "clip_ratio/high_mean": 0.00507326218066737,
      "clip_ratio/low_mean": 0.010504274675622583,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015577536821365357,
      "completion_length": 79.96750030517578,
      "epoch": 0.48602439727211605,
      "grad_norm": 2.6766583919525146,
      "kl": 0.4622874528169632,
      "learning_rate": 5.700961265689434e-07,
      "loss": -0.0011,
      "reward": 1.8167934179306031,
      "reward_std": 0.30146218538284303,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.6621467113494873,
      "step": 2530,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.07196793179027736,
      "clip_ratio/high_mean": 0.013576928357360884,
      "clip_ratio/low_mean": 0.0018521397636504845,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015429067742661572,
      "completion_length": 87.03000030517578,
      "epoch": 0.4879454423206224,
      "grad_norm": 1.347899317741394,
      "kl": 0.7047492057085037,
      "learning_rate": 5.673818908657644e-07,
      "loss": -0.0079,
      "reward": 1.6893932342529296,
      "reward_std": 0.24144218415021895,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.5981341004371643,
      "step": 2540,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.03861275149974972,
      "clip_ratio/high_mean": 0.005072360605117865,
      "clip_ratio/low_mean": 0.0013027720240643248,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0063751326058991255,
      "completion_length": 78.80750122070313,
      "epoch": 0.4898664873691288,
      "grad_norm": 1.7946380376815796,
      "kl": 0.7765734851360321,
      "learning_rate": 5.646670217001451e-07,
      "loss": 0.004,
      "reward": 1.8638887882232666,
      "reward_std": 0.1732952728867531,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.6831943988800049,
      "step": 2550,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.044772564456798135,
      "clip_ratio/high_mean": 0.008199371959199198,
      "clip_ratio/low_mean": 0.007188984929234721,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015388356836047024,
      "completion_length": 92.03750305175781,
      "epoch": 0.4917875324176352,
      "grad_norm": 8241.9619140625,
      "kl": 3.7090125039219854,
      "learning_rate": 5.619516180122789e-07,
      "loss": 0.2194,
      "reward": 1.346347188949585,
      "reward_std": 0.3114967554807663,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.4303610801696777,
      "step": 2560,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.136108908476308,
      "clip_ratio/high_mean": 0.01777363264700398,
      "clip_ratio/low_mean": 0.0005986301795928739,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018372263037599625,
      "completion_length": 77.66500091552734,
      "epoch": 0.4937085774661416,
      "grad_norm": 2.8724048137664795,
      "kl": 0.30402788892388344,
      "learning_rate": 5.592357787618398e-07,
      "loss": -0.0095,
      "reward": 1.235116672515869,
      "reward_std": 0.16121466904878617,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.3706833332777023,
      "step": 2570,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.21411730786785482,
      "clip_ratio/high_mean": 0.02751181152416393,
      "clip_ratio/low_mean": 0.005141469169757329,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03265328073175624,
      "completion_length": 77.39250030517579,
      "epoch": 0.495629622514648,
      "grad_norm": 3.1119463443756104,
      "kl": 0.516096468269825,
      "learning_rate": 5.565196029243746e-07,
      "loss": -0.0097,
      "reward": 1.7056148529052735,
      "reward_std": 0.26717675626277926,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.6065573751926422,
      "step": 2580,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.06655758274719119,
      "clip_ratio/high_mean": 0.00869752592407167,
      "clip_ratio/low_mean": 0.0007154849590733647,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009413010929711163,
      "completion_length": 78.71000061035156,
      "epoch": 0.49755066756315436,
      "grad_norm": 9.566883087158203,
      "kl": 6.985853771865368,
      "learning_rate": 5.538031894876971e-07,
      "loss": 0.0154,
      "reward": 1.8047074317932128,
      "reward_std": 0.2406391829252243,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.655791187286377,
      "step": 2590,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.04153572088107467,
      "clip_ratio/high_mean": 0.0076960999285802245,
      "clip_ratio/low_mean": 0.00053562533139484,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008231725194491446,
      "completion_length": 87.35249938964844,
      "epoch": 0.49947171261166073,
      "grad_norm": 4.163487911224365,
      "kl": 3.02228729724884,
      "learning_rate": 5.510866374482799e-07,
      "loss": 0.0014,
      "reward": 1.7271404266357422,
      "reward_std": 0.20059744864702225,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6170076906681061,
      "step": 2600,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.09242721796035766,
      "clip_ratio/high_mean": 0.01333312913775444,
      "clip_ratio/low_mean": 0.0022988114287727512,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01563194077461958,
      "completion_length": 86.66250152587891,
      "epoch": 0.5013927576601671,
      "grad_norm": 1.7816847562789917,
      "kl": 2.135231140255928,
      "learning_rate": 5.48370045807647e-07,
      "loss": -0.0043,
      "reward": 1.5687429666519166,
      "reward_std": 0.22490316033363342,
      "rewards/code_format_reward": 0.9524999976158142,
      "rewards/code_reward": 0.5462464988231659,
      "step": 2610,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.10432031177915632,
      "clip_ratio/high_mean": 0.01704162026871927,
      "clip_ratio/low_mean": 0.0019667694039526397,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01900838967994787,
      "completion_length": 98.58000183105469,
      "epoch": 0.5033138027086735,
      "grad_norm": 2.1069369316101074,
      "kl": 2.131927290558815,
      "learning_rate": 5.456535135687656e-07,
      "loss": -0.0069,
      "reward": 1.6628828048706055,
      "reward_std": 0.23133169412612914,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.5858163475990296,
      "step": 2620,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.04196612173691392,
      "clip_ratio/high_mean": 0.0064837948535569016,
      "clip_ratio/low_mean": 0.0025595034239813685,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009043298207689076,
      "completion_length": 86.6,
      "epoch": 0.5052348477571799,
      "grad_norm": 15.670801162719727,
      "kl": 2.1330361180007458,
      "learning_rate": 5.429371397324378e-07,
      "loss": -0.0054,
      "reward": 1.4884859561920165,
      "reward_std": 0.3388957381248474,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.497055447101593,
      "step": 2630,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.036724881688132885,
      "clip_ratio/high_mean": 0.005309284973191097,
      "clip_ratio/low_mean": 0.003665669827023521,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00897495478275232,
      "completion_length": 84.73250122070313,
      "epoch": 0.5071558928056863,
      "grad_norm": 6.480928421020508,
      "kl": 0.9241176024079323,
      "learning_rate": 5.402210232936934e-07,
      "loss": -0.0009,
      "reward": 1.792254877090454,
      "reward_std": 0.29597480297088624,
      "rewards/code_format_reward": 0.9974999904632569,
      "rewards/code_reward": 0.646752405166626,
      "step": 2640,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.109601711621508,
      "clip_ratio/high_mean": 0.015215938963228837,
      "clip_ratio/low_mean": 0.0034228902019094675,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018638829072006046,
      "completion_length": 88.82750244140625,
      "epoch": 0.5090769378541927,
      "grad_norm": 5.080334186553955,
      "kl": 0.6404796183109284,
      "learning_rate": 5.37505263238181e-07,
      "loss": -0.0032,
      "reward": 1.7266260623931884,
      "reward_std": 0.27733459770679475,
      "rewards/code_format_reward": 0.993749988079071,
      "rewards/code_reward": 0.6148755311965942,
      "step": 2650,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.0589832967845723,
      "clip_ratio/high_mean": 0.009531341239926406,
      "clip_ratio/low_mean": 0.00046608211705461143,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00999742336862255,
      "completion_length": 88.69250183105468,
      "epoch": 0.510997982902699,
      "grad_norm": 7.949027061462402,
      "kl": 0.6428510576486588,
      "learning_rate": 5.347899585385619e-07,
      "loss": -0.0028,
      "reward": 1.8208046436309815,
      "reward_std": 0.32592435777187345,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6647772669792176,
      "step": 2660,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.15465617645531893,
      "clip_ratio/high_mean": 0.022943795099854468,
      "clip_ratio/low_mean": 0.0016213681577937678,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024565163621446118,
      "completion_length": 87.13250274658203,
      "epoch": 0.5129190279512055,
      "grad_norm": 34.059959411621094,
      "kl": 0.5654895901679993,
      "learning_rate": 5.320752081509019e-07,
      "loss": -0.0048,
      "reward": 1.7013320207595826,
      "reward_std": 0.27322621941566466,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6037909984588623,
      "step": 2670,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.04982744911685586,
      "clip_ratio/high_mean": 0.007483145385049283,
      "clip_ratio/low_mean": 0.0010201202865573577,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008503265725448728,
      "completion_length": 91.09750213623047,
      "epoch": 0.5148400729997118,
      "grad_norm": 3.5055086612701416,
      "kl": 0.5736653476953506,
      "learning_rate": 5.293611110110661e-07,
      "loss": -0.0032,
      "reward": 1.672940969467163,
      "reward_std": 0.24722242057323457,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.5902204990386963,
      "step": 2680,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.06388061246834695,
      "clip_ratio/high_mean": 0.008427212300011888,
      "clip_ratio/low_mean": 0.000504276818537619,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008931488974485546,
      "completion_length": 84.06750183105468,
      "epoch": 0.5167611180482182,
      "grad_norm": 1.1749032735824585,
      "kl": 0.6257337100803853,
      "learning_rate": 5.266477660311123e-07,
      "loss": -0.0049,
      "reward": 1.883350706100464,
      "reward_std": 0.1923319399356842,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.6929253697395324,
      "step": 2690,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.032716090651229025,
      "clip_ratio/high_mean": 0.004722256149398163,
      "clip_ratio/low_mean": 0.00025295682498835956,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0049752129940316085,
      "completion_length": 101.05250244140625,
      "epoch": 0.5186821630967247,
      "grad_norm": 2.250870704650879,
      "kl": 0.3336128618568182,
      "learning_rate": 5.239352720956869e-07,
      "loss": -0.0014,
      "reward": 1.803996729850769,
      "reward_std": 0.3182943195104599,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.6548108696937561,
      "step": 2700,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.057230279734358194,
      "clip_ratio/high_mean": 0.01016470161266625,
      "clip_ratio/low_mean": 0.001601585964090191,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01176628761459142,
      "completion_length": 92.2125015258789,
      "epoch": 0.520603208145231,
      "grad_norm": 1.7528822422027588,
      "kl": 0.30482072457671167,
      "learning_rate": 5.212237280584214e-07,
      "loss": -0.0012,
      "reward": 1.6862072706222535,
      "reward_std": 0.2419889122247696,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5971661269664764,
      "step": 2710,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.060608417179901154,
      "clip_ratio/high_mean": 0.00894762706157053,
      "clip_ratio/low_mean": 0.0007068538383464329,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009654480942117516,
      "completion_length": 92.53000183105469,
      "epoch": 0.5225242531937374,
      "grad_norm": 274.7859802246094,
      "kl": 1.1552282243967056,
      "learning_rate": 5.185132327383284e-07,
      "loss": 0.1157,
      "reward": 1.7673757076263428,
      "reward_std": 0.3102965742349625,
      "rewards/code_format_reward": 0.9887499809265137,
      "rewards/code_reward": 0.6365003228187561,
      "step": 2720,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.15797494337894022,
      "clip_ratio/high_mean": 0.02083307456341572,
      "clip_ratio/low_mean": 0.009061275536078028,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02989435092313215,
      "completion_length": 88.4500015258789,
      "epoch": 0.5244452982422437,
      "grad_norm": 4.456059455871582,
      "kl": 1.3563814774155616,
      "learning_rate": 5.158038849162024e-07,
      "loss": 0.0014,
      "reward": 1.5090751886367797,
      "reward_std": 0.23531495928764343,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.5098500728607178,
      "step": 2730,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.054899439518339935,
      "clip_ratio/high_mean": 0.008722224002121947,
      "clip_ratio/low_mean": 0.0002903619286371395,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009012586006429046,
      "completion_length": 85.53750305175781,
      "epoch": 0.5263663432907502,
      "grad_norm": 1.951745867729187,
      "kl": 0.5144835211336612,
      "learning_rate": 5.130957833310177e-07,
      "loss": -0.0017,
      "reward": 1.7648489713668822,
      "reward_std": 0.1646851196885109,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6355494737625123,
      "step": 2740,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.10675688227638602,
      "clip_ratio/high_mean": 0.016114802553784103,
      "clip_ratio/low_mean": 0.0011672286826069466,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0172820313135162,
      "completion_length": 97.54000244140624,
      "epoch": 0.5282873883392566,
      "grad_norm": 2.7782888412475586,
      "kl": 0.484642443805933,
      "learning_rate": 5.103890266763317e-07,
      "loss": -0.0017,
      "reward": 1.7005881071090698,
      "reward_std": 0.17179570347070694,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6046690165996551,
      "step": 2750,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.04718098001321778,
      "clip_ratio/high_mean": 0.0069286267200368455,
      "clip_ratio/low_mean": 0.0022616338639636522,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009190260457398836,
      "completion_length": 91.71500091552734,
      "epoch": 0.5302084333877629,
      "grad_norm": 1.6982417106628418,
      "kl": 0.40430613309144975,
      "learning_rate": 5.076837135966868e-07,
      "loss": -0.0001,
      "reward": 1.7166170120239257,
      "reward_std": 0.12425057031214237,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.6111209750175476,
      "step": 2760,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.01563614197075367,
      "clip_ratio/high_mean": 0.0026440696616191416,
      "clip_ratio/low_mean": 0.0005518664722330869,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031959360814653335,
      "completion_length": 93.39000091552734,
      "epoch": 0.5321294784362693,
      "grad_norm": 0.12160471081733704,
      "kl": 0.3728202864527702,
      "learning_rate": 5.049799426840166e-07,
      "loss": -0.0008,
      "reward": 1.8690509557724,
      "reward_std": 0.20764816105365752,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6889004349708557,
      "step": 2770,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.08729656506329775,
      "clip_ratio/high_mean": 0.013787648268043995,
      "clip_ratio/low_mean": 0.0016946192088653333,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015482267551124095,
      "completion_length": 83.49000091552735,
      "epoch": 0.5340505234847757,
      "grad_norm": 2.061514377593994,
      "kl": 0.2805942878127098,
      "learning_rate": 5.02277812474052e-07,
      "loss": -0.0005,
      "reward": 1.5558062076568604,
      "reward_std": 0.18851915150880813,
      "rewards/code_format_reward": 0.9924999833106994,
      "rewards/code_reward": 0.529778128862381,
      "step": 2780,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.047503374773077665,
      "clip_ratio/high_mean": 0.007121381178149022,
      "clip_ratio/low_mean": 0.003943280148087069,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011064661786076613,
      "completion_length": 90.33500213623047,
      "epoch": 0.5359715685332821,
      "grad_norm": 2.8578014373779297,
      "kl": 0.9348004341125489,
      "learning_rate": 4.995774214427299e-07,
      "loss": -0.0083,
      "reward": 1.5787676095962524,
      "reward_std": 0.24208036959171295,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.5412587821483612,
      "step": 2790,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.0682177669601515,
      "clip_ratio/high_mean": 0.010770523789688013,
      "clip_ratio/low_mean": 0.0030192029429599644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01378972665697802,
      "completion_length": 97.35500030517578,
      "epoch": 0.5378926135817885,
      "grad_norm": 3.7523272037506104,
      "kl": 0.49133365601301193,
      "learning_rate": 4.968788680026062e-07,
      "loss": 0.0019,
      "reward": 1.8675085306167603,
      "reward_std": 0.3084888607263565,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.6887542605400085,
      "step": 2800,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.051895615691319105,
      "clip_ratio/high_mean": 0.007092137623112648,
      "clip_ratio/low_mean": 0.0009049189888173714,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007997056643944234,
      "completion_length": 84.84000244140626,
      "epoch": 0.5398136586302948,
      "grad_norm": 6879.29296875,
      "kl": 41.48030465692282,
      "learning_rate": 4.941822504992665e-07,
      "loss": 0.3058,
      "reward": 1.8456867456436157,
      "reward_std": 0.17148398756980895,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.6750308394432067,
      "step": 2810,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.055018832255154845,
      "clip_ratio/high_mean": 0.009382021031342447,
      "clip_ratio/low_mean": 0.0012206103128846735,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0106026312103495,
      "completion_length": 92.16250152587891,
      "epoch": 0.5417347036788013,
      "grad_norm": 1.7369046211242676,
      "kl": 39.203208688646555,
      "learning_rate": 4.914876672077444e-07,
      "loss": 0.0739,
      "reward": 1.7605399131774901,
      "reward_std": 0.22667703181505203,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6337074398994446,
      "step": 2820,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.07783090919256211,
      "clip_ratio/high_mean": 0.013414820143952965,
      "clip_ratio/low_mean": 0.004346576618263498,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017761396686546506,
      "completion_length": 86.75750122070312,
      "epoch": 0.5436557487273077,
      "grad_norm": 1.3669426441192627,
      "kl": 0.6254852950572968,
      "learning_rate": 4.887952163289387e-07,
      "loss": -0.0037,
      "reward": 1.7524815320968627,
      "reward_std": 0.18003067299723624,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.6271782517433167,
      "step": 2830,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.032230034773238006,
      "clip_ratio/high_mean": 0.005299290179391391,
      "clip_ratio/low_mean": 0.002357826306251809,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007657116468180902,
      "completion_length": 92.63250122070312,
      "epoch": 0.545576793775814,
      "grad_norm": 6.607495307922363,
      "kl": 0.6308505192399025,
      "learning_rate": 4.861049959860352e-07,
      "loss": -0.0026,
      "reward": 1.879476284980774,
      "reward_std": 0.21936110258102418,
      "rewards/code_format_reward": 0.9787499785423279,
      "rewards/code_reward": 0.6950506567955017,
      "step": 2840,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.07762792855501174,
      "clip_ratio/high_mean": 0.012513539101928473,
      "clip_ratio/low_mean": 0.0019065381304244511,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014420077262911946,
      "completion_length": 79.06750183105468,
      "epoch": 0.5474978388243205,
      "grad_norm": 2.1427297592163086,
      "kl": 0.7649303644895553,
      "learning_rate": 4.834171042209299e-07,
      "loss": -0.0016,
      "reward": 1.7679643869400024,
      "reward_std": 0.2242477983236313,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.640857207775116,
      "step": 2850,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.10583647433668375,
      "clip_ratio/high_mean": 0.015110303135588764,
      "clip_ratio/low_mean": 0.0026529163093073293,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017763219110202046,
      "completion_length": 89.09499969482422,
      "epoch": 0.5494188838728268,
      "grad_norm": 5.39391565322876,
      "kl": 1.1404950305819512,
      "learning_rate": 4.807316389906573e-07,
      "loss": 0.0011,
      "reward": 1.6588359355926514,
      "reward_std": 0.23765334486961365,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.5822304427623749,
      "step": 2860,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.13843556838110088,
      "clip_ratio/high_mean": 0.022050847904756664,
      "clip_ratio/low_mean": 0.006549120438285172,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028599968180060387,
      "completion_length": 85.02750091552734,
      "epoch": 0.5513399289213332,
      "grad_norm": 6.328859329223633,
      "kl": 1.3457766875624657,
      "learning_rate": 4.780486981638194e-07,
      "loss": 0.004,
      "reward": 1.4554174661636352,
      "reward_std": 0.291735103726387,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.4839587390422821,
      "step": 2870,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.03891797037795186,
      "clip_ratio/high_mean": 0.005045192840043455,
      "clip_ratio/low_mean": 0.0029750549525488167,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008020247751846909,
      "completion_length": 85.00500183105468,
      "epoch": 0.5532609739698396,
      "grad_norm": 3.746497392654419,
      "kl": 1.5130328834056854,
      "learning_rate": 4.75368379517019e-07,
      "loss": -0.0033,
      "reward": 1.8564167737960815,
      "reward_std": 0.14603331089019775,
      "rewards/code_format_reward": 0.9987499952316284,
      "rewards/code_reward": 0.6785208344459533,
      "step": 2880,
      "zero_std_ratio": 0.7
    },
    {
      "clip_ratio/high_max": 0.23034826815128326,
      "clip_ratio/high_mean": 0.037423617928288876,
      "clip_ratio/low_mean": 0.0014674156729597599,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03889103210531175,
      "completion_length": 81.40750122070312,
      "epoch": 0.555182019018346,
      "grad_norm": 7.282290458679199,
      "kl": 0.5326755799353122,
      "learning_rate": 4.7269078073129696e-07,
      "loss": 0.0032,
      "reward": 1.700506567955017,
      "reward_std": 0.3424434006214142,
      "rewards/code_format_reward": 0.9700000047683716,
      "rewards/code_reward": 0.6077533006668091,
      "step": 2890,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.06711816978640854,
      "clip_ratio/high_mean": 0.008929741784231737,
      "clip_ratio/low_mean": 0.0021304662863258273,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011060208058916032,
      "completion_length": 75.07750244140625,
      "epoch": 0.5571030640668524,
      "grad_norm": 3.718710422515869,
      "kl": 0.3807241953909397,
      "learning_rate": 4.7001599938857204e-07,
      "loss": -0.0016,
      "reward": 1.6593467235565185,
      "reward_std": 0.2742844566702843,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.5821733415126801,
      "step": 2900,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.10134089784696698,
      "clip_ratio/high_mean": 0.014033923938404769,
      "clip_ratio/low_mean": 0.0036910680413711817,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01772499195067212,
      "completion_length": 74.57250061035157,
      "epoch": 0.5590241091153587,
      "grad_norm": 18.590866088867188,
      "kl": 0.9125221639871597,
      "learning_rate": 4.673441329680844e-07,
      "loss": 0.0044,
      "reward": 1.6198436498641968,
      "reward_std": 0.1470041409134865,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.5621092915534973,
      "step": 2910,
      "zero_std_ratio": 0.7
    },
    {
      "clip_ratio/high_max": 0.042256729071959855,
      "clip_ratio/high_mean": 0.007948629459133372,
      "clip_ratio/low_mean": 0.001496748169302009,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009445377223892137,
      "completion_length": 77.4625015258789,
      "epoch": 0.5609451541638651,
      "grad_norm": 0.18645010888576508,
      "kl": 0.4780749522149563,
      "learning_rate": 4.6467527884284365e-07,
      "loss": 0.0006,
      "reward": 1.8204985857009888,
      "reward_std": 0.19856311585754155,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.6649368166923523,
      "step": 2920,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.1271044396329671,
      "clip_ratio/high_mean": 0.016186495171859862,
      "clip_ratio/low_mean": 0.0011034044640837238,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01728989938274026,
      "completion_length": 82.72500305175781,
      "epoch": 0.5628661992123716,
      "grad_norm": 6.4396162033081055,
      "kl": 0.30610462203621863,
      "learning_rate": 4.6200953427607927e-07,
      "loss": -0.0021,
      "reward": 1.7915108680725098,
      "reward_std": 0.22729050666093825,
      "rewards/code_format_reward": 0.9700000047683716,
      "rewards/code_reward": 0.6532554149627685,
      "step": 2930,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.23429102210793645,
      "clip_ratio/high_mean": 0.03006269016477745,
      "clip_ratio/low_mean": 0.001874277341994457,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0319369669421576,
      "completion_length": 88.97500152587891,
      "epoch": 0.5647872442608779,
      "grad_norm": 43.83531951904297,
      "kl": 0.5952823750674725,
      "learning_rate": 4.5934699641769747e-07,
      "loss": -0.0032,
      "reward": 1.837431001663208,
      "reward_std": 0.3392215400934219,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.6730904817581177,
      "step": 2940,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.06733583421446383,
      "clip_ratio/high_mean": 0.00863017894444056,
      "clip_ratio/low_mean": 0.005618994176620618,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014249173022108153,
      "completion_length": 77.96000061035156,
      "epoch": 0.5667082893093843,
      "grad_norm": 2.642043352127075,
      "kl": 0.56968834400177,
      "learning_rate": 4.566877623007389e-07,
      "loss": 0.0049,
      "reward": 1.7328413248062133,
      "reward_std": 0.21620932817459107,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.6229831516742707,
      "step": 2950,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.04462944087572396,
      "clip_ratio/high_mean": 0.007475414098007604,
      "clip_ratio/low_mean": 0.002004683316772571,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009480097430059686,
      "completion_length": 85.0875015258789,
      "epoch": 0.5686293343578906,
      "grad_norm": 3.8512065410614014,
      "kl": 0.33709155321121215,
      "learning_rate": 4.540319288378439e-07,
      "loss": -0.0057,
      "reward": 1.6900140762329101,
      "reward_std": 0.21961634010076522,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.5978195071220398,
      "step": 2960,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.06556220971979201,
      "clip_ratio/high_mean": 0.01001431758631952,
      "clip_ratio/low_mean": 0.003507485325098969,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01352180291141849,
      "completion_length": 92.67500152587891,
      "epoch": 0.5705503794063971,
      "grad_norm": 2.966658592224121,
      "kl": 0.5968067184090614,
      "learning_rate": 4.513795928177193e-07,
      "loss": 0.0007,
      "reward": 1.4343469619750977,
      "reward_std": 0.16000542044639587,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.4681109845638275,
      "step": 2970,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.006220952537842095,
      "clip_ratio/high_mean": 0.0009992636245442555,
      "clip_ratio/low_mean": 0.0029718225210672244,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0039710860582999885,
      "completion_length": 92.65750274658203,
      "epoch": 0.5724714244549035,
      "grad_norm": 9.493338584899902,
      "kl": 0.5875692501664161,
      "learning_rate": 4.4873085090161266e-07,
      "loss": -0.0009,
      "reward": 1.4061829090118407,
      "reward_std": 0.20027331858873368,
      "rewards/code_format_reward": 0.9762499928474426,
      "rewards/code_reward": 0.45902894139289857,
      "step": 2980,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.0330589919583872,
      "clip_ratio/high_mean": 0.004416179939289578,
      "clip_ratio/low_mean": 0.002115111546299886,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006531291425926611,
      "completion_length": 79.80750274658203,
      "epoch": 0.5743924695034098,
      "grad_norm": 1.592423915863037,
      "kl": 0.6846940219402313,
      "learning_rate": 4.460857996197879e-07,
      "loss": -0.0088,
      "reward": 1.8656628370285033,
      "reward_std": 0.24907293021678925,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.6850189208984375,
      "step": 2990,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.10685318629257382,
      "clip_ratio/high_mean": 0.014238242123974487,
      "clip_ratio/low_mean": 0.0005060926268924959,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014744334877468646,
      "completion_length": 75.28500213623047,
      "epoch": 0.5763135145519163,
      "grad_norm": 11.023285865783691,
      "kl": 1.773244822025299,
      "learning_rate": 4.434445353680084e-07,
      "loss": -0.0004,
      "reward": 1.6719849348068236,
      "reward_std": 0.23447352051734924,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.5888049364089966,
      "step": 3000,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.09909297195263207,
      "clip_ratio/high_mean": 0.014624686987372116,
      "clip_ratio/low_mean": 0.0008105992455966771,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015435286390129477,
      "completion_length": 80.88750228881835,
      "epoch": 0.5782345596004226,
      "grad_norm": 3.5932817459106445,
      "kl": 1.2866470351815225,
      "learning_rate": 4.4080715440402417e-07,
      "loss": 0.0028,
      "reward": 1.7477641582489014,
      "reward_std": 0.27256832718849183,
      "rewards/code_format_reward": 0.9800000071525574,
      "rewards/code_reward": 0.628882086277008,
      "step": 3010,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.061281491769477725,
      "clip_ratio/high_mean": 0.008347922342363746,
      "clip_ratio/low_mean": 0.00354889674927108,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011896818911191076,
      "completion_length": 75.85250091552734,
      "epoch": 0.580155604648929,
      "grad_norm": 4.849332809448242,
      "kl": 0.476963010430336,
      "learning_rate": 4.381737528440624e-07,
      "loss": -0.0002,
      "reward": 1.5080678462982178,
      "reward_std": 0.1984383262693882,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.5099714159965515,
      "step": 3020,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.02014698493294418,
      "clip_ratio/high_mean": 0.0029024946445133535,
      "clip_ratio/low_mean": 0.001273224765463965,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0041757194034289565,
      "completion_length": 86.35750122070313,
      "epoch": 0.5820766496974354,
      "grad_norm": 5.408311367034912,
      "kl": 1.1033611692488194,
      "learning_rate": 4.3554442665932664e-07,
      "loss": -0.0044,
      "reward": 1.7480007410049438,
      "reward_std": 0.20548871904611588,
      "rewards/code_format_reward": 0.9674999952316284,
      "rewards/code_reward": 0.6321253478527069,
      "step": 3030,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.07207567039877176,
      "clip_ratio/high_mean": 0.010315603285562247,
      "clip_ratio/low_mean": 0.0024311804067110644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012746783741749822,
      "completion_length": 87.45250091552734,
      "epoch": 0.5839976947459418,
      "grad_norm": 5.45907735824585,
      "kl": 0.7388446770608426,
      "learning_rate": 4.329192716724974e-07,
      "loss": -0.0134,
      "reward": 1.617799663543701,
      "reward_std": 0.28184359073638915,
      "rewards/code_format_reward": 0.9900000095367432,
      "rewards/code_reward": 0.5613998055458069,
      "step": 3040,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.03178482772782445,
      "clip_ratio/high_mean": 0.00484326797304675,
      "clip_ratio/low_mean": 0.0010359384352341295,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005879206501413136,
      "completion_length": 83.70250091552734,
      "epoch": 0.5859187397944482,
      "grad_norm": 6.244964122772217,
      "kl": 0.8223805136978626,
      "learning_rate": 4.3029838355424165e-07,
      "loss": -0.0028,
      "reward": 1.5551699638366698,
      "reward_std": 0.23868169337511064,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5300849676132202,
      "step": 3050,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.031123768421821296,
      "clip_ratio/high_mean": 0.0042093763331649825,
      "clip_ratio/low_mean": 0.00023920949752209708,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0044485858496045695,
      "completion_length": 90.6500015258789,
      "epoch": 0.5878397848429545,
      "grad_norm": 1.844166874885559,
      "kl": 0.9453303083777428,
      "learning_rate": 4.2768185781972433e-07,
      "loss": 0.0038,
      "reward": 1.7277095794677735,
      "reward_std": 0.22161270976066588,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.6176047682762146,
      "step": 3060,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.06112067271023989,
      "clip_ratio/high_mean": 0.008206171146593989,
      "clip_ratio/low_mean": 0.0006491162814199925,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008855287660844624,
      "completion_length": 81.27750091552734,
      "epoch": 0.589760829891461,
      "grad_norm": 3.0321500301361084,
      "kl": 0.4705409877002239,
      "learning_rate": 4.2506978982512964e-07,
      "loss": -0.0002,
      "reward": 1.9011548519134522,
      "reward_std": 0.2363950289785862,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.7037024021148681,
      "step": 3070,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.028480308945290744,
      "clip_ratio/high_mean": 0.00514335140469484,
      "clip_ratio/low_mean": 0.0035089968900138047,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008652348211035133,
      "completion_length": 88.05750274658203,
      "epoch": 0.5916818749399674,
      "grad_norm": 4.498425483703613,
      "kl": 0.9383749194443226,
      "learning_rate": 4.224622747641835e-07,
      "loss": -0.0068,
      "reward": 1.2419449806213378,
      "reward_std": 0.1959183931350708,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.37597247362136843,
      "step": 3080,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.02165755571331829,
      "clip_ratio/high_mean": 0.003493850605445914,
      "clip_ratio/low_mean": 0.0001163623295724392,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003610212981584482,
      "completion_length": 83.10500030517578,
      "epoch": 0.5936029199884737,
      "grad_norm": 1.0221151113510132,
      "kl": 1.614695566892624,
      "learning_rate": 4.1985940766468663e-07,
      "loss": 0.1048,
      "reward": 1.8437815666198731,
      "reward_std": 0.12033854126930237,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.6731407642364502,
      "step": 3090,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.05757335813250393,
      "clip_ratio/high_mean": 0.0107182093168376,
      "clip_ratio/low_mean": 0.004042259410198312,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014760468708118424,
      "completion_length": 86.6625,
      "epoch": 0.5955239650369801,
      "grad_norm": 3.0221967697143555,
      "kl": 0.4662696644663811,
      "learning_rate": 4.1726128338504997e-07,
      "loss": 0.0059,
      "reward": 1.6797678232192994,
      "reward_std": 0.23598156571388246,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.5930089056491852,
      "step": 3100,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.1632944119395688,
      "clip_ratio/high_mean": 0.02386658971372526,
      "clip_ratio/low_mean": 0.000367270597780589,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02423386004229542,
      "completion_length": 87.04000244140624,
      "epoch": 0.5974450100854864,
      "grad_norm": 3124.911865234375,
      "kl": 1.4825018651783466,
      "learning_rate": 4.146679966108374e-07,
      "loss": 0.109,
      "reward": 1.7368038177490235,
      "reward_std": 0.2290027320384979,
      "rewards/code_format_reward": 0.9912499785423279,
      "rewards/code_reward": 0.620589405298233,
      "step": 3110,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.011806602030992508,
      "clip_ratio/high_mean": 0.00222149578621611,
      "clip_ratio/low_mean": 0.001782867594738491,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004004363450803794,
      "completion_length": 76.50500183105468,
      "epoch": 0.5993660551339929,
      "grad_norm": 5.609122276306152,
      "kl": 1.2381610602140427,
      "learning_rate": 4.120796418513165e-07,
      "loss": 0.0687,
      "reward": 1.6538613319396973,
      "reward_std": 0.2478315144777298,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.5813056170940399,
      "step": 3120,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.04111457797698677,
      "clip_ratio/high_mean": 0.006102612579707056,
      "clip_ratio/low_mean": 0.0006678692123387008,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006770481838611886,
      "completion_length": 90.63500213623047,
      "epoch": 0.6012871001824993,
      "grad_norm": 1.7537983655929565,
      "kl": 0.8379382207989693,
      "learning_rate": 4.094963134360129e-07,
      "loss": 3.0713,
      "reward": 1.8111864566802978,
      "reward_std": 0.23444892466068268,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6599682211875916,
      "step": 3130,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.07487462717108428,
      "clip_ratio/high_mean": 0.009757341054501012,
      "clip_ratio/low_mean": 0.002470593445468694,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012227934325346723,
      "completion_length": 84.99250183105468,
      "epoch": 0.6032081452310056,
      "grad_norm": 7.498387336730957,
      "kl": 0.5894037500023842,
      "learning_rate": 4.0691810551127327e-07,
      "loss": 0.0462,
      "reward": 1.6221882104873657,
      "reward_std": 0.25462802946567537,
      "rewards/code_format_reward": 0.9975000023841858,
      "rewards/code_reward": 0.5617190957069397,
      "step": 3140,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.015196678857319058,
      "clip_ratio/high_mean": 0.0022096226894063875,
      "clip_ratio/low_mean": 0.002686911600176245,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004896534324507229,
      "completion_length": 88.44750213623047,
      "epoch": 0.6051291902795121,
      "grad_norm": 0.7371006011962891,
      "kl": 1.5165767412632705,
      "learning_rate": 4.0434511203683386e-07,
      "loss": 0.0113,
      "reward": 1.958918571472168,
      "reward_std": 0.17050198167562486,
      "rewards/code_format_reward": 0.9962499856948852,
      "rewards/code_reward": 0.7303967714309693,
      "step": 3150,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.047509356867522,
      "clip_ratio/high_mean": 0.0060635729460045695,
      "clip_ratio/low_mean": 0.0037405278504593297,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009804100578185171,
      "completion_length": 93.71500091552734,
      "epoch": 0.6070502353280185,
      "grad_norm": 4.062532424926758,
      "kl": 164.7577206812799,
      "learning_rate": 4.017774267823967e-07,
      "loss": 0.3479,
      "reward": 1.8433427095413208,
      "reward_std": 0.20897280275821686,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6760463416576385,
      "step": 3160,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.007103513111360371,
      "clip_ratio/high_mean": 0.0009442454349482432,
      "clip_ratio/low_mean": 0.0005656339257257059,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015098793461220338,
      "completion_length": 97.03500061035156,
      "epoch": 0.6089712803765248,
      "grad_norm": 0.3194718658924103,
      "kl": 19.38877977654338,
      "learning_rate": 3.9921514332421193e-07,
      "loss": 0.1279,
      "reward": 1.3801440358161927,
      "reward_std": 0.26880781557410954,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.44757200181484225,
      "step": 3170,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.027826181857381015,
      "clip_ratio/high_mean": 0.004423137854610104,
      "clip_ratio/low_mean": 0.000519216748944018,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004942354625381995,
      "completion_length": 99.6375015258789,
      "epoch": 0.6108923254250312,
      "grad_norm": 133.27151489257812,
      "kl": 91.96420569866896,
      "learning_rate": 3.966583550416676e-07,
      "loss": 284.3821,
      "reward": 1.6065278768539428,
      "reward_std": 0.2671674907207489,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.5598264217376709,
      "step": 3180,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.03810381339862943,
      "clip_ratio/high_mean": 0.005511091940570622,
      "clip_ratio/low_mean": 0.00701818183879368,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01252927360474132,
      "completion_length": 90.77000122070312,
      "epoch": 0.6128133704735376,
      "grad_norm": 2.931155204772949,
      "kl": 4.587994083762169,
      "learning_rate": 3.9410715511388647e-07,
      "loss": 28143.1688,
      "reward": 1.7186223268508911,
      "reward_std": 0.2031429558992386,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.6139986515045166,
      "step": 3190,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.18900979291647674,
      "clip_ratio/high_mean": 0.025313075329177082,
      "clip_ratio/low_mean": 0.00013794690457871183,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02545102240983397,
      "completion_length": 88.60750122070313,
      "epoch": 0.614734415522044,
      "grad_norm": 3.9914708137512207,
      "kl": 0.678571529686451,
      "learning_rate": 3.915616365163304e-07,
      "loss": 0.0002,
      "reward": 1.818918228149414,
      "reward_std": 0.24608779847621917,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.6653966069221496,
      "step": 3200,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.05510081194806844,
      "clip_ratio/high_mean": 0.008429678474203683,
      "clip_ratio/low_mean": 0.0015389235399197788,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0099686019733781,
      "completion_length": 85.05250091552735,
      "epoch": 0.6166554605705504,
      "grad_norm": 2.0297534465789795,
      "kl": 0.5190044179558754,
      "learning_rate": 3.890218920174122e-07,
      "loss": -0.0056,
      "reward": 1.938026785850525,
      "reward_std": 0.2829041987657547,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.7218258857727051,
      "step": 3210,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.04617450258228928,
      "clip_ratio/high_mean": 0.007303895291988738,
      "clip_ratio/low_mean": 0.002542783234093804,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009846678806934506,
      "completion_length": 92.52000122070312,
      "epoch": 0.6185765056190567,
      "grad_norm": 3.2283730506896973,
      "kl": 0.5362374372780323,
      "learning_rate": 3.86488014175114e-07,
      "loss": 0.0003,
      "reward": 1.7741312742233277,
      "reward_std": 0.20447308868169783,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6395656108856201,
      "step": 3220,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.059750709845684466,
      "clip_ratio/high_mean": 0.00790787541482132,
      "clip_ratio/low_mean": 0.0012954409321537241,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009203316466300748,
      "completion_length": 90.4375,
      "epoch": 0.6204975506675632,
      "grad_norm": 2.409045934677124,
      "kl": 0.553566773980856,
      "learning_rate": 3.8396009533361486e-07,
      "loss": -0.0,
      "reward": 1.6513851642608643,
      "reward_std": 0.24081393480300903,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.580692571401596,
      "step": 3230,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.03564520282670856,
      "clip_ratio/high_mean": 0.004964679945260286,
      "clip_ratio/low_mean": 0.004444090686592972,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009408770385198295,
      "completion_length": 79.08000183105469,
      "epoch": 0.6224185957160695,
      "grad_norm": 7.759763717651367,
      "kl": 1.2998816877603532,
      "learning_rate": 3.814382276199251e-07,
      "loss": -0.0006,
      "reward": 1.6336610555648803,
      "reward_std": 0.1691926121711731,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.5680804908275604,
      "step": 3240,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.011579358880408109,
      "clip_ratio/high_mean": 0.002202258622855879,
      "clip_ratio/low_mean": 0.0003946456956327893,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0025969043519580735,
      "completion_length": 88.7375,
      "epoch": 0.6243396407645759,
      "grad_norm": 9.489768981933594,
      "kl": 4.286054483056068,
      "learning_rate": 3.7892250294052853e-07,
      "loss": 31.2761,
      "reward": 1.8622464895248414,
      "reward_std": 0.2547990471124649,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.6858106970787048,
      "step": 3250,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.07659143296186813,
      "clip_ratio/high_mean": 0.010122461079299682,
      "clip_ratio/low_mean": 0.0019954566974774933,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012117917880095775,
      "completion_length": 99.80750274658203,
      "epoch": 0.6262606858130824,
      "grad_norm": 2.884183168411255,
      "kl": 1.2840011775493623,
      "learning_rate": 3.764130129780341e-07,
      "loss": 0.0383,
      "reward": 1.6670962572097778,
      "reward_std": 0.34920003414154055,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5907356142997742,
      "step": 3260,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.029844516195589678,
      "clip_ratio/high_mean": 0.004244843772175955,
      "clip_ratio/low_mean": 0.0002169124811189249,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004461756230011815,
      "completion_length": 100.70250091552734,
      "epoch": 0.6281817308615887,
      "grad_norm": 4.036985397338867,
      "kl": 2.1118960954248904,
      "learning_rate": 3.7390984918783286e-07,
      "loss": 0.9419,
      "reward": 1.6084105730056764,
      "reward_std": 0.17128639966249465,
      "rewards/code_format_reward": 0.9712500095367431,
      "rewards/code_reward": 0.5613927602767944,
      "step": 3270,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.07067356873303651,
      "clip_ratio/high_mean": 0.00971948360092938,
      "clip_ratio/low_mean": 0.0006290240438829642,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010348507657181472,
      "completion_length": 88.9000015258789,
      "epoch": 0.6301027759100951,
      "grad_norm": 1.543152928352356,
      "kl": 0.5742107287049294,
      "learning_rate": 3.714131027947669e-07,
      "loss": 0.0006,
      "reward": 1.808586883544922,
      "reward_std": 0.20984979271888732,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.6564809083938599,
      "step": 3280,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.060896387742832306,
      "clip_ratio/high_mean": 0.00765781793743372,
      "clip_ratio/low_mean": 0.01029690281720832,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017954721208661796,
      "completion_length": 80.08500213623047,
      "epoch": 0.6320238209586014,
      "grad_norm": 2.127617359161377,
      "kl": 0.6725200928747654,
      "learning_rate": 3.689228647898034e-07,
      "loss": 0.1143,
      "reward": 1.678031039237976,
      "reward_std": 0.19750893712043763,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.5921404898166657,
      "step": 3290,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.05414980174973607,
      "clip_ratio/high_mean": 0.007520435960032046,
      "clip_ratio/low_mean": 0.00011696565634338185,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007637401651300025,
      "completion_length": 92.725,
      "epoch": 0.6339448660071079,
      "grad_norm": 8.315914154052734,
      "kl": 0.30459046363830566,
      "learning_rate": 3.6643922592671904e-07,
      "loss": -0.0066,
      "reward": 1.5898099780082702,
      "reward_std": 0.1832955375313759,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.5464674949645996,
      "step": 3300,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.02745349882170558,
      "clip_ratio/high_mean": 0.004275670822244138,
      "clip_ratio/low_mean": 0.001036624335392844,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005312295141629874,
      "completion_length": 86.80250091552735,
      "epoch": 0.6358659110556143,
      "grad_norm": 4.2797441482543945,
      "kl": 2.398578557372093,
      "learning_rate": 3.6396227671879267e-07,
      "loss": 0.028,
      "reward": 1.7730424404144287,
      "reward_std": 0.3175764262676239,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.6387087047100067,
      "step": 3310,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.03558065614197403,
      "clip_ratio/high_mean": 0.004970578508800827,
      "clip_ratio/low_mean": 0.0008951228694058955,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0058657014247728515,
      "completion_length": 91.0875015258789,
      "epoch": 0.6377869561041206,
      "grad_norm": 5.376333713531494,
      "kl": 1.4305558323860168,
      "learning_rate": 3.614921074355067e-07,
      "loss": 0.0034,
      "reward": 1.7029305696487427,
      "reward_std": 0.34837333858013153,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.6052152514457703,
      "step": 3320,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.04346057323855348,
      "clip_ratio/high_mean": 0.005737839776702458,
      "clip_ratio/low_mean": 0.001675139949657023,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00741297956337803,
      "completion_length": 88.75250396728515,
      "epoch": 0.639708001152627,
      "grad_norm": 2.969228744506836,
      "kl": 0.7607076019048691,
      "learning_rate": 3.5902880809925704e-07,
      "loss": -0.0001,
      "reward": 1.6762405157089233,
      "reward_std": 0.2515918217599392,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.5909327387809753,
      "step": 3330,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.05080293011851609,
      "clip_ratio/high_mean": 0.006427765643456951,
      "clip_ratio/low_mean": 0.00040708604792598634,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006834851735038683,
      "completion_length": 88.0750015258789,
      "epoch": 0.6416290462011334,
      "grad_norm": 12.137472152709961,
      "kl": 0.31881698705255984,
      "learning_rate": 3.565724684820727e-07,
      "loss": 3.6118,
      "reward": 1.8916306495666504,
      "reward_std": 0.1850387692451477,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.6973778128623962,
      "step": 3340,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.14287711144424975,
      "clip_ratio/high_mean": 0.019231261435197666,
      "clip_ratio/low_mean": 0.0020263168029487134,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021257578127551822,
      "completion_length": 94.19000091552735,
      "epoch": 0.6435500912496398,
      "grad_norm": 6.10810661315918,
      "kl": 0.8296034529805183,
      "learning_rate": 3.541231781023436e-07,
      "loss": -0.0004,
      "reward": 1.6248144626617431,
      "reward_std": 0.2219874605536461,
      "rewards/code_format_reward": 0.9887499809265137,
      "rewards/code_reward": 0.5652197122573852,
      "step": 3350,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.07329725201707334,
      "clip_ratio/high_mean": 0.009671362905646675,
      "clip_ratio/low_mean": 0.005342914546781685,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015014277724549174,
      "completion_length": 97.74500274658203,
      "epoch": 0.6454711362981462,
      "grad_norm": 2.801866054534912,
      "kl": 0.5770246163010597,
      "learning_rate": 3.5168102622155894e-07,
      "loss": 0.0,
      "reward": 1.6838999271392823,
      "reward_std": 0.2707583636045456,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.594137442111969,
      "step": 3360,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.005975091701839119,
      "clip_ratio/high_mean": 0.0011488659016322344,
      "clip_ratio/low_mean": 0.001098146109143272,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022470120195066555,
      "completion_length": 89.36750030517578,
      "epoch": 0.6473921813466526,
      "grad_norm": 34.13050079345703,
      "kl": 2.2693535044789312,
      "learning_rate": 3.492461018410535e-07,
      "loss": 0.0028,
      "reward": 1.8977232933044434,
      "reward_std": 0.2937870219349861,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.7022991299629211,
      "step": 3370,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.04565345844021067,
      "clip_ratio/high_mean": 0.009023689541209023,
      "clip_ratio/low_mean": 0.00031946374219842254,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009343153254303616,
      "completion_length": 84.38249969482422,
      "epoch": 0.649313226395159,
      "grad_norm": 0.9170461893081665,
      "kl": 108.80695619434118,
      "learning_rate": 3.468184936987645e-07,
      "loss": 920.5057,
      "reward": 1.7496967315673828,
      "reward_std": 0.2916144669055939,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.626410859823227,
      "step": 3380,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.04991705315187574,
      "clip_ratio/high_mean": 0.007881995162460954,
      "clip_ratio/low_mean": 0.00031314246589317916,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008195137570146472,
      "completion_length": 88.86000061035156,
      "epoch": 0.6512342714436653,
      "grad_norm": 3.084516763687134,
      "kl": 1331.8980419039726,
      "learning_rate": 3.4439829026599765e-07,
      "loss": 2.6994,
      "reward": 1.7110779523849486,
      "reward_std": 0.22298349142074586,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.6071014523506164,
      "step": 3390,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.03301922780228779,
      "clip_ratio/high_mean": 0.005802097530977335,
      "clip_ratio/low_mean": 0.002479181956732646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00828127931599738,
      "completion_length": 77.6500015258789,
      "epoch": 0.6531553164921717,
      "grad_norm": 3643.742919921875,
      "kl": 629.580971956253,
      "learning_rate": 3.4198557974420236e-07,
      "loss": 1.3601,
      "reward": 1.9027020692825318,
      "reward_std": 0.23692196756601333,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.7038509964942932,
      "step": 3400,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.026931732892990112,
      "clip_ratio/high_mean": 0.004060871619731188,
      "clip_ratio/low_mean": 0.0013453641964588313,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00540623576962389,
      "completion_length": 81.77250213623047,
      "epoch": 0.6550763615406782,
      "grad_norm": 3.2221176624298096,
      "kl": 17.398655989021062,
      "learning_rate": 3.3958045006175804e-07,
      "loss": 0.0552,
      "reward": 1.7479909420013429,
      "reward_std": 0.22741918563842772,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.6308704853057862,
      "step": 3410,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.06737710665911437,
      "clip_ratio/high_mean": 0.008767830353463069,
      "clip_ratio/low_mean": 0.0005067014892119915,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009274532069684937,
      "completion_length": 92.88500061035157,
      "epoch": 0.6569974065891845,
      "grad_norm": 4.063995838165283,
      "kl": 2.0011128395795823,
      "learning_rate": 3.3718298887077003e-07,
      "loss": 0.0159,
      "reward": 1.7235053777694702,
      "reward_std": 0.2168472334742546,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.616440212726593,
      "step": 3420,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.019622295489534737,
      "clip_ratio/high_mean": 0.003191170998616144,
      "clip_ratio/low_mean": 0.0015002752974396572,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0046914463368011635,
      "completion_length": 80.16000213623047,
      "epoch": 0.6589184516376909,
      "grad_norm": 1.253300428390503,
      "kl": 0.48067781031131745,
      "learning_rate": 3.3479328354387286e-07,
      "loss": 0.0008,
      "reward": 1.7450715541839599,
      "reward_std": 0.1590050458908081,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6244107484817505,
      "step": 3430,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.03181373123079538,
      "clip_ratio/high_mean": 0.0046242739539593455,
      "clip_ratio/low_mean": 0.012107005770667456,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016731279762461783,
      "completion_length": 84.32750244140625,
      "epoch": 0.6608394966861972,
      "grad_norm": 1.5854672193527222,
      "kl": 0.42518851198256014,
      "learning_rate": 3.324114211710498e-07,
      "loss": 0.0,
      "reward": 1.6541699171066284,
      "reward_std": 0.1113172210752964,
      "rewards/code_format_reward": 0.9962499856948852,
      "rewards/code_reward": 0.5780224561691284,
      "step": 3440,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.13535575959831475,
      "clip_ratio/high_mean": 0.018421862670220435,
      "clip_ratio/low_mean": 0.0012572539155371488,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019679116318002343,
      "completion_length": 91.63250122070312,
      "epoch": 0.6627605417347037,
      "grad_norm": 4.593750476837158,
      "kl": 0.7388513803482055,
      "learning_rate": 3.300374885564553e-07,
      "loss": -0.0,
      "reward": 1.5408308625221252,
      "reward_std": 0.29571940898895266,
      "rewards/code_format_reward": 0.9749999880790711,
      "rewards/code_reward": 0.5266654074192048,
      "step": 3450,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.04671933995559811,
      "clip_ratio/high_mean": 0.0062255718978121875,
      "clip_ratio/low_mean": 0.003391482085862663,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009617053843976464,
      "completion_length": 78.45250091552734,
      "epoch": 0.6646815867832101,
      "grad_norm": 2.5849409103393555,
      "kl": 10.90581871420145,
      "learning_rate": 3.2767157221525437e-07,
      "loss": 0.0178,
      "reward": 1.5087457418441772,
      "reward_std": 0.19353876560926436,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.5074978828430176,
      "step": 3460,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.0318772604689002,
      "clip_ratio/high_mean": 0.004644899175036699,
      "clip_ratio/low_mean": 0.0032211030862526967,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007866002165246754,
      "completion_length": 75.36500091552735,
      "epoch": 0.6666026318317164,
      "grad_norm": 1.8117616176605225,
      "kl": 187030.33910432606,
      "learning_rate": 3.253137583704673e-07,
      "loss": 374.1458,
      "reward": 1.6825225114822389,
      "reward_std": 0.2058879092335701,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5921987533569336,
      "step": 3470,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.09497236199676991,
      "clip_ratio/high_mean": 0.015650217607617378,
      "clip_ratio/low_mean": 0.0006928690614586231,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016343086725100875,
      "completion_length": 89.13250198364258,
      "epoch": 0.6685236768802229,
      "grad_norm": 5.850868225097656,
      "kl": 0.5080707125365734,
      "learning_rate": 3.229641329498296e-07,
      "loss": 0.0463,
      "reward": 1.6678599119186401,
      "reward_std": 0.2830047011375427,
      "rewards/code_format_reward": 0.9724999904632569,
      "rewards/code_reward": 0.5908049464225769,
      "step": 3480,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.050425410037860274,
      "clip_ratio/high_mean": 0.006465365196345374,
      "clip_ratio/low_mean": 0.0006157927738968283,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0070811579586006704,
      "completion_length": 81.40749969482422,
      "epoch": 0.6704447219287293,
      "grad_norm": 10.526844024658203,
      "kl": 1.5019532606005668,
      "learning_rate": 3.2062278158265866e-07,
      "loss": -0.0021,
      "reward": 1.7323597908020019,
      "reward_std": 0.15349715426564217,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6202423751354218,
      "step": 3490,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.07051093662157655,
      "clip_ratio/high_mean": 0.009594869159627706,
      "clip_ratio/low_mean": 0.001997971232049167,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011592840391676873,
      "completion_length": 96.4625,
      "epoch": 0.6723657669772356,
      "grad_norm": 12.833992958068848,
      "kl": 0.37389371246099473,
      "learning_rate": 3.182897895967338e-07,
      "loss": 0.0008,
      "reward": 1.6037346363067626,
      "reward_std": 0.329493448138237,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.556554788351059,
      "step": 3500,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.017038265755400062,
      "clip_ratio/high_mean": 0.0027443476661574095,
      "clip_ratio/low_mean": 0.000347714369854657,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003092062100768089,
      "completion_length": 84.05,
      "epoch": 0.674286812025742,
      "grad_norm": 6.119350910186768,
      "kl": 0.4559432238340378,
      "learning_rate": 3.15965242015187e-07,
      "loss": 0.0298,
      "reward": 1.6935490131378175,
      "reward_std": 0.26772548258304596,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6002120196819305,
      "step": 3510,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.03435098186600953,
      "clip_ratio/high_mean": 0.005973302901838906,
      "clip_ratio/low_mean": 0.0006568559459992684,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006630158764892258,
      "completion_length": 95.0,
      "epoch": 0.6762078570742484,
      "grad_norm": 4.796656608581543,
      "kl": 0.3851431407034397,
      "learning_rate": 3.1364922355340346e-07,
      "loss": 0.0214,
      "reward": 1.8059131860733033,
      "reward_std": 0.18592590391635894,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6554565787315368,
      "step": 3520,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.024073917022906243,
      "clip_ratio/high_mean": 0.0035194387339288367,
      "clip_ratio/low_mean": 0.0002464063392835669,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0037658450222807006,
      "completion_length": 86.9800018310547,
      "epoch": 0.6781289021227548,
      "grad_norm": 7.799978256225586,
      "kl": 0.2617302156984806,
      "learning_rate": 3.113418186159349e-07,
      "loss": -0.0088,
      "reward": 1.515157699584961,
      "reward_std": 0.2593328535556793,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5138288617134095,
      "step": 3530,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.014256173744797707,
      "clip_ratio/high_mean": 0.002001363394083455,
      "clip_ratio/low_mean": 0.001286455297667999,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0032878186670131982,
      "completion_length": 93.04500274658203,
      "epoch": 0.6800499471712612,
      "grad_norm": 1.323721170425415,
      "kl": 0.32287237197160723,
      "learning_rate": 3.090431112934235e-07,
      "loss": -0.0056,
      "reward": 1.8219903230667114,
      "reward_std": 0.28862411081790923,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6641201436519623,
      "step": 3540,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.0389601900940761,
      "clip_ratio/high_mean": 0.005975415915600024,
      "clip_ratio/low_mean": 0.0006638197373831645,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006639235676266253,
      "completion_length": 95.84250183105469,
      "epoch": 0.6819709922197675,
      "grad_norm": 4.850042819976807,
      "kl": 1.8627108559012413,
      "learning_rate": 3.067531853595369e-07,
      "loss": 1.6968,
      "reward": 1.8796481132507323,
      "reward_std": 0.13976119682192803,
      "rewards/code_format_reward": 0.9837500095367432,
      "rewards/code_reward": 0.6938865780830383,
      "step": 3550,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.01971529610455036,
      "clip_ratio/high_mean": 0.002580236754147336,
      "clip_ratio/low_mean": 0.0005060694311396219,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030863061954732986,
      "completion_length": 85.99000091552735,
      "epoch": 0.683892037268274,
      "grad_norm": 499.5800476074219,
      "kl": 3.8079170405864717,
      "learning_rate": 3.0447212426791546e-07,
      "loss": 0.0153,
      "reward": 1.73906729221344,
      "reward_std": 0.21255102157592773,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.622658634185791,
      "step": 3560,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.03546578506939113,
      "clip_ratio/high_mean": 0.005373837990919128,
      "clip_ratio/low_mean": 0.0011442803021054714,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0065181183628737925,
      "completion_length": 93.75249938964843,
      "epoch": 0.6858130823167803,
      "grad_norm": 3.144973039627075,
      "kl": 0.7828342400491237,
      "learning_rate": 3.022000111491309e-07,
      "loss": 0.0001,
      "reward": 1.8471190690994264,
      "reward_std": 0.27725095450878146,
      "rewards/code_format_reward": 0.9487499952316284,
      "rewards/code_reward": 0.686372023820877,
      "step": 3570,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.01367393396794796,
      "clip_ratio/high_mean": 0.001831050164764747,
      "clip_ratio/low_mean": 0.0008013980732357595,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002632448251824826,
      "completion_length": 96.29500122070313,
      "epoch": 0.6877341273652867,
      "grad_norm": 3.9027657508850098,
      "kl": 0.8669951900839805,
      "learning_rate": 2.99936928807657e-07,
      "loss": -0.0007,
      "reward": 1.6410433769226074,
      "reward_std": 0.25681858956813813,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.5736466705799103,
      "step": 3580,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.028569919406436384,
      "clip_ratio/high_mean": 0.0037314103537937626,
      "clip_ratio/low_mean": 0.0012738955876557157,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005005306130624376,
      "completion_length": 84.3875015258789,
      "epoch": 0.6896551724137931,
      "grad_norm": 1.8412340879440308,
      "kl": 0.6606554225087166,
      "learning_rate": 2.976829597188506e-07,
      "loss": -0.0007,
      "reward": 1.6131571292877198,
      "reward_std": 0.15807003602385522,
      "rewards/code_format_reward": 0.9950000047683716,
      "rewards/code_reward": 0.5578285574913024,
      "step": 3590,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.045477401558309795,
      "clip_ratio/high_mean": 0.007051247591152787,
      "clip_ratio/low_mean": 0.00021358822996262461,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007264835678506642,
      "completion_length": 92.69250030517578,
      "epoch": 0.6915762174622995,
      "grad_norm": 4.787570953369141,
      "kl": 0.2786871612071991,
      "learning_rate": 2.9543818602594826e-07,
      "loss": 0.0001,
      "reward": 1.6197675943374634,
      "reward_std": 0.2863120764493942,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.5651962697505951,
      "step": 3600,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.047238136362284425,
      "clip_ratio/high_mean": 0.006483453582040966,
      "clip_ratio/low_mean": 0.0015064548759255558,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007989908382296563,
      "completion_length": 83.11750183105468,
      "epoch": 0.6934972625108059,
      "grad_norm": 1.5795401334762573,
      "kl": 0.512858135998249,
      "learning_rate": 2.932026895370697e-07,
      "loss": 0.0021,
      "reward": 1.6763751983642579,
      "reward_std": 0.12559455148875714,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.5903751432895661,
      "step": 3610,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.031613614642992616,
      "clip_ratio/high_mean": 0.00453935784753412,
      "clip_ratio/low_mean": 0.0026672417909139766,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007206599647179246,
      "completion_length": 89.80250244140625,
      "epoch": 0.6954183075593122,
      "grad_norm": 0.9828081130981445,
      "kl": 2.053369848430157,
      "learning_rate": 2.909765517222392e-07,
      "loss": -0.0015,
      "reward": 1.6560463190078736,
      "reward_std": 0.2526627391576767,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5820856630802155,
      "step": 3620,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.00962083850754425,
      "clip_ratio/high_mean": 0.0013845860186847859,
      "clip_ratio/low_mean": 0.00100141861839802,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0023860046290792524,
      "completion_length": 93.0125015258789,
      "epoch": 0.6973393526078187,
      "grad_norm": 1.4326051473617554,
      "kl": 0.7425350762903691,
      "learning_rate": 2.887598537104141e-07,
      "loss": 0.017,
      "reward": 1.608488416671753,
      "reward_std": 0.18181688338518143,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.557056725025177,
      "step": 3630,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.10694800971541554,
      "clip_ratio/high_mean": 0.016866487907827833,
      "clip_ratio/low_mean": 0.000146488708560355,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017012976511614396,
      "completion_length": 86.96750183105469,
      "epoch": 0.6992603976563251,
      "grad_norm": 5.3714776039123535,
      "kl": 0.5909165881574154,
      "learning_rate": 2.8655267628653044e-07,
      "loss": 0.0005,
      "reward": 1.6461472749710082,
      "reward_std": 0.22788509875535964,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.5765111327171326,
      "step": 3640,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.015923467138782142,
      "clip_ratio/high_mean": 0.0022047571546863765,
      "clip_ratio/low_mean": 0.0014544774603564292,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036592346790712328,
      "completion_length": 91.53250122070312,
      "epoch": 0.7011814427048314,
      "grad_norm": 7.581000328063965,
      "kl": 3.2652564592659474,
      "learning_rate": 2.8435509988855683e-07,
      "loss": -0.0019,
      "reward": 1.6843700885772706,
      "reward_std": 0.20299706608057022,
      "rewards/code_format_reward": 0.993749988079071,
      "rewards/code_reward": 0.5937475442886353,
      "step": 3650,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.04569347179494798,
      "clip_ratio/high_mean": 0.00580012007849291,
      "clip_ratio/low_mean": 0.003195645064988639,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008995765156578272,
      "completion_length": 82.49500122070313,
      "epoch": 0.7031024877533378,
      "grad_norm": 10.031012535095215,
      "kl": 0.3446802504360676,
      "learning_rate": 2.821672046045642e-07,
      "loss": -0.003,
      "reward": 1.9148546934127808,
      "reward_std": 0.15906044691801072,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.7089898109436035,
      "step": 3660,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.01652005296200514,
      "clip_ratio/high_mean": 0.0032194001134485005,
      "clip_ratio/low_mean": 0.0004348491333075799,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003654249230748974,
      "completion_length": 87.48000030517578,
      "epoch": 0.7050235328018442,
      "grad_norm": 4.5817551612854,
      "kl": 0.5381794683635235,
      "learning_rate": 2.799890701698068e-07,
      "loss": -0.0018,
      "reward": 1.4432553768157959,
      "reward_std": 0.19258553311228752,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.4744401514530182,
      "step": 3670,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.03230769606307149,
      "clip_ratio/high_mean": 0.004254726751241833,
      "clip_ratio/low_mean": 0.0003341716161230579,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0045888983644545075,
      "completion_length": 91.96000366210937,
      "epoch": 0.7069445778503506,
      "grad_norm": 3.1825077533721924,
      "kl": 0.5493438571691514,
      "learning_rate": 2.7782077596381596e-07,
      "loss": 0.0032,
      "reward": 1.8943065643310546,
      "reward_std": 0.22485891729593277,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6990282416343689,
      "step": 3680,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.017006529681384563,
      "clip_ratio/high_mean": 0.0026059710187837483,
      "clip_ratio/low_mean": 0.00022266755404416473,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0028286385582759976,
      "completion_length": 92.6875015258789,
      "epoch": 0.708865622898857,
      "grad_norm": 3.126534938812256,
      "kl": 2.302929486706853,
      "learning_rate": 2.7566240100750794e-07,
      "loss": 0.0024,
      "reward": 1.6279277324676513,
      "reward_std": 0.3058730036020279,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.568026351928711,
      "step": 3690,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.020053896540775894,
      "clip_ratio/high_mean": 0.0029980215302202852,
      "clip_ratio/low_mean": 0.0004860887274844572,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003484110155841336,
      "completion_length": 97.92250061035156,
      "epoch": 0.7107866679473633,
      "grad_norm": 4.224461555480957,
      "kl": 4.42233342602849,
      "learning_rate": 2.735140239603034e-07,
      "loss": -0.0003,
      "reward": 1.960454559326172,
      "reward_std": 0.24239360094070433,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.7349147796630859,
      "step": 3700,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.02752018291503191,
      "clip_ratio/high_mean": 0.005125764373224229,
      "clip_ratio/low_mean": 0.00023403638042509555,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005359800613950938,
      "completion_length": 101.37250061035157,
      "epoch": 0.7127077129958698,
      "grad_norm": 4.285885334014893,
      "kl": 0.952894814312458,
      "learning_rate": 2.713757231172611e-07,
      "loss": -0.0013,
      "reward": 1.6773537874221802,
      "reward_std": 0.2778655707836151,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5927394092082977,
      "step": 3710,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.021348989009857176,
      "clip_ratio/high_mean": 0.0030060237273573875,
      "clip_ratio/low_mean": 0.0013181588088627904,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004324182611890137,
      "completion_length": 95.14250183105469,
      "epoch": 0.7146287580443761,
      "grad_norm": 2.7202091217041016,
      "kl": 2.8931914918124675,
      "learning_rate": 2.692475764062245e-07,
      "loss": -0.0021,
      "reward": 1.8867613315582275,
      "reward_std": 0.18746355026960373,
      "rewards/code_format_reward": 0.9987499952316284,
      "rewards/code_reward": 0.6936931371688843,
      "step": 3720,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.007143327506491914,
      "clip_ratio/high_mean": 0.0009208801442582626,
      "clip_ratio/low_mean": 0.00037053466949146243,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001291414822480874,
      "completion_length": 94.1875015258789,
      "epoch": 0.7165498030928825,
      "grad_norm": 2.7853496074676514,
      "kl": 0.6755535811185837,
      "learning_rate": 2.6712966138498174e-07,
      "loss": -0.003,
      "reward": 1.723927640914917,
      "reward_std": 0.2750594407320023,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.6163387894630432,
      "step": 3730,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.019027433777227997,
      "clip_ratio/high_mean": 0.002618219889700413,
      "clip_ratio/low_mean": 0.0018478537182090803,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004466073628282175,
      "completion_length": 102.04000091552734,
      "epoch": 0.718470848141389,
      "grad_norm": 5.998534202575684,
      "kl": 0.9062080264091492,
      "learning_rate": 2.650220552384391e-07,
      "loss": 0.0289,
      "reward": 1.8737354516983031,
      "reward_std": 0.34540517926216124,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6912427186965943,
      "step": 3740,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.056439303827937694,
      "clip_ratio/high_mean": 0.007310985976073425,
      "clip_ratio/low_mean": 0.0005420514833531342,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007853037484164816,
      "completion_length": 92.48250122070313,
      "epoch": 0.7203918931898953,
      "grad_norm": 5.3343424797058105,
      "kl": 0.3819971337914467,
      "learning_rate": 2.6292483477580816e-07,
      "loss": -0.011,
      "reward": 1.672910475730896,
      "reward_std": 0.2516419067978859,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5920802116394043,
      "step": 3750,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.13834233868401496,
      "clip_ratio/high_mean": 0.018591971611022017,
      "clip_ratio/low_mean": 0.0006771487518562935,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019269120390526952,
      "completion_length": 99.33000030517579,
      "epoch": 0.7223129382384017,
      "grad_norm": 1.4892189502716064,
      "kl": 0.9441468060016632,
      "learning_rate": 2.6083807642780644e-07,
      "loss": -0.0084,
      "reward": 1.5579908847808839,
      "reward_std": 0.272139647603035,
      "rewards/code_format_reward": 0.9787500023841857,
      "rewards/code_reward": 0.5343079507350922,
      "step": 3760,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.023900310718454422,
      "clip_ratio/high_mean": 0.005545906673069112,
      "clip_ratio/low_mean": 0.0007872088695876301,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006333115603774786,
      "completion_length": 90.84000396728516,
      "epoch": 0.724233983286908,
      "grad_norm": 12.181316375732422,
      "kl": 8.179486125707626,
      "learning_rate": 2.5876185624387225e-07,
      "loss": 0.0398,
      "reward": 1.743166995048523,
      "reward_std": 0.3216101437807083,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6259585380554199,
      "step": 3770,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.00846199265215546,
      "clip_ratio/high_mean": 0.0012625553936231881,
      "clip_ratio/low_mean": 0.00030621195983258074,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001568767352728173,
      "completion_length": 118.35750122070313,
      "epoch": 0.7261550283354145,
      "grad_norm": 1.6517783403396606,
      "kl": 0.968211068212986,
      "learning_rate": 2.5669624988939287e-07,
      "loss": 0.1551,
      "reward": 1.7871047019958497,
      "reward_std": 0.21420088410377502,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.644802349805832,
      "step": 3780,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.02817701958119869,
      "clip_ratio/high_mean": 0.0037564294645562766,
      "clip_ratio/low_mean": 0.011859719056519679,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015616148672415875,
      "completion_length": 93.14750213623047,
      "epoch": 0.7280760733839209,
      "grad_norm": 11.322369575500488,
      "kl": 0.45075275003910065,
      "learning_rate": 2.5464133264294705e-07,
      "loss": -0.0008,
      "reward": 1.662767267227173,
      "reward_std": 0.24967537969350814,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.5848211228847504,
      "step": 3790,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.05006286900024861,
      "clip_ratio/high_mean": 0.007249254969065077,
      "clip_ratio/low_mean": 0.00040258544613607227,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007651840391918086,
      "completion_length": 110.32750396728515,
      "epoch": 0.7299971184324272,
      "grad_norm": 16.862590789794922,
      "kl": 0.3901309326291084,
      "learning_rate": 2.5259717939356175e-07,
      "loss": -0.0019,
      "reward": 1.7777814149856568,
      "reward_std": 0.25982470586895945,
      "rewards/code_format_reward": 0.987500011920929,
      "rewards/code_reward": 0.6420157194137573,
      "step": 3800,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.007159786019474268,
      "clip_ratio/high_mean": 0.0011859470629133283,
      "clip_ratio/low_mean": 0.0021440873795654626,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033300343551672996,
      "completion_length": 96.07000122070312,
      "epoch": 0.7319181634809336,
      "grad_norm": 2.4953460693359375,
      "kl": 0.3146058402955532,
      "learning_rate": 2.505638646379831e-07,
      "loss": -0.0042,
      "reward": 1.7296765804290772,
      "reward_std": 0.3011175274848938,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6189007639884949,
      "step": 3810,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.03548359724227339,
      "clip_ratio/high_mean": 0.004679994014441036,
      "clip_ratio/low_mean": 0.00017329893162241206,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004853292935877107,
      "completion_length": 101.32000122070312,
      "epoch": 0.7338392085294401,
      "grad_norm": 3.954063892364502,
      "kl": 0.34448319524526594,
      "learning_rate": 2.485414624779603e-07,
      "loss": -0.0051,
      "reward": 1.690654444694519,
      "reward_std": 0.24299487322568894,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.6000146985054016,
      "step": 3820,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.00636189088691026,
      "clip_ratio/high_mean": 0.0008543322241166606,
      "clip_ratio/low_mean": 0.00028777473780792204,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0011421069371863267,
      "completion_length": 94.74000244140625,
      "epoch": 0.7357602535779464,
      "grad_norm": 1.0420587062835693,
      "kl": 0.28902386128902435,
      "learning_rate": 2.4653004661754703e-07,
      "loss": 0.0021,
      "reward": 1.929768443107605,
      "reward_std": 0.19695264101028442,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.7173841595649719,
      "step": 3830,
      "zero_std_ratio": 0.7
    },
    {
      "clip_ratio/high_max": 0.0385974693344906,
      "clip_ratio/high_mean": 0.0054892279236810285,
      "clip_ratio/low_mean": 0.0004371934803202748,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005926421421463601,
      "completion_length": 100.23250122070313,
      "epoch": 0.7376812986264528,
      "grad_norm": 6.22709846496582,
      "kl": 0.39053357392549515,
      "learning_rate": 2.445296903604131e-07,
      "loss": -0.0123,
      "reward": 1.7683161497116089,
      "reward_std": 0.4236398935317993,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.6413455486297608,
      "step": 3840,
      "zero_std_ratio": 0.3
    },
    {
      "clip_ratio/high_max": 0.013776408764533699,
      "clip_ratio/high_mean": 0.0019065461441641674,
      "clip_ratio/low_mean": 0.0035487653221935034,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005455311315017752,
      "completion_length": 91.36000213623046,
      "epoch": 0.7396023436749591,
      "grad_norm": 3.84639573097229,
      "kl": 9.267435324192046,
      "learning_rate": 2.4254046660717555e-07,
      "loss": 0.0107,
      "reward": 1.7194789409637452,
      "reward_std": 0.23012096285820008,
      "rewards/code_format_reward": 0.98125,
      "rewards/code_reward": 0.6144269347190857,
      "step": 3850,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.019276025268482044,
      "clip_ratio/high_mean": 0.0034578723403683397,
      "clip_ratio/low_mean": 0.0028569042566232382,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006314776389626786,
      "completion_length": 96.3250015258789,
      "epoch": 0.7415233887234656,
      "grad_norm": 4.765519142150879,
      "kl": 0.5375766545534134,
      "learning_rate": 2.4056244785273895e-07,
      "loss": -0.0038,
      "reward": 1.713827419281006,
      "reward_std": 0.28884910941123965,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6112887144088746,
      "step": 3860,
      "zero_std_ratio": 0.35
    },
    {
      "clip_ratio/high_max": 0.06692883024225012,
      "clip_ratio/high_mean": 0.008779459849756676,
      "clip_ratio/low_mean": 0.0002708235711907037,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009050283460237552,
      "completion_length": 103.41250152587891,
      "epoch": 0.743444433771972,
      "grad_norm": 2.68007493019104,
      "kl": 0.34222877621650694,
      "learning_rate": 2.3859570618365614e-07,
      "loss": -0.0009,
      "reward": 1.74418466091156,
      "reward_std": 0.20953620076179505,
      "rewards/code_format_reward": 0.9912499785423279,
      "rewards/code_reward": 0.6242798089981079,
      "step": 3870,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.01482260066550225,
      "clip_ratio/high_mean": 0.0023997865355340764,
      "clip_ratio/low_mean": 0.00038790585967944934,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002787692387937568,
      "completion_length": 98.42250061035156,
      "epoch": 0.7453654788204783,
      "grad_norm": 4.816893100738525,
      "kl": 0.4661983668804169,
      "learning_rate": 2.366403132754995e-07,
      "loss": -0.0019,
      "reward": 1.6338875532150268,
      "reward_std": 0.21452725008130075,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.5697562634944916,
      "step": 3880,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.02494101980701089,
      "clip_ratio/high_mean": 0.003492716047912836,
      "clip_ratio/low_mean": 0.00024301124794874341,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0037357273045927285,
      "completion_length": 97.44000091552735,
      "epoch": 0.7472865238689848,
      "grad_norm": 82.46282958984375,
      "kl": 0.5981974095106125,
      "learning_rate": 2.3469634039024927e-07,
      "loss": 0.0024,
      "reward": 1.8161945581436156,
      "reward_std": 0.17759706005454062,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6621597528457641,
      "step": 3890,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.0019452353473752737,
      "clip_ratio/high_mean": 0.00039936143439263106,
      "clip_ratio/low_mean": 0.0002315789126441814,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0006309403397608548,
      "completion_length": 94.07750091552734,
      "epoch": 0.7492075689174911,
      "grad_norm": 6.090396404266357,
      "kl": 0.8421477146446705,
      "learning_rate": 2.3276385837369632e-07,
      "loss": 0.014,
      "reward": 1.4471250534057618,
      "reward_std": 0.25895166750997306,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.4773125171661377,
      "step": 3900,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.02143551183398813,
      "clip_ratio/high_mean": 0.002903820894425735,
      "clip_ratio/low_mean": 0.00011704202042892576,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030208629119442775,
      "completion_length": 89.32750091552734,
      "epoch": 0.7511286139659975,
      "grad_norm": 7.675207614898682,
      "kl": 4.630686198174954,
      "learning_rate": 2.3084293765286074e-07,
      "loss": 0.0109,
      "reward": 1.7639801740646361,
      "reward_std": 0.32505679726600645,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6360525727272034,
      "step": 3910,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.007216949050780385,
      "clip_ratio/high_mean": 0.0012314463703660295,
      "clip_ratio/low_mean": 0.000596191274235025,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018276376475114375,
      "completion_length": 93.16250152587891,
      "epoch": 0.7530496590145039,
      "grad_norm": 3.4967644214630127,
      "kl": 0.9979558669030666,
      "learning_rate": 2.2893364823342454e-07,
      "loss": 0.0016,
      "reward": 1.5569410085678101,
      "reward_std": 0.2807903170585632,
      "rewards/code_format_reward": 0.9674999952316284,
      "rewards/code_reward": 0.5365955173969269,
      "step": 3920,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.01850514723919332,
      "clip_ratio/high_mean": 0.003044746146770194,
      "clip_ratio/low_mean": 0.0006967324326978997,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0037414785125292837,
      "completion_length": 95.95500183105469,
      "epoch": 0.7549707040630103,
      "grad_norm": 2.8742544651031494,
      "kl": 0.44021010398864746,
      "learning_rate": 2.270360596971809e-07,
      "loss": -0.0037,
      "reward": 1.823073673248291,
      "reward_std": 0.24968771934509276,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.663411819934845,
      "step": 3930,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.015126590803265571,
      "clip_ratio/high_mean": 0.0023361636558547616,
      "clip_ratio/low_mean": 0.00015787699958309532,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002494040655437857,
      "completion_length": 91.19500122070312,
      "epoch": 0.7568917491115167,
      "grad_norm": 3.40413236618042,
      "kl": 0.386103405430913,
      "learning_rate": 2.2515024119949826e-07,
      "loss": -0.011,
      "reward": 1.5718731164932251,
      "reward_std": 0.2807211749255657,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.5415615499019623,
      "step": 3940,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.015597179555334151,
      "clip_ratio/high_mean": 0.0027747701620683073,
      "clip_ratio/low_mean": 0.00042166481143794953,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003196435049176216,
      "completion_length": 98.425,
      "epoch": 0.758812794160023,
      "grad_norm": 4.560734272003174,
      "kl": 0.4831135801970959,
      "learning_rate": 2.2327626146679974e-07,
      "loss": -0.0022,
      "reward": 1.7759766340255738,
      "reward_std": 0.2547271862626076,
      "rewards/code_format_reward": 0.9625,
      "rewards/code_reward": 0.6473633050918579,
      "step": 3950,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.008683394081890583,
      "clip_ratio/high_mean": 0.0011145618045702577,
      "clip_ratio/low_mean": 0.0009394719265401364,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020540336496196686,
      "completion_length": 102.31250305175782,
      "epoch": 0.7607338392085294,
      "grad_norm": 0.1577247530221939,
      "kl": 1.2770531885325909,
      "learning_rate": 2.2141418879405855e-07,
      "loss": 0.0032,
      "reward": 1.7324957370758056,
      "reward_std": 0.19914634823799132,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.6199978470802308,
      "step": 3960,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.004086668835952878,
      "clip_ratio/high_mean": 0.0005708287237212062,
      "clip_ratio/low_mean": 2.821670495904982e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0005990454228594899,
      "completion_length": 95.21750335693359,
      "epoch": 0.7626548842570359,
      "grad_norm": 268.0164794921875,
      "kl": 3.985953611135483,
      "learning_rate": 2.1956409104230986e-07,
      "loss": 0.0127,
      "reward": 1.7277408480644225,
      "reward_std": 0.19516595900058747,
      "rewards/code_format_reward": 0.9737500071525573,
      "rewards/code_reward": 0.6204329133033752,
      "step": 3970,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.02115430913399905,
      "clip_ratio/high_mean": 0.003100222998182289,
      "clip_ratio/low_mean": 0.00045731081045232715,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0035575337868067438,
      "completion_length": 99.47250213623047,
      "epoch": 0.7645759293055422,
      "grad_norm": 4.087578773498535,
      "kl": 0.2619202695786953,
      "learning_rate": 2.1772603563617603e-07,
      "loss": -0.0024,
      "reward": 1.6976868152618407,
      "reward_std": 0.31094631999731065,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.6041558861732483,
      "step": 3980,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.022111613873858006,
      "clip_ratio/high_mean": 0.0033171431292430497,
      "clip_ratio/low_mean": 0.00019350402581039817,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003510647150687873,
      "completion_length": 93.09000091552734,
      "epoch": 0.7664969743540486,
      "grad_norm": 2.557553291320801,
      "kl": 0.4590821463614702,
      "learning_rate": 2.1590008956141137e-07,
      "loss": -0.0014,
      "reward": 1.7825278520584107,
      "reward_std": 0.26515288949012755,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.6440764307975769,
      "step": 3990,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.03076116186566651,
      "clip_ratio/high_mean": 0.004437833256088197,
      "clip_ratio/low_mean": 0.0004819675668841228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004919800782226957,
      "completion_length": 89.73500061035156,
      "epoch": 0.7684180194025549,
      "grad_norm": 2.5422067642211914,
      "kl": 0.26607592329382895,
      "learning_rate": 2.1408631936245908e-07,
      "loss": 0.0026,
      "reward": 1.8288384914398192,
      "reward_std": 0.2508297085762024,
      "rewards/code_format_reward": 0.9837499856948853,
      "rewards/code_reward": 0.6684817314147949,
      "step": 4000,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.020826040930114687,
      "clip_ratio/high_mean": 0.0040985049330629405,
      "clip_ratio/low_mean": 0.000369196553947404,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004467701492831111,
      "completion_length": 97.69500122070312,
      "epoch": 0.7703390644510614,
      "grad_norm": 2.079371929168701,
      "kl": 0.3304180882871151,
      "learning_rate": 2.122847911400278e-07,
      "loss": 0.0019,
      "reward": 1.693557620048523,
      "reward_std": 0.21333991810679437,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5977162718772888,
      "step": 4010,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.009364375309087337,
      "clip_ratio/high_mean": 0.0013745424774242565,
      "clip_ratio/low_mean": 0.0020853754234849476,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034599179547512905,
      "completion_length": 94.00750274658203,
      "epoch": 0.7722601094995678,
      "grad_norm": 3.2660512924194336,
      "kl": 0.6432372182607651,
      "learning_rate": 2.1049557054868082e-07,
      "loss": 0.0073,
      "reward": 1.8483120203018188,
      "reward_std": 0.316910046339035,
      "rewards/code_format_reward": 0.9649999856948852,
      "rewards/code_reward": 0.6829060018062592,
      "step": 4020,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.08980275879148394,
      "clip_ratio/high_mean": 0.011746273408061825,
      "clip_ratio/low_mean": 0.000331929670937825,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012078202966949902,
      "completion_length": 92.7925018310547,
      "epoch": 0.7741811545480741,
      "grad_norm": 3.004549503326416,
      "kl": 0.74478175714612,
      "learning_rate": 2.0871872279444554e-07,
      "loss": -0.0021,
      "reward": 1.7010861873626708,
      "reward_std": 0.25111902356147764,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.6071055889129638,
      "step": 4030,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.0778543038177304,
      "clip_ratio/high_mean": 0.00988141688721953,
      "clip_ratio/low_mean": 0.0002543082577176392,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01013572499359725,
      "completion_length": 105.63250122070312,
      "epoch": 0.7761021995965806,
      "grad_norm": 6.268821716308594,
      "kl": 0.32837071269750595,
      "learning_rate": 2.0695431263243512e-07,
      "loss": -0.0003,
      "reward": 1.716653084754944,
      "reward_std": 0.2870768278837204,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6108265280723572,
      "step": 4040,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.07360692555084825,
      "clip_ratio/high_mean": 0.009302017895970493,
      "clip_ratio/low_mean": 0.0003425976261496544,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009644615522120148,
      "completion_length": 91.73750152587891,
      "epoch": 0.7780232446450869,
      "grad_norm": 4.801341533660889,
      "kl": 13.291237189993263,
      "learning_rate": 2.052024043644897e-07,
      "loss": 0.0294,
      "reward": 1.7232446193695068,
      "reward_std": 0.24269133806228638,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6134972870349884,
      "step": 4050,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.012731208954937756,
      "clip_ratio/high_mean": 0.00180651948612649,
      "clip_ratio/low_mean": 0.00015854310477152466,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019650626258226112,
      "completion_length": 92.22000274658203,
      "epoch": 0.7799442896935933,
      "grad_norm": 0.6561126112937927,
      "kl": 0.4966626279056072,
      "learning_rate": 2.0346306183683254e-07,
      "loss": 0.0001,
      "reward": 1.8969059467315674,
      "reward_std": 0.33292114436626435,
      "rewards/code_format_reward": 0.9800000071525574,
      "rewards/code_reward": 0.7034529447555542,
      "step": 4060,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.015003547444939614,
      "clip_ratio/high_mean": 0.002088976529194042,
      "clip_ratio/low_mean": 0.0003269152017310262,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002415891730925068,
      "completion_length": 88.61250152587891,
      "epoch": 0.7818653347420997,
      "grad_norm": 3.062511920928955,
      "kl": 27.40203034952283,
      "learning_rate": 2.0173634843774363e-07,
      "loss": 0.0554,
      "reward": 1.7011754512786865,
      "reward_std": 0.3188599109649658,
      "rewards/code_format_reward": 0.981250011920929,
      "rewards/code_reward": 0.6052752196788788,
      "step": 4070,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.006837105128215626,
      "clip_ratio/high_mean": 0.0008925169277063105,
      "clip_ratio/low_mean": 0.0005512935545993969,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0014438104728469626,
      "completion_length": 91.84750213623047,
      "epoch": 0.7837863797906061,
      "grad_norm": 3.0254440307617188,
      "kl": 1.3981286019086838,
      "learning_rate": 2.0002232709524897e-07,
      "loss": 0.0033,
      "reward": 1.6401101350784302,
      "reward_std": 0.26853239685297015,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.5738050699234009,
      "step": 4080,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.03983018643921241,
      "clip_ratio/high_mean": 0.005185264609463047,
      "clip_ratio/low_mean": 0.0019072047754889355,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007092469278723002,
      "completion_length": 88.79250030517578,
      "epoch": 0.7857074248391125,
      "grad_norm": 2.8119072914123535,
      "kl": 0.41205914914608,
      "learning_rate": 1.983210602748279e-07,
      "loss": -0.0029,
      "reward": 1.9083050966262818,
      "reward_std": 0.29446094632148745,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.7085274815559387,
      "step": 4090,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.013765955006238072,
      "clip_ratio/high_mean": 0.0018926289907540196,
      "clip_ratio/low_mean": 0.0033484802523162218,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00524110905098496,
      "completion_length": 85.72500305175781,
      "epoch": 0.7876284698876188,
      "grad_norm": 9.436022758483887,
      "kl": 0.5864221028983593,
      "learning_rate": 1.966326099771361e-07,
      "loss": -0.0013,
      "reward": 1.8478533029556274,
      "reward_std": 0.2244624227285385,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6770516157150268,
      "step": 4100,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.008409230364486575,
      "clip_ratio/high_mean": 0.0011749810015317052,
      "clip_ratio/low_mean": 0.00043775633239420133,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001612737326649949,
      "completion_length": 91.16000213623047,
      "epoch": 0.7895495149361252,
      "grad_norm": 6.15724515914917,
      "kl": 19.288723162561656,
      "learning_rate": 1.9495703773574628e-07,
      "loss": 0.0383,
      "reward": 1.6099607944488525,
      "reward_std": 0.30300846993923186,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.5599803984165191,
      "step": 4110,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.009142859559506177,
      "clip_ratio/high_mean": 0.001581054090638645,
      "clip_ratio/low_mean": 0.0003552554393536411,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019363095459993928,
      "completion_length": 91.99500122070313,
      "epoch": 0.7914705599846317,
      "grad_norm": 6.634824752807617,
      "kl": 6.53539779484272,
      "learning_rate": 1.9329440461490576e-07,
      "loss": 0.0342,
      "reward": 1.647179627418518,
      "reward_std": 0.2863168239593506,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5770273089408875,
      "step": 4120,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.002549535338766873,
      "clip_ratio/high_mean": 0.0003399143257411197,
      "clip_ratio/low_mean": 0.00017536718805786223,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.000515281516709365,
      "completion_length": 90.89250183105469,
      "epoch": 0.793391605033138,
      "grad_norm": 2.817605972290039,
      "kl": 2.417458937317133,
      "learning_rate": 1.9164477120731066e-07,
      "loss": 0.0066,
      "reward": 1.7660948038101196,
      "reward_std": 0.2769928514957428,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.640859854221344,
      "step": 4130,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.037738511635689066,
      "clip_ratio/high_mean": 0.0050403060296957845,
      "clip_ratio/low_mean": 0.0007518758837250061,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005792181929427898,
      "completion_length": 96.35750122070313,
      "epoch": 0.7953126500816444,
      "grad_norm": 4.240172386169434,
      "kl": 0.28425633125007155,
      "learning_rate": 1.900081976318983e-07,
      "loss": 0.002,
      "reward": 1.6942025184631349,
      "reward_std": 0.3146607309579849,
      "rewards/code_format_reward": 0.9737499833106995,
      "rewards/code_reward": 0.6036637306213379,
      "step": 4140,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.005694918753579259,
      "clip_ratio/high_mean": 0.0007544978521764279,
      "clip_ratio/low_mean": 0.000571403895446565,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001325901737436652,
      "completion_length": 91.79500122070313,
      "epoch": 0.7972336951301509,
      "grad_norm": 3.9649434089660645,
      "kl": 0.5314306125044823,
      "learning_rate": 1.8838474353165547e-07,
      "loss": -0.0054,
      "reward": 1.7638010501861572,
      "reward_std": 0.2793388396501541,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.6362755179405213,
      "step": 4150,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.07577090607956052,
      "clip_ratio/high_mean": 0.009897856542374938,
      "clip_ratio/low_mean": 0.00011142043076688424,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010009276978962589,
      "completion_length": 94.04000244140624,
      "epoch": 0.7991547401786572,
      "grad_norm": 2.2340188026428223,
      "kl": 0.524626237899065,
      "learning_rate": 1.8677446807144554e-07,
      "loss": -0.0045,
      "reward": 1.7472325563430786,
      "reward_std": 0.3027869775891304,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.6289287328720092,
      "step": 4160,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.012572024948894978,
      "clip_ratio/high_mean": 0.0020916348788887263,
      "clip_ratio/low_mean": 0.00022255638323258607,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0023141912854043765,
      "completion_length": 94.53000183105469,
      "epoch": 0.8010757852271636,
      "grad_norm": 10.561907768249512,
      "kl": 2.102495136484504,
      "learning_rate": 1.8517742993585178e-07,
      "loss": 0.0137,
      "reward": 1.7456205368041993,
      "reward_std": 0.2167625606060028,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.6262477397918701,
      "step": 4170,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.06855184989399277,
      "clip_ratio/high_mean": 0.008778795686521335,
      "clip_ratio/low_mean": 5.122950533404946e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008830025191855384,
      "completion_length": 101.0425018310547,
      "epoch": 0.8029968302756699,
      "grad_norm": 5.673184871673584,
      "kl": 0.428597304970026,
      "learning_rate": 1.835936873270389e-07,
      "loss": -0.0078,
      "reward": 1.818405318260193,
      "reward_std": 0.23994216322898865,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6626401782035828,
      "step": 4180,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.002845590282231569,
      "clip_ratio/high_mean": 0.0004983038117643446,
      "clip_ratio/low_mean": 0.00037282529519870876,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008711291156942025,
      "completion_length": 92.31250305175782,
      "epoch": 0.8049178753241764,
      "grad_norm": 6.281589508056641,
      "kl": 0.4346353754401207,
      "learning_rate": 1.8202329796263172e-07,
      "loss": -0.0009,
      "reward": 1.8768694639205932,
      "reward_std": 0.21767425537109375,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6918722629547119,
      "step": 4190,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.003876271191984415,
      "clip_ratio/high_mean": 0.0004845338989980519,
      "clip_ratio/low_mean": 0.0001560977878398262,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0006406316824723035,
      "completion_length": 75.7275016784668,
      "epoch": 0.8068389203726828,
      "grad_norm": 1.0423272848129272,
      "kl": 0.9193772681057453,
      "learning_rate": 1.8046631907361226e-07,
      "loss": 0.0041,
      "reward": 1.8756553649902343,
      "reward_std": 0.18836807161569596,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.6893901348114013,
      "step": 4200,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.004609162057749927,
      "clip_ratio/high_mean": 0.0007380300055956468,
      "clip_ratio/low_mean": 0.00015018127305665985,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.000888211271376349,
      "completion_length": 86.45750122070312,
      "epoch": 0.8087599654211891,
      "grad_norm": 4.096966743469238,
      "kl": 0.45643181502819063,
      "learning_rate": 1.7892280740223303e-07,
      "loss": -0.004,
      "reward": 1.5836501359939574,
      "reward_std": 0.2258547842502594,
      "rewards/code_format_reward": 0.9799999952316284,
      "rewards/code_reward": 0.5468250632286071,
      "step": 4210,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.007562826108187437,
      "clip_ratio/high_mean": 0.0010166528285481037,
      "clip_ratio/low_mean": 0.000701455632224679,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0017181085073389112,
      "completion_length": 90.34250335693359,
      "epoch": 0.8106810104696955,
      "grad_norm": 0.29423439502716064,
      "kl": 0.2636001568287611,
      "learning_rate": 1.7739281919995045e-07,
      "loss": 0.0161,
      "reward": 1.5646157741546631,
      "reward_std": 0.12648468129336835,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5363703727722168,
      "step": 4220,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.02073557274416089,
      "clip_ratio/high_mean": 0.002738419675733894,
      "clip_ratio/low_mean": 0.001565844019933138,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004304263507947326,
      "completion_length": 85.92750091552735,
      "epoch": 0.8126020555182019,
      "grad_norm": 3.8796801567077637,
      "kl": 0.6587013073265553,
      "learning_rate": 1.7587641022537335e-07,
      "loss": -0.0031,
      "reward": 1.598485040664673,
      "reward_std": 0.23664331436157227,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5526800036430359,
      "step": 4230,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.0027086240705102684,
      "clip_ratio/high_mean": 0.0003605463745770976,
      "clip_ratio/low_mean": 0.0004666288397856988,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008271752245491371,
      "completion_length": 86.6875,
      "epoch": 0.8145231005667083,
      "grad_norm": 6.270168781280518,
      "kl": 3.9651204235851765,
      "learning_rate": 1.7437363574223244e-07,
      "loss": 0.0141,
      "reward": 1.8213656187057494,
      "reward_std": 0.2221561223268509,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.66474529504776,
      "step": 4240,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.009644670388661325,
      "clip_ratio/high_mean": 0.0013658979878528044,
      "clip_ratio/low_mean": 0.0006750999338692055,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002040997930453159,
      "completion_length": 86.42250061035156,
      "epoch": 0.8164441456152147,
      "grad_norm": 4.402440071105957,
      "kl": 0.27487861886620524,
      "learning_rate": 1.7288455051736474e-07,
      "loss": -0.0005,
      "reward": 1.6581492662429809,
      "reward_std": 0.14444592781364918,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5825121104717255,
      "step": 4250,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.017426467640325426,
      "clip_ratio/high_mean": 0.0023936200188472865,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0023936200188472865,
      "completion_length": 88.66250152587891,
      "epoch": 0.818365190663721,
      "grad_norm": 15.625293731689453,
      "kl": 0.5453658372163772,
      "learning_rate": 1.7140920881871927e-07,
      "loss": 0.0001,
      "reward": 1.9025921821594238,
      "reward_std": 0.1951783686876297,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.7037960886955261,
      "step": 4260,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.024276501801796257,
      "clip_ratio/high_mean": 0.0037041545001557097,
      "clip_ratio/low_mean": 0.0004929742426611483,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004197128777741454,
      "completion_length": 94.525,
      "epoch": 0.8202862357122275,
      "grad_norm": 19.728607177734375,
      "kl": 3.983573118597269,
      "learning_rate": 1.699476644133778e-07,
      "loss": 0.0122,
      "reward": 1.7488954544067383,
      "reward_std": 0.2558127373456955,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6269477069377899,
      "step": 4270,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.00913497168221511,
      "clip_ratio/high_mean": 0.0011987716374278535,
      "clip_ratio/low_mean": 0.0007998564062290825,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019986280538432767,
      "completion_length": 87.73999938964843,
      "epoch": 0.8222072807607338,
      "grad_norm": 4.567457675933838,
      "kl": 0.6975361555814743,
      "learning_rate": 1.6849997056559662e-07,
      "loss": -0.0116,
      "reward": 1.7202219009399413,
      "reward_std": 0.27057143300771713,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.6169859290122985,
      "step": 4280,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.016106344643048942,
      "clip_ratio/high_mean": 0.002376156343962066,
      "clip_ratio/low_mean": 5.540161509998143e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0024315579648828134,
      "completion_length": 94.21750030517578,
      "epoch": 0.8241283258092402,
      "grad_norm": 17.505773544311523,
      "kl": 1.1381098613142968,
      "learning_rate": 1.670661800348644e-07,
      "loss": -0.0006,
      "reward": 1.7664429664611816,
      "reward_std": 0.283676877617836,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.6369715094566345,
      "step": 4290,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.05613061334006488,
      "clip_ratio/high_mean": 0.0073514855874236675,
      "clip_ratio/low_mean": 0.00012296391359996052,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0074744494573678825,
      "completion_length": 94.24500122070313,
      "epoch": 0.8260493708577467,
      "grad_norm": 36.729576110839844,
      "kl": 2.2444246262311935,
      "learning_rate": 1.656463450739801e-07,
      "loss": 0.0024,
      "reward": 1.7431164741516114,
      "reward_std": 0.29661422967910767,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.6268707036972045,
      "step": 4300,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.003962649451568723,
      "clip_ratio/high_mean": 0.0005655559070874006,
      "clip_ratio/low_mean": 0.00023454214970115573,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008000980655197054,
      "completion_length": 91.94250030517578,
      "epoch": 0.827970415906253,
      "grad_norm": 5.331088066101074,
      "kl": 0.6558065637946129,
      "learning_rate": 1.6424051742714851e-07,
      "loss": 0.0002,
      "reward": 1.76786208152771,
      "reward_std": 0.17127570807933806,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.6361185550689697,
      "step": 4310,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.007816581195220352,
      "clip_ratio/high_mean": 0.001516599569004029,
      "clip_ratio/low_mean": 4.7630388871766625e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015642299549654126,
      "completion_length": 82.7000015258789,
      "epoch": 0.8298914609547594,
      "grad_norm": 9.622750282287598,
      "kl": 0.9509772717952728,
      "learning_rate": 1.6284874832809436e-07,
      "loss": 0.0023,
      "reward": 1.9346927881240845,
      "reward_std": 0.3074748650193214,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.7189089298248291,
      "step": 4320,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.02933923137607053,
      "clip_ratio/high_mean": 0.004640504893905018,
      "clip_ratio/low_mean": 0.00011013215407729149,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004750637047982309,
      "completion_length": 88.08000183105469,
      "epoch": 0.8318125060032657,
      "grad_norm": 1.8961539268493652,
      "kl": 1.2761327236890794,
      "learning_rate": 1.614710884981951e-07,
      "loss": 0.0002,
      "reward": 1.5815791606903076,
      "reward_std": 0.24661691784858703,
      "rewards/code_format_reward": 0.9862499833106995,
      "rewards/code_reward": 0.5442270636558533,
      "step": 4330,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.035589413810521366,
      "clip_ratio/high_mean": 0.005689902242738754,
      "clip_ratio/low_mean": 0.00015636042662663384,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005846262606792152,
      "completion_length": 89.16500244140624,
      "epoch": 0.8337335510517722,
      "grad_norm": 1.6006284952163696,
      "kl": 0.6420656457543373,
      "learning_rate": 1.6010758814463287e-07,
      "loss": 0.0027,
      "reward": 1.643228530883789,
      "reward_std": 0.2129346549510956,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.5741142451763153,
      "step": 4340,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.007347659638617188,
      "clip_ratio/high_mean": 0.001005946182704065,
      "clip_ratio/low_mean": 0.00028780620195902886,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012937523904838599,
      "completion_length": 98.85000152587891,
      "epoch": 0.8356545961002786,
      "grad_norm": 5.479083061218262,
      "kl": 0.3409851986914873,
      "learning_rate": 1.5875829695856406e-07,
      "loss": -0.0007,
      "reward": 1.882705855369568,
      "reward_std": 0.22037020921707154,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.6938528895378113,
      "step": 4350,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.032100778096355496,
      "clip_ratio/high_mean": 0.004409284892608412,
      "clip_ratio/low_mean": 4.9924499762710184e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004459209358901717,
      "completion_length": 90.73500213623046,
      "epoch": 0.8375756411487849,
      "grad_norm": 56.100852966308594,
      "kl": 0.22566271349787712,
      "learning_rate": 1.5742326411330942e-07,
      "loss": 0.0011,
      "reward": 1.8064903020858765,
      "reward_std": 0.1691088706254959,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.65418261885643,
      "step": 4360,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.005500979837961495,
      "clip_ratio/high_mean": 0.0008934643206885085,
      "clip_ratio/low_mean": 0.0005694760067854077,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001462940318742767,
      "completion_length": 93.03750305175781,
      "epoch": 0.8394966861972913,
      "grad_norm": 7.828958034515381,
      "kl": 0.6565275602042675,
      "learning_rate": 1.5610253826256036e-07,
      "loss": 0.003,
      "reward": 1.7732144832611083,
      "reward_std": 0.33924323320388794,
      "rewards/code_format_reward": 0.9825000047683716,
      "rewards/code_reward": 0.6409822225570678,
      "step": 4370,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.0038046793546527625,
      "clip_ratio/high_mean": 0.0004755849193315953,
      "clip_ratio/low_mean": 0.0006387827248545364,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0011143676441861317,
      "completion_length": 85.95250244140625,
      "epoch": 0.8414177312457977,
      "grad_norm": 3.0064802169799805,
      "kl": 9.46174124404788,
      "learning_rate": 1.5479616753860792e-07,
      "loss": 0.0195,
      "reward": 1.8130270481109618,
      "reward_std": 0.1679749459028244,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6583885312080383,
      "step": 4380,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.020394568890333177,
      "clip_ratio/high_mean": 0.002549321111291647,
      "clip_ratio/low_mean": 0.0012285682838410138,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003777889395132661,
      "completion_length": 94.72500305175781,
      "epoch": 0.8433387762943041,
      "grad_norm": 8.23426342010498,
      "kl": 0.3538756832480431,
      "learning_rate": 1.5350419955058645e-07,
      "loss": -0.0046,
      "reward": 1.6075192928314208,
      "reward_std": 0.16927714347839357,
      "rewards/code_format_reward": 0.9962499976158142,
      "rewards/code_reward": 0.5546970963478088,
      "step": 4390,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.06588131491444074,
      "clip_ratio/high_mean": 0.009102540424646578,
      "clip_ratio/low_mean": 0.0007730728961178101,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009875613666372374,
      "completion_length": 90.04500274658203,
      "epoch": 0.8452598213428105,
      "grad_norm": 7.7215776443481445,
      "kl": 0.2362464390695095,
      "learning_rate": 1.522266813827407e-07,
      "loss": 0.0036,
      "reward": 1.8586368560791016,
      "reward_std": 0.2194239765405655,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.6805683970451355,
      "step": 4400,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.004174966411665082,
      "clip_ratio/high_mean": 0.0007423789938911796,
      "clip_ratio/low_mean": 7.375134955509566e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008161303412634879,
      "completion_length": 88.90750122070312,
      "epoch": 0.8471808663913168,
      "grad_norm": 2.829716920852661,
      "kl": 1.5581397600471973,
      "learning_rate": 1.509636595927078e-07,
      "loss": 0.003,
      "reward": 1.9052275657653808,
      "reward_std": 0.256375952064991,
      "rewards/code_format_reward": 0.9787499904632568,
      "rewards/code_reward": 0.7079262495040893,
      "step": 4410,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.07640773041639477,
      "clip_ratio/high_mean": 0.009898414360941387,
      "clip_ratio/low_mean": 9.467430354561656e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00999308866157662,
      "completion_length": 95.14500122070312,
      "epoch": 0.8491019114398233,
      "grad_norm": 0.3017069101333618,
      "kl": 0.8497596487402916,
      "learning_rate": 1.4971518020982232e-07,
      "loss": -0.0017,
      "reward": 1.5574845552444458,
      "reward_std": 0.1220448928885162,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5321797609329224,
      "step": 4420,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.02955477687064558,
      "clip_ratio/high_mean": 0.00414732932113111,
      "clip_ratio/low_mean": 4.42216987721622e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00419155100826174,
      "completion_length": 99.7,
      "epoch": 0.8510229564883296,
      "grad_norm": 5.942404747009277,
      "kl": 0.5168043114244938,
      "learning_rate": 1.4848128873343773e-07,
      "loss": -0.0003,
      "reward": 1.6633994817733764,
      "reward_std": 0.2619109332561493,
      "rewards/code_format_reward": 0.9762499928474426,
      "rewards/code_reward": 0.5876372039318085,
      "step": 4430,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.014663098810706288,
      "clip_ratio/high_mean": 0.0024809099428239278,
      "clip_ratio/low_mean": 3.1672295881435276e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002512582238705363,
      "completion_length": 100.20750122070312,
      "epoch": 0.852944001536836,
      "grad_norm": 3.1078836917877197,
      "kl": 0.39873379915952684,
      "learning_rate": 1.4726203013126844e-07,
      "loss": 0.006,
      "reward": 1.7631917238235473,
      "reward_std": 0.22433922737836837,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.6350333511829376,
      "step": 4440,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.014706605696119368,
      "clip_ratio/high_mean": 0.00257694432802964,
      "clip_ratio/low_mean": 0.00032811136916279795,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002905055697192438,
      "completion_length": 99.6875,
      "epoch": 0.8548650465853425,
      "grad_norm": 8.708415985107422,
      "kl": 0.4677444875240326,
      "learning_rate": 1.4605744883775122e-07,
      "loss": -0.0036,
      "reward": 1.8840698957443238,
      "reward_std": 0.2510286644101143,
      "rewards/code_format_reward": 0.9850000143051147,
      "rewards/code_reward": 0.6957849264144897,
      "step": 4450,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.008664844953455032,
      "clip_ratio/high_mean": 0.0019024941400857642,
      "clip_ratio/low_mean": 0.002259151160251349,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004161645277054049,
      "completion_length": 89.9000015258789,
      "epoch": 0.8567860916338488,
      "grad_norm": 7.514847755432129,
      "kl": 0.3879747323691845,
      "learning_rate": 1.4486758875242557e-07,
      "loss": -0.0046,
      "reward": 1.9147763013839723,
      "reward_std": 0.2857444554567337,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.7102006316184998,
      "step": 4460,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.01213214877061546,
      "clip_ratio/high_mean": 0.0017360628451569937,
      "clip_ratio/low_mean": 0.0008027118048630655,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002538774654385634,
      "completion_length": 100.34250183105469,
      "epoch": 0.8587071366823552,
      "grad_norm": 4.4957380294799805,
      "kl": 0.7120470233261585,
      "learning_rate": 1.436924932383341e-07,
      "loss": -0.0029,
      "reward": 1.7210463523864745,
      "reward_std": 0.348609185218811,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.6164606809616089,
      "step": 4470,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio/high_max": 0.04909939672797918,
      "clip_ratio/high_mean": 0.006639666750561446,
      "clip_ratio/low_mean": 0.0001961415633559227,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006835808313917368,
      "completion_length": 89.65500030517578,
      "epoch": 0.8606281817308616,
      "grad_norm": 0.6291245818138123,
      "kl": 0.914973171055317,
      "learning_rate": 1.4253220512044194e-07,
      "loss": 0.0052,
      "reward": 1.5310453414916991,
      "reward_std": 0.2040669571608305,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.519272655248642,
      "step": 4480,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.023789329756982624,
      "clip_ratio/high_mean": 0.0034216867323266344,
      "clip_ratio/low_mean": 6.479026051238179e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003486476981197484,
      "completion_length": 94.68500061035157,
      "epoch": 0.862549226779368,
      "grad_norm": 3.6435203552246094,
      "kl": 0.24871882200241088,
      "learning_rate": 1.4138676668407637e-07,
      "loss": -0.004,
      "reward": 1.7728254079818726,
      "reward_std": 0.21846108362078667,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.6386001646518707,
      "step": 4490,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.02858473571250215,
      "clip_ratio/high_mean": 0.004445229801058303,
      "clip_ratio/low_mean": 0.005347848287783563,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009793078135407996,
      "completion_length": 94.14000091552734,
      "epoch": 0.8644702718278744,
      "grad_norm": 7.250815391540527,
      "kl": 1.268965845555067,
      "learning_rate": 1.402562196733855e-07,
      "loss": 0.1222,
      "reward": 1.6482325553894044,
      "reward_std": 0.321136474609375,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5813037693500519,
      "step": 4500,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.0016861034324392675,
      "clip_ratio/high_mean": 0.00025714511721162123,
      "clip_ratio/low_mean": 8.047257215366699e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0003376176857273094,
      "completion_length": 89.82750244140625,
      "epoch": 0.8663913168763807,
      "grad_norm": 1.5697243213653564,
      "kl": 0.3187939524650574,
      "learning_rate": 1.3914060528981713e-07,
      "loss": -0.0008,
      "reward": 1.6549904108047486,
      "reward_std": 0.15924324840307236,
      "rewards/code_format_reward": 0.9912499785423279,
      "rewards/code_reward": 0.5796826839447021,
      "step": 4510,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.005441831634379923,
      "clip_ratio/high_mean": 0.0007462791429134086,
      "clip_ratio/low_mean": 0.0008039395906962454,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015502187496167607,
      "completion_length": 97.60250091552734,
      "epoch": 0.8683123619248871,
      "grad_norm": 2.864607334136963,
      "kl": 0.36184127181768416,
      "learning_rate": 1.38039964190617e-07,
      "loss": -0.0068,
      "reward": 1.5000358819961548,
      "reward_std": 0.22264644205570222,
      "rewards/code_format_reward": 0.9850000023841858,
      "rewards/code_reward": 0.5037679553031922,
      "step": 4520,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.045888486225157975,
      "clip_ratio/high_mean": 0.006488210440147668,
      "clip_ratio/low_mean": 4.380840982776135e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006532018849975429,
      "completion_length": 107.06000061035157,
      "epoch": 0.8702334069733936,
      "grad_norm": 3.5723934173583984,
      "kl": 0.21280892938375473,
      "learning_rate": 1.369543364873474e-07,
      "loss": 0.0008,
      "reward": 1.8976154088974,
      "reward_std": 0.22375442534685136,
      "rewards/code_format_reward": 0.9737499952316284,
      "rewards/code_reward": 0.7053701996803283,
      "step": 4530,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.021734172268770634,
      "clip_ratio/high_mean": 0.00285033899708651,
      "clip_ratio/low_mean": 0.00015430593703058548,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003004644898464903,
      "completion_length": 90.5125,
      "epoch": 0.8721544520218999,
      "grad_norm": 26.33332633972168,
      "kl": 16.64756402745843,
      "learning_rate": 1.3588376174442495e-07,
      "loss": 0.0407,
      "reward": 1.8465018033981324,
      "reward_std": 0.26863393038511274,
      "rewards/code_format_reward": 0.9900000095367432,
      "rewards/code_reward": 0.6757509171962738,
      "step": 4540,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.01540006476570852,
      "clip_ratio/high_mean": 0.00195431642132462,
      "clip_ratio/low_mean": 0.0003294220077805221,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022837384400190785,
      "completion_length": 91.96750183105469,
      "epoch": 0.8740754970704063,
      "grad_norm": 5.637061595916748,
      "kl": 0.5615961387753486,
      "learning_rate": 1.348282789776792e-07,
      "loss": 0.0006,
      "reward": 1.7335857629776001,
      "reward_std": 0.16677757501602172,
      "rewards/code_format_reward": 0.9712500095367431,
      "rewards/code_reward": 0.6239803791046142,
      "step": 4550,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.012607228197157382,
      "clip_ratio/high_mean": 0.0017772652208805084,
      "clip_ratio/low_mean": 0.00020470973395276816,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019819749519228934,
      "completion_length": 90.42000122070313,
      "epoch": 0.8759965421189126,
      "grad_norm": 4.87404727935791,
      "kl": 0.5051522366702557,
      "learning_rate": 1.3378792665293032e-07,
      "loss": -0.0007,
      "reward": 1.8114176988601685,
      "reward_std": 0.27143858969211576,
      "rewards/code_format_reward": 0.9687499880790711,
      "rewards/code_reward": 0.6635213375091553,
      "step": 4560,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.003521555650513619,
      "clip_ratio/high_mean": 0.0005311336179147474,
      "clip_ratio/low_mean": 0.00039719248161418363,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009283260951633565,
      "completion_length": 96.31000061035157,
      "epoch": 0.8779175871674191,
      "grad_norm": 3.5294971466064453,
      "kl": 0.44891551434993743,
      "learning_rate": 1.3276274268458749e-07,
      "loss": -0.0011,
      "reward": 1.8015916109085084,
      "reward_std": 0.23535949736833572,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.65454580783844,
      "step": 4570,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.016813984792679548,
      "clip_ratio/high_mean": 0.0026305554260034115,
      "clip_ratio/low_mean": 0.00013278115511639044,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0027633365796646105,
      "completion_length": 92.12000122070313,
      "epoch": 0.8798386322159255,
      "grad_norm": 3.3064281940460205,
      "kl": 147.6638460204005,
      "learning_rate": 1.3175276443426704e-07,
      "loss": 0.3018,
      "reward": 1.8557111263275146,
      "reward_std": 0.21927002370357512,
      "rewards/code_format_reward": 0.9924999833106994,
      "rewards/code_reward": 0.6797305464744567,
      "step": 4580,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.004338507051579654,
      "clip_ratio/high_mean": 0.0005835221760207787,
      "clip_ratio/low_mean": 9.975638386094943e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0006832785555161535,
      "completion_length": 96.14250183105469,
      "epoch": 0.8817596772644318,
      "grad_norm": 5.933443546295166,
      "kl": 0.7469953082501888,
      "learning_rate": 1.3075802870943102e-07,
      "loss": -0.0005,
      "reward": 1.7140401601791382,
      "reward_std": 0.32567469477653505,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.6145200908184052,
      "step": 4590,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.008221420878544449,
      "clip_ratio/high_mean": 0.0010466745734447613,
      "clip_ratio/low_mean": 0.00021989296365063638,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012665675370953978,
      "completion_length": 96.91000213623047,
      "epoch": 0.8836807223129383,
      "grad_norm": 3.6585068702697754,
      "kl": 0.2884219281375408,
      "learning_rate": 1.2977857176204554e-07,
      "loss": -0.0014,
      "reward": 1.745366358757019,
      "reward_std": 0.28437634110450744,
      "rewards/code_format_reward": 0.9612499952316285,
      "rewards/code_reward": 0.6323706865310669,
      "step": 4600,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.013106013461947442,
      "clip_ratio/high_mean": 0.001983167743310332,
      "clip_ratio/low_mean": 0.0010545071098022162,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030376748647540806,
      "completion_length": 95.46000366210937,
      "epoch": 0.8856017673614446,
      "grad_norm": 3.166572332382202,
      "kl": 0.7999920375645161,
      "learning_rate": 1.2881442928725997e-07,
      "loss": 0.0024,
      "reward": 1.7604058027267455,
      "reward_std": 0.1588110476732254,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6342653870582581,
      "step": 4610,
      "zero_std_ratio": 0.725
    },
    {
      "clip_ratio/high_max": 0.03971324802841991,
      "clip_ratio/high_mean": 0.005240282195154577,
      "clip_ratio/low_mean": 0.00012449334171833472,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005364775560155977,
      "completion_length": 91.37250213623047,
      "epoch": 0.887522812409951,
      "grad_norm": 1.2688926458358765,
      "kl": 52.058202140033245,
      "learning_rate": 1.2786563642210536e-07,
      "loss": 0.1059,
      "reward": 1.6578764081001283,
      "reward_std": 0.1922210179269314,
      "rewards/code_format_reward": 0.9724999904632569,
      "rewards/code_reward": 0.5858131945133209,
      "step": 4620,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.014312215382233262,
      "clip_ratio/high_mean": 0.002295189391588792,
      "clip_ratio/low_mean": 0.0010927254450507462,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033879148249980062,
      "completion_length": 92.44750061035157,
      "epoch": 0.8894438574584574,
      "grad_norm": 1.0473991632461548,
      "kl": 0.48051133900880816,
      "learning_rate": 1.269322277442151e-07,
      "loss": 0.0015,
      "reward": 1.8454564094543457,
      "reward_std": 0.23949076235294342,
      "rewards/code_format_reward": 0.9824999809265137,
      "rewards/code_reward": 0.6771032094955445,
      "step": 4630,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.040262592025101185,
      "clip_ratio/high_mean": 0.005372756696306169,
      "clip_ratio/low_mean": 0.0007898360927356407,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006162592757027597,
      "completion_length": 84.6050018310547,
      "epoch": 0.8913649025069638,
      "grad_norm": 6.553028106689453,
      "kl": 0.6895815744996071,
      "learning_rate": 1.2601423727056346e-07,
      "loss": -0.0001,
      "reward": 1.6561978340148926,
      "reward_std": 0.36703028678894045,
      "rewards/code_format_reward": 0.975,
      "rewards/code_reward": 0.5843489110469818,
      "step": 4640,
      "zero_std_ratio": 0.325
    },
    {
      "clip_ratio/high_max": 0.06538669131696224,
      "clip_ratio/high_mean": 0.009138646663632243,
      "clip_ratio/low_mean": 0.0017342485502013006,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010872895480133593,
      "completion_length": 88.39750061035156,
      "epoch": 0.8932859475554702,
      "grad_norm": 4.167427062988281,
      "kl": 1.728559673577547,
      "learning_rate": 1.2511169845622699e-07,
      "loss": 0.0019,
      "reward": 1.6277015209197998,
      "reward_std": 0.21625073552131652,
      "rewards/code_format_reward": 0.975000011920929,
      "rewards/code_reward": 0.5701007604598999,
      "step": 4650,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.042488472175318745,
      "clip_ratio/high_mean": 0.005791870540997479,
      "clip_ratio/low_mean": 2.0525451691355557e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005812396005785559,
      "completion_length": 92.57250213623047,
      "epoch": 0.8952069926039765,
      "grad_norm": 6.026858806610107,
      "kl": 0.7588046140968799,
      "learning_rate": 1.2422464419316432e-07,
      "loss": 0.0034,
      "reward": 1.7008742094039917,
      "reward_std": 0.27438378930091856,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.6079370617866516,
      "step": 4660,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.013316971366293728,
      "clip_ratio/high_mean": 0.0019765587523579596,
      "clip_ratio/low_mean": 8.563735173083842e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002062196109909564,
      "completion_length": 93.35000305175781,
      "epoch": 0.897128037652483,
      "grad_norm": 4.863064765930176,
      "kl": 7.6627843722701074,
      "learning_rate": 1.233531068090184e-07,
      "loss": 0.011,
      "reward": 1.8806322813034058,
      "reward_std": 0.28162118047475815,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.6931286633014679,
      "step": 4670,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.004586372757330537,
      "clip_ratio/high_mean": 0.0006299379543634132,
      "clip_ratio/low_mean": 1.7313018906861545e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0006472509849118069,
      "completion_length": 92.36500091552735,
      "epoch": 0.8990490827009894,
      "grad_norm": 2.1931703090667725,
      "kl": 0.2519014351069927,
      "learning_rate": 1.2249711806593762e-07,
      "loss": 0.0034,
      "reward": 1.8040930509567261,
      "reward_std": 0.24223610311746596,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.6561090111732483,
      "step": 4680,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.006883417209610343,
      "clip_ratio/high_mean": 0.0009808192204218357,
      "clip_ratio/low_mean": 0.00029269491278682835,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001273514133208664,
      "completion_length": 91.60500030517578,
      "epoch": 0.9009701277494957,
      "grad_norm": 21.0294132232666,
      "kl": 0.25964570268988607,
      "learning_rate": 1.2165670915941866e-07,
      "loss": -0.0043,
      "reward": 1.9244711637496947,
      "reward_std": 0.1629927098751068,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.7156730651855469,
      "step": 4690,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.010230390657670795,
      "clip_ratio/high_mean": 0.0014585633180104196,
      "clip_ratio/low_mean": 3.0266345129348336e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001488829671870917,
      "completion_length": 87.84500274658203,
      "epoch": 0.9028911727980021,
      "grad_norm": 1.7218002080917358,
      "kl": 16.447690600901844,
      "learning_rate": 1.2083191071716937e-07,
      "loss": 0.0339,
      "reward": 1.940086579322815,
      "reward_std": 0.16455088555812836,
      "rewards/code_format_reward": 0.993749988079071,
      "rewards/code_reward": 0.7216057777404785,
      "step": 4700,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.025706328079104425,
      "clip_ratio/high_mean": 0.003573437442537397,
      "clip_ratio/low_mean": 3.392130311112851e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036073587427381424,
      "completion_length": 81.84500274658203,
      "epoch": 0.9048122178465084,
      "grad_norm": 0.22543705999851227,
      "kl": 0.31741214692592623,
      "learning_rate": 1.2002275279799288e-07,
      "loss": -0.0056,
      "reward": 1.8292718410491944,
      "reward_std": 0.12828939855098725,
      "rewards/code_format_reward": 0.9987499952316284,
      "rewards/code_reward": 0.6649484276771546,
      "step": 4710,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.009241180948447437,
      "clip_ratio/high_mean": 0.0013442957555525937,
      "clip_ratio/low_mean": 4.643963038688526e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013907353932154365,
      "completion_length": 95.63500213623047,
      "epoch": 0.9067332628950149,
      "grad_norm": 5.23514986038208,
      "kl": 0.804936108738184,
      "learning_rate": 1.192292648906918e-07,
      "loss": 0.0031,
      "reward": 1.925449275970459,
      "reward_std": 0.2213977299630642,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.7152246475219727,
      "step": 4720,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.021669640118489042,
      "clip_ratio/high_mean": 0.003889294656983111,
      "clip_ratio/low_mean": 0.00044275675172684715,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004332051414530724,
      "completion_length": 92.25250244140625,
      "epoch": 0.9086543079435213,
      "grad_norm": 66.00515747070312,
      "kl": 2.1086502872407435,
      "learning_rate": 1.1845147591299378e-07,
      "loss": 0.0162,
      "reward": 1.5327723979949952,
      "reward_std": 0.2872114762663841,
      "rewards/code_format_reward": 0.9725000023841858,
      "rewards/code_reward": 0.5232611894607544,
      "step": 4730,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.006079713994404301,
      "clip_ratio/high_mean": 0.0010439059922646265,
      "clip_ratio/low_mean": 0.0013928397551353556,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002436745767045068,
      "completion_length": 97.43000030517578,
      "epoch": 0.9105753529920276,
      "grad_norm": 2.8770546913146973,
      "kl": 3.1866038836538793,
      "learning_rate": 1.1768941421049768e-07,
      "loss": 0.0069,
      "reward": 1.7776832818984984,
      "reward_std": 0.29561240673065187,
      "rewards/code_format_reward": 0.9949999928474427,
      "rewards/code_reward": 0.6400915861129761,
      "step": 4740,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.005499497149139642,
      "clip_ratio/high_mean": 0.0006874371436424553,
      "clip_ratio/low_mean": 0.000482194940559566,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0011696320783812554,
      "completion_length": 88.9500015258789,
      "epoch": 0.9124963980405341,
      "grad_norm": 8.540057182312012,
      "kl": 0.9072364956140518,
      "learning_rate": 1.1694310755564014e-07,
      "loss": -0.0021,
      "reward": 1.6791202545166015,
      "reward_std": 0.326928648352623,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.595185148715973,
      "step": 4750,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.008684736292343587,
      "clip_ratio/high_mean": 0.001148225087672472,
      "clip_ratio/low_mean": 0.0005504319502506405,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016986570524750277,
      "completion_length": 95.68250122070313,
      "epoch": 0.9144174430890404,
      "grad_norm": 4.539205551147461,
      "kl": 0.860013198107481,
      "learning_rate": 1.1621258314668402e-07,
      "loss": 0.0,
      "reward": 1.7214089155197143,
      "reward_std": 0.1847836285829544,
      "rewards/code_format_reward": 0.9674999952316284,
      "rewards/code_reward": 0.6188294351100921,
      "step": 4760,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.012012088089250028,
      "clip_ratio/high_mean": 0.0020424059097422288,
      "clip_ratio/low_mean": 7.006222731433808e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0021124681399669496,
      "completion_length": 93.21750183105469,
      "epoch": 0.9163384881375468,
      "grad_norm": 6.60590124130249,
      "kl": 0.45315413996577264,
      "learning_rate": 1.1549786760672676e-07,
      "loss": -0.0013,
      "reward": 1.7664082288742065,
      "reward_std": 0.24015129953622819,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.6369540929794312,
      "step": 4770,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.028257530624978246,
      "clip_ratio/high_mean": 0.00573968501703348,
      "clip_ratio/low_mean": 0.00028595919138751924,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006025644272449426,
      "completion_length": 93.59000244140626,
      "epoch": 0.9182595331860532,
      "grad_norm": 3.693448781967163,
      "kl": 0.5863466400653123,
      "learning_rate": 1.1479898698273037e-07,
      "loss": 0.0001,
      "reward": 1.7522862911224366,
      "reward_std": 0.24038469642400742,
      "rewards/code_format_reward": 0.9762500047683715,
      "rewards/code_reward": 0.6320806205272674,
      "step": 4780,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.006044295988976956,
      "clip_ratio/high_mean": 0.0008545084856450558,
      "clip_ratio/low_mean": 0.0004956311546266079,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013501396053470672,
      "completion_length": 100.22000122070312,
      "epoch": 0.9201805782345596,
      "grad_norm": 17.894886016845703,
      "kl": 0.33935268595814705,
      "learning_rate": 1.1411596674457193e-07,
      "loss": -0.0019,
      "reward": 1.697510004043579,
      "reward_std": 0.16087576895952224,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6018799901008606,
      "step": 4790,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.005959878279827535,
      "clip_ratio/high_mean": 0.0009170519857434556,
      "clip_ratio/low_mean": 7.972503372002392e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009967770281946286,
      "completion_length": 98.54750213623046,
      "epoch": 0.922101623283066,
      "grad_norm": 3.242460250854492,
      "kl": 0.46977903619408606,
      "learning_rate": 1.1344883178411565e-07,
      "loss": -0.0036,
      "reward": 1.7927821159362793,
      "reward_std": 0.24044746458530425,
      "rewards/code_format_reward": 0.9699999809265136,
      "rewards/code_reward": 0.6538910627365112,
      "step": 4800,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.00756604690104723,
      "clip_ratio/high_mean": 0.0010185762541368604,
      "clip_ratio/low_mean": 0.00016034738000598737,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001178923639236018,
      "completion_length": 99.49000091552735,
      "epoch": 0.9240226683315724,
      "grad_norm": 7.225472927093506,
      "kl": 0.2285786397755146,
      "learning_rate": 1.1279760641430568e-07,
      "loss": 0.0001,
      "reward": 1.7233760595321654,
      "reward_std": 0.22306990921497344,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6148130118846893,
      "step": 4810,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.011060118256136776,
      "clip_ratio/high_mean": 0.0017047788191121072,
      "clip_ratio/low_mean": 0.00028281604463700207,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019875948812114073,
      "completion_length": 92.36500091552735,
      "epoch": 0.9259437133800787,
      "grad_norm": 4.390386581420898,
      "kl": 0.8032988727092742,
      "learning_rate": 1.1216231436827974e-07,
      "loss": 0.0005,
      "reward": 1.7829072952270508,
      "reward_std": 0.21434771865606309,
      "rewards/code_format_reward": 0.9862500071525574,
      "rewards/code_reward": 0.6448911607265473,
      "step": 4820,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.013947398256277665,
      "clip_ratio/high_mean": 0.0018392194229818414,
      "clip_ratio/low_mean": 0.0004060613617184572,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002245280790521065,
      "completion_length": 103.5125015258789,
      "epoch": 0.9278647584285852,
      "grad_norm": 6.774899482727051,
      "kl": 0.34710453301668165,
      "learning_rate": 1.1154297879850462e-07,
      "loss": 0.0003,
      "reward": 1.7023445606231689,
      "reward_std": 0.23593612909317016,
      "rewards/code_format_reward": 0.96875,
      "rewards/code_reward": 0.60898477435112,
      "step": 4830,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.00909699429757893,
      "clip_ratio/high_mean": 0.0014705892943311482,
      "clip_ratio/low_mean": 0.0004355724740889855,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019061617698753253,
      "completion_length": 91.47000274658203,
      "epoch": 0.9297858034770915,
      "grad_norm": 1.7924318313598633,
      "kl": 0.5235365644097328,
      "learning_rate": 1.1093962227593214e-07,
      "loss": 0.0017,
      "reward": 1.823938512802124,
      "reward_std": 0.18318418860435487,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.6650941967964172,
      "step": 4840,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.008868952537886799,
      "clip_ratio/high_mean": 0.0013198618631577118,
      "clip_ratio/low_mean": 6.896776030771435e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013888296205550432,
      "completion_length": 97.04750061035156,
      "epoch": 0.9317068485255979,
      "grad_norm": 5.492427825927734,
      "kl": 0.27957614585757257,
      "learning_rate": 1.1035226678917662e-07,
      "loss": 0.0001,
      "reward": 1.7743586778640748,
      "reward_std": 0.19067177027463914,
      "rewards/code_format_reward": 0.9699999928474426,
      "rewards/code_reward": 0.6446793019771576,
      "step": 4850,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.00021865542512387038,
      "clip_ratio/high_mean": 2.7331928140483797e-05,
      "clip_ratio/low_mean": 0.00022580694640055298,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0002531388745410368,
      "completion_length": 91.65750274658203,
      "epoch": 0.9336278935741044,
      "grad_norm": 8.045164108276367,
      "kl": 0.20759812816977502,
      "learning_rate": 1.0978093374371373e-07,
      "loss": -0.0004,
      "reward": 1.7663999795913696,
      "reward_std": 0.281513449549675,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.6353874802589417,
      "step": 4860,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.02832627217285335,
      "clip_ratio/high_mean": 0.0035600741393864155,
      "clip_ratio/low_mean": 0.00011176664993399754,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036718408693559466,
      "completion_length": 84.46500244140626,
      "epoch": 0.9355489386226107,
      "grad_norm": 4.819484233856201,
      "kl": 0.5664212189614772,
      "learning_rate": 1.0922564396109993e-07,
      "loss": -0.0008,
      "reward": 1.7755849838256836,
      "reward_std": 0.20761601328849794,
      "rewards/code_format_reward": 0.9899999856948852,
      "rewards/code_reward": 0.640292489528656,
      "step": 4870,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.006872700434178114,
      "clip_ratio/high_mean": 0.0009693403088022023,
      "clip_ratio/low_mean": 3.415665923967026e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0010034969680418726,
      "completion_length": 92.47500152587891,
      "epoch": 0.9374699836711171,
      "grad_norm": 2.605060338973999,
      "kl": 0.6489929877221584,
      "learning_rate": 1.0868641767821432e-07,
      "loss": -0.0041,
      "reward": 1.9151075601577758,
      "reward_std": 0.2566168040037155,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.7113037467002868,
      "step": 4880,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.018258474441245197,
      "clip_ratio/high_mean": 0.003355332469800487,
      "clip_ratio/low_mean": 0.000607103164657019,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003962435649009421,
      "completion_length": 90.51000366210937,
      "epoch": 0.9393910287196234,
      "grad_norm": 4.408846378326416,
      "kl": 0.35625301077961924,
      "learning_rate": 1.0816327454652044e-07,
      "loss": -0.0018,
      "reward": 1.7154739379882813,
      "reward_std": 0.2987362504005432,
      "rewards/code_format_reward": 0.9612499952316285,
      "rewards/code_reward": 0.6174244284629822,
      "step": 4890,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.010325380798894912,
      "clip_ratio/high_mean": 0.0015298718310077675,
      "clip_ratio/low_mean": 0.000294900168228196,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018247719475766645,
      "completion_length": 100.19250183105468,
      "epoch": 0.9413120737681299,
      "grad_norm": 9.08279037475586,
      "kl": 0.23486268445849418,
      "learning_rate": 1.0765623363135061e-07,
      "loss": -0.0011,
      "reward": 1.5800267338752747,
      "reward_std": 0.26311944872140886,
      "rewards/code_format_reward": 0.9862499952316284,
      "rewards/code_reward": 0.5434508502483368,
      "step": 4900,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.004708675656002015,
      "clip_ratio/high_mean": 0.0008911975004593842,
      "clip_ratio/low_mean": 0.0001348661200609058,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001026063623430673,
      "completion_length": 85.6300033569336,
      "epoch": 0.9432331188166363,
      "grad_norm": 2.5798628330230713,
      "kl": 0.5353534445166588,
      "learning_rate": 1.071653134112109e-07,
      "loss": -0.0018,
      "reward": 1.7293733358383179,
      "reward_std": 0.23426424115896224,
      "rewards/code_format_reward": 0.9862499833106995,
      "rewards/code_reward": 0.6181241631507873,
      "step": 4910,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.031931064534001054,
      "clip_ratio/high_mean": 0.004416047394624911,
      "clip_ratio/low_mean": 0.00037934551510261373,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0047953929373761636,
      "completion_length": 93.08500061035156,
      "epoch": 0.9451541638651426,
      "grad_norm": 3.0407347679138184,
      "kl": 0.3617399115115404,
      "learning_rate": 1.0669053177710766e-07,
      "loss": -0.0023,
      "reward": 1.602178120613098,
      "reward_std": 0.23843889832496643,
      "rewards/code_format_reward": 0.987499988079071,
      "rewards/code_reward": 0.5542140543460846,
      "step": 4920,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.008546069997828453,
      "clip_ratio/high_mean": 0.0011838132908451372,
      "clip_ratio/low_mean": 6.596306338906288e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012497763542341999,
      "completion_length": 102.79500122070313,
      "epoch": 0.947075208913649,
      "grad_norm": 5.987438678741455,
      "kl": 0.28761252388358116,
      "learning_rate": 1.0623190603189566e-07,
      "loss": 0.0011,
      "reward": 1.5471005201339723,
      "reward_std": 0.28855718672275543,
      "rewards/code_format_reward": 0.9674999952316284,
      "rewards/code_reward": 0.5316752552986145,
      "step": 4930,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.03786106104962528,
      "clip_ratio/high_mean": 0.005264365172479302,
      "clip_ratio/low_mean": 0.001157468621386215,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00642183352902066,
      "completion_length": 95.2625015258789,
      "epoch": 0.9489962539621554,
      "grad_norm": 4.088647842407227,
      "kl": 9114.393886435031,
      "learning_rate": 1.0578945288964734e-07,
      "loss": 18.226,
      "reward": 1.5625978589057923,
      "reward_std": 0.22688832581043245,
      "rewards/code_format_reward": 0.9762499928474426,
      "rewards/code_reward": 0.5372364044189453,
      "step": 4940,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.009068883489817381,
      "clip_ratio/high_mean": 0.0015044378931634128,
      "clip_ratio/low_mean": 8.003948896657675e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015844773850403726,
      "completion_length": 88.91999969482421,
      "epoch": 0.9509172990106618,
      "grad_norm": 4.558302879333496,
      "kl": 0.322134206071496,
      "learning_rate": 1.0536318847504383e-07,
      "loss": 0.0008,
      "reward": 1.683999252319336,
      "reward_std": 0.15837213546037673,
      "rewards/code_format_reward": 0.9887500047683716,
      "rewards/code_reward": 0.5948120951652527,
      "step": 4950,
      "zero_std_ratio": 0.65
    },
    {
      "clip_ratio/high_max": 0.004135725944070146,
      "clip_ratio/high_mean": 0.0006349837080051657,
      "clip_ratio/low_mean": 0.0002028582151979208,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008378419290238526,
      "completion_length": 88.58000030517579,
      "epoch": 0.9528383440591682,
      "grad_norm": 1.3143569231033325,
      "kl": 0.32492467686533927,
      "learning_rate": 1.0495312832278721e-07,
      "loss": 0.001,
      "reward": 1.757376217842102,
      "reward_std": 0.18446292728185654,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.6318130671977997,
      "step": 4960,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.004577422246802599,
      "clip_ratio/high_mean": 0.00067823924619006,
      "clip_ratio/low_mean": 0.00020590101485140622,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0008841402595862746,
      "completion_length": 91.77249908447266,
      "epoch": 0.9547593891076745,
      "grad_norm": 2.7616970539093018,
      "kl": 0.6282597549259663,
      "learning_rate": 1.0455928737703441e-07,
      "loss": 0.0001,
      "reward": 1.665701198577881,
      "reward_std": 0.1566584974527359,
      "rewards/code_format_reward": 0.99375,
      "rewards/code_reward": 0.5844130754470825,
      "step": 4970,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.012442531622946262,
      "clip_ratio/high_mean": 0.0018589732819236815,
      "clip_ratio/low_mean": 6.720430101267994e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019261775829363613,
      "completion_length": 90.92250213623046,
      "epoch": 0.956680434156181,
      "grad_norm": 2.84462308883667,
      "kl": 0.3018207371234894,
      "learning_rate": 1.0418167999085259e-07,
      "loss": 0.0041,
      "reward": 1.7472755432128906,
      "reward_std": 0.24319706559181214,
      "rewards/code_format_reward": 0.9774999856948853,
      "rewards/code_reward": 0.6292627692222595,
      "step": 4980,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.01709002295974642,
      "clip_ratio/high_mean": 0.002679444645764306,
      "clip_ratio/low_mean": 0.0003698979213368148,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003049342567101121,
      "completion_length": 92.97750091552734,
      "epoch": 0.9586014792046873,
      "grad_norm": 11.978320121765137,
      "kl": 1.2294385731220245,
      "learning_rate": 1.0382031992569592e-07,
      "loss": 0.0036,
      "reward": 1.739167046546936,
      "reward_std": 0.29275294244289396,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.622708535194397,
      "step": 4990,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.007942511793226003,
      "clip_ratio/high_mean": 0.001185902243014425,
      "clip_ratio/low_mean": 5.571418441832066e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012416164390742779,
      "completion_length": 93.31250305175782,
      "epoch": 0.9605225242531937,
      "grad_norm": 3.364788055419922,
      "kl": 0.35085868686437605,
      "learning_rate": 1.0347522035090446e-07,
      "loss": -0.0003,
      "reward": 1.9564055442810058,
      "reward_std": 0.2229623466730118,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.7303902268409729,
      "step": 5000,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.015932422177866102,
      "clip_ratio/high_mean": 0.0028564550855662675,
      "clip_ratio/low_mean": 0.00020086783915758134,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003057322936365381,
      "completion_length": 96.12750091552735,
      "epoch": 0.9624435693017002,
      "grad_norm": 5.283419609069824,
      "kl": 0.3115640334784985,
      "learning_rate": 1.0314639384322356e-07,
      "loss": -0.0037,
      "reward": 1.6293291807174684,
      "reward_std": 0.2581008836627007,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.5693520545959473,
      "step": 5010,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.003986756759695708,
      "clip_ratio/high_mean": 0.0006319725507637486,
      "clip_ratio/low_mean": 0.000603693921584636,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012356664577964694,
      "completion_length": 86.04750366210938,
      "epoch": 0.9643646143502065,
      "grad_norm": 8.70874309539795,
      "kl": 0.47548493221402166,
      "learning_rate": 1.0283385238634632e-07,
      "loss": 0.0041,
      "reward": 1.622909712791443,
      "reward_std": 0.2179076835513115,
      "rewards/code_format_reward": 0.9712499976158142,
      "rewards/code_reward": 0.5686423420906067,
      "step": 5020,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.005893218703567982,
      "clip_ratio/high_mean": 0.000819433806464076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.000819433806464076,
      "completion_length": 88.9375015258789,
      "epoch": 0.9662856593987129,
      "grad_norm": 6.6337199211120605,
      "kl": 0.5933880299329758,
      "learning_rate": 1.0253760737047606e-07,
      "loss": -0.0043,
      "reward": 1.7307970523834229,
      "reward_std": 0.1557233951985836,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6172735095024109,
      "step": 5030,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.010898534208536148,
      "clip_ratio/high_mean": 0.0015226851450279356,
      "clip_ratio/low_mean": 0.009100449224933981,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010623134509660303,
      "completion_length": 87.65500183105469,
      "epoch": 0.9682067044472192,
      "grad_norm": 12.837902069091797,
      "kl": 0.1521947119385004,
      "learning_rate": 1.0225766959191187e-07,
      "loss": 0.0007,
      "reward": 1.766017746925354,
      "reward_std": 0.1697022169828415,
      "rewards/code_format_reward": 0.9924999952316285,
      "rewards/code_reward": 0.6348838567733764,
      "step": 5040,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.039930257271043955,
      "clip_ratio/high_mean": 0.005228024450480007,
      "clip_ratio/low_mean": 0.0012023555900668725,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006430380133679137,
      "completion_length": 99.63500213623047,
      "epoch": 0.9701277494957257,
      "grad_norm": 3.0135834217071533,
      "kl": 0.5389343507587909,
      "learning_rate": 1.0199404925265473e-07,
      "loss": -0.0011,
      "reward": 1.5655887126922607,
      "reward_std": 0.1425598829984665,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.5365443468093872,
      "step": 5050,
      "zero_std_ratio": 0.575
    },
    {
      "clip_ratio/high_max": 0.013292990019544959,
      "clip_ratio/high_mean": 0.0019570814620237797,
      "clip_ratio/low_mean": 0.0004139953598496504,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0023710768204182387,
      "completion_length": 92.15750122070312,
      "epoch": 0.9720487945442321,
      "grad_norm": 8.622629165649414,
      "kl": 0.3708019584417343,
      "learning_rate": 1.0174675596003588e-07,
      "loss": -0.0037,
      "reward": 1.6285043001174926,
      "reward_std": 0.21171441301703453,
      "rewards/code_format_reward": 0.9675000071525574,
      "rewards/code_reward": 0.5723771452903748,
      "step": 5060,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.011086594103835523,
      "clip_ratio/high_mean": 0.001482552892412059,
      "clip_ratio/low_mean": 7.31003499822691e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001555653239483945,
      "completion_length": 92.72000122070312,
      "epoch": 0.9739698395927384,
      "grad_norm": 10.519503593444824,
      "kl": 0.42225370053201916,
      "learning_rate": 1.0151579872636673e-07,
      "loss": 0.0073,
      "reward": 1.9428821086883545,
      "reward_std": 0.2824172407388687,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.7261285543441772,
      "step": 5070,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.02070889645256102,
      "clip_ratio/high_mean": 0.0035216436022892593,
      "clip_ratio/low_mean": 0.0003085655207542004,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0038302090688375756,
      "completion_length": 105.0050048828125,
      "epoch": 0.9758908846412448,
      "grad_norm": 4.139841079711914,
      "kl": 0.3159520372748375,
      "learning_rate": 1.0130118596861028e-07,
      "loss": -0.0044,
      "reward": 1.6708447217941285,
      "reward_std": 0.30501508712768555,
      "rewards/code_format_reward": 0.9837499976158142,
      "rewards/code_reward": 0.5894848227500915,
      "step": 5080,
      "zero_std_ratio": 0.4
    },
    {
      "clip_ratio/high_max": 0.008058706868905575,
      "clip_ratio/high_mean": 0.0012073565638274885,
      "clip_ratio/low_mean": 0.00031636476196581496,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015237213257933036,
      "completion_length": 84.28750152587891,
      "epoch": 0.9778119296897512,
      "grad_norm": 4.015879154205322,
      "kl": 0.2918614260852337,
      "learning_rate": 1.0110292550807451e-07,
      "loss": -0.0012,
      "reward": 1.7721335172653199,
      "reward_std": 0.286711610853672,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.6385667800903321,
      "step": 5090,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.019471552316099407,
      "clip_ratio/high_mean": 0.0026317643467336895,
      "clip_ratio/low_mean": 0.0003316317946882918,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029633961850777267,
      "completion_length": 90.81000213623047,
      "epoch": 0.9797329747382576,
      "grad_norm": 1.132954716682434,
      "kl": 0.2704964060336351,
      "learning_rate": 1.0092102457012717e-07,
      "loss": -0.0022,
      "reward": 1.6570582151412965,
      "reward_std": 0.21210518777370452,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5810291051864624,
      "step": 5100,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio/high_max": 0.011018617497757077,
      "clip_ratio/high_mean": 0.0013860319217201323,
      "clip_ratio/low_mean": 3.4722223062999547e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0014207541418727488,
      "completion_length": 93.61250305175781,
      "epoch": 0.981654019786764,
      "grad_norm": 16.08737564086914,
      "kl": 0.26382347345352175,
      "learning_rate": 1.0075548978393277e-07,
      "loss": -0.0002,
      "reward": 1.8070130348205566,
      "reward_std": 0.1673865035176277,
      "rewards/code_format_reward": 0.9912500023841858,
      "rewards/code_reward": 0.6556940078735352,
      "step": 5110,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio/high_max": 0.010004310857038946,
      "clip_ratio/high_mean": 0.0012777127660228871,
      "clip_ratio/low_mean": 0.0003342236072057858,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016119363936013542,
      "completion_length": 89.3,
      "epoch": 0.9835750648352704,
      "grad_norm": 0.4556010961532593,
      "kl": 0.4934497371315956,
      "learning_rate": 1.0060632718221066e-07,
      "loss": 0.0026,
      "reward": 1.3408710062503815,
      "reward_std": 0.16168890111148357,
      "rewards/code_format_reward": 0.9875,
      "rewards/code_reward": 0.42356050610542295,
      "step": 5120,
      "zero_std_ratio": 0.7
    },
    {
      "clip_ratio/high_max": 0.05311971204355359,
      "clip_ratio/high_mean": 0.0075716287479735914,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0075716287479735914,
      "completion_length": 102.92249908447266,
      "epoch": 0.9854961098837768,
      "grad_norm": 3.9305222034454346,
      "kl": 0.27781638093292715,
      "learning_rate": 1.0047354220101518e-07,
      "loss": -0.0011,
      "reward": 1.630450439453125,
      "reward_std": 0.18297318816185,
      "rewards/code_format_reward": 0.9887499928474426,
      "rewards/code_reward": 0.5680376827716828,
      "step": 5130,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.004692732833791524,
      "clip_ratio/high_mean": 0.0006299943852354772,
      "clip_ratio/low_mean": 0.00031122941145440565,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009412237734068186,
      "completion_length": 88.23250122070313,
      "epoch": 0.9874171549322832,
      "grad_norm": 4.31157112121582,
      "kl": 0.2751577727496624,
      "learning_rate": 1.0035713967953797e-07,
      "loss": -0.0038,
      "reward": 1.635274839401245,
      "reward_std": 0.29494107216596605,
      "rewards/code_format_reward": 0.9849999904632568,
      "rewards/code_reward": 0.5713874340057373,
      "step": 5140,
      "zero_std_ratio": 0.45
    },
    {
      "clip_ratio/high_max": 0.012987824180163443,
      "clip_ratio/high_mean": 0.0019960356265073644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019960356265073644,
      "completion_length": 86.49750061035157,
      "epoch": 0.9893381999807895,
      "grad_norm": 7.45393705368042,
      "kl": 0.3619408316910267,
      "learning_rate": 1.0025712385993115e-07,
      "loss": 0.0012,
      "reward": 1.687432312965393,
      "reward_std": 0.2386924833059311,
      "rewards/code_format_reward": 0.9912499904632568,
      "rewards/code_reward": 0.5959036707878113,
      "step": 5150,
      "zero_std_ratio": 0.475
    },
    {
      "clip_ratio/high_max": 0.014351918507600203,
      "clip_ratio/high_mean": 0.002000216278975131,
      "clip_ratio/low_mean": 7.898250914877281e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020791988128621595,
      "completion_length": 89.34250030517578,
      "epoch": 0.991259245029296,
      "grad_norm": 35.51432418823242,
      "kl": 0.2617587223649025,
      "learning_rate": 1.0017349838715278e-07,
      "loss": -0.004,
      "reward": 1.2408424496650696,
      "reward_std": 0.21315770447254181,
      "rewards/code_format_reward": 0.9774999976158142,
      "rewards/code_reward": 0.3760462045669556,
      "step": 5160,
      "zero_std_ratio": 0.525
    },
    {
      "clip_ratio/high_max": 0.003973034140653908,
      "clip_ratio/high_mean": 0.0004966292675817385,
      "clip_ratio/low_mean": 6.530825339723379e-06,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0005031600929214619,
      "completion_length": 99.66499938964844,
      "epoch": 0.9931802900778023,
      "grad_norm": 2.5418131351470947,
      "kl": 0.1747375037521124,
      "learning_rate": 1.0010626630883432e-07,
      "loss": 0.003,
      "reward": 1.421428418159485,
      "reward_std": 0.09218620862811804,
      "rewards/code_format_reward": 0.9612499952316285,
      "rewards/code_reward": 0.4704016923904419,
      "step": 5170,
      "zero_std_ratio": 0.675
    },
    {
      "clip_ratio/high_max": 0.02674068254418671,
      "clip_ratio/high_mean": 0.003380646219011396,
      "clip_ratio/low_mean": 9.682812378741801e-05,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034774743369780483,
      "completion_length": 92.72500152587891,
      "epoch": 0.9951013351263087,
      "grad_norm": 6.10137414932251,
      "kl": 0.41192906014621256,
      "learning_rate": 1.0005543007516928e-07,
      "loss": -0.0051,
      "reward": 1.5263760328292846,
      "reward_std": 0.28926219046115875,
      "rewards/code_format_reward": 0.9899999976158143,
      "rewards/code_reward": 0.5156879663467407,
      "step": 5180,
      "zero_std_ratio": 0.425
    },
    {
      "clip_ratio/high_max": 0.1273454572306946,
      "clip_ratio/high_mean": 0.016399703072966076,
      "clip_ratio/low_mean": 0.0004187120386632159,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016818415274610744,
      "completion_length": 81.02250213623047,
      "epoch": 0.9970223801748151,
      "grad_norm": 7.77527379989624,
      "kl": 0.7098278045654297,
      "learning_rate": 1.0002099153882402e-07,
      "loss": -0.0041,
      "reward": 1.6053562879562377,
      "reward_std": 0.16601394787430762,
      "rewards/code_format_reward": 0.9824999928474426,
      "rewards/code_reward": 0.557053166627884,
      "step": 5190,
      "zero_std_ratio": 0.6
    },
    {
      "clip_ratio/high_max": 0.0028753917664289474,
      "clip_ratio/high_mean": 0.00045008738234173504,
      "clip_ratio/low_mean": 0.00016858125454746186,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.000618668642709963,
      "completion_length": 95.49250183105468,
      "epoch": 0.9989434252233215,
      "grad_norm": 6.507387638092041,
      "kl": 0.9340068377554417,
      "learning_rate": 1.0000295195487024e-07,
      "loss": -0.0018,
      "reward": 1.4542541027069091,
      "reward_std": 0.20283248797059059,
      "rewards/code_format_reward": 0.981249988079071,
      "rewards/code_reward": 0.4818145722150803,
      "step": 5200,
      "zero_std_ratio": 0.55
    },
    {
      "clip_ratio/high_max": 0.010171899455599487,
      "clip_ratio/high_mean": 0.0014317149762064219,
      "clip_ratio/low_mean": 0.00016592920292168856,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015976441791281104,
      "completion_length": 90.05000305175781,
      "epoch": 0.999711843242724,
      "kl": 0.5218422394245863,
      "reward": 1.0329873859882355,
      "reward_std": 0.19616412371397018,
      "rewards/code_format_reward": 0.934374988079071,
      "rewards/code_reward": 0.28289994597435,
      "step": 5204,
      "total_flos": 0.0,
      "train_loss": 1756184.5472393532,
      "train_runtime": 149594.4727,
      "train_samples_per_second": 0.139,
      "train_steps_per_second": 0.035,
      "zero_std_ratio": 0.5625
    }
  ],
  "logging_steps": 10,
  "max_steps": 5205,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 2000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 5,
  "trial_name": null,
  "trial_params": null
}