| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.27938535222510474, |
| "eval_steps": 500, |
| "global_step": 200, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6335.173723493303, |
| "epoch": 0.0013969267611255239, |
| "grad_norm": 0.1530565852782136, |
| "learning_rate": 0.0, |
| "loss": 0.0119, |
| "num_tokens": 1330201.0, |
| "reward": 0.3924093656241894, |
| "reward_std": 0.32559600685323986, |
| "rewards/reward_func": 0.3924093528517655, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "epoch": 0.0027938535222510478, |
| "grad_norm": 0.15303408357738274, |
| "learning_rate": 1.3888888888888887e-08, |
| "loss": 0.0119, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.010408175710056509, |
| "epoch": 0.004190780283376571, |
| "grad_norm": 0.1764633199425111, |
| "learning_rate": 2.7777777777777774e-08, |
| "loss": 0.0125, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.010680466823812042, |
| "epoch": 0.0055877070445020955, |
| "grad_norm": 0.2275277428050235, |
| "learning_rate": 4.166666666666666e-08, |
| "loss": 0.0126, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5737.668526785715, |
| "epoch": 0.006984633805627619, |
| "grad_norm": 0.19107039038527468, |
| "learning_rate": 5.555555555555555e-08, |
| "loss": 0.0293, |
| "num_tokens": 2545372.0, |
| "reward": 0.4294457712343761, |
| "reward_std": 0.37008823986564365, |
| "rewards/reward_func": 0.4294457531401089, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.010828850524766105, |
| "epoch": 0.008381560566753142, |
| "grad_norm": 0.2240354781456917, |
| "learning_rate": 6.944444444444444e-08, |
| "loss": 0.0301, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.010841753533376115, |
| "epoch": 0.009778487327878668, |
| "grad_norm": 0.2767916742737982, |
| "learning_rate": 8.333333333333333e-08, |
| "loss": 0.0301, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.010758176312914916, |
| "epoch": 0.011175414089004191, |
| "grad_norm": 0.2668705057271519, |
| "learning_rate": 9.722222222222222e-08, |
| "loss": 0.0301, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6511.857439313616, |
| "epoch": 0.012572340850129714, |
| "grad_norm": 0.19266419886931196, |
| "learning_rate": 1.111111111111111e-07, |
| "loss": 0.0051, |
| "num_tokens": 3911895.0, |
| "reward": 0.32038674876093864, |
| "reward_std": 0.3298808889729636, |
| "rewards/reward_func": 0.32038673226322445, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.012134518141725234, |
| "epoch": 0.013969267611255238, |
| "grad_norm": 0.21591989333017872, |
| "learning_rate": 1.25e-07, |
| "loss": 0.0058, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.012228245115173715, |
| "epoch": 0.015366194372380763, |
| "grad_norm": 0.20676433229073893, |
| "learning_rate": 1.3888888888888888e-07, |
| "loss": 0.0058, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.012062976469418831, |
| "epoch": 0.016763121133506285, |
| "grad_norm": 0.2100849809088676, |
| "learning_rate": 1.527777777777778e-07, |
| "loss": 0.0058, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5776.398141043527, |
| "epoch": 0.01816004789463181, |
| "grad_norm": 0.21639349074679098, |
| "learning_rate": 1.6666666666666665e-07, |
| "loss": 0.0233, |
| "num_tokens": 5132268.0, |
| "reward": 0.4814984883580889, |
| "reward_std": 0.3754643425345421, |
| "rewards/reward_func": 0.48149845749139786, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.010207080947501319, |
| "epoch": 0.019556974655757335, |
| "grad_norm": 0.22611722201173162, |
| "learning_rate": 1.8055555555555554e-07, |
| "loss": 0.0241, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.010003214618856353, |
| "epoch": 0.02095390141688286, |
| "grad_norm": 0.21387408972131952, |
| "learning_rate": 1.9444444444444445e-07, |
| "loss": 0.024, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.010145484004169703, |
| "epoch": 0.022350828178008382, |
| "grad_norm": 0.2965931721160837, |
| "learning_rate": 2.0833333333333333e-07, |
| "loss": 0.024, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6396.923740931919, |
| "epoch": 0.023747754939133905, |
| "grad_norm": 0.17812701051388663, |
| "learning_rate": 2.222222222222222e-07, |
| "loss": 0.0143, |
| "num_tokens": 6474962.0, |
| "reward": 0.4179248466555561, |
| "reward_std": 0.3605238295027188, |
| "rewards/reward_func": 0.41792482949261156, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.010267588176897593, |
| "epoch": 0.02514468170025943, |
| "grad_norm": 0.18587492539762027, |
| "learning_rate": 2.361111111111111e-07, |
| "loss": 0.015, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.010148471055020179, |
| "epoch": 0.026541608461384952, |
| "grad_norm": 0.1810944495339226, |
| "learning_rate": 2.5e-07, |
| "loss": 0.0149, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.01020466532957341, |
| "epoch": 0.027938535222510476, |
| "grad_norm": 0.1924193708866086, |
| "learning_rate": 2.638888888888889e-07, |
| "loss": 0.015, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5085.755362374442, |
| "epoch": 0.029335461983636, |
| "grad_norm": 0.3165497451368565, |
| "learning_rate": 2.7777777777777776e-07, |
| "loss": 0.0104, |
| "num_tokens": 7560523.0, |
| "reward": 0.464439152606896, |
| "reward_std": 0.3689843277846064, |
| "rewards/reward_func": 0.46443913131952286, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.010546388570219278, |
| "epoch": 0.030732388744761526, |
| "grad_norm": 0.3395774959095009, |
| "learning_rate": 2.916666666666667e-07, |
| "loss": 0.0113, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.010336563357019, |
| "epoch": 0.032129315505887046, |
| "grad_norm": 0.31582946500885495, |
| "learning_rate": 3.055555555555556e-07, |
| "loss": 0.0112, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.010099471979109305, |
| "epoch": 0.03352624226701257, |
| "grad_norm": 0.36247326697578075, |
| "learning_rate": 3.194444444444444e-07, |
| "loss": 0.0112, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6461.342215401785, |
| "epoch": 0.03492316902813809, |
| "grad_norm": 0.15567466235487185, |
| "learning_rate": 3.333333333333333e-07, |
| "loss": 0.0193, |
| "num_tokens": 8914764.0, |
| "reward": 0.38339252876383917, |
| "reward_std": 0.3311073939715113, |
| "rewards/reward_func": 0.3833925181201526, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.009515652732391442, |
| "epoch": 0.03632009578926362, |
| "grad_norm": 0.25805173133838866, |
| "learning_rate": 3.472222222222222e-07, |
| "loss": 0.0199, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.009263422406677688, |
| "epoch": 0.03771702255038915, |
| "grad_norm": 0.4090162080972072, |
| "learning_rate": 3.6111111111111107e-07, |
| "loss": 0.0199, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.009094293162758862, |
| "epoch": 0.03911394931151467, |
| "grad_norm": 0.14691760483745145, |
| "learning_rate": 3.75e-07, |
| "loss": 0.0198, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5968.250279017857, |
| "epoch": 0.040510876072640194, |
| "grad_norm": 0.16406943154337036, |
| "learning_rate": 3.888888888888889e-07, |
| "loss": 0.0115, |
| "num_tokens": 10173741.0, |
| "reward": 0.3594382884246962, |
| "reward_std": 0.32283720054796766, |
| "rewards/reward_func": 0.35943826926606043, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.01090411888435483, |
| "epoch": 0.04190780283376572, |
| "grad_norm": 0.21927196182565817, |
| "learning_rate": 4.027777777777778e-07, |
| "loss": 0.0124, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.010733458446338773, |
| "epoch": 0.04330472959489124, |
| "grad_norm": 0.3447213393816689, |
| "learning_rate": 4.1666666666666667e-07, |
| "loss": 0.0121, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.010630057725523199, |
| "epoch": 0.044701656356016764, |
| "grad_norm": 0.3620568701207513, |
| "learning_rate": 4.3055555555555555e-07, |
| "loss": 0.0121, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5812.648210797991, |
| "epoch": 0.04609858311714229, |
| "grad_norm": 0.16987419224516487, |
| "learning_rate": 4.444444444444444e-07, |
| "loss": 0.0179, |
| "num_tokens": 11403427.0, |
| "reward": 0.3713971259338515, |
| "reward_std": 0.3195132836699486, |
| "rewards/reward_func": 0.3713971110326903, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.010423385953929807, |
| "epoch": 0.04749550987826781, |
| "grad_norm": 0.16329217057193554, |
| "learning_rate": 4.5833333333333327e-07, |
| "loss": 0.0185, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.010347368278806763, |
| "epoch": 0.048892436639393334, |
| "grad_norm": 0.15422959333602954, |
| "learning_rate": 4.722222222222222e-07, |
| "loss": 0.0185, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.010140369646251202, |
| "epoch": 0.05028936340051886, |
| "grad_norm": 0.14689712083262277, |
| "learning_rate": 4.861111111111111e-07, |
| "loss": 0.0184, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5546.617606026785, |
| "epoch": 0.05168629016164438, |
| "grad_norm": 0.23439115342799705, |
| "learning_rate": 5e-07, |
| "loss": 0.0161, |
| "num_tokens": 12578819.0, |
| "reward": 0.4297656629766737, |
| "reward_std": 0.357943703021322, |
| "rewards/reward_func": 0.42976564009274754, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.010506943932601384, |
| "epoch": 0.053083216922769905, |
| "grad_norm": 0.28362735010406454, |
| "learning_rate": 5.138888888888889e-07, |
| "loss": 0.0168, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.010338681018246072, |
| "epoch": 0.05448014368389543, |
| "grad_norm": 0.22707404637725537, |
| "learning_rate": 5.277777777777777e-07, |
| "loss": 0.0167, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.010024449788033962, |
| "epoch": 0.05587707044502095, |
| "grad_norm": 0.17811231341381648, |
| "learning_rate": 5.416666666666666e-07, |
| "loss": 0.0165, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6501.444231305803, |
| "epoch": 0.057273997206146475, |
| "grad_norm": 0.19673275542337104, |
| "learning_rate": 5.555555555555555e-07, |
| "loss": 0.0203, |
| "num_tokens": 13942392.0, |
| "reward": 0.39034509126629147, |
| "reward_std": 0.3664030634931156, |
| "rewards/reward_func": 0.3903450625283377, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.010402361529746227, |
| "epoch": 0.058670923967272, |
| "grad_norm": 0.46687244963414065, |
| "learning_rate": 5.694444444444444e-07, |
| "loss": 0.0211, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.010496154893189669, |
| "epoch": 0.06006785072839753, |
| "grad_norm": 0.19428194405187207, |
| "learning_rate": 5.833333333333334e-07, |
| "loss": 0.0209, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.010158670766811286, |
| "epoch": 0.06146477748952305, |
| "grad_norm": 0.15306555166745645, |
| "learning_rate": 5.972222222222222e-07, |
| "loss": 0.0208, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6369.643048967634, |
| "epoch": 0.06286170425064858, |
| "grad_norm": 0.1871015916141642, |
| "learning_rate": 6.111111111111112e-07, |
| "loss": 0.0161, |
| "num_tokens": 15279078.0, |
| "reward": 0.36025070399045944, |
| "reward_std": 0.35143060450042996, |
| "rewards/reward_func": 0.360250677381243, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.01079715442444597, |
| "epoch": 0.06425863101177409, |
| "grad_norm": 0.6294866160536174, |
| "learning_rate": 6.249999999999999e-07, |
| "loss": 0.0169, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.010736021545848675, |
| "epoch": 0.06565555777289962, |
| "grad_norm": 2.137657977104326, |
| "learning_rate": 6.388888888888888e-07, |
| "loss": 0.0171, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.010864090407267213, |
| "epoch": 0.06705248453402514, |
| "grad_norm": 0.18428434234054034, |
| "learning_rate": 6.527777777777777e-07, |
| "loss": 0.0166, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6655.908412388393, |
| "epoch": 0.06844941129515067, |
| "grad_norm": 0.16953542603782387, |
| "learning_rate": 6.666666666666666e-07, |
| "loss": 0.0148, |
| "num_tokens": 16672949.0, |
| "reward": 0.35046685327376637, |
| "reward_std": 0.30536604832325664, |
| "rewards/reward_func": 0.3504668373082365, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.010905002310339893, |
| "epoch": 0.06984633805627619, |
| "grad_norm": 1.9607105946099048, |
| "learning_rate": 6.805555555555556e-07, |
| "loss": 0.0153, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.011137549964977162, |
| "epoch": 0.07124326481740172, |
| "grad_norm": 0.16619610980435887, |
| "learning_rate": 6.944444444444444e-07, |
| "loss": 0.0153, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.011474365035870246, |
| "epoch": 0.07264019157852725, |
| "grad_norm": 0.13731779782413311, |
| "learning_rate": 7.083333333333334e-07, |
| "loss": 0.0151, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6259.326799665178, |
| "epoch": 0.07403711833965276, |
| "grad_norm": 0.18005417578145988, |
| "learning_rate": 7.222222222222221e-07, |
| "loss": 0.0124, |
| "num_tokens": 17987081.0, |
| "reward": 0.3450017161667347, |
| "reward_std": 0.3385175902928625, |
| "rewards/reward_func": 0.34500169487936155, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.012061000069869416, |
| "epoch": 0.0754340451007783, |
| "grad_norm": 0.3653574862365132, |
| "learning_rate": 7.361111111111111e-07, |
| "loss": 0.0132, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.01210923127031752, |
| "epoch": 0.07683097186190381, |
| "grad_norm": 0.1572526876806962, |
| "learning_rate": 7.5e-07, |
| "loss": 0.013, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.012545851857534476, |
| "epoch": 0.07822789862302934, |
| "grad_norm": 0.1348151811096135, |
| "learning_rate": 7.638888888888888e-07, |
| "loss": 0.0128, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 6945.081944056919, |
| "epoch": 0.07962482538415486, |
| "grad_norm": 0.1355243529384343, |
| "learning_rate": 7.777777777777778e-07, |
| "loss": 0.0203, |
| "num_tokens": 19437527.0, |
| "reward": 0.2853341962077788, |
| "reward_std": 0.28026825402464184, |
| "rewards/reward_func": 0.28533417971006464, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.010294904433456915, |
| "epoch": 0.08102175214528039, |
| "grad_norm": 1.4215854683138216, |
| "learning_rate": 7.916666666666666e-07, |
| "loss": 0.0211, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.010296880932790893, |
| "epoch": 0.0824186789064059, |
| "grad_norm": 0.1328390444888187, |
| "learning_rate": 8.055555555555556e-07, |
| "loss": 0.0208, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.010402131692639418, |
| "epoch": 0.08381560566753143, |
| "grad_norm": 0.10630427559017747, |
| "learning_rate": 8.194444444444443e-07, |
| "loss": 0.0207, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5832.984967912947, |
| "epoch": 0.08521253242865695, |
| "grad_norm": 0.2568979639198368, |
| "learning_rate": 8.333333333333333e-07, |
| "loss": 0.0135, |
| "num_tokens": 20669861.0, |
| "reward": 0.37696379157049315, |
| "reward_std": 0.3524509519338608, |
| "rewards/reward_func": 0.37696376868656706, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.011475847900978156, |
| "epoch": 0.08660945918978248, |
| "grad_norm": 0.26568198717380825, |
| "learning_rate": 8.472222222222222e-07, |
| "loss": 0.0142, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.011013164584125792, |
| "epoch": 0.088006385950908, |
| "grad_norm": 0.5160311893436309, |
| "learning_rate": 8.611111111111111e-07, |
| "loss": 0.014, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.010899153238694583, |
| "epoch": 0.08940331271203353, |
| "grad_norm": 0.16319985648001487, |
| "learning_rate": 8.75e-07, |
| "loss": 0.0138, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5130.444091796875, |
| "epoch": 0.09080023947315904, |
| "grad_norm": 0.28767641683692746, |
| "learning_rate": 8.888888888888888e-07, |
| "loss": 0.0025, |
| "num_tokens": 21764127.0, |
| "reward": 0.46981193338121685, |
| "reward_std": 0.3410501836666039, |
| "rewards/reward_func": 0.4698119152869497, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.011167634411581926, |
| "epoch": 0.09219716623428457, |
| "grad_norm": 1786685.0973493713, |
| "learning_rate": 9.027777777777778e-07, |
| "loss": 31.3025, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.011178254083331143, |
| "epoch": 0.09359409299541009, |
| "grad_norm": 0.2699319084211939, |
| "learning_rate": 9.166666666666665e-07, |
| "loss": 0.0029, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.010996093108717884, |
| "epoch": 0.09499101975653562, |
| "grad_norm": 0.1796173018225043, |
| "learning_rate": 9.305555555555555e-07, |
| "loss": 0.0026, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5374.097150530134, |
| "epoch": 0.09638794651766115, |
| "grad_norm": 0.29248413368918635, |
| "learning_rate": 9.444444444444444e-07, |
| "loss": 0.0204, |
| "num_tokens": 22905462.0, |
| "reward": 0.42480308030332836, |
| "reward_std": 0.3633535067949976, |
| "rewards/reward_func": 0.4248030515653746, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.010870669968426228, |
| "epoch": 0.09778487327878667, |
| "grad_norm": 0.2916766016433164, |
| "learning_rate": 9.583333333333334e-07, |
| "loss": 0.0211, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.010642989072948694, |
| "epoch": 0.0991818000399122, |
| "grad_norm": 0.2574669164587784, |
| "learning_rate": 9.722222222222222e-07, |
| "loss": 0.0209, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.010603911941871047, |
| "epoch": 0.10057872680103772, |
| "grad_norm": 0.3923293286285201, |
| "learning_rate": 9.861111111111112e-07, |
| "loss": 0.0206, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4780.178763253348, |
| "epoch": 0.10197565356216325, |
| "grad_norm": 0.3357433852104651, |
| "learning_rate": 1e-06, |
| "loss": 0.0111, |
| "num_tokens": 23928872.0, |
| "reward": 0.49998574065310614, |
| "reward_std": 0.3773101898176329, |
| "rewards/reward_func": 0.4999857119151524, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.012297127628698945, |
| "epoch": 0.10337258032328876, |
| "grad_norm": 0.24331462448033314, |
| "learning_rate": 1e-06, |
| "loss": 0.0117, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.012708199177203434, |
| "epoch": 0.10476950708441429, |
| "grad_norm": 0.6107245136629853, |
| "learning_rate": 1e-06, |
| "loss": 0.0114, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.013628917180800013, |
| "epoch": 0.10616643384553981, |
| "grad_norm": 0.1571665107730879, |
| "learning_rate": 1e-06, |
| "loss": 0.0112, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5568.459463936942, |
| "epoch": 0.10756336060666534, |
| "grad_norm": 0.3303048447870775, |
| "learning_rate": 1e-06, |
| "loss": 0.0085, |
| "num_tokens": 25108534.0, |
| "reward": 0.3940093676958765, |
| "reward_std": 0.32062976009079386, |
| "rewards/reward_func": 0.39400935066597803, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.012926211846726281, |
| "epoch": 0.10896028736779086, |
| "grad_norm": 0.4586726422820888, |
| "learning_rate": 1e-06, |
| "loss": 0.0092, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.013429554013003196, |
| "epoch": 0.11035721412891639, |
| "grad_norm": 0.46850871504202196, |
| "learning_rate": 1e-06, |
| "loss": 0.0089, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.01421830172850085, |
| "epoch": 0.1117541408900419, |
| "grad_norm": 0.2694884119514331, |
| "learning_rate": 1e-06, |
| "loss": 0.0087, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5048.066615513393, |
| "epoch": 0.11315106765116743, |
| "grad_norm": 0.4818094588291497, |
| "learning_rate": 1e-06, |
| "loss": 0.0003, |
| "num_tokens": 26188824.0, |
| "reward": 0.4587076764021601, |
| "reward_std": 0.3670096014227186, |
| "rewards/reward_func": 0.45870765511478695, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.014440144545265607, |
| "epoch": 0.11454799441229295, |
| "grad_norm": 39.00637105645605, |
| "learning_rate": 1e-06, |
| "loss": 0.0041, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.014427063853612967, |
| "epoch": 0.11594492117341848, |
| "grad_norm": 0.9150065270465777, |
| "learning_rate": 1e-06, |
| "loss": 0.0009, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.015046841731028897, |
| "epoch": 0.117341847934544, |
| "grad_norm": 0.24009263771304865, |
| "learning_rate": 1e-06, |
| "loss": 0.0007, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4776.74507359096, |
| "epoch": 0.11873877469566953, |
| "grad_norm": 0.4772391378994009, |
| "learning_rate": 1e-06, |
| "loss": 0.0175, |
| "num_tokens": 27213245.0, |
| "reward": 0.46243097526686533, |
| "reward_std": 0.3514525145292282, |
| "rewards/reward_func": 0.4624309518507549, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.013508185378408857, |
| "epoch": 0.12013570145679506, |
| "grad_norm": 166639.4843736821, |
| "learning_rate": 1e-06, |
| "loss": 7.3176, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.013775874627754092, |
| "epoch": 0.12153262821792057, |
| "grad_norm": 27.738301939542676, |
| "learning_rate": 1e-06, |
| "loss": 0.019, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.014050438667514495, |
| "epoch": 0.1229295549790461, |
| "grad_norm": 2.3910898137109857, |
| "learning_rate": 1e-06, |
| "loss": 0.0182, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4914.035984584263, |
| "epoch": 0.12432648174017162, |
| "grad_norm": 0.5313282819608155, |
| "learning_rate": 1e-06, |
| "loss": 0.0136, |
| "num_tokens": 28265422.0, |
| "reward": 0.493380460355963, |
| "reward_std": 0.39978101530245375, |
| "rewards/reward_func": 0.4933804316180093, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.017403810085462674, |
| "epoch": 0.12572340850129715, |
| "grad_norm": 2.5053609302596263, |
| "learning_rate": 1e-06, |
| "loss": 0.0153, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.018204916534679278, |
| "epoch": 0.12712033526242267, |
| "grad_norm": 1.3673142623484746, |
| "learning_rate": 1e-06, |
| "loss": 0.0147, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.019155576026865413, |
| "epoch": 0.12851726202354818, |
| "grad_norm": 0.285245341653243, |
| "learning_rate": 1e-06, |
| "loss": 0.0145, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4607.285940987723, |
| "epoch": 0.12991418878467373, |
| "grad_norm": 0.5382478014891808, |
| "learning_rate": 1e-06, |
| "loss": 0.0251, |
| "num_tokens": 29256889.0, |
| "reward": 0.4975069910287857, |
| "reward_std": 0.39376414673668997, |
| "rewards/reward_func": 0.4975069612264633, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.017131159414670298, |
| "epoch": 0.13131111554579925, |
| "grad_norm": 277.06893594236533, |
| "learning_rate": 1e-06, |
| "loss": 0.041, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.017470890456544503, |
| "epoch": 0.13270804230692476, |
| "grad_norm": 0.595127984757126, |
| "learning_rate": 1e-06, |
| "loss": 0.0264, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.017918821956430162, |
| "epoch": 0.13410496906805028, |
| "grad_norm": 0.3727932311400719, |
| "learning_rate": 1e-06, |
| "loss": 0.026, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 5105.030866350447, |
| "epoch": 0.13550189582917582, |
| "grad_norm": 0.5204808748332284, |
| "learning_rate": 1e-06, |
| "loss": 0.0166, |
| "num_tokens": 30346587.0, |
| "reward": 0.40901294563497814, |
| "reward_std": 0.3415069899388722, |
| "rewards/reward_func": 0.4090129222188677, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0174449899766062, |
| "epoch": 0.13689882259030134, |
| "grad_norm": 1.1528181627310667, |
| "learning_rate": 1e-06, |
| "loss": 0.0183, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.01785454393497535, |
| "epoch": 0.13829574935142686, |
| "grad_norm": 0.6193098655808096, |
| "learning_rate": 1e-06, |
| "loss": 0.0177, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.018185677073363746, |
| "epoch": 0.13969267611255237, |
| "grad_norm": 0.29035255969562146, |
| "learning_rate": 1e-06, |
| "loss": 0.0174, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3904.9848981584823, |
| "epoch": 0.14108960287367792, |
| "grad_norm": 0.709202941284989, |
| "learning_rate": 1e-06, |
| "loss": 0.0147, |
| "num_tokens": 31202372.0, |
| "reward": 0.529426036136491, |
| "reward_std": 0.35220117547682356, |
| "rewards/reward_func": 0.5294260127203805, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.01801375742070377, |
| "epoch": 0.14248652963480343, |
| "grad_norm": 1.0665575599442703, |
| "learning_rate": 1e-06, |
| "loss": 0.0166, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.017245101110477532, |
| "epoch": 0.14388345639592895, |
| "grad_norm": 0.5821159923874696, |
| "learning_rate": 1e-06, |
| "loss": 0.0158, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.01721734370637153, |
| "epoch": 0.1452803831570545, |
| "grad_norm": 0.4938223790926132, |
| "learning_rate": 1e-06, |
| "loss": 0.0153, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4391.734933035715, |
| "epoch": 0.14667730991818, |
| "grad_norm": 0.6621058895011234, |
| "learning_rate": 1e-06, |
| "loss": 0.0067, |
| "num_tokens": 32151129.0, |
| "reward": 0.5155340305396489, |
| "reward_std": 0.36261362050260815, |
| "rewards/reward_func": 0.5155340198959623, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.021650432209883417, |
| "epoch": 0.14807423667930553, |
| "grad_norm": 7.221717083371915, |
| "learning_rate": 1e-06, |
| "loss": 0.0095, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.022131933498063257, |
| "epoch": 0.14947116344043104, |
| "grad_norm": 3.9217845571015726, |
| "learning_rate": 1e-06, |
| "loss": 0.0087, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.023090457543730736, |
| "epoch": 0.1508680902015566, |
| "grad_norm": 1.0915253624674428, |
| "learning_rate": 1e-06, |
| "loss": 0.0083, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4265.617579868862, |
| "epoch": 0.1522650169626821, |
| "grad_norm": 0.7674370802581663, |
| "learning_rate": 1e-06, |
| "loss": 0.0076, |
| "num_tokens": 33074935.0, |
| "reward": 0.49383693294865744, |
| "reward_std": 0.3673490881919861, |
| "rewards/reward_func": 0.49383691059691565, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.01965414373470204, |
| "epoch": 0.15366194372380762, |
| "grad_norm": 3.114603095736279, |
| "learning_rate": 1e-06, |
| "loss": 0.0099, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.020012260481183018, |
| "epoch": 0.15505887048493314, |
| "grad_norm": 4.049634875724923, |
| "learning_rate": 1e-06, |
| "loss": 0.0094, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.02028396732306906, |
| "epoch": 0.15645579724605868, |
| "grad_norm": 0.5429450253769021, |
| "learning_rate": 1e-06, |
| "loss": 0.0089, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3556.7655116489955, |
| "epoch": 0.1578527240071842, |
| "grad_norm": 0.8213126340122913, |
| "learning_rate": 1e-06, |
| "loss": 0.0142, |
| "num_tokens": 33860148.0, |
| "reward": 0.520222473357405, |
| "reward_std": 0.3601721642272813, |
| "rewards/reward_func": 0.5202224499412945, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.021029458713850806, |
| "epoch": 0.15924965076830971, |
| "grad_norm": 511.99698568152866, |
| "learning_rate": 1e-06, |
| "loss": 0.0391, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.020824986709547893, |
| "epoch": 0.16064657752943523, |
| "grad_norm": 1.6554233851238163, |
| "learning_rate": 1e-06, |
| "loss": 0.0163, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.021296620901141847, |
| "epoch": 0.16204350429056077, |
| "grad_norm": 5.389449179546646, |
| "learning_rate": 1e-06, |
| "loss": 0.016, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4172.214425223215, |
| "epoch": 0.1634404310516863, |
| "grad_norm": 0.7951777611207881, |
| "learning_rate": 1e-06, |
| "loss": 0.0188, |
| "num_tokens": 34766754.0, |
| "reward": 0.43932128697633743, |
| "reward_std": 0.36100248460258755, |
| "rewards/reward_func": 0.4393212656889643, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.024627858506781713, |
| "epoch": 0.1648373578128118, |
| "grad_norm": 5.497957714291199, |
| "learning_rate": 1e-06, |
| "loss": 0.0267, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.024561051279306412, |
| "epoch": 0.16623428457393732, |
| "grad_norm": 4.006479624309579, |
| "learning_rate": 1e-06, |
| "loss": 0.0259, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.02518816019541451, |
| "epoch": 0.16763121133506287, |
| "grad_norm": 5.696382550533168, |
| "learning_rate": 1e-06, |
| "loss": 0.0217, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3450.438938685826, |
| "epoch": 0.16902813809618839, |
| "grad_norm": 0.8756816112591876, |
| "learning_rate": 1e-06, |
| "loss": 0.0128, |
| "num_tokens": 35530451.0, |
| "reward": 0.4647199107067926, |
| "reward_std": 0.35261689871549606, |
| "rewards/reward_func": 0.46471990112747463, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.023490584788045714, |
| "epoch": 0.1704250648573139, |
| "grad_norm": 8.184182062225446, |
| "learning_rate": 1e-06, |
| "loss": 0.0176, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.022877583213682686, |
| "epoch": 0.17182199161843945, |
| "grad_norm": 2.337796998952532, |
| "learning_rate": 1e-06, |
| "loss": 0.0158, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.022725746002314345, |
| "epoch": 0.17321891837956496, |
| "grad_norm": 1.204853002031202, |
| "learning_rate": 1e-06, |
| "loss": 0.0149, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3755.143040248326, |
| "epoch": 0.17461584514069048, |
| "grad_norm": 1.1667266762961181, |
| "learning_rate": 1e-06, |
| "loss": 0.0009, |
| "num_tokens": 36355620.0, |
| "reward": 0.48860155258859905, |
| "reward_std": 0.3995990295495306, |
| "rewards/reward_func": 0.4886015323655946, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.024759572331926653, |
| "epoch": 0.176012771901816, |
| "grad_norm": 11.549731072093149, |
| "learning_rate": 1e-06, |
| "loss": 0.0075, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.023941728619060347, |
| "epoch": 0.17740969866294154, |
| "grad_norm": 1.5478778519158778, |
| "learning_rate": 1e-06, |
| "loss": 0.0043, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.023272603883274963, |
| "epoch": 0.17880662542406706, |
| "grad_norm": 1.0384027782585576, |
| "learning_rate": 1e-06, |
| "loss": 0.0034, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3561.3930053710938, |
| "epoch": 0.18020355218519257, |
| "grad_norm": 1.203850582780552, |
| "learning_rate": 1e-06, |
| "loss": 0.0059, |
| "num_tokens": 37143444.0, |
| "reward": 0.4855222978762218, |
| "reward_std": 0.38453598746231626, |
| "rewards/reward_func": 0.4855222872325352, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.024004271253943443, |
| "epoch": 0.1816004789463181, |
| "grad_norm": 11.686104385649747, |
| "learning_rate": 1e-06, |
| "loss": 0.012, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.02320788980328611, |
| "epoch": 0.18299740570744363, |
| "grad_norm": 51.97949282308589, |
| "learning_rate": 1e-06, |
| "loss": 0.0129, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.023830042220652103, |
| "epoch": 0.18439433246856915, |
| "grad_norm": 77704.4679582965, |
| "learning_rate": 1e-06, |
| "loss": 0.929, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3375.382795061384, |
| "epoch": 0.18579125922969467, |
| "grad_norm": 1.1905136253909123, |
| "learning_rate": 1e-06, |
| "loss": 0.0195, |
| "num_tokens": 37893570.0, |
| "reward": 0.4746183859450476, |
| "reward_std": 0.41642780708415167, |
| "rewards/reward_func": 0.47461836465767454, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.03054676218224423, |
| "epoch": 0.18718818599082018, |
| "grad_norm": 47.607522363933455, |
| "learning_rate": 1e-06, |
| "loss": 0.0306, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.03216503720198359, |
| "epoch": 0.18858511275194573, |
| "grad_norm": 6.197384439543361, |
| "learning_rate": 1e-06, |
| "loss": 0.0265, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.03178960005087512, |
| "epoch": 0.18998203951307124, |
| "grad_norm": 2.236284664240462, |
| "learning_rate": 1e-06, |
| "loss": 0.0236, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3824.760410853795, |
| "epoch": 0.19137896627419676, |
| "grad_norm": 1.0181749371365818, |
| "learning_rate": 1e-06, |
| "loss": 0.0301, |
| "num_tokens": 38731414.0, |
| "reward": 0.4555432179144451, |
| "reward_std": 0.371168036546026, |
| "rewards/reward_func": 0.45554319130522863, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.02426370605826378, |
| "epoch": 0.1927758930353223, |
| "grad_norm": 15.973977289606324, |
| "learning_rate": 1e-06, |
| "loss": 2.1765, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.023818871338984797, |
| "epoch": 0.19417281979644782, |
| "grad_norm": 125.42621828076582, |
| "learning_rate": 1e-06, |
| "loss": 0.0544, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0235800734455032, |
| "epoch": 0.19556974655757334, |
| "grad_norm": 5.708397430999111, |
| "learning_rate": 1e-06, |
| "loss": 0.0358, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3121.5970720563614, |
| "epoch": 0.19696667331869885, |
| "grad_norm": 0.9463162061122932, |
| "learning_rate": 1e-06, |
| "loss": 0.0074, |
| "num_tokens": 39432472.0, |
| "reward": 0.3878828393561499, |
| "reward_std": 0.32353205340249197, |
| "rewards/reward_func": 0.3878828233906201, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.031209798529744148, |
| "epoch": 0.1983636000798244, |
| "grad_norm": 19.178703569706425, |
| "learning_rate": 1e-06, |
| "loss": 0.0158, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.031645855334188254, |
| "epoch": 0.19976052684094991, |
| "grad_norm": 73.92547009994209, |
| "learning_rate": 1e-06, |
| "loss": 0.0183, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.032449710848076005, |
| "epoch": 0.20115745360207543, |
| "grad_norm": 5.770213423174226, |
| "learning_rate": 1e-06, |
| "loss": 0.0164, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3645.8165196010045, |
| "epoch": 0.20255438036320095, |
| "grad_norm": 1.2228735898177363, |
| "learning_rate": 1e-06, |
| "loss": 0.0139, |
| "num_tokens": 40234738.0, |
| "reward": 0.4959501911486898, |
| "reward_std": 0.3776449923004423, |
| "rewards/reward_func": 0.4959501826337406, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.029987079756600515, |
| "epoch": 0.2039513071243265, |
| "grad_norm": 45.612373429574255, |
| "learning_rate": 1e-06, |
| "loss": 0.0274, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.029181892717523233, |
| "epoch": 0.205348233885452, |
| "grad_norm": 61.829029657734615, |
| "learning_rate": 1e-06, |
| "loss": 0.0238, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.028760566908333982, |
| "epoch": 0.20674516064657752, |
| "grad_norm": 6.854234290541286, |
| "learning_rate": 1e-06, |
| "loss": 0.0212, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3616.0256783621653, |
| "epoch": 0.20814208740770304, |
| "grad_norm": 1.1080044572109065, |
| "learning_rate": 1e-06, |
| "loss": 0.0188, |
| "num_tokens": 41031965.0, |
| "reward": 0.37510563326733454, |
| "reward_std": 0.35230499718870434, |
| "rewards/reward_func": 0.37510561889835764, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.032221541074769836, |
| "epoch": 0.20953901416882859, |
| "grad_norm": 22328.844087735877, |
| "learning_rate": 1e-06, |
| "loss": 0.7198, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.03243154074464526, |
| "epoch": 0.2109359409299541, |
| "grad_norm": 11006.630246836325, |
| "learning_rate": 1e-06, |
| "loss": 1.3521, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.032671846981559484, |
| "epoch": 0.21233286769107962, |
| "grad_norm": 52.44059977916511, |
| "learning_rate": 1e-06, |
| "loss": 0.0309, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3217.7909197126114, |
| "epoch": 0.21372979445220514, |
| "grad_norm": 1.3203188901357052, |
| "learning_rate": 1e-06, |
| "loss": 0.0184, |
| "num_tokens": 41750731.0, |
| "reward": 0.4333155059388706, |
| "reward_std": 0.3546490435089384, |
| "rewards/reward_func": 0.43331548997334074, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.032571413687297275, |
| "epoch": 0.21512672121333068, |
| "grad_norm": 56932.32164959619, |
| "learning_rate": 1e-06, |
| "loss": 1.4737, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.03264636732637882, |
| "epoch": 0.2165236479744562, |
| "grad_norm": 569.4118154787187, |
| "learning_rate": 1e-06, |
| "loss": 0.0464, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.033495722870741575, |
| "epoch": 0.2179205747355817, |
| "grad_norm": 14.372919673445498, |
| "learning_rate": 1e-06, |
| "loss": 0.0297, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 2904.224591936384, |
| "epoch": 0.21931750149670726, |
| "grad_norm": 1.4884091255614038, |
| "learning_rate": 1e-06, |
| "loss": 0.0027, |
| "num_tokens": 42407374.0, |
| "reward": 0.46237824857234955, |
| "reward_std": 0.38831384958965437, |
| "rewards/reward_func": 0.4623782304780824, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.03049316949078015, |
| "epoch": 0.22071442825783277, |
| "grad_norm": 753.2718719394906, |
| "learning_rate": 1e-06, |
| "loss": 0.0615, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.028818647909377302, |
| "epoch": 0.2221113550189583, |
| "grad_norm": 193.7516952886048, |
| "learning_rate": 1e-06, |
| "loss": 0.03, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.028797397390007973, |
| "epoch": 0.2235082817800838, |
| "grad_norm": 86.17864803711156, |
| "learning_rate": 1e-06, |
| "loss": 0.0187, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 2706.137860979353, |
| "epoch": 0.22490520854120935, |
| "grad_norm": 1.437051783056985, |
| "learning_rate": 1e-06, |
| "loss": 0.004, |
| "num_tokens": 43026529.0, |
| "reward": 0.40501056824411663, |
| "reward_std": 0.3891604415008, |
| "rewards/reward_func": 0.4050105522785868, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.03492548343326364, |
| "epoch": 0.22630213530233487, |
| "grad_norm": 1637.6323089099733, |
| "learning_rate": 1e-06, |
| "loss": 0.0796, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.034652630399380414, |
| "epoch": 0.22769906206346038, |
| "grad_norm": 294.8538016270806, |
| "learning_rate": 1e-06, |
| "loss": 0.0397, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.03514658712915012, |
| "epoch": 0.2290959888245859, |
| "grad_norm": 1729.0881090208584, |
| "learning_rate": 1e-06, |
| "loss": 0.1081, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3302.3215855189733, |
| "epoch": 0.23049291558571144, |
| "grad_norm": 1.3478961700451182, |
| "learning_rate": 1e-06, |
| "loss": 0.0095, |
| "num_tokens": 43761689.0, |
| "reward": 0.354110734803336, |
| "reward_std": 0.3442910239100456, |
| "rewards/reward_func": 0.35411071511251585, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.03175032351698194, |
| "epoch": 0.23188984234683696, |
| "grad_norm": 34.2352104129209, |
| "learning_rate": 1e-06, |
| "loss": 0.0236, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.031168780555682524, |
| "epoch": 0.23328676910796248, |
| "grad_norm": 3479.355422028937, |
| "learning_rate": 1e-06, |
| "loss": 0.3311, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.031411009441529004, |
| "epoch": 0.234683695869088, |
| "grad_norm": 211.96486688673437, |
| "learning_rate": 1e-06, |
| "loss": 0.0428, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3288.581760951451, |
| "epoch": 0.23608062263021354, |
| "grad_norm": 1.5809513920036224, |
| "learning_rate": 1e-06, |
| "loss": 0.0206, |
| "num_tokens": 44495433.0, |
| "reward": 0.3336522286491735, |
| "reward_std": 0.35370165216071264, |
| "rewards/reward_func": 0.3336522126836436, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.03352949349209666, |
| "epoch": 0.23747754939133905, |
| "grad_norm": 131.25750741813656, |
| "learning_rate": 1e-06, |
| "loss": 0.0411, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.033907221497169564, |
| "epoch": 0.23887447615246457, |
| "grad_norm": 38.44427070292949, |
| "learning_rate": 1e-06, |
| "loss": 0.0333, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.034381040039339235, |
| "epoch": 0.24027140291359012, |
| "grad_norm": 17107451499810.188, |
| "learning_rate": 1e-06, |
| "loss": 549430272.0, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3361.0919799804688, |
| "epoch": 0.24166832967471563, |
| "grad_norm": 1.637145998911673, |
| "learning_rate": 1e-06, |
| "loss": 0.0187, |
| "num_tokens": 45243898.0, |
| "reward": 0.394156134554318, |
| "reward_std": 0.40689716062375475, |
| "rewards/reward_func": 0.3941561228462628, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.032513607027275224, |
| "epoch": 0.24306525643584115, |
| "grad_norm": 800.8287273060004, |
| "learning_rate": 1e-06, |
| "loss": 1.6832, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.030495245408798968, |
| "epoch": 0.24446218319696666, |
| "grad_norm": 729.365666028725, |
| "learning_rate": 1e-06, |
| "loss": 0.1281, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.03067155448453767, |
| "epoch": 0.2458591099580922, |
| "grad_norm": 257.8929935202217, |
| "learning_rate": 1e-06, |
| "loss": 0.0783, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 2669.938886369978, |
| "epoch": 0.24725603671921773, |
| "grad_norm": 1.6984457904054797, |
| "learning_rate": 1e-06, |
| "loss": -0.0017, |
| "num_tokens": 45856081.0, |
| "reward": 0.3272132841604097, |
| "reward_std": 0.36115063620465143, |
| "rewards/reward_func": 0.3272132767098291, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.03918117896786758, |
| "epoch": 0.24865296348034324, |
| "grad_norm": 459.0007426096092, |
| "learning_rate": 1e-06, |
| "loss": 0.0378, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.037612543175263066, |
| "epoch": 0.2500498902414688, |
| "grad_norm": 198.16000110545232, |
| "learning_rate": 1e-06, |
| "loss": 0.0357, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.03760030067392758, |
| "epoch": 0.2514468170025943, |
| "grad_norm": 193.31294839005466, |
| "learning_rate": 1e-06, |
| "loss": 0.0318, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3767.30623953683, |
| "epoch": 0.2528437437637198, |
| "grad_norm": 1.4505504547674242, |
| "learning_rate": 1e-06, |
| "loss": 0.0004, |
| "num_tokens": 46683920.0, |
| "reward": 0.3422575050166675, |
| "reward_std": 0.3818031592028482, |
| "rewards/reward_func": 0.3422574890511377, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.03315202798694372, |
| "epoch": 0.25424067052484534, |
| "grad_norm": 238.64962645549264, |
| "learning_rate": 1e-06, |
| "loss": 0.0462, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.03287409871284451, |
| "epoch": 0.25563759728597085, |
| "grad_norm": 6799.648792237924, |
| "learning_rate": 1e-06, |
| "loss": 0.0493, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.03349324116217239, |
| "epoch": 0.25703452404709637, |
| "grad_norm": 1395.3440659332787, |
| "learning_rate": 1e-06, |
| "loss": 0.1532, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3256.025630405971, |
| "epoch": 0.2584314508082219, |
| "grad_norm": 1.8519382626688905, |
| "learning_rate": 1e-06, |
| "loss": 0.0091, |
| "num_tokens": 47409833.0, |
| "reward": 0.32464135118893217, |
| "reward_std": 0.3450521251985005, |
| "rewards/reward_func": 0.324641336287771, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.034153553524187634, |
| "epoch": 0.25982837756934746, |
| "grad_norm": 55.33553677644012, |
| "learning_rate": 1e-06, |
| "loss": 0.0322, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.03325902218265193, |
| "epoch": 0.261225304330473, |
| "grad_norm": 807.7840491437855, |
| "learning_rate": 1e-06, |
| "loss": 0.0771, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.032821542982544215, |
| "epoch": 0.2626222310915985, |
| "grad_norm": 1395.5436294619333, |
| "learning_rate": 1e-06, |
| "loss": 0.0888, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3214.1480887276784, |
| "epoch": 0.264019157852724, |
| "grad_norm": 1.7683494272507614, |
| "learning_rate": 1e-06, |
| "loss": 0.0112, |
| "num_tokens": 48129036.0, |
| "reward": 0.31755988619157244, |
| "reward_std": 0.3383456287639482, |
| "rewards/reward_func": 0.31755986117890905, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.03975264209189585, |
| "epoch": 0.2654160846138495, |
| "grad_norm": 45184.22194084358, |
| "learning_rate": 1e-06, |
| "loss": 0.4443, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.03915290268404143, |
| "epoch": 0.26681301137497504, |
| "grad_norm": 1242.8816256939144, |
| "learning_rate": 1e-06, |
| "loss": 0.1447, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.03953878129167216, |
| "epoch": 0.26820993813610056, |
| "grad_norm": 1479.8321627428613, |
| "learning_rate": 1e-06, |
| "loss": 0.1168, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 3272.2399553571427, |
| "epoch": 0.26960686489722613, |
| "grad_norm": 2.177506823211759, |
| "learning_rate": 1e-06, |
| "loss": 0.0092, |
| "num_tokens": 48860342.0, |
| "reward": 0.34522383979388643, |
| "reward_std": 0.3796129695006779, |
| "rewards/reward_func": 0.34522382276398794, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.03925856229450021, |
| "epoch": 0.27100379165835164, |
| "grad_norm": 513.8290574797221, |
| "learning_rate": 1e-06, |
| "loss": 0.0613, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.03799283770578248, |
| "epoch": 0.27240071841947716, |
| "grad_norm": 8983.081819712857, |
| "learning_rate": 1e-06, |
| "loss": 0.3161, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.036926204205623696, |
| "epoch": 0.2737976451806027, |
| "grad_norm": 1027.0780848048523, |
| "learning_rate": 1e-06, |
| "loss": 0.1274, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4028.9797712053573, |
| "epoch": 0.2751945719417282, |
| "grad_norm": 1.3698484721094724, |
| "learning_rate": 1e-06, |
| "loss": 0.018, |
| "num_tokens": 49737682.0, |
| "reward": 0.2579219543508121, |
| "reward_std": 0.32817436435392927, |
| "rewards/reward_func": 0.25792194423930986, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.03714690676757267, |
| "epoch": 0.2765914987028537, |
| "grad_norm": 759.0060595947047, |
| "learning_rate": 1e-06, |
| "loss": 0.0657, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0380573890038899, |
| "epoch": 0.2779884254639792, |
| "grad_norm": 7489.486866588998, |
| "learning_rate": 1e-06, |
| "loss": 0.9186, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.03898970350357039, |
| "epoch": 0.27938535222510474, |
| "grad_norm": 403.4437322167033, |
| "learning_rate": 1e-06, |
| "loss": 0.1357, |
| "step": 200 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 715, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
