{ "schema_version": 1, "model": "Qwen3-0.6B", "revision": "6130ef31402718485ca4d80a6234f70d9a4cf362", "experiments": { "full": { "initial": { "all": { "count": 240, "bce": 2.797879710263199, "brier": 0.44017959982352844, "accuracy": 0.5125 }, "singles": { "count": 80, "bce": 2.5989240627105574, "brier": 0.4524544972244879, "accuracy": 0.4875 }, "compounds": { "count": 160, "bce": 2.89735753403952, "brier": 0.4340421511230487, "accuracy": 0.525 }, "consistency_mean_squared_residual": 0.26970594965789746 }, "arms": { "trained_pairs": [ { "epoch": 1, "metrics": { "all": { "count": 240, "bce": 0.6912725896884998, "brier": 0.24877889092481087, "accuracy": 0.55 }, "singles": { "count": 80, "bce": 0.698501494526863, "brier": 0.2525475204182717, "accuracy": 0.5125 }, "compounds": { "count": 160, "bce": 0.6876581372693181, "brier": 0.2468945761780804, "accuracy": 0.56875 }, "consistency_mean_squared_residual": 0.01645940415757584 }, "online_train_bce": 0.8700355769090893, "online_train_consistency": 0.088222957701116, "reload_logit_delta": 0.0 }, { "epoch": 2, "metrics": { "all": { "count": 240, "bce": 0.6784292731434107, "brier": 0.24272720767306039, "accuracy": 0.6 }, "singles": { "count": 80, "bce": 0.6728713203221559, "brier": 0.24021277695266074, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 0.681208249554038, "brier": 0.24398442303326023, "accuracy": 0.60625 }, "consistency_mean_squared_residual": 0.01550376633727353 }, "online_train_bce": 0.7090116314291954, "online_train_consistency": 0.03302131691408704, "reload_logit_delta": 0.0 }, { "epoch": 3, "metrics": { "all": { "count": 240, "bce": 0.6651852332055569, "brier": 0.23659983209814903, "accuracy": 0.5916666666666667 }, "singles": { "count": 80, "bce": 0.6690574835985899, "brier": 0.23830150932401684, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 0.6632491080090404, "brier": 0.2357489934852151, "accuracy": 0.59375 }, "consistency_mean_squared_residual": 0.020538321759299587 }, "online_train_bce": 0.6738235643357039, "online_train_consistency": 0.03182658189082972, "reload_logit_delta": 0.0 }, { "epoch": 4, "metrics": { "all": { "count": 240, "bce": 0.6751832305143277, "brier": 0.23994941651292487, "accuracy": 0.6083333333333333 }, "singles": { "count": 80, "bce": 0.655123658105731, "brier": 0.2322081918452985, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 0.685213016718626, "brier": 0.24382002884673803, "accuracy": 0.6 }, "consistency_mean_squared_residual": 0.027540484558916056 }, "online_train_bce": 0.6533930205106735, "online_train_consistency": 0.03511735076684033, "reload_logit_delta": 0.0 }, { "epoch": 5, "metrics": { "all": { "count": 240, "bce": 0.6531197127265235, "brier": 0.2287134168793762, "accuracy": 0.65 }, "singles": { "count": 80, "bce": 0.635483156889677, "brier": 0.22299667810619123, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 0.6619379906449467, "brier": 0.2315717862659687, "accuracy": 0.6625 }, "consistency_mean_squared_residual": 0.041875045605937765 }, "online_train_bce": 0.6314454612191767, "online_train_consistency": 0.05292515419042748, "reload_logit_delta": 0.0 }, { "epoch": 6, "metrics": { "all": { "count": 240, "bce": 0.6725992381883165, "brier": 0.23722440110093773, "accuracy": 0.625 }, "singles": { "count": 80, "bce": 0.7505759821273387, "brier": 0.2736695511014346, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 0.6336108662188054, "brier": 0.21900182610068927, "accuracy": 0.66875 }, "consistency_mean_squared_residual": 0.07645798480923348 }, "online_train_bce": 0.6047581194564701, "online_train_consistency": 0.07274628277885009, "reload_logit_delta": 0.0 }, { "epoch": 7, "metrics": { "all": { "count": 240, "bce": 0.7048459393903613, "brier": 0.23784925716932676, "accuracy": 0.6583333333333333 }, "singles": { "count": 80, "bce": 0.7435728173702956, "brier": 0.262903201108282, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 0.6854825004003942, "brier": 0.2253222851998491, "accuracy": 0.69375 }, "consistency_mean_squared_residual": 0.07514777584253617 }, "online_train_bce": 0.5180205555278807, "online_train_consistency": 0.08576734493623683, "reload_logit_delta": 0.0 }, { "epoch": 8, "metrics": { "all": { "count": 240, "bce": 0.7606193039294643, "brier": 0.24685987639988863, "accuracy": 0.6541666666666667 }, "singles": { "count": 80, "bce": 0.7999645276577212, "brier": 0.27941148536961136, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 0.7409466920653358, "brier": 0.23058407191502725, "accuracy": 0.6875 }, "consistency_mean_squared_residual": 0.1472353122534201 }, "online_train_bce": 0.4365049054492265, "online_train_consistency": 0.08533142963776276, "reload_logit_delta": 0.0 }, { "epoch": 9, "metrics": { "all": { "count": 240, "bce": 0.8374117475197029, "brier": 0.2547511736700749, "accuracy": 0.6458333333333334 }, "singles": { "count": 80, "bce": 0.7873681813129224, "brier": 0.2599854128793661, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 0.8624335306230932, "brier": 0.2521340540654292, "accuracy": 0.65625 }, "consistency_mean_squared_residual": 0.12587043743369974 }, "online_train_bce": 0.37341824287746567, "online_train_consistency": 0.11927109681679819, "reload_logit_delta": 0.0 }, { "epoch": 10, "metrics": { "all": { "count": 240, "bce": 1.1411160637516635, "brier": 0.2954688537069601, "accuracy": 0.625 }, "singles": { "count": 80, "bce": 1.280222963484175, "brier": 0.34725964297420925, "accuracy": 0.5625 }, "compounds": { "count": 160, "bce": 1.0715626138854077, "brier": 0.2695734590733355, "accuracy": 0.65625 }, "consistency_mean_squared_residual": 0.21782895864386997 }, "online_train_bce": 0.28744937091447353, "online_train_consistency": 0.1045376866994593, "reload_logit_delta": 0.0 }, { "epoch": 11, "metrics": { "all": { "count": 240, "bce": 1.539519040103257, "brier": 0.3434825518842539, "accuracy": 0.5791666666666667 }, "singles": { "count": 80, "bce": 1.7007498369224776, "brier": 0.3855036006946173, "accuracy": 0.525 }, "compounds": { "count": 160, "bce": 1.4589036416936465, "brier": 0.3224720274790722, "accuracy": 0.60625 }, "consistency_mean_squared_residual": 0.2150597183996397 }, "online_train_bce": 0.2556025594144885, "online_train_consistency": 0.10757993735091967, "reload_logit_delta": 0.0 }, { "epoch": 12, "metrics": { "all": { "count": 240, "bce": 1.3024355706501212, "brier": 0.26407574298461106, "accuracy": 0.6666666666666666 }, "singles": { "count": 80, "bce": 1.1136987867859716, "brier": 0.26471320534121323, "accuracy": 0.6625 }, "compounds": { "count": 160, "bce": 1.396803962582196, "brier": 0.26375701180630995, "accuracy": 0.66875 }, "consistency_mean_squared_residual": 0.24028849681657774 }, "online_train_bce": 0.18135354359155487, "online_train_consistency": 0.09216108997542381, "reload_logit_delta": 0.0 }, { "epoch": 13, "metrics": { "all": { "count": 240, "bce": 2.011862748199651, "brier": 0.33487785634724204, "accuracy": 0.625 }, "singles": { "count": 80, "bce": 1.9086057923994304, "brier": 0.3868805046599705, "accuracy": 0.5625 }, "compounds": { "count": 160, "bce": 2.063491226099761, "brier": 0.30887653219087785, "accuracy": 0.65625 }, "consistency_mean_squared_residual": 0.2722556116199634 }, "online_train_bce": 0.15222653928454694, "online_train_consistency": 0.07521749778730186, "reload_logit_delta": 0.0 }, { "epoch": 14, "metrics": { "all": { "count": 240, "bce": 1.390415902547034, "brier": 0.282861645778398, "accuracy": 0.65 }, "singles": { "count": 80, "bce": 1.226921251896647, "brier": 0.3043845800859416, "accuracy": 0.6125 }, "compounds": { "count": 160, "bce": 1.4721632278722274, "brier": 0.27210017862462615, "accuracy": 0.66875 }, "consistency_mean_squared_residual": 0.2392183324627149 }, "online_train_bce": 0.1680347861685418, "online_train_consistency": 0.07957240797947651, "reload_logit_delta": 0.0 }, { "epoch": 15, "metrics": { "all": { "count": 240, "bce": 1.7976772821084377, "brier": 0.3056101568516621, "accuracy": 0.6583333333333333 }, "singles": { "count": 80, "bce": 1.9261352901449755, "brier": 0.3827216131025323, "accuracy": 0.575 }, "compounds": { "count": 160, "bce": 1.7334482780901688, "brier": 0.267054428726227, "accuracy": 0.7 }, "consistency_mean_squared_residual": 0.3061342606902169 }, "online_train_bce": 0.11935167519251501, "online_train_consistency": 0.06227658872920817, "reload_logit_delta": 0.0 }, { "epoch": 16, "metrics": { "all": { "count": 240, "bce": 2.216978316529371, "brier": 0.3263631367244487, "accuracy": 0.6416666666666667 }, "singles": { "count": 80, "bce": 2.334272910506131, "brier": 0.3833361148851177, "accuracy": 0.575 }, "compounds": { "count": 160, "bce": 2.1583310195409906, "brier": 0.29787664764411426, "accuracy": 0.675 }, "consistency_mean_squared_residual": 0.2889504754615689 }, "online_train_bce": 0.18937085365203918, "online_train_consistency": 0.07923085255648017, "reload_logit_delta": 0.0 }, { "epoch": 17, "metrics": { "all": { "count": 240, "bce": 2.0337006579400376, "brier": 0.2893893477411182, "accuracy": 0.6875 }, "singles": { "count": 80, "bce": 1.6776074487620172, "brier": 0.32218811872062575, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 2.211747262529048, "brier": 0.2729899622513644, "accuracy": 0.71875 }, "consistency_mean_squared_residual": 0.25932546314791427 }, "online_train_bce": 0.12789308705505126, "online_train_consistency": 0.07580642961221552, "reload_logit_delta": 0.0 }, { "epoch": 18, "metrics": { "all": { "count": 240, "bce": 2.1392702062173, "brier": 0.30205941002263575, "accuracy": 0.6666666666666666 }, "singles": { "count": 80, "bce": 2.734093774735495, "brier": 0.4152853826370665, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 1.841858421958203, "brier": 0.2454464237154203, "accuracy": 0.73125 }, "consistency_mean_squared_residual": 0.2658120575522531 }, "online_train_bce": 0.12123284367694338, "online_train_consistency": 0.06060933452014107, "reload_logit_delta": 0.0 }, { "epoch": 19, "metrics": { "all": { "count": 240, "bce": 2.5141952279698327, "brier": 0.30136402168150983, "accuracy": 0.6708333333333333 }, "singles": { "count": 80, "bce": 2.634057778830233, "brier": 0.36332109704761273, "accuracy": 0.6 }, "compounds": { "count": 160, "bce": 2.4542639525396326, "brier": 0.2703854839984584, "accuracy": 0.70625 }, "consistency_mean_squared_residual": 0.25181865851921714 }, "online_train_bce": 0.08209334940898169, "online_train_consistency": 0.048548789508771784, "reload_logit_delta": 0.0 }, { "epoch": 20, "metrics": { "all": { "count": 240, "bce": 2.605278405919127, "brier": 0.31971765023482707, "accuracy": 0.6583333333333333 }, "singles": { "count": 80, "bce": 3.0607192401844143, "brier": 0.4254081346431642, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 2.3775579887864833, "brier": 0.26687240803065854, "accuracy": 0.7125 }, "consistency_mean_squared_residual": 0.3133243185931374 }, "online_train_bce": 0.10451867091792386, "online_train_consistency": 0.04574096209959903, "reload_logit_delta": 0.0 } ], "trained_pairs_consistency": [ { "epoch": 1, "metrics": { "all": { "count": 240, "bce": 0.6858528750638168, "brier": 0.2463882498890775, "accuracy": 0.5375 }, "singles": { "count": 80, "bce": 0.6908663302659989, "brier": 0.24889129065276636, "accuracy": 0.4875 }, "compounds": { "count": 160, "bce": 0.6833461474627256, "brier": 0.24513672950723303, "accuracy": 0.5625 }, "consistency_mean_squared_residual": 0.007210918521293033 }, "online_train_bce": 0.8513709255217181, "online_train_consistency": 0.07308507667824089, "reload_logit_delta": 0.0 }, { "epoch": 2, "metrics": { "all": { "count": 240, "bce": 0.6724238649010659, "brier": 0.23976203751181738, "accuracy": 0.5875 }, "singles": { "count": 80, "bce": 0.6673522826284171, "brier": 0.23729190332594968, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 0.6749596560373903, "brier": 0.2409971046047512, "accuracy": 0.56875 }, "consistency_mean_squared_residual": 0.005447118445927236 }, "online_train_bce": 0.722751635171473, "online_train_consistency": 0.028075928483527406, "reload_logit_delta": 0.0 }, { "epoch": 3, "metrics": { "all": { "count": 240, "bce": 0.6747750665992498, "brier": 0.24113884032507477, "accuracy": 0.5666666666666667 }, "singles": { "count": 80, "bce": 0.6829064652323723, "brier": 0.24507056670088806, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 0.6707093672826886, "brier": 0.2391729771371681, "accuracy": 0.575 }, "consistency_mean_squared_residual": 0.013628334178358391 }, "online_train_bce": 0.6851360881850124, "online_train_consistency": 0.020495293355528996, "reload_logit_delta": 0.0 }, { "epoch": 4, "metrics": { "all": { "count": 240, "bce": 0.6489413851251205, "brier": 0.22846922880875978, "accuracy": 0.625 }, "singles": { "count": 80, "bce": 0.6525195680558682, "brier": 0.23021355990804335, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 0.6471522936597467, "brier": 0.22759706325911794, "accuracy": 0.625 }, "consistency_mean_squared_residual": 0.009804644444044138 }, "online_train_bce": 0.6565514976158738, "online_train_consistency": 0.011086096619681693, "reload_logit_delta": 0.0 }, { "epoch": 5, "metrics": { "all": { "count": 240, "bce": 0.6460438410441081, "brier": 0.22714625560912174, "accuracy": 0.6291666666666667 }, "singles": { "count": 80, "bce": 0.656113475561142, "brier": 0.23219886384296923, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 0.6410090237855911, "brier": 0.224619951492198, "accuracy": 0.65 }, "consistency_mean_squared_residual": 0.020266167739119937 }, "online_train_bce": 0.64965324402228, "online_train_consistency": 0.023403367251309534, "reload_logit_delta": 0.0 }, { "epoch": 6, "metrics": { "all": { "count": 240, "bce": 0.638258077390492, "brier": 0.2233481963810852, "accuracy": 0.6375 }, "singles": { "count": 80, "bce": 0.6710412975400686, "brier": 0.24032834425364094, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 0.6218664673157036, "brier": 0.21485812244480731, "accuracy": 0.68125 }, "consistency_mean_squared_residual": 0.028045463512721844 }, "online_train_bce": 0.6165920368488877, "online_train_consistency": 0.02856850684503267, "reload_logit_delta": 0.0 }, { "epoch": 7, "metrics": { "all": { "count": 240, "bce": 0.6329470505937934, "brier": 0.21673087140437353, "accuracy": 0.6875 }, "singles": { "count": 80, "bce": 0.68070667386055, "brier": 0.24366062243083153, "accuracy": 0.575 }, "compounds": { "count": 160, "bce": 0.6090672389604151, "brier": 0.20326599589114452, "accuracy": 0.74375 }, "consistency_mean_squared_residual": 0.038309397476144626 }, "online_train_bce": 0.5474252840783447, "online_train_consistency": 0.03524779673407367, "reload_logit_delta": 0.0 }, { "epoch": 8, "metrics": { "all": { "count": 240, "bce": 0.621567866485566, "brier": 0.21212000405131048, "accuracy": 0.6791666666666667 }, "singles": { "count": 80, "bce": 0.6405022306367755, "brier": 0.2253450168998179, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 0.6121006844099611, "brier": 0.20550749762705678, "accuracy": 0.70625 }, "consistency_mean_squared_residual": 0.05360753123809744 }, "online_train_bce": 0.4973309539053589, "online_train_consistency": 0.04242562274610843, "reload_logit_delta": 0.0 }, { "epoch": 9, "metrics": { "all": { "count": 240, "bce": 0.6790092518553138, "brier": 0.22455367843424912, "accuracy": 0.6708333333333333 }, "singles": { "count": 80, "bce": 0.6958753482438624, "brier": 0.24937678462009383, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 0.6705762036610394, "brier": 0.21214212534132676, "accuracy": 0.7375 }, "consistency_mean_squared_residual": 0.12768090701636162 }, "online_train_bce": 0.4422362478082068, "online_train_consistency": 0.05031257512326079, "reload_logit_delta": 0.0 }, { "epoch": 10, "metrics": { "all": { "count": 240, "bce": 0.6697997413381623, "brier": 0.2192244617018041, "accuracy": 0.7 }, "singles": { "count": 80, "bce": 0.7213038931600749, "brier": 0.25210429398908873, "accuracy": 0.6125 }, "compounds": { "count": 160, "bce": 0.644047665427206, "brier": 0.20278454555816178, "accuracy": 0.74375 }, "consistency_mean_squared_residual": 0.09868803738650252 }, "online_train_bce": 0.3797718913895078, "online_train_consistency": 0.0495080318712896, "reload_logit_delta": 0.0 }, { "epoch": 11, "metrics": { "all": { "count": 240, "bce": 0.8995163983792736, "brier": 0.2644899090938223, "accuracy": 0.625 }, "singles": { "count": 80, "bce": 1.0569489147106652, "brier": 0.31507798998626874, "accuracy": 0.5625 }, "compounds": { "count": 160, "bce": 0.8208001402135778, "brier": 0.23919586864759906, "accuracy": 0.65625 }, "consistency_mean_squared_residual": 0.17528139156108002 }, "online_train_bce": 0.3171051122047938, "online_train_consistency": 0.0626928274463581, "reload_logit_delta": 0.0 }, { "epoch": 12, "metrics": { "all": { "count": 240, "bce": 0.8132625088782031, "brier": 0.2438452127748381, "accuracy": 0.6791666666666667 }, "singles": { "count": 80, "bce": 0.8260781587974634, "brier": 0.2640426056251968, "accuracy": 0.6375 }, "compounds": { "count": 160, "bce": 0.8068546839185728, "brier": 0.23374651634965873, "accuracy": 0.7 }, "consistency_mean_squared_residual": 0.17527264125969097 }, "online_train_bce": 0.3168611138663255, "online_train_consistency": 0.06573254934138982, "reload_logit_delta": 0.0 }, { "epoch": 13, "metrics": { "all": { "count": 240, "bce": 1.117344486264877, "brier": 0.2881781153328163, "accuracy": 0.6375 }, "singles": { "count": 80, "bce": 1.1464489900310582, "brier": 0.3309140776887192, "accuracy": 0.575 }, "compounds": { "count": 160, "bce": 1.1027922343817864, "brier": 0.2668101341548648, "accuracy": 0.66875 }, "consistency_mean_squared_residual": 0.21248332473492915 }, "online_train_bce": 0.2424991555675515, "online_train_consistency": 0.07297411633268412, "reload_logit_delta": 0.0 }, { "epoch": 14, "metrics": { "all": { "count": 240, "bce": 1.2727445882821788, "brier": 0.27370571440160496, "accuracy": 0.65 }, "singles": { "count": 80, "bce": 1.130817635008134, "brier": 0.3042492681075169, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 1.3437080649192013, "brier": 0.258433937548649, "accuracy": 0.7 }, "consistency_mean_squared_residual": 0.2218148207245331 }, "online_train_bce": 0.18102024189819008, "online_train_consistency": 0.047730903066453965, "reload_logit_delta": 0.0 }, { "epoch": 15, "metrics": { "all": { "count": 240, "bce": 1.1097636039459, "brier": 0.27486663200653116, "accuracy": 0.6708333333333333 }, "singles": { "count": 80, "bce": 1.2262635563539335, "brier": 0.33361499778477094, "accuracy": 0.575 }, "compounds": { "count": 160, "bce": 1.0515136277418833, "brier": 0.2454924491174113, "accuracy": 0.71875 }, "consistency_mean_squared_residual": 0.23615640481660313 }, "online_train_bce": 0.17651366597613583, "online_train_consistency": 0.06464040100282992, "reload_logit_delta": 0.0 }, { "epoch": 16, "metrics": { "all": { "count": 240, "bce": 1.18084035784885, "brier": 0.2622163066980739, "accuracy": 0.6791666666666667 }, "singles": { "count": 80, "bce": 1.2152634781219604, "brier": 0.3140623296561463, "accuracy": 0.575 }, "compounds": { "count": 160, "bce": 1.1636287977122948, "brier": 0.23629329521903766, "accuracy": 0.73125 }, "consistency_mean_squared_residual": 0.22513600830525213 }, "online_train_bce": 0.13013237538144448, "online_train_consistency": 0.060617828063138784, "reload_logit_delta": 0.0 }, { "epoch": 17, "metrics": { "all": { "count": 240, "bce": 1.5957437954519642, "brier": 0.2712891177227644, "accuracy": 0.6833333333333333 }, "singles": { "count": 80, "bce": 1.3804146400202626, "brier": 0.29523745341779123, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 1.7034083731678151, "brier": 0.259314949875251, "accuracy": 0.7125 }, "consistency_mean_squared_residual": 0.26284974813264467 }, "online_train_bce": 0.10342015212699505, "online_train_consistency": 0.029866594306349612, "reload_logit_delta": 0.0 }, { "epoch": 18, "metrics": { "all": { "count": 240, "bce": 1.7686559001978392, "brier": 0.30682622265836207, "accuracy": 0.65 }, "singles": { "count": 80, "bce": 2.2141526985094173, "brier": 0.382759161622086, "accuracy": 0.5625 }, "compounds": { "count": 160, "bce": 1.5459075010420502, "brier": 0.26885975317650007, "accuracy": 0.69375 }, "consistency_mean_squared_residual": 0.2229922901541698 }, "online_train_bce": 0.12822506481158272, "online_train_consistency": 0.06613326209064405, "reload_logit_delta": 0.0 }, { "epoch": 19, "metrics": { "all": { "count": 240, "bce": 1.8190542272046595, "brier": 0.30026328412983344, "accuracy": 0.6541666666666667 }, "singles": { "count": 80, "bce": 2.00841132036237, "brier": 0.3619352586944317, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 1.7243756806258042, "brier": 0.2694272968475343, "accuracy": 0.6875 }, "consistency_mean_squared_residual": 0.3212853905745128 }, "online_train_bce": 0.14386428100445808, "online_train_consistency": 0.08116597757507614, "reload_logit_delta": 0.0 }, { "epoch": 20, "metrics": { "all": { "count": 240, "bce": 1.9560574391636658, "brier": 0.276058614377741, "accuracy": 0.6916666666666667 }, "singles": { "count": 80, "bce": 1.8511595179635691, "brier": 0.3117603796372024, "accuracy": 0.625 }, "compounds": { "count": 160, "bce": 2.0085063997637143, "brier": 0.25820773174801037, "accuracy": 0.725 }, "consistency_mean_squared_residual": 0.22255777878123456 }, "online_train_bce": 0.12104937226158265, "online_train_consistency": 0.06513511843489582, "reload_logit_delta": 0.0 } ] } }, "one": { "initial": { "all": { "count": 240, "bce": 2.797879710263199, "brier": 0.44017959982352844, "accuracy": 0.5125 }, "singles": { "count": 80, "bce": 2.5989240627105574, "brier": 0.4524544972244879, "accuracy": 0.4875 }, "compounds": { "count": 160, "bce": 2.89735753403952, "brier": 0.4340421511230487, "accuracy": 0.525 }, "consistency_mean_squared_residual": 0.26970594965789746 }, "arms": { "trained_pairs": [ { "epoch": 1, "metrics": { "all": { "count": 240, "bce": 0.7282227860142787, "brier": 0.2657061639866628, "accuracy": 0.5 }, "singles": { "count": 80, "bce": 0.75650639757514, "brier": 0.2777940890114784, "accuracy": 0.525 }, "compounds": { "count": 160, "bce": 0.7140809802338481, "brier": 0.259662201474255, "accuracy": 0.4875 }, "consistency_mean_squared_residual": 0.029989932527007854 }, "masked_training": { "count": 500, "bce": 0.7119221907258034, "brier": 0.2577294744566241, "accuracy": 0.56 }, "online_train_bce": 1.0269259251215772, "online_train_consistency": 0.1146136517453499, "reload_logit_delta": 0.0 }, { "epoch": 2, "metrics": { "all": { "count": 240, "bce": 0.7315568106248975, "brier": 0.2657594524754393, "accuracy": 0.55 }, "singles": { "count": 80, "bce": 0.7879516130313278, "brier": 0.2896249304008255, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 0.7033594094216824, "brier": 0.2538267135127462, "accuracy": 0.55 }, "consistency_mean_squared_residual": 0.0803360281452156 }, "masked_training": { "count": 500, "bce": 0.7302600430548191, "brier": 0.26462164698075563, "accuracy": 0.558 }, "online_train_bce": 0.7937408776953816, "online_train_consistency": 0.0322341072679567, "reload_logit_delta": 0.0 }, { "epoch": 3, "metrics": { "all": { "count": 240, "bce": 0.72397865789632, "brier": 0.26325659303341425, "accuracy": 0.5333333333333333 }, "singles": { "count": 80, "bce": 0.7737860463559627, "brier": 0.28521362078129286, "accuracy": 0.525 }, "compounds": { "count": 160, "bce": 0.6990749636664987, "brier": 0.25227807915947487, "accuracy": 0.5375 }, "consistency_mean_squared_residual": 0.030178769831859065 }, "masked_training": { "count": 500, "bce": 0.6950730970799923, "brier": 0.2501191895954406, "accuracy": 0.548 }, "online_train_bce": 0.624194153137505, "online_train_consistency": 0.05972870982289351, "reload_logit_delta": 0.0 }, { "epoch": 4, "metrics": { "all": { "count": 240, "bce": 0.7801739279801647, "brier": 0.27877279159225427, "accuracy": 0.55 }, "singles": { "count": 80, "bce": 0.8090580098330975, "brier": 0.29385801327760985, "accuracy": 0.5 }, "compounds": { "count": 160, "bce": 0.7657318870536983, "brier": 0.27123018074957644, "accuracy": 0.575 }, "consistency_mean_squared_residual": 0.0739478893276534 }, "masked_training": { "count": 500, "bce": 0.766945435628295, "brier": 0.27323828919261123, "accuracy": 0.564 }, "online_train_bce": 0.5875598385185004, "online_train_consistency": 0.04691442930574283, "reload_logit_delta": 0.0 }, { "epoch": 5, "metrics": { "all": { "count": 240, "bce": 0.9292507452579836, "brier": 0.31282426307359995, "accuracy": 0.525 }, "singles": { "count": 80, "bce": 1.1986268103588373, "brier": 0.37674274542409203, "accuracy": 0.4875 }, "compounds": { "count": 160, "bce": 0.7945627127075567, "brier": 0.28086502189835383, "accuracy": 0.54375 }, "consistency_mean_squared_residual": 0.16148922410213346 }, "masked_training": { "count": 500, "bce": 1.0097588803563267, "brier": 0.32746562704899596, "accuracy": 0.554 }, "online_train_bce": 0.47036977142794056, "online_train_consistency": 0.1107922143466795, "reload_logit_delta": 0.0 }, { "epoch": 6, "metrics": { "all": { "count": 240, "bce": 0.8768239394761622, "brier": 0.3037314039063046, "accuracy": 0.5208333333333334 }, "singles": { "count": 80, "bce": 0.9134990317746997, "brier": 0.32262044124808625, "accuracy": 0.4375 }, "compounds": { "count": 160, "bce": 0.8584863933268935, "brier": 0.2942868852354138, "accuracy": 0.5625 }, "consistency_mean_squared_residual": 0.11476548073196027 }, "masked_training": { "count": 500, "bce": 0.8557767989039421, "brier": 0.29674358422064323, "accuracy": 0.542 }, "online_train_bce": 0.5113462197757326, "online_train_consistency": 0.1494703031620575, "reload_logit_delta": 0.0 }, { "epoch": 7, "metrics": { "all": { "count": 240, "bce": 1.4102849007584155, "brier": 0.3777781617053449, "accuracy": 0.5375 }, "singles": { "count": 80, "bce": 1.1072347325272858, "brier": 0.33436356994734917, "accuracy": 0.5875 }, "compounds": { "count": 160, "bce": 1.5618099848739804, "brier": 0.3994854575843427, "accuracy": 0.5125 }, "consistency_mean_squared_residual": 0.26606797332728704 }, "masked_training": { "count": 500, "bce": 1.2391046124193816, "brier": 0.3648829223850491, "accuracy": 0.504 }, "online_train_bce": 0.45669589593671844, "online_train_consistency": 0.16436309924817213, "reload_logit_delta": 0.0 } ], "trained_pairs_consistency": [ { "epoch": 1, "metrics": { "all": { "count": 240, "bce": 0.7107202423736453, "brier": 0.25814376593713245, "accuracy": 0.5083333333333333 }, "singles": { "count": 80, "bce": 0.7262479418888688, "brier": 0.26492585236903854, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 0.7029563926160336, "brier": 0.2547527227211793, "accuracy": 0.49375 }, "consistency_mean_squared_residual": 0.027181599264313884 }, "masked_training": { "count": 500, "bce": 0.6962558002471924, "brier": 0.2510286668733105, "accuracy": 0.558 }, "online_train_bce": 1.009421896916283, "online_train_consistency": 0.06715826545309211, "reload_logit_delta": 0.0 }, { "epoch": 2, "metrics": { "all": { "count": 240, "bce": 0.7805559674898783, "brier": 0.28707026041833844, "accuracy": 0.4625 }, "singles": { "count": 80, "bce": 0.7521415695548057, "brier": 0.27672019563464817, "accuracy": 0.3875 }, "compounds": { "count": 160, "bce": 0.7947631664574146, "brier": 0.2922452928101836, "accuracy": 0.5 }, "consistency_mean_squared_residual": 0.044777818783774105 }, "masked_training": { "count": 500, "bce": 0.7725240466594696, "brier": 0.2850269883753478, "accuracy": 0.458 }, "online_train_bce": 0.69074115306139, "online_train_consistency": 0.03284830819258161, "reload_logit_delta": 0.0 }, { "epoch": 3, "metrics": { "all": { "count": 240, "bce": 0.7103739934042096, "brier": 0.2577143170716877, "accuracy": 0.5291666666666667 }, "singles": { "count": 80, "bce": 0.7357348725199699, "brier": 0.26901120422728847, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 0.6976935538463295, "brier": 0.2520658734938873, "accuracy": 0.51875 }, "consistency_mean_squared_residual": 0.019074471635780407 }, "masked_training": { "count": 500, "bce": 0.6881927189528942, "brier": 0.2476577166460212, "accuracy": 0.544 }, "online_train_bce": 0.6496737927570939, "online_train_consistency": 0.012579378286293377, "reload_logit_delta": 0.0 }, { "epoch": 4, "metrics": { "all": { "count": 240, "bce": 0.7167795719578862, "brier": 0.2589590317049423, "accuracy": 0.5666666666666667 }, "singles": { "count": 80, "bce": 0.7480419300496578, "brier": 0.2737711183710509, "accuracy": 0.5125 }, "compounds": { "count": 160, "bce": 0.7011483929120004, "brier": 0.25155298837188794, "accuracy": 0.59375 }, "consistency_mean_squared_residual": 0.03299840542573568 }, "masked_training": { "count": 500, "bce": 0.7087644826769829, "brier": 0.2553443526388318, "accuracy": 0.584 }, "online_train_bce": 0.6132138941623271, "online_train_consistency": 0.018274644489184765, "reload_logit_delta": 0.0 }, { "epoch": 5, "metrics": { "all": { "count": 240, "bce": 0.7180139424589773, "brier": 0.25988417865884633, "accuracy": 0.575 }, "singles": { "count": 80, "bce": 0.7733029527589679, "brier": 0.2844266195596313, "accuracy": 0.5125 }, "compounds": { "count": 160, "bce": 0.690369437308982, "brier": 0.24761295820845386, "accuracy": 0.60625 }, "consistency_mean_squared_residual": 0.059048227857188176 }, "masked_training": { "count": 500, "bce": 0.6998500396162272, "brier": 0.25155228181231853, "accuracy": 0.562 }, "online_train_bce": 0.5324606975913048, "online_train_consistency": 0.02593888111816213, "reload_logit_delta": 0.0 }, { "epoch": 6, "metrics": { "all": { "count": 240, "bce": 0.7215013353619725, "brier": 0.25835048947608735, "accuracy": 0.6041666666666666 }, "singles": { "count": 80, "bce": 0.8004489971324802, "brier": 0.2931572948386245, "accuracy": 0.525 }, "compounds": { "count": 160, "bce": 0.6820275044767186, "brier": 0.24094708679481874, "accuracy": 0.64375 }, "consistency_mean_squared_residual": 0.06846306707978272 }, "masked_training": { "count": 500, "bce": 0.6861591701284051, "brier": 0.2427640531302066, "accuracy": 0.602 }, "online_train_bce": 0.5142884172452614, "online_train_consistency": 0.035828589515401174, "reload_logit_delta": 0.0 }, { "epoch": 7, "metrics": { "all": { "count": 240, "bce": 0.788197694416158, "brier": 0.27781122428633886, "accuracy": 0.575 }, "singles": { "count": 80, "bce": 0.8192545361816883, "brier": 0.2931450861582746, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 0.7726692735333927, "brier": 0.27014429335037105, "accuracy": 0.5875 }, "consistency_mean_squared_residual": 0.09659540351682121 }, "masked_training": { "count": 500, "bce": 0.7112594054266811, "brier": 0.24792827244358215, "accuracy": 0.606 }, "online_train_bce": 0.38974821574927776, "online_train_consistency": 0.07554774080675088, "reload_logit_delta": 0.0 }, { "epoch": 8, "metrics": { "all": { "count": 240, "bce": 0.8181726142463351, "brier": 0.27955445623409325, "accuracy": 0.6083333333333333 }, "singles": { "count": 80, "bce": 0.9562138020992279, "brier": 0.3339641675881435, "accuracy": 0.525 }, "compounds": { "count": 160, "bce": 0.7491520203198888, "brier": 0.2523496005570681, "accuracy": 0.65 }, "consistency_mean_squared_residual": 0.12007789851546498 }, "masked_training": { "count": 500, "bce": 0.7156787494653836, "brier": 0.2437013476828792, "accuracy": 0.636 }, "online_train_bce": 0.30587712115142496, "online_train_consistency": 0.060739139624225516, "reload_logit_delta": 0.0 }, { "epoch": 9, "metrics": { "all": { "count": 240, "bce": 0.8999742304324172, "brier": 0.28885066232994683, "accuracy": 0.6125 }, "singles": { "count": 80, "bce": 1.1655285601038485, "brier": 0.3758261277109626, "accuracy": 0.4875 }, "compounds": { "count": 160, "bce": 0.7671970655967015, "brier": 0.24536292963943898, "accuracy": 0.675 }, "consistency_mean_squared_residual": 0.19574424954290537 }, "masked_training": { "count": 500, "bce": 0.7585776502140797, "brier": 0.24866379055129934, "accuracy": 0.644 }, "online_train_bce": 0.2344151316105854, "online_train_consistency": 0.08899258340424239, "reload_logit_delta": 0.0 }, { "epoch": 10, "metrics": { "all": { "count": 240, "bce": 0.9014485469674886, "brier": 0.2848464856843408, "accuracy": 0.6125 }, "singles": { "count": 80, "bce": 1.0786516824737191, "brier": 0.34038882350335553, "accuracy": 0.5 }, "compounds": { "count": 160, "bce": 0.8128469792143733, "brier": 0.2570753167748335, "accuracy": 0.66875 }, "consistency_mean_squared_residual": 0.12261563281858123 }, "masked_training": { "count": 500, "bce": 1.0085119497890118, "brier": 0.2943725747417964, "accuracy": 0.608 }, "online_train_bce": 0.22096310122477006, "online_train_consistency": 0.07040396038442975, "reload_logit_delta": 0.0 }, { "epoch": 11, "metrics": { "all": { "count": 240, "bce": 1.040046605115155, "brier": 0.31376920718952367, "accuracy": 0.5958333333333333 }, "singles": { "count": 80, "bce": 1.2338341071270407, "brier": 0.3842562419358117, "accuracy": 0.5 }, "compounds": { "count": 160, "bce": 0.943152854109212, "brier": 0.2785256898163796, "accuracy": 0.64375 }, "consistency_mean_squared_residual": 0.14672082638149478 }, "masked_training": { "count": 500, "bce": 0.8331332823993871, "brier": 0.25794591831536684, "accuracy": 0.642 }, "online_train_bce": 0.39740959772403583, "online_train_consistency": 0.09368932328550728, "reload_logit_delta": 0.0 }, { "epoch": 12, "metrics": { "all": { "count": 240, "bce": 1.3549420283858733, "brier": 0.344807181703685, "accuracy": 0.5625 }, "singles": { "count": 80, "bce": 1.689118728655012, "brier": 0.39126679756933325, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 1.1878536782513038, "brier": 0.3215773737708609, "accuracy": 0.56875 }, "consistency_mean_squared_residual": 0.1947597416020275 }, "masked_training": { "count": 500, "bce": 1.2620596601419747, "brier": 0.32424097947613306, "accuracy": 0.596 }, "online_train_bce": 0.16317406494625175, "online_train_consistency": 0.08114757343256909, "reload_logit_delta": 0.0 }, { "epoch": 13, "metrics": { "all": { "count": 240, "bce": 1.2133905063181374, "brier": 0.31870322399369816, "accuracy": 0.6041666666666666 }, "singles": { "count": 80, "bce": 1.319540628680261, "brier": 0.3735331041323055, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 1.1603154451370756, "brier": 0.2912882839243945, "accuracy": 0.63125 }, "consistency_mean_squared_residual": 0.1980687515992789 }, "masked_training": { "count": 500, "bce": 0.9859466492788997, "brier": 0.26338458237486806, "accuracy": 0.662 }, "online_train_bce": 0.15389185555219229, "online_train_consistency": 0.07428259644554296, "reload_logit_delta": 0.0 }, { "epoch": 14, "metrics": { "all": { "count": 240, "bce": 1.5245554151290739, "brier": 0.33587329733530846, "accuracy": 0.6041666666666666 }, "singles": { "count": 80, "bce": 1.8687765699149168, "brier": 0.39964924684830155, "accuracy": 0.55 }, "compounds": { "count": 160, "bce": 1.3524448377361522, "brier": 0.30398532257881195, "accuracy": 0.63125 }, "consistency_mean_squared_residual": 0.2305133335704446 }, "masked_training": { "count": 500, "bce": 1.1717442425510198, "brier": 0.27992163607842196, "accuracy": 0.656 }, "online_train_bce": 0.06258920108410165, "online_train_consistency": 0.07861407037868554, "reload_logit_delta": 0.0 }, { "epoch": 15, "metrics": { "all": { "count": 240, "bce": 1.3966942172141785, "brier": 0.3262418718117441, "accuracy": 0.6041666666666666 }, "singles": { "count": 80, "bce": 1.587022002399317, "brier": 0.3807795404239699, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 1.3015303246216092, "brier": 0.29897303750563115, "accuracy": 0.6375 }, "consistency_mean_squared_residual": 0.2655190471723328 }, "masked_training": { "count": 500, "bce": 1.1212483402269435, "brier": 0.2647117574216445, "accuracy": 0.676 }, "online_train_bce": 0.0740258354580726, "online_train_consistency": 0.07944470788344535, "reload_logit_delta": 0.0 }, { "epoch": 16, "metrics": { "all": { "count": 240, "bce": 1.6147496980303306, "brier": 0.359158758063724, "accuracy": 0.5875 }, "singles": { "count": 80, "bce": 1.549438691372052, "brier": 0.37599630849525023, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 1.64740520135947, "brier": 0.35073998284796093, "accuracy": 0.6125 }, "consistency_mean_squared_residual": 0.20317697648947852 }, "masked_training": { "count": 500, "bce": 1.4066114703121129, "brier": 0.3145951148759321, "accuracy": 0.622 }, "online_train_bce": 0.08742677452136377, "online_train_consistency": 0.09385904415530882, "reload_logit_delta": 0.0 }, { "epoch": 17, "metrics": { "all": { "count": 240, "bce": 1.9598490126623234, "brier": 0.3551545006313761, "accuracy": 0.6 }, "singles": { "count": 80, "bce": 2.233700388665761, "brier": 0.43881505996272485, "accuracy": 0.4875 }, "compounds": { "count": 160, "bce": 1.8229233246606042, "brier": 0.3133242209657018, "accuracy": 0.65625 }, "consistency_mean_squared_residual": 0.22654918719977638 }, "masked_training": { "count": 500, "bce": 1.6143440279081196, "brier": 0.28752953545736293, "accuracy": 0.668 }, "online_train_bce": 0.05441721308417414, "online_train_consistency": 0.06829261733286393, "reload_logit_delta": 0.0 }, { "epoch": 18, "metrics": { "all": { "count": 240, "bce": 1.250802196820905, "brier": 0.3195511470109326, "accuracy": 0.6041666666666666 }, "singles": { "count": 80, "bce": 1.351317834376823, "brier": 0.36359770654916523, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 1.200544378042946, "brier": 0.29752786724181635, "accuracy": 0.6375 }, "consistency_mean_squared_residual": 0.25004737232744056 }, "masked_training": { "count": 500, "bce": 1.0913713516092711, "brier": 0.2623462583388506, "accuracy": 0.676 }, "online_train_bce": 0.10486247220574114, "online_train_consistency": 0.11599385921554567, "reload_logit_delta": 0.0 }, { "epoch": 19, "metrics": { "all": { "count": 240, "bce": 1.4979513462675924, "brier": 0.3360776258614249, "accuracy": 0.6166666666666667 }, "singles": { "count": 80, "bce": 1.7675331288606686, "brier": 0.4104814967143807, "accuracy": 0.5375 }, "compounds": { "count": 160, "bce": 1.3631604549710545, "brier": 0.29887569043494694, "accuracy": 0.65625 }, "consistency_mean_squared_residual": 0.2566793765024351 }, "masked_training": { "count": 500, "bce": 1.339972244543538, "brier": 0.2765089893821906, "accuracy": 0.684 }, "online_train_bce": 0.032552987146998474, "online_train_consistency": 0.06973058009111092, "reload_logit_delta": 0.0 }, { "epoch": 20, "metrics": { "all": { "count": 240, "bce": 2.225523833705834, "brier": 0.3732415991983974, "accuracy": 0.5833333333333334 }, "singles": { "count": 80, "bce": 2.844045914252669, "brier": 0.4349075681429956, "accuracy": 0.525 }, "compounds": { "count": 160, "bce": 1.9162627934324168, "brier": 0.3424086147260982, "accuracy": 0.6125 }, "consistency_mean_squared_residual": 0.26645934850564557 }, "masked_training": { "count": 500, "bce": 1.52833389474198, "brier": 0.28304756640629336, "accuracy": 0.674 }, "online_train_bce": 0.17919961594132067, "online_train_consistency": 0.09475719073234327, "reload_logit_delta": 0.0 } ] } } }, "sources": { "pilot375-full20-complete": "81244c454205c45919bbcf416f37510524119602e4e7b82a0263c9c7e63e9fce", "pilot375-one-label-stopped-v1": "7064b26a660beec6e5e4276fcd59aa2d820645cea64fb5ea84b8ed68f904d62a", "pilot375-one-label-consistency-complete": "274b1224ef6e30658fe8c3db1501d765f987f5cce57633a92ceb615c6b64d72c" }, "smoke": { "source_document": "docs/fixed-kv-smoke.md", "source_sha256": "d0662bf77cdb15e49f984b43efb277af2d770b5549f23b8f5fc1cf6324836b7a", "train": [ { "arm": "singles", "queries": 10, "updates": 1, "bce_before": 5.13617, "bce_after": 3.34181 }, { "arm": "pairs", "queries": 120, "updates": 1, "bce_before": 3.042312, "bce_after": 2.161691, "consistency_before": 0.361743, "consistency_after": 0.360404 }, { "arm": "pairs_consistency", "queries": 120, "updates": 1, "bce_before": 3.042312, "bce_after": 2.162989, "consistency_before": 0.361743, "consistency_after": 0.358503 } ], "validation": [ { "checkpoint": "initial", "singles20_bce": 3.232157, "pairs60_bce": 2.908728, "pairs60_brier": 0.448511, "pairs_consistency": 0.423986 }, { "checkpoint": "singles", "singles20_bce": 2.101527, "pairs60_bce": 2.106276, "pairs60_brier": 0.455081, "pairs_consistency": 0.457002 }, { "checkpoint": "pairs", "singles20_bce": 2.134721, "pairs60_bce": 2.006804, "pairs60_brier": 0.445607, "pairs_consistency": 0.429788 }, { "checkpoint": "pairs_consistency", "singles20_bce": 2.137763, "pairs60_bce": 2.011799, "pairs60_brier": 0.44593, "pairs_consistency": 0.428787 } ] } }