632 lines
19 KiB
JSON
632 lines
19 KiB
JSON
{
|
|
"cells": {
|
|
"tp2_rho0p0025": {
|
|
"structured_attention": {
|
|
"legacy_pooled_distribution_bias": {
|
|
"e2e": {
|
|
"mean": 0.12696146154030977,
|
|
"p50": 0.14796251184179712,
|
|
"p90": 0.12767235986604622,
|
|
"p99": -0.03965040123988098
|
|
},
|
|
"tpot": {
|
|
"mean": 0.15073709978689778,
|
|
"p50": 0.18527285925405948,
|
|
"p90": 0.07863996768588354,
|
|
"p99": 0.1661995397125918
|
|
},
|
|
"ttft": {
|
|
"mean": -0.045433491316312524,
|
|
"p50": -0.0875990583320057,
|
|
"p90": -0.1142200855839675,
|
|
"p99": -0.07674087497700505
|
|
}
|
|
},
|
|
"trialwise_distribution_bias": [
|
|
{
|
|
"e2e": {
|
|
"mean": 0.12073678040141796,
|
|
"p50": 0.14420262806746403,
|
|
"p90": 0.11915070375608593,
|
|
"p99": -0.022788557992646938
|
|
},
|
|
"tpot": {
|
|
"mean": 0.14386158526258666,
|
|
"p50": 0.1787588067010671,
|
|
"p90": 0.071419448886029,
|
|
"p99": 0.17844287277146695
|
|
},
|
|
"ttft": {
|
|
"mean": -0.044844475267623835,
|
|
"p50": -0.08906525404741798,
|
|
"p90": -0.11412907883561804,
|
|
"p99": -0.07227786537885415
|
|
}
|
|
},
|
|
{
|
|
"e2e": {
|
|
"mean": 0.13325567383154438,
|
|
"p50": 0.15920445684060733,
|
|
"p90": 0.12953600816854335,
|
|
"p99": -0.009483068681846096
|
|
},
|
|
"tpot": {
|
|
"mean": 0.15769576871598226,
|
|
"p50": 0.196444372264794,
|
|
"p90": 0.08604019319402298,
|
|
"p99": 0.19311481061458743
|
|
},
|
|
"ttft": {
|
|
"mean": -0.046021781355297144,
|
|
"p50": -0.08612813517467069,
|
|
"p90": -0.11510233828040589,
|
|
"p99": -0.07592149320892366
|
|
}
|
|
}
|
|
],
|
|
"trialwise_distribution_bias_summary": {
|
|
"e2e": {
|
|
"mean": {
|
|
"max": 0.13325567383154438,
|
|
"mean": 0.12699622711648118,
|
|
"min": 0.12073678040141796
|
|
},
|
|
"p50": {
|
|
"max": 0.15920445684060733,
|
|
"mean": 0.15170354245403567,
|
|
"min": 0.14420262806746403
|
|
},
|
|
"p90": {
|
|
"max": 0.12953600816854335,
|
|
"mean": 0.12434335596231463,
|
|
"min": 0.11915070375608593
|
|
},
|
|
"p99": {
|
|
"max": -0.009483068681846096,
|
|
"mean": -0.016135813337246518,
|
|
"min": -0.022788557992646938
|
|
}
|
|
},
|
|
"tpot": {
|
|
"mean": {
|
|
"max": 0.15769576871598226,
|
|
"mean": 0.15077867698928446,
|
|
"min": 0.14386158526258666
|
|
},
|
|
"p50": {
|
|
"max": 0.196444372264794,
|
|
"mean": 0.18760158948293054,
|
|
"min": 0.1787588067010671
|
|
},
|
|
"p90": {
|
|
"max": 0.08604019319402298,
|
|
"mean": 0.07872982104002599,
|
|
"min": 0.071419448886029
|
|
},
|
|
"p99": {
|
|
"max": 0.19311481061458743,
|
|
"mean": 0.1857788416930272,
|
|
"min": 0.17844287277146695
|
|
}
|
|
},
|
|
"ttft": {
|
|
"mean": {
|
|
"max": -0.044844475267623835,
|
|
"mean": -0.045433128311460486,
|
|
"min": -0.046021781355297144
|
|
},
|
|
"p50": {
|
|
"max": -0.08612813517467069,
|
|
"mean": -0.08759669461104433,
|
|
"min": -0.08906525404741798
|
|
},
|
|
"p90": {
|
|
"max": -0.11412907883561804,
|
|
"mean": -0.11461570855801197,
|
|
"min": -0.11510233828040589
|
|
},
|
|
"p99": {
|
|
"max": -0.07227786537885415,
|
|
"mean": -0.07409967929388891,
|
|
"min": -0.07592149320892366
|
|
}
|
|
}
|
|
},
|
|
"waiting_p99_ms": 777.5911879793948
|
|
},
|
|
"structured_attention_plus_prefill_moe": {
|
|
"legacy_pooled_distribution_bias": {
|
|
"e2e": {
|
|
"mean": 0.15546889966298288,
|
|
"p50": 0.160521791081281,
|
|
"p90": 0.14949973986858572,
|
|
"p99": 0.063501869119007
|
|
},
|
|
"tpot": {
|
|
"mean": 0.1725158165141388,
|
|
"p50": 0.19940581724371484,
|
|
"p90": 0.1197438720681557,
|
|
"p99": 0.16591937069699617
|
|
},
|
|
"ttft": {
|
|
"mean": 0.052253838015785536,
|
|
"p50": 0.011133339751522397,
|
|
"p90": -0.05603209782959642,
|
|
"p99": 0.008949154409837737
|
|
}
|
|
},
|
|
"paired_relative_error": [
|
|
{
|
|
"e2e": {
|
|
"mean": 0.19703071945786896,
|
|
"p50": 0.16651188156434055,
|
|
"p90": 0.3164763187383559,
|
|
"p99": 0.828609074267838
|
|
},
|
|
"tpot": {
|
|
"mean": 0.19036780497451272,
|
|
"p50": 0.17682334345090506,
|
|
"p90": 0.25886898169288464,
|
|
"p99": 0.7758893841183759
|
|
},
|
|
"ttft": {
|
|
"mean": 0.45065657187067015,
|
|
"p50": 0.10957981115881457,
|
|
"p90": 1.0421699185581517,
|
|
"p99": 5.507647228917042
|
|
}
|
|
},
|
|
{
|
|
"e2e": {
|
|
"mean": 0.20969149313321356,
|
|
"p50": 0.17863746376866904,
|
|
"p90": 0.32770775080455516,
|
|
"p99": 0.8245897427826396
|
|
},
|
|
"tpot": {
|
|
"mean": 0.20429023428176118,
|
|
"p50": 0.19259101506782267,
|
|
"p90": 0.2812533323043662,
|
|
"p99": 0.7385621812554619
|
|
},
|
|
"ttft": {
|
|
"mean": 0.4457497323795849,
|
|
"p50": 0.10597514413253763,
|
|
"p90": 1.0162200916250081,
|
|
"p99": 5.496662131059387
|
|
}
|
|
}
|
|
],
|
|
"trialwise_distribution_bias": [
|
|
{
|
|
"e2e": {
|
|
"mean": 0.14908675997874046,
|
|
"p50": 0.1567207723136509,
|
|
"p90": 0.14081313742067922,
|
|
"p99": 0.08217486261364416
|
|
},
|
|
"tpot": {
|
|
"mean": 0.16551017680032373,
|
|
"p50": 0.1928140924228096,
|
|
"p90": 0.11224819981281003,
|
|
"p99": 0.1781597624045383
|
|
},
|
|
"ttft": {
|
|
"mean": 0.052903132111322415,
|
|
"p50": 0.00950848459985634,
|
|
"p90": -0.05593511273439991,
|
|
"p99": 0.013826387288597744
|
|
}
|
|
},
|
|
{
|
|
"e2e": {
|
|
"mean": 0.16192232934855333,
|
|
"p50": 0.1718867285340943,
|
|
"p90": 0.15139946120181186,
|
|
"p99": 0.09690951004812934
|
|
},
|
|
"tpot": {
|
|
"mean": 0.17960618440333612,
|
|
"p50": 0.21071053715514645,
|
|
"p90": 0.1274260991438306,
|
|
"p99": 0.19282817544573386
|
|
},
|
|
"ttft": {
|
|
"mean": 0.051605344227498705,
|
|
"p50": 0.012763433909333126,
|
|
"p90": -0.05697230680643937,
|
|
"p99": 0.009844585085406704
|
|
}
|
|
}
|
|
],
|
|
"trialwise_distribution_bias_summary": {
|
|
"e2e": {
|
|
"mean": {
|
|
"max": 0.16192232934855333,
|
|
"mean": 0.15550454466364688,
|
|
"min": 0.14908675997874046
|
|
},
|
|
"p50": {
|
|
"max": 0.1718867285340943,
|
|
"mean": 0.1643037504238726,
|
|
"min": 0.1567207723136509
|
|
},
|
|
"p90": {
|
|
"max": 0.15139946120181186,
|
|
"mean": 0.14610629931124552,
|
|
"min": 0.14081313742067922
|
|
},
|
|
"p99": {
|
|
"max": 0.09690951004812934,
|
|
"mean": 0.08954218633088676,
|
|
"min": 0.08217486261364416
|
|
}
|
|
},
|
|
"tpot": {
|
|
"mean": {
|
|
"max": 0.17960618440333612,
|
|
"mean": 0.17255818060182992,
|
|
"min": 0.16551017680032373
|
|
},
|
|
"p50": {
|
|
"max": 0.21071053715514645,
|
|
"mean": 0.20176231478897805,
|
|
"min": 0.1928140924228096
|
|
},
|
|
"p90": {
|
|
"max": 0.1274260991438306,
|
|
"mean": 0.11983714947832032,
|
|
"min": 0.11224819981281003
|
|
},
|
|
"p99": {
|
|
"max": 0.19282817544573386,
|
|
"mean": 0.1854939689251361,
|
|
"min": 0.1781597624045383
|
|
}
|
|
},
|
|
"ttft": {
|
|
"mean": {
|
|
"max": 0.052903132111322415,
|
|
"mean": 0.05225423816941056,
|
|
"min": 0.051605344227498705
|
|
},
|
|
"p50": {
|
|
"max": 0.012763433909333126,
|
|
"mean": 0.011135959254594732,
|
|
"min": 0.00950848459985634
|
|
},
|
|
"p90": {
|
|
"max": -0.05593511273439991,
|
|
"mean": -0.056453709770419636,
|
|
"min": -0.05697230680643937
|
|
},
|
|
"p99": {
|
|
"max": 0.013826387288597744,
|
|
"mean": 0.011835486187002224,
|
|
"min": 0.009844585085406704
|
|
}
|
|
}
|
|
},
|
|
"waiting_p99_ms": 872.1814900223759
|
|
}
|
|
},
|
|
"tp2_rho0p005": {
|
|
"structured_attention": {
|
|
"legacy_pooled_distribution_bias": {
|
|
"e2e": {
|
|
"mean": 0.050164236759908075,
|
|
"p50": 0.0912493994395978,
|
|
"p90": -0.0004979191794830456,
|
|
"p99": -0.0014369075488634014
|
|
},
|
|
"tpot": {
|
|
"mean": 0.05457336599027876,
|
|
"p50": 0.0872512146277895,
|
|
"p90": 0.01132352325010614,
|
|
"p99": -0.05948329733667248
|
|
},
|
|
"ttft": {
|
|
"mean": -0.07082500585057615,
|
|
"p50": -0.1284088888361358,
|
|
"p90": -0.111136573344055,
|
|
"p99": -0.09012556161973535
|
|
}
|
|
},
|
|
"trialwise_distribution_bias": [
|
|
{
|
|
"e2e": {
|
|
"mean": 0.04462683161260099,
|
|
"p50": 0.0878889042422101,
|
|
"p90": -0.006843953727060337,
|
|
"p99": -0.0036641600308731643
|
|
},
|
|
"tpot": {
|
|
"mean": 0.048585629713612626,
|
|
"p50": 0.08234652581183928,
|
|
"p90": 0.003101343693636994,
|
|
"p99": -0.05952673172238401
|
|
},
|
|
"ttft": {
|
|
"mean": -0.07113138559444453,
|
|
"p50": -0.1294796632798084,
|
|
"p90": -0.11063584772248519,
|
|
"p99": -0.08982438463776078
|
|
}
|
|
},
|
|
{
|
|
"e2e": {
|
|
"mean": 0.05576066061144686,
|
|
"p50": 0.09176507061640418,
|
|
"p90": 0.003187406757202862,
|
|
"p99": 0.004813545234813467
|
|
},
|
|
"tpot": {
|
|
"mean": 0.060629878515092876,
|
|
"p50": 0.09375745746657616,
|
|
"p90": 0.019190079238111446,
|
|
"p99": -0.05840233314016777
|
|
},
|
|
"ttft": {
|
|
"mean": -0.07051842392629604,
|
|
"p50": -0.12771160295843229,
|
|
"p90": -0.11011498510890981,
|
|
"p99": -0.08861845103384182
|
|
}
|
|
}
|
|
],
|
|
"trialwise_distribution_bias_summary": {
|
|
"e2e": {
|
|
"mean": {
|
|
"max": 0.05576066061144686,
|
|
"mean": 0.05019374611202393,
|
|
"min": 0.04462683161260099
|
|
},
|
|
"p50": {
|
|
"max": 0.09176507061640418,
|
|
"mean": 0.08982698742930714,
|
|
"min": 0.0878889042422101
|
|
},
|
|
"p90": {
|
|
"max": 0.003187406757202862,
|
|
"mean": -0.0018282734849287376,
|
|
"min": -0.006843953727060337
|
|
},
|
|
"p99": {
|
|
"max": 0.004813545234813467,
|
|
"mean": 0.0005746926019701514,
|
|
"min": -0.0036641600308731643
|
|
}
|
|
},
|
|
"tpot": {
|
|
"mean": {
|
|
"max": 0.060629878515092876,
|
|
"mean": 0.05460775411435275,
|
|
"min": 0.048585629713612626
|
|
},
|
|
"p50": {
|
|
"max": 0.09375745746657616,
|
|
"mean": 0.08805199163920771,
|
|
"min": 0.08234652581183928
|
|
},
|
|
"p90": {
|
|
"max": 0.019190079238111446,
|
|
"mean": 0.01114571146587422,
|
|
"min": 0.003101343693636994
|
|
},
|
|
"p99": {
|
|
"max": -0.05840233314016777,
|
|
"mean": -0.05896453243127589,
|
|
"min": -0.05952673172238401
|
|
}
|
|
},
|
|
"ttft": {
|
|
"mean": {
|
|
"max": -0.07051842392629604,
|
|
"mean": -0.07082490476037029,
|
|
"min": -0.07113138559444453
|
|
},
|
|
"p50": {
|
|
"max": -0.12771160295843229,
|
|
"mean": -0.12859563311912034,
|
|
"min": -0.1294796632798084
|
|
},
|
|
"p90": {
|
|
"max": -0.11011498510890981,
|
|
"mean": -0.11037541641569751,
|
|
"min": -0.11063584772248519
|
|
},
|
|
"p99": {
|
|
"max": -0.08861845103384182,
|
|
"mean": -0.0892214178358013,
|
|
"min": -0.08982438463776078
|
|
}
|
|
}
|
|
},
|
|
"waiting_p99_ms": 1166.1196883368564
|
|
},
|
|
"structured_attention_plus_prefill_moe": {
|
|
"legacy_pooled_distribution_bias": {
|
|
"e2e": {
|
|
"mean": 0.09135599701015996,
|
|
"p50": 0.12135156828958114,
|
|
"p90": 0.046904152670232756,
|
|
"p99": 0.05267198541612694
|
|
},
|
|
"tpot": {
|
|
"mean": 0.09686033389444083,
|
|
"p50": 0.10920022438478016,
|
|
"p90": 0.055959658792935615,
|
|
"p99": 0.03647201411457465
|
|
},
|
|
"ttft": {
|
|
"mean": 0.03133826519177879,
|
|
"p50": -0.002404965655955949,
|
|
"p90": -0.03671038522787783,
|
|
"p99": -0.003047423826180572
|
|
}
|
|
},
|
|
"paired_relative_error": [
|
|
{
|
|
"e2e": {
|
|
"mean": 0.1373684769346999,
|
|
"p50": 0.10813242136980938,
|
|
"p90": 0.27973507165610256,
|
|
"p99": 0.6892985502092539
|
|
},
|
|
"tpot": {
|
|
"mean": 0.12750409894388354,
|
|
"p50": 0.10486952080742089,
|
|
"p90": 0.2347358304319833,
|
|
"p99": 0.7326960496792728
|
|
},
|
|
"ttft": {
|
|
"mean": 0.48959967245129044,
|
|
"p50": 0.11864693915356177,
|
|
"p90": 1.3203954670852478,
|
|
"p99": 5.486884563706639
|
|
}
|
|
},
|
|
{
|
|
"e2e": {
|
|
"mean": 0.15015620590606898,
|
|
"p50": 0.11828746639583118,
|
|
"p90": 0.29344565738935435,
|
|
"p99": 0.7209062198737742
|
|
},
|
|
"tpot": {
|
|
"mean": 0.14093466935703095,
|
|
"p50": 0.11608679958107468,
|
|
"p90": 0.24292321061978972,
|
|
"p99": 0.7622579889720426
|
|
},
|
|
"ttft": {
|
|
"mean": 0.4989603034128753,
|
|
"p50": 0.11800815758794723,
|
|
"p90": 1.394773550999063,
|
|
"p99": 5.683406566684342
|
|
}
|
|
}
|
|
],
|
|
"trialwise_distribution_bias": [
|
|
{
|
|
"e2e": {
|
|
"mean": 0.08560139205995335,
|
|
"p50": 0.11789837366536833,
|
|
"p90": 0.040257153080803736,
|
|
"p99": 0.05032404535106296
|
|
},
|
|
"tpot": {
|
|
"mean": 0.09063249747878419,
|
|
"p50": 0.10419652159558444,
|
|
"p90": 0.04737458218848663,
|
|
"p99": 0.03642414837746145
|
|
},
|
|
"ttft": {
|
|
"mean": 0.03099819883665655,
|
|
"p50": -0.0036305394992732502,
|
|
"p90": -0.03616773291859882,
|
|
"p99": -0.0027174231630140012
|
|
}
|
|
},
|
|
{
|
|
"e2e": {
|
|
"mean": 0.09717193562089739,
|
|
"p50": 0.12188146428139615,
|
|
"p90": 0.05076425771763047,
|
|
"p99": 0.05926112994918994
|
|
},
|
|
"tpot": {
|
|
"mean": 0.10315970439292232,
|
|
"p50": 0.11583781275404642,
|
|
"p90": 0.06417341590032893,
|
|
"p99": 0.037663262642925396
|
|
},
|
|
"ttft": {
|
|
"mean": 0.031678555957153555,
|
|
"p50": -0.001606874761951341,
|
|
"p90": -0.03560325750951616,
|
|
"p99": -0.0013960775328830752
|
|
}
|
|
}
|
|
],
|
|
"trialwise_distribution_bias_summary": {
|
|
"e2e": {
|
|
"mean": {
|
|
"max": 0.09717193562089739,
|
|
"mean": 0.09138666384042538,
|
|
"min": 0.08560139205995335
|
|
},
|
|
"p50": {
|
|
"max": 0.12188146428139615,
|
|
"mean": 0.11988991897338225,
|
|
"min": 0.11789837366536833
|
|
},
|
|
"p90": {
|
|
"max": 0.05076425771763047,
|
|
"mean": 0.0455107053992171,
|
|
"min": 0.040257153080803736
|
|
},
|
|
"p99": {
|
|
"max": 0.05926112994918994,
|
|
"mean": 0.05479258765012645,
|
|
"min": 0.05032404535106296
|
|
}
|
|
},
|
|
"tpot": {
|
|
"mean": {
|
|
"max": 0.10315970439292232,
|
|
"mean": 0.09689610093585325,
|
|
"min": 0.09063249747878419
|
|
},
|
|
"p50": {
|
|
"max": 0.11583781275404642,
|
|
"mean": 0.11001716717481544,
|
|
"min": 0.10419652159558444
|
|
},
|
|
"p90": {
|
|
"max": 0.06417341590032893,
|
|
"mean": 0.05577399904440778,
|
|
"min": 0.04737458218848663
|
|
},
|
|
"p99": {
|
|
"max": 0.037663262642925396,
|
|
"mean": 0.037043705510193425,
|
|
"min": 0.03642414837746145
|
|
}
|
|
},
|
|
"ttft": {
|
|
"mean": {
|
|
"max": 0.031678555957153555,
|
|
"mean": 0.03133837739690505,
|
|
"min": 0.03099819883665655
|
|
},
|
|
"p50": {
|
|
"max": -0.001606874761951341,
|
|
"mean": -0.002618707130612296,
|
|
"min": -0.0036305394992732502
|
|
},
|
|
"p90": {
|
|
"max": -0.03560325750951616,
|
|
"mean": -0.03588549521405749,
|
|
"min": -0.03616773291859882
|
|
},
|
|
"p99": {
|
|
"max": -0.0013960775328830752,
|
|
"mean": -0.0020567503479485385,
|
|
"min": -0.0027174231630140012
|
|
}
|
|
}
|
|
},
|
|
"waiting_p99_ms": 1296.3971075780032
|
|
}
|
|
}
|
|
},
|
|
"decision": "reject_global_scale_and_fit_shape_conditioned_curve",
|
|
"gates": {
|
|
"subcritical_mean_ttft_abs_bias_improves_3pp": false,
|
|
"subcritical_ttft_e2e_abs_bias_not_worse": false,
|
|
"subcritical_waiting_below_1s": true
|
|
},
|
|
"schema": "frontier-tp2-prefill-serving-replay-verdict.v1"
|
|
}
|