gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record code-trace canary fidelity gates

Browse Source
This commit is contained in:
Gahow Wang
2026-07-24 02:42:29 +08:00
parent e1f2557a0c
commit e0811e95b4
8 changed files with 1526 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
runs/frontier-code-trace-v0/results/canary-analysis-tp2-v1.json Normal file
View File

@@ -0,0 +1,319 @@
{
"drain": {
"interpretation": "Report the max-completion request explicitly; a response that arrived well before the cutoff can create a long drain tail without implying queue accumulation.",
"real_per_trial": [
{
"last_arrival_s": 597.0069999999832,
"last_completion_s": 714.3421455998905,
"tail_after_last_arrival_s": 117.33514559990726,
"tail_driver": {
"admission_lag_ms": 12.391218915581703,
"arrival_before_cutoff_s": 62.03099999995902,
"arrival_s": 534.9760000000242,
"e2e_ms": 179353.7543809507,
"input_tokens": 61976,
"output_tokens": 21361,
"request_index": 40
}
}
],
"sim": {
"last_arrival_s": 597.007,
"last_completion_s": 713.4565521329489,
"tail_after_last_arrival_s": 116.44955213294895,
"tail_driver": {
"arrival_before_cutoff_s": 62.03099999999995,
"arrival_s": 534.976,
"e2e_ms": 178480.5521329489,
"input_tokens": 61976,
"output_tokens": 21361,
"request_index": 40,
"waiting_ms": 0.0
}
}
},
"input": {
"frontier_csv_sha256": "624bc983d3c9fba58cbfa3bb5126304b99b19883da4c78f63cd8622b856dbc1f",
"manifest": "runs/frontier-code-trace-v0/results/canary-input-r0p0004-v1/manifest.json",
"paired_row_vector_sha256": "72b09903851f5684838da23498a068efc409e6481d1e84efeeefddd8e431c3e7"
},
"latency_ms": {
"e2e": {
"real": {
"count": 53,
"max": 179353.7543809507,
"mean": 15936.508002247274,
"p50": 3162.7415760885924,
"p90": 31124.16882226244,
"p95": 62989.56617997021,
"p99": 165236.24419869846
},
"real_per_trial": [
{
"count": 53,
"max": 179353.7543809507,
"mean": 15936.508002247274,
"p50": 3162.7415760885924,
"p90": 31124.16882226244,
"p95": 62989.56617997021,
"p99": 165236.24419869846
}
],
"relative_bias_percent": {
"mean": -8.269543527339733,
"p50": -8.321761801888824,
"p90": -18.368943324171262,
"p95": -23.79834859601531,
"p99": -2.75020118151507
},
"sim": {
"count": 53,
"max": 178480.5521329489,
"mean": 14618.631536263456,
"p50": 2899.5457557171953,
"p90": 25406.98789118167,
"p95": 47999.08964134313,
"p99": 160691.91505845473
}
},
"tpot": {
"real": {
"count": 53,
"max": 641.5267530052612,
"mean": 27.41422879865018,
"p50": 8.106854443332919,
"p90": 23.880988038291846,
"p95": 92.03784805296183,
"p99": 397.1900480028069
},
"real_per_trial": [
{
"count": 53,
"max": 641.5267530052612,
"mean": 27.41422879865018,
"p50": 8.106854443332919,
"p90": 23.880988038291846,
"p95": 92.03784805296183,
"p99": 397.1900480028069
}
],
"relative_bias_percent": {
"mean": -9.878641617364694,
"p50": -2.3180300975258694,
"p90": 26.54184469239794,
"p95": -24.72214442719661,
"p99": -13.722225303935645
},
"sim": {
"count": 53,
"max": 556.599034356792,
"mean": 24.706075383467144,
"p50": 7.918935117373849,
"p90": 30.219442794425397,
"p95": 69.28411832962485,
"p99": 342.6867347310516
}
},
"ttft": {
"real": {
"count": 53,
"max": 23546.024559997022,
"mean": 3438.4982450513767,
"p50": 918.7854011543095,
"p90": 10432.362293498598,
"p95": 13642.840061988678,
"p99": 22228.129854062565
},
"real_per_trial": [
{
"count": 53,
"max": 23546.024559997022,
"mean": 3438.4982450513767,
"p50": 918.7854011543095,
"p90": 10432.362293498598,
"p95": 13642.840061988678,
"p99": 22228.129854062565
}
],
"relative_bias_percent": {
"mean": -21.116133763518285,
"p50": 13.780393777591794,
"p90": -32.09252788469729,
"p95": -13.833140262007847,
"p99": -14.006169043239403
},
"sim": {
"count": 53,
"max": 19564.258977976864,
"mean": 2712.4203561700992,
"p50": 1045.3976474043998,
"p90": 7084.353515424915,
"p95": 11755.606860492386,
"p99": 19114.8204115518
}
}
},
"prefix_cache": {
"real_hit_ratio_mean": 0.2399084848137731,
"real_per_trial": [
{
"hit_ratio": 0.2399084848137731,
"hit_tokens": 309968.0,
"query_tokens": 1292026.0
}
],
"sim": {
"block_size_tokens": 16,
"hit_ratio": 0.23997274866840085,
"mean_cached_prefill_tokens": 5848.452830188679,
"requests": 53,
"requests_with_hits": 9,
"total_cached_prefill_tokens": 309968,
"total_hit_blocks": 19373,
"total_query_blocks": 80730
}
},
"real_artifacts": [
{
"result_sha256": "54ebf30bb66fd845d0d6bbdb15674784989070ec12aa8b785f1dd19886707269",
"root": "runs/frontier-code-trace-v0/results/real-canary/r0p0004-tp2-t1-v3"
}
],
"real_decode_batch_per_trial": [
{
"histogram": {
"1": 53281,
"2": 5701,
"3": 2919,
"4": 476,
"5": 110,
"6": 62
},
"max": 6,
"mixed_steps_excluded": 152,
"share_gt_1": 0.14817183328270636,
"steps": 62549
}
],
"requests_per_trial": 53,
"schema": "frontier-code-trace-canary-analysis-v1",
"sim_decode_batch": {
"histogram": {
"1": 53233,
"2": 6466,
"3": 2626,
"4": 395,
"5": 103,
"6": 29
},
"max": 6,
"share_gt_1": 0.1530420670782155,
"share_gt_4": 0.0021001718322408197,
"stages": 62852
},
"slo": {
"definition": {
"target_pass_rate": 0.95,
"tpot_ms": 150,
"ttft_ms": "1000 + 1000 * input_tokens / 8000"
},
"feasibility_flip": false,
"real_per_trial": [
{
"feasible": false,
"pass_rate": 0.7735849056603774,
"passed": 41
}
],
"sim": {
"feasible": false,
"pass_rate": 0.8679245283018868,
"passed": 46
}
},
"topology": "tp2_mns16",
"tpot_by_min_output_tokens": {
"2": {
"real": {
"count": 53,
"max": 641.5267530052612,
"mean": 27.41422879865018,
"p50": 8.106854443332919,
"p90": 23.880988038291846,
"p95": 92.03784805296183,
"p99": 397.1900480028069
},
"relative_bias_percent": {
"mean": -9.878641617364694,
"p50": -2.3180300975258694,
"p90": 26.54184469239794,
"p95": -24.72214442719661,
"p99": -13.722225303935645
},
"sim": {
"count": 53,
"max": 556.599034356792,
"mean": 24.706075383467144,
"p50": 7.918935117373849,
"p90": 30.219442794425397,
"p95": 69.28411832962485,
"p99": 342.6867347310516
}
},
"32": {
"real": {
"count": 45,
"max": 171.6484741543905,
"mean": 16.593114914019644,
"p50": 8.106854443332919,
"p90": 19.44677900381304,
"p95": 66.47742398097495,
"p99": 148.17240555253153
},
"relative_bias_percent": {
"mean": -9.940572863733157,
"p50": -2.3180300975258694,
"p90": 10.853794302071227,
"p95": -22.75922601837412,
"p99": -18.755468241526987
},
"sim": {
"count": 45,
"max": 145.22922738421678,
"mean": 14.943664235628548,
"p50": 7.918935117373849,
"p90": 21.557492395265285,
"p95": 51.34767680595202,
"p99": 120.38197708641991
}
},
"8": {
"real": {
"count": 52,
"max": 171.6484741543905,
"mean": 15.604372563907663,
"p50": 8.031664372446695,
"p90": 21.733123898935485,
"p95": 52.378594044159485,
"p99": 144.4375764567812
},
"relative_bias_percent": {
"mean": -7.222385426340451,
"p50": -1.403560331275464,
"p90": 10.95444148849469,
"p95": -10.968821104857895,
"p99": -19.391471177881424
},
"sim": {
"count": 52,
"max": 145.22922738421678,
"mean": 14.477364633980129,
"p50": 7.918935117373849,
"p90": 24.11386624006643,
"p95": 46.63327976621588,
"p99": 116.42900544813402
}
}
},
"trials": 1
}