Record code-trace canary fidelity gates

runs/frontier-code-trace-v0/README.md

@@ -1,9 +1,9 @@
 # Frontier code-trace campaign handoff
 
-Phase A code prefill+decode 已开始执行。data/profile/max-length gate 和
+Phase A code prefill+decode 已进入 61min real matrix。data/profile、
-Frontier rho calibration 已完成；TP2/TP4 paired 10min real canary 正在
+max-length、Frontier rho calibration 和 TP2/TP4 paired canary 均已完成；
-运行。61min full paired inputs 与 code prefill-only calibration cache
+第一批 TP4 三个 load 与 TP2 low-rho diagnostic 正在 dash1–dash4 并行
-均已物化，尚未越过 canary gate 启动正式 1h real matrix。
+运行。code prefill-only 的独立 sim calibration 也已完成。
 
 完整设计与 gate 见 [`experiment-card.md`](experiment-card.md)。
 
@@ -16,11 +16,15 @@ Frontier rho calibration 已完成；TP2/TP4 paired 10min real canary 正在
 - full paired inputs（CPFS）：
   `runs/frontier-code-trace-v0/inputs/full-r0p{0002,0004,0008,0016}-v1/`；
 - compact provenance：`results/calibration-summary.json`、
+  `results/prefill-only-calibration-summary.json`、
+  `results/canary-analysis-tp{2,4}-v*.json` 和
   `results/paired-input-manifests/`；
-- 下一步：完成 paired canary 分析；若 real 零失败、digest/hit ratio
+- 当前 A4 wave 1：TP4 `rho={0.0002,0.0008,0.0016}` trial 1，以及
-  对齐且 queue 不发散，启动 TP4
+  TP2 `rho=0.0002` trial 1 diagnostic；
-  `rho={0.0002,0.0008,0.0016}` 与 TP2
+- TP4 canary 的 TTFT/E2E、prefix hit 与 decode batch 通过；TP2
-  `rho={0.0002,0.0004,0.0008}` 的 1h jobs。
+  TTFT p90 低估 32.1%，因此 TP2 其余 cell 暂不扩展；
+- prefill-only：TP2 已冻结 `rho={0.0004,0.0008,0.0016}`；
+  TP4 到 `0.0032` 仍亚临界，需追加更高 rho 后冻结 near-knee。
 
 source trace 的远端位置是：
 

runs/frontier-code-trace-v0/analyze_canary.py

@@ -16,7 +16,10 @@ from typing import Any
 
 
 METRICS = ("ttft", "tpot", "e2e")
+TPOT_MIN_OUTPUT_TOKENS = (2, 8, 32)
+SLO_TARGET_PASS_RATE = 0.95
 PROM_COUNTERS = ("vllm:prefix_cache_queries_total", "vllm:prefix_cache_hits_total")
+csv.field_size_limit(16 * 1024 * 1024)
 ITERATION_RE = re.compile(
     r"Iteration.*?:\s+"
     r"(?P<context_requests>\d+) context requests, "
@@ -165,8 +168,35 @@ def load_sim(root: Path, trace: list[dict[str, str]], trace_sha: str) -> dict[st
     tail_index = max(range(len(completions)), key=completions.__getitem__)
     last_arrival = max(float(row["arrived_at"]) for row in trace)
     summary = json.loads((root / "summary.json").read_text())
+    slo_pass = []
+    for trace_row, metric_row in zip(trace, rows):
+        input_tokens = int(trace_row["num_prefill_tokens"])
+        ttft_threshold_ms = 1000 + 1000 * input_tokens / 8000
+        tpot = (
+            float(metric_row["tpot"])
+            if metric_row["tpot"].strip()
+            else None
+        )
+        slo_pass.append(
+            float(metric_row["ttft"]) <= ttft_threshold_ms
+            and (tpot is None or tpot <= 150)
+        )
     return {
         "values": values,
+        "tpot_by_min_output_tokens": {
+            str(threshold): [
+                float(metric_row["tpot"])
+                for trace_row, metric_row in zip(trace, rows)
+                if int(trace_row["num_decode_tokens"]) >= threshold
+                and metric_row["tpot"].strip()
+            ]
+            for threshold in TPOT_MIN_OUTPUT_TOKENS
+        },
+        "slo": {
+            "passed": sum(slo_pass),
+            "pass_rate": sum(slo_pass) / len(slo_pass),
+            "feasible": sum(slo_pass) / len(slo_pass) >= SLO_TARGET_PASS_RATE,
+        },
         "summary": summary,
         "drain": {
             "last_arrival_s": last_arrival,
@@ -224,6 +254,23 @@ def load_real(
         "root": str(root),
         "result_sha256": sha256(result_path),
         "values": values,
+        "tpot_by_min_output_tokens": {
+            str(threshold): [
+                float(request["tpot_ms"])
+                for request in requests
+                if int(request["requested_output_tokens"]) >= threshold
+                and request.get("tpot_ms") is not None
+            ]
+            for threshold in TPOT_MIN_OUTPUT_TOKENS
+        },
+        "slo": {
+            "passed": sum(bool(request["slo_pass"]) for request in requests),
+            "pass_rate": sum(bool(request["slo_pass"]) for request in requests)
+            / len(requests),
+            "feasible": sum(bool(request["slo_pass"]) for request in requests)
+            / len(requests)
+            >= SLO_TARGET_PASS_RATE,
+        },
         "summary": result["summary"],
         "prefix_cache": prefix_cache_delta(root),
         "decode_batch": real_decode_batch(root),
@@ -262,19 +309,60 @@ def main() -> None:
     latency = {}
     for metric in METRICS:
         real_dist = distribution(pooled[metric])
+        real_per_trial = [
+            distribution(real["values"][metric]) for real in reals
+        ]
+        real_reference = {
+            statistic: statistics.fmean(
+                float(trial[statistic]) for trial in real_per_trial
+            )
+            for statistic in ("mean", "p50", "p90", "p95", "p99")
+        }
         sim_dist = distribution(sim["values"][metric])
         latency[metric] = {
             "real": real_dist,
+            "real_trial_statistic_mean": real_reference,
             "sim": sim_dist,
             "relative_bias_percent": {
                 statistic: 100
-                * (float(sim_dist[statistic]) - float(real_dist[statistic]))
+                * (float(sim_dist[statistic]) - real_reference[statistic])
-                / float(real_dist[statistic])
+                / real_reference[statistic]
+                for statistic in ("mean", "p50", "p90", "p95", "p99")
+            },
+            "real_per_trial": real_per_trial,
+        }
+    tpot_sensitivity = {}
+    for threshold in TPOT_MIN_OUTPUT_TOKENS:
+        key = str(threshold)
+        real_per_trial = [
+            distribution(real["tpot_by_min_output_tokens"][key])
+            for real in reals
+        ]
+        real_values = [
+            value
+            for real in reals
+            for value in real["tpot_by_min_output_tokens"][key]
+        ]
+        sim_values = sim["tpot_by_min_output_tokens"][key]
+        real_dist = distribution(real_values)
+        real_reference = {
+            statistic: statistics.fmean(
+                float(trial[statistic]) for trial in real_per_trial
+            )
+            for statistic in ("mean", "p50", "p90", "p95", "p99")
+        }
+        sim_dist = distribution(sim_values)
+        tpot_sensitivity[key] = {
+            "real": real_dist,
+            "real_trial_statistic_mean": real_reference,
+            "real_per_trial": real_per_trial,
+            "sim": sim_dist,
+            "relative_bias_percent": {
+                statistic: 100
+                * (float(sim_dist[statistic]) - real_reference[statistic])
+                / real_reference[statistic]
                 for statistic in ("mean", "p50", "p90", "p95", "p99")
             },
-            "real_per_trial": [
-                distribution(real["values"][metric]) for real in reals
-            ],
         }
     payload = {
         "schema": "frontier-code-trace-canary-analysis-v1",
@@ -287,6 +375,20 @@ def main() -> None:
             "frontier_csv_sha256": trace_sha,
         },
         "latency_ms": latency,
+        "tpot_by_min_output_tokens": tpot_sensitivity,
+        "slo": {
+            "definition": {
+                "ttft_ms": "1000 + 1000 * input_tokens / 8000",
+                "tpot_ms": 150,
+                "target_pass_rate": SLO_TARGET_PASS_RATE,
+            },
+            "real_per_trial": [real["slo"] for real in reals],
+            "sim": sim["slo"],
+            "feasibility_flip": any(
+                real["slo"]["feasible"] != sim["slo"]["feasible"]
+                for real in reals
+            ),
+        },
         "prefix_cache": {
             "real_per_trial": [real["prefix_cache"] for real in reals],
             "real_hit_ratio_mean": statistics.fmean(

runs/frontier-code-trace-v0/experiment-card.md

@@ -1,8 +1,8 @@
 # EXP-CODE-TRACE：从 chat 1h trace 扩展到 code 与 phase-separated replay
 
-> **状态：RUNNING（Phase A code P+D）。** A0 数据/profile 与 A1 TP4
+> **状态：RUNNING（Phase A code P+D）。** A0 数据/profile、A1
-> max-length smoke 已完成；A2 paired canary 正在运行；A3 sim calibration
+> max-length smoke、A2 paired canary 与 A3 sim calibration 已完成；
-> 已冻结 topology-specific rho。只使用 `dash1`--`dash4`，禁止使用 `dash0`。
+> A4 第一批 61min real jobs 正在 `dash1`--`dash4` 运行。禁止使用 `dash0`。
 
 ## 目标与成功定义
 
@@ -17,10 +17,10 @@
 
 | 项目 | 当前结论 | 下一 gate |
 |---|---|---|
-| code P+D 数据 | 61min development window、long-context profile-v6、四个 paired full 输入已冻结 | paired canary real-vs-sim |
+| code P+D 数据 | 61min development window、long-context profile-v6、四个 paired full 输入已冻结 | 第一批 1h real 运行中 |
-| TP4 负载 | low/mid/near-knee=`rho 0.0002/0.0008/0.0016`；三点均亚临界 | canary 通过后启动 1h |
+| TP4 负载 | low/mid/near-knee=`rho 0.0002/0.0008/0.0016`；paired canary 的 TTFT/E2E、KV、batch 通过 | 三个 load 的 trial 1 运行中 |
-| TP2 负载 | low/mid/near-knee=`rho 0.0002/0.0004/0.0008`；`0.0016` 明确过载 | TP2 canary/KV gate |
+| TP2 负载 | `rho 0.0004` canary 暴露 TTFT p90 `-32.1%` bad case | 只跑 `rho 0.0002` 1h diagnostic，暂不铺满 |
-| code prefill-only | `rho<=0.0032` 的 paired remap cache 已生成，OSL 全为 1 | 独立 sim rho calibration |
+| code prefill-only | TP2 已冻结 `rho 0.0004/0.0008/0.0016`，`0.0032` 过载；TP4 到 `0.0032` 仍亚临界 | TP4 追加更高 rho 边界 |
 | strict decode-only | vLLM 0.20.0 有 `DecodeBenchConnector`；Frontier trace generator 尚不能注入 initial computed tokens | C0 contract canary，未进入正式结果 |
 
 ## 三种 workload mode 的冻结定义
@@ -173,9 +173,10 @@ tail driver 分解共同决定，不能只用一个 drain 秒数。
 
 若某 topology 的 near-knee 过载，像现有 chat TP2/ρ0.01 一样排除，不为凑齐矩阵强跑。
 
-当前状态：A0 完成；A1 的 TP4 完成、TP2 由 paired canary 同时验证；
+当前状态：A0–A3 完成。A4 第一批为 TP4
-A2 运行中；A3 完成；A4 的 paired input 已物化但尚未在 canary gate
+`rho={0.0002,0.0008,0.0016}` trial 1，以及 TP2 `rho=0.0002`
-前启动。
+trial 1 diagnostic；其余 TP2 cell 等该 diagnostic 验证 canary bad case
+后再决定是否扩展。
 
 ### Phase B：chat/code prefill-only
 
@@ -308,3 +309,38 @@ mode-specific：
   FlashInfer JIT 的 apparatus cost。它发生在 readiness 前，不进入 TTFT；
   runner commit `d5bb974` 改为长上下文默认使用 host-local vLLM cache，
   并按 topology 复用 FlashInfer workspace。
+- 第一轮 paired real canary 的 3 次旧 client 运行都只在同一个
+  `106709+197` 请求失败，根因是 `return_token_ids` 把 100k+ prompt
+  vector 放进单条 SSE event，超过 aiohttp 默认 512KiB line limit。
+  commit `e1f2557` 把 exact client read buffer 提到 8MiB；700KiB
+  单-event runtime 对照和随后 TP4×2、TP2×1 的 53/53 replay 均通过。
+- TP4 canary 的 real-vs-sim prefix hit ratio =
+  `0.239908/0.239973`；pure-decode batch max 都为 4，
+  `share(b>1)=15.87%/15.69%`（real 两 trial）vs `16.35%`（sim）。
+  TTFT mean/p50/p90/p95/p99 bias =
+  `-8.3/-4.1/-8.8/-13.1/-6.3%`，E2E =
+  `+2.2/+6.1/+11.0/+1.4/-1.3%`。长 drain 的同一
+  `61976+21361` 请求 real=92.61/92.34s、sim=91.37s，不是 backlog。
+- TP4 若把 OSL=4 请求纳入 TPOT，mean/p99 bias 会被单个
+  `~213ms/token` 样本放大到 `-29.9%/-71.5%`；OSL≥8 后
+  mean/p50/p90/p95/p99 bias =
+  `+8.7/+8.2/+0.6/+13.4/+3.7%`。因此 raw TPOT 仍保留，但正式报告必须
+  同时给 OSL threshold sensitivity，不能把短输出的三段 inter-token
+  interval 当作稳定 decode service。
+- TP4 canary 两 trial 的 real SLO pass rate 都是 `50/53=94.34%`，
+  sim 为 `52/53=98.11%`，在 95% feasibility threshold 上发生翻转；
+  这由两个临界 TTFT 请求和上述 OSL=4 请求共同造成，作为明确 bad case
+  进入 1h 检验，不能被总体 latency gap 掩盖。
+- TP2 canary 的 cache/batch/drain 仍对齐，但 TTFT p90 bias=`-32.1%`，
+  E2E p90/p95=`-18.4%/-23.8%`。因此先只启动 low-rho 1h diagnostic，
+  不直接铺满 TP2 六个正式 jobs。
+- code prefill-only 已完成 10-cell Frontier calibration。TP2
+  `rho=0.0032` drain=1384.59s、waiting p50=751.01s，明确过载；
+  `0.0004/0.0008/0.0016` 冻结为 low/mid/near-knee。TP4 到
+  `rho=0.0032` 仍只有 9.09s drain，暂称 highest-tested，追加更高 rho
+  后才冻结 near-knee。compact table 在
+  `results/prefill-only-calibration-summary.json`。
+- 2026-07-23 18:37 UTC 启动 A4 wave 1：dash1=`TP4/rho0.0002/t1`、
+  dash3=`TP4/rho0.0008/t1`、dash4=`TP4/rho0.0016/t1`、
+  dash2=`TP2/rho0.0002/t1 diagnostic`；四台启动前再次确认 8×H20
+  memory/utilization=0、无 compute process、uncorrected ECC=0。

runs/frontier-code-trace-v0/results/calibration-summary.json

@@ -1,5 +1,6 @@
 {
   "attention_profile_sha256": "fbcf7e1f95789a6f6d771e24d1fc60958b7daf04eb0db260d27869a19d71d550",
+  "cell_version": "v3",
   "cells": [
     {
       "decode_batch": {
@@ -628,5 +629,6 @@
     }
   ],
   "schema": "frontier-code-trace-calibration-summary-v1",
-  "subcritical_rule": "drain tail <= 10% of the 3660s arrival window"
+  "subcritical_rule": "drain tail <= 10% of the 3660s arrival window",
+  "workload_mode": "prefill_decode"
 }

runs/frontier-code-trace-v0/results/canary-analysis-tp2-v1.json

@@ -0,0 +1,319 @@
+{
+  "drain": {
+    "interpretation": "Report the max-completion request explicitly; a response that arrived well before the cutoff can create a long drain tail without implying queue accumulation.",
+    "real_per_trial": [
+      {
+        "last_arrival_s": 597.0069999999832,
+        "last_completion_s": 714.3421455998905,
+        "tail_after_last_arrival_s": 117.33514559990726,
+        "tail_driver": {
+          "admission_lag_ms": 12.391218915581703,
+          "arrival_before_cutoff_s": 62.03099999995902,
+          "arrival_s": 534.9760000000242,
+          "e2e_ms": 179353.7543809507,
+          "input_tokens": 61976,
+          "output_tokens": 21361,
+          "request_index": 40
+        }
+      }
+    ],
+    "sim": {
+      "last_arrival_s": 597.007,
+      "last_completion_s": 713.4565521329489,
+      "tail_after_last_arrival_s": 116.44955213294895,
+      "tail_driver": {
+        "arrival_before_cutoff_s": 62.03099999999995,
+        "arrival_s": 534.976,
+        "e2e_ms": 178480.5521329489,
+        "input_tokens": 61976,
+        "output_tokens": 21361,
+        "request_index": 40,
+        "waiting_ms": 0.0
+      }
+    }
+  },
+  "input": {
+    "frontier_csv_sha256": "624bc983d3c9fba58cbfa3bb5126304b99b19883da4c78f63cd8622b856dbc1f",
+    "manifest": "runs/frontier-code-trace-v0/results/canary-input-r0p0004-v1/manifest.json",
+    "paired_row_vector_sha256": "72b09903851f5684838da23498a068efc409e6481d1e84efeeefddd8e431c3e7"
+  },
+  "latency_ms": {
+    "e2e": {
+      "real": {
+        "count": 53,
+        "max": 179353.7543809507,
+        "mean": 15936.508002247274,
+        "p50": 3162.7415760885924,
+        "p90": 31124.16882226244,
+        "p95": 62989.56617997021,
+        "p99": 165236.24419869846
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 179353.7543809507,
+          "mean": 15936.508002247274,
+          "p50": 3162.7415760885924,
+          "p90": 31124.16882226244,
+          "p95": 62989.56617997021,
+          "p99": 165236.24419869846
+        }
+      ],
+      "relative_bias_percent": {
+        "mean": -8.269543527339733,
+        "p50": -8.321761801888824,
+        "p90": -18.368943324171262,
+        "p95": -23.79834859601531,
+        "p99": -2.75020118151507
+      },
+      "sim": {
+        "count": 53,
+        "max": 178480.5521329489,
+        "mean": 14618.631536263456,
+        "p50": 2899.5457557171953,
+        "p90": 25406.98789118167,
+        "p95": 47999.08964134313,
+        "p99": 160691.91505845473
+      }
+    },
+    "tpot": {
+      "real": {
+        "count": 53,
+        "max": 641.5267530052612,
+        "mean": 27.41422879865018,
+        "p50": 8.106854443332919,
+        "p90": 23.880988038291846,
+        "p95": 92.03784805296183,
+        "p99": 397.1900480028069
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 641.5267530052612,
+          "mean": 27.41422879865018,
+          "p50": 8.106854443332919,
+          "p90": 23.880988038291846,
+          "p95": 92.03784805296183,
+          "p99": 397.1900480028069
+        }
+      ],
+      "relative_bias_percent": {
+        "mean": -9.878641617364694,
+        "p50": -2.3180300975258694,
+        "p90": 26.54184469239794,
+        "p95": -24.72214442719661,
+        "p99": -13.722225303935645
+      },
+      "sim": {
+        "count": 53,
+        "max": 556.599034356792,
+        "mean": 24.706075383467144,
+        "p50": 7.918935117373849,
+        "p90": 30.219442794425397,
+        "p95": 69.28411832962485,
+        "p99": 342.6867347310516
+      }
+    },
+    "ttft": {
+      "real": {
+        "count": 53,
+        "max": 23546.024559997022,
+        "mean": 3438.4982450513767,
+        "p50": 918.7854011543095,
+        "p90": 10432.362293498598,
+        "p95": 13642.840061988678,
+        "p99": 22228.129854062565
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 23546.024559997022,
+          "mean": 3438.4982450513767,
+          "p50": 918.7854011543095,
+          "p90": 10432.362293498598,
+          "p95": 13642.840061988678,
+          "p99": 22228.129854062565
+        }
+      ],
+      "relative_bias_percent": {
+        "mean": -21.116133763518285,
+        "p50": 13.780393777591794,
+        "p90": -32.09252788469729,
+        "p95": -13.833140262007847,
+        "p99": -14.006169043239403
+      },
+      "sim": {
+        "count": 53,
+        "max": 19564.258977976864,
+        "mean": 2712.4203561700992,
+        "p50": 1045.3976474043998,
+        "p90": 7084.353515424915,
+        "p95": 11755.606860492386,
+        "p99": 19114.8204115518
+      }
+    }
+  },
+  "prefix_cache": {
+    "real_hit_ratio_mean": 0.2399084848137731,
+    "real_per_trial": [
+      {
+        "hit_ratio": 0.2399084848137731,
+        "hit_tokens": 309968.0,
+        "query_tokens": 1292026.0
+      }
+    ],
+    "sim": {
+      "block_size_tokens": 16,
+      "hit_ratio": 0.23997274866840085,
+      "mean_cached_prefill_tokens": 5848.452830188679,
+      "requests": 53,
+      "requests_with_hits": 9,
+      "total_cached_prefill_tokens": 309968,
+      "total_hit_blocks": 19373,
+      "total_query_blocks": 80730
+    }
+  },
+  "real_artifacts": [
+    {
+      "result_sha256": "54ebf30bb66fd845d0d6bbdb15674784989070ec12aa8b785f1dd19886707269",
+      "root": "runs/frontier-code-trace-v0/results/real-canary/r0p0004-tp2-t1-v3"
+    }
+  ],
+  "real_decode_batch_per_trial": [
+    {
+      "histogram": {
+        "1": 53281,
+        "2": 5701,
+        "3": 2919,
+        "4": 476,
+        "5": 110,
+        "6": 62
+      },
+      "max": 6,
+      "mixed_steps_excluded": 152,
+      "share_gt_1": 0.14817183328270636,
+      "steps": 62549
+    }
+  ],
+  "requests_per_trial": 53,
+  "schema": "frontier-code-trace-canary-analysis-v1",
+  "sim_decode_batch": {
+    "histogram": {
+      "1": 53233,
+      "2": 6466,
+      "3": 2626,
+      "4": 395,
+      "5": 103,
+      "6": 29
+    },
+    "max": 6,
+    "share_gt_1": 0.1530420670782155,
+    "share_gt_4": 0.0021001718322408197,
+    "stages": 62852
+  },
+  "slo": {
+    "definition": {
+      "target_pass_rate": 0.95,
+      "tpot_ms": 150,
+      "ttft_ms": "1000 + 1000 * input_tokens / 8000"
+    },
+    "feasibility_flip": false,
+    "real_per_trial": [
+      {
+        "feasible": false,
+        "pass_rate": 0.7735849056603774,
+        "passed": 41
+      }
+    ],
+    "sim": {
+      "feasible": false,
+      "pass_rate": 0.8679245283018868,
+      "passed": 46
+    }
+  },
+  "topology": "tp2_mns16",
+  "tpot_by_min_output_tokens": {
+    "2": {
+      "real": {
+        "count": 53,
+        "max": 641.5267530052612,
+        "mean": 27.41422879865018,
+        "p50": 8.106854443332919,
+        "p90": 23.880988038291846,
+        "p95": 92.03784805296183,
+        "p99": 397.1900480028069
+      },
+      "relative_bias_percent": {
+        "mean": -9.878641617364694,
+        "p50": -2.3180300975258694,
+        "p90": 26.54184469239794,
+        "p95": -24.72214442719661,
+        "p99": -13.722225303935645
+      },
+      "sim": {
+        "count": 53,
+        "max": 556.599034356792,
+        "mean": 24.706075383467144,
+        "p50": 7.918935117373849,
+        "p90": 30.219442794425397,
+        "p95": 69.28411832962485,
+        "p99": 342.6867347310516
+      }
+    },
+    "32": {
+      "real": {
+        "count": 45,
+        "max": 171.6484741543905,
+        "mean": 16.593114914019644,
+        "p50": 8.106854443332919,
+        "p90": 19.44677900381304,
+        "p95": 66.47742398097495,
+        "p99": 148.17240555253153
+      },
+      "relative_bias_percent": {
+        "mean": -9.940572863733157,
+        "p50": -2.3180300975258694,
+        "p90": 10.853794302071227,
+        "p95": -22.75922601837412,
+        "p99": -18.755468241526987
+      },
+      "sim": {
+        "count": 45,
+        "max": 145.22922738421678,
+        "mean": 14.943664235628548,
+        "p50": 7.918935117373849,
+        "p90": 21.557492395265285,
+        "p95": 51.34767680595202,
+        "p99": 120.38197708641991
+      }
+    },
+    "8": {
+      "real": {
+        "count": 52,
+        "max": 171.6484741543905,
+        "mean": 15.604372563907663,
+        "p50": 8.031664372446695,
+        "p90": 21.733123898935485,
+        "p95": 52.378594044159485,
+        "p99": 144.4375764567812
+      },
+      "relative_bias_percent": {
+        "mean": -7.222385426340451,
+        "p50": -1.403560331275464,
+        "p90": 10.95444148849469,
+        "p95": -10.968821104857895,
+        "p99": -19.391471177881424
+      },
+      "sim": {
+        "count": 52,
+        "max": 145.22922738421678,
+        "mean": 14.477364633980129,
+        "p50": 7.918935117373849,
+        "p90": 24.11386624006643,
+        "p95": 46.63327976621588,
+        "p99": 116.42900544813402
+      }
+    }
+  },
+  "trials": 1
+}

runs/frontier-code-trace-v0/results/canary-analysis-tp4-v2.json

@@ -0,0 +1,484 @@
+{
+  "drain": {
+    "interpretation": "Report the max-completion request explicitly; a response that arrived well before the cutoff can create a long drain tail without implying queue accumulation.",
+    "real_per_trial": [
+      {
+        "last_arrival_s": 597.0069999999832,
+        "last_completion_s": 689.6167901020963,
+        "tail_after_last_arrival_s": 92.60979010211304,
+        "tail_driver": {
+          "admission_lag_ms": 11.249033967033029,
+          "arrival_before_cutoff_s": 62.03099999995902,
+          "arrival_s": 534.9760000000242,
+          "e2e_ms": 154629.54106810503,
+          "input_tokens": 61976,
+          "output_tokens": 21361,
+          "request_index": 40
+        }
+      },
+      {
+        "last_arrival_s": 597.0069999999832,
+        "last_completion_s": 689.344073610846,
+        "tail_after_last_arrival_s": 92.33707361086272,
+        "tail_driver": {
+          "admission_lag_ms": 12.91605201549828,
+          "arrival_before_cutoff_s": 62.03099999995902,
+          "arrival_s": 534.9760000000242,
+          "e2e_ms": 154355.15755880624,
+          "input_tokens": 61976,
+          "output_tokens": 21361,
+          "request_index": 40
+        }
+      }
+    ],
+    "sim": {
+      "last_arrival_s": 597.007,
+      "last_completion_s": 688.377327361797,
+      "tail_after_last_arrival_s": 91.37032736179708,
+      "tail_driver": {
+        "arrival_before_cutoff_s": 62.03099999999995,
+        "arrival_s": 534.976,
+        "e2e_ms": 153401.32736179704,
+        "input_tokens": 61976,
+        "output_tokens": 21361,
+        "request_index": 40,
+        "waiting_ms": 0.0
+      }
+    }
+  },
+  "input": {
+    "frontier_csv_sha256": "624bc983d3c9fba58cbfa3bb5126304b99b19883da4c78f63cd8622b856dbc1f",
+    "manifest": "runs/frontier-code-trace-v0/results/canary-input-r0p0004-v1/manifest.json",
+    "paired_row_vector_sha256": "72b09903851f5684838da23498a068efc409e6481d1e84efeeefddd8e431c3e7"
+  },
+  "latency_ms": {
+    "e2e": {
+      "real": {
+        "count": 106,
+        "max": 154629.54106810503,
+        "mean": 10734.692601765997,
+        "p50": 1698.6070264829323,
+        "p90": 16955.372447962873,
+        "p95": 40098.011117079295,
+        "p99": 152916.82229990847
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 154629.54106810503,
+          "mean": 10752.865282098219,
+          "p50": 1699.7638561297208,
+          "p90": 16719.75081679412,
+          "p95": 32787.88763261397,
+          "p99": 139528.17495073183
+        },
+        {
+          "count": 53,
+          "max": 154355.15755880624,
+          "mean": 10716.519921433777,
+          "p50": 1697.8129579219967,
+          "p90": 16614.017988741405,
+          "p95": 32681.116299051762,
+          "p99": 139096.4567810855
+        }
+      ],
+      "real_trial_statistic_mean": {
+        "mean": 10734.692601765997,
+        "p50": 1698.7884070258588,
+        "p90": 16666.884402767762,
+        "p95": 32734.501965832867,
+        "p99": 139312.31586590866
+      },
+      "relative_bias_percent": {
+        "mean": 2.2014558826783013,
+        "p50": 6.0982380145415895,
+        "p90": 11.04630840126803,
+        "p95": 1.3549410756255433,
+        "p99": -1.3257901787372948
+      },
+      "sim": {
+        "count": 53,
+        "max": 153401.32736179704,
+        "mean": 10971.012123535007,
+        "p50": 1802.3845674497352,
+        "p90": 18507.95985478033,
+        "p95": 33178.03517886939,
+        "p99": 137465.32686438697
+      }
+    },
+    "tpot": {
+      "real": {
+        "count": 106,
+        "max": 212.7724236343056,
+        "mean": 10.903347021010111,
+        "p50": 6.172303046257078,
+        "p90": 8.493485802941574,
+        "p95": 9.056036081054664,
+        "p99": 203.35920991970733
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 212.7724236343056,
+          "mean": 10.931995250979348,
+          "p50": 6.17668606558597,
+          "p90": 8.407062460850472,
+          "p95": 8.930405185071265,
+          "p99": 137.66455429146993
+        },
+        {
+          "count": 53,
+          "max": 210.46578868602714,
+          "mean": 10.874698791040872,
+          "p50": 6.167920026928186,
+          "p90": 8.391951717689471,
+          "p95": 8.880314963829257,
+          "p99": 136.53404073389595
+        }
+      ],
+      "real_trial_statistic_mean": {
+        "mean": 10.90334702101011,
+        "p50": 6.172303046257078,
+        "p90": 8.39950708926997,
+        "p95": 8.905360074450261,
+        "p99": 137.09929751268294
+      },
+      "relative_bias_percent": {
+        "mean": -29.893478835305263,
+        "p50": 8.372972857395876,
+        "p90": -4.849833918394976,
+        "p95": 9.343177342624926,
+        "p99": -71.54426929141353
+      },
+      "sim": {
+        "count": 53,
+        "max": 68.73522813087318,
+        "mean": 7.643957286944565,
+        "p50": 6.6891083049964015,
+        "p90": 7.992144945476565,
+        "p95": 9.737403659205464,
+        "p99": 39.01260690357286
+      }
+    },
+    "ttft": {
+      "real": {
+        "count": 106,
+        "max": 12213.839336996898,
+        "mean": 1499.2377276352238,
+        "p50": 523.7399585312232,
+        "p90": 3843.0782224750146,
+        "p95": 7372.6646098075435,
+        "p99": 12137.850578338843
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 12213.839336996898,
+          "mean": 1501.9183055506492,
+          "p50": 524.1345730610192,
+          "p90": 3807.3078704066584,
+          "p95": 6591.913985600691,
+          "p99": 11615.677419919517
+        },
+        {
+          "count": 53,
+          "max": 12194.393874146044,
+          "mean": 1496.5571497197984,
+          "p50": 523.3453440014273,
+          "p90": 3795.468537742273,
+          "p95": 6574.768227571618,
+          "p99": 11596.839222898703
+        }
+      ],
+      "real_trial_statistic_mean": {
+        "mean": 1499.2377276352238,
+        "p50": 523.7399585312232,
+        "p90": 3801.3882040744656,
+        "p95": 6583.341106586155,
+        "p99": 11606.25832140911
+      },
+      "relative_bias_percent": {
+        "mean": -8.33138601964073,
+        "p50": -4.13255547614917,
+        "p90": -8.820129670118508,
+        "p95": -13.08726396978494,
+        "p99": -6.311340671626045
+      },
+      "sim": {
+        "count": 53,
+        "max": 11454.943556960472,
+        "mean": 1374.3304451938434,
+        "p50": 502.0961141941598,
+        "p90": 3466.1008352105086,
+        "p95": 5721.761877935864,
+        "p99": 10873.747819516035
+      }
+    }
+  },
+  "prefix_cache": {
+    "real_hit_ratio_mean": 0.2399084848137731,
+    "real_per_trial": [
+      {
+        "hit_ratio": 0.2399084848137731,
+        "hit_tokens": 309968.0,
+        "query_tokens": 1292026.0
+      },
+      {
+        "hit_ratio": 0.2399084848137731,
+        "hit_tokens": 309968.0,
+        "query_tokens": 1292026.0
+      }
+    ],
+    "sim": {
+      "block_size_tokens": 16,
+      "hit_ratio": 0.23997274866840085,
+      "mean_cached_prefill_tokens": 5848.452830188679,
+      "requests": 53,
+      "requests_with_hits": 9,
+      "total_cached_prefill_tokens": 309968,
+      "total_hit_blocks": 19373,
+      "total_query_blocks": 80730
+    }
+  },
+  "real_artifacts": [
+    {
+      "result_sha256": "c1105d6ba182be86fcd85f4eaf6889d1e788d48f923779b742ab95b4d3bd46ca",
+      "root": "runs/frontier-code-trace-v0/results/real-canary/r0p0004-tp4-t1-v2"
+    },
+    {
+      "result_sha256": "8e9092ac94b9d84d13aca41bd41d33db67d9917c8cedc8b61854aa2963a124ce",
+      "root": "runs/frontier-code-trace-v0/results/real-canary/r0p0004-tp4-t2-v3"
+    }
+  ],
+  "real_decode_batch_per_trial": [
+    {
+      "histogram": {
+        "1": 54699,
+        "2": 9377,
+        "3": 867,
+        "4": 77
+      },
+      "max": 4,
+      "mixed_steps_excluded": 155,
+      "share_gt_1": 0.15873577360812058,
+      "steps": 65020
+    },
+    {
+      "histogram": {
+        "1": 54893,
+        "2": 9265,
+        "3": 877,
+        "4": 77
+      },
+      "max": 4,
+      "mixed_steps_excluded": 155,
+      "share_gt_1": 0.15694495638284803,
+      "steps": 65112
+    }
+  ],
+  "requests_per_trial": 53,
+  "schema": "frontier-code-trace-canary-analysis-v1",
+  "sim_decode_batch": {
+    "histogram": {
+      "1": 54192,
+      "2": 9673,
+      "3": 840,
+      "4": 77
+    },
+    "max": 4,
+    "share_gt_1": 0.1634713346299898,
+    "share_gt_4": 0.0,
+    "stages": 64782
+  },
+  "slo": {
+    "definition": {
+      "target_pass_rate": 0.95,
+      "tpot_ms": 150,
+      "ttft_ms": "1000 + 1000 * input_tokens / 8000"
+    },
+    "feasibility_flip": true,
+    "real_per_trial": [
+      {
+        "feasible": false,
+        "pass_rate": 0.9433962264150944,
+        "passed": 50
+      },
+      {
+        "feasible": false,
+        "pass_rate": 0.9433962264150944,
+        "passed": 50
+      }
+    ],
+    "sim": {
+      "feasible": true,
+      "pass_rate": 0.9811320754716981,
+      "passed": 52
+    }
+  },
+  "topology": "tp4_mns16",
+  "tpot_by_min_output_tokens": {
+    "2": {
+      "real": {
+        "count": 106,
+        "max": 212.7724236343056,
+        "mean": 10.903347021010111,
+        "p50": 6.172303046257078,
+        "p90": 8.493485802941574,
+        "p95": 9.056036081054664,
+        "p99": 203.35920991970733
+      },
+      "real_per_trial": [
+        {
+          "count": 53,
+          "max": 212.7724236343056,
+          "mean": 10.931995250979348,
+          "p50": 6.17668606558597,
+          "p90": 8.407062460850472,
+          "p95": 8.930405185071265,
+          "p99": 137.66455429146993
+        },
+        {
+          "count": 53,
+          "max": 210.46578868602714,
+          "mean": 10.874698791040872,
+          "p50": 6.167920026928186,
+          "p90": 8.391951717689471,
+          "p95": 8.880314963829257,
+          "p99": 136.53404073389595
+        }
+      ],
+      "real_trial_statistic_mean": {
+        "mean": 10.90334702101011,
+        "p50": 6.172303046257078,
+        "p90": 8.39950708926997,
+        "p95": 8.905360074450261,
+        "p99": 137.09929751268294
+      },
+      "relative_bias_percent": {
+        "mean": -29.893478835305263,
+        "p50": 8.372972857395876,
+        "p90": -4.849833918394976,
+        "p95": 9.343177342624926,
+        "p99": -71.54426929141353
+      },
+      "sim": {
+        "count": 53,
+        "max": 68.73522813087318,
+        "mean": 7.643957286944565,
+        "p50": 6.6891083049964015,
+        "p90": 7.992144945476565,
+        "p95": 9.737403659205464,
+        "p99": 39.01260690357286
+      }
+    },
+    "32": {
+      "real": {
+        "count": 90,
+        "max": 68.33421335962247,
+        "mean": 7.270237487352443,
+        "p50": 6.167292870073156,
+        "p90": 8.490015433923487,
+        "p95": 8.695471711641535,
+        "p99": 68.2942852512835
+      },
+      "real_per_trial": [
+        {
+          "count": 45,
+          "max": 68.33421335962247,
+          "mean": 7.2794157345844095,
+          "p50": 6.17668606558597,
+          "p90": 8.316301157486755,
+          "p95": 8.667796532745909,
+          "p99": 42.34591449575732
+        },
+        {
+          "count": 45,
+          "max": 68.28935031654497,
+          "mean": 7.261059240120478,
+          "p50": 6.157899674560342,
+          "p90": 8.294755593709974,
+          "p95": 8.647785231561759,
+          "p99": 42.27831274721521
+        }
+      ],
+      "real_trial_statistic_mean": {
+        "mean": 7.270237487352444,
+        "p50": 6.167292870073156,
+        "p90": 8.305528375598364,
+        "p95": 8.657790882153833,
+        "p99": 42.31211362148626
+      },
+      "relative_bias_percent": {
+        "mean": 8.827198903080898,
+        "p50": 10.602910786790718,
+        "p90": 3.6523081450671953,
+        "p95": 13.443742144167812,
+        "p99": 3.009077380352242
+      },
+      "sim": {
+        "count": 45,
+        "max": 68.73522813087318,
+        "mean": 7.911995811087395,
+        "p50": 6.821205431047118,
+        "p90": 8.60887186495121,
+        "p95": 9.821721963731866,
+        "p99": 43.585317861619345
+      }
+    },
+    "8": {
+      "real": {
+        "count": 104,
+        "max": 68.33421335962247,
+        "mean": 7.043428576026337,
+        "p50": 6.162909850744264,
+        "p90": 8.031762239708375,
+        "p95": 8.660612540767117,
+        "p99": 66.51876673984641
+      },
+      "real_per_trial": [
+        {
+          "count": 52,
+          "max": 68.33421335962247,
+          "mean": 7.050448551299998,
+          "p50": 6.167180339419747,
+          "p90": 8.00607090381441,
+          "p95": 8.604306061843479,
+          "p99": 38.21141240377873
+        },
+        {
+          "count": 52,
+          "max": 68.28935031654497,
+          "mean": 7.036408600752676,
+          "p50": 6.162909850744264,
+          "p90": 7.971146045984491,
+          "p95": 8.582739223220587,
+          "p99": 38.14019313391272
+        }
+      ],
+      "real_trial_statistic_mean": {
+        "mean": 7.043428576026336,
+        "p50": 6.1650450950820055,
+        "p90": 7.988608474899451,
+        "p95": 8.593522642532033,
+        "p99": 38.17580276884573
+      },
+      "relative_bias_percent": {
+        "mean": 8.736391650572934,
+        "p50": 8.20225561474644,
+        "p90": 0.591743534737434,
+        "p95": 13.433615674969445,
+        "p99": 3.6892295703930555
+      },
+      "sim": {
+        "count": 52,
+        "max": 68.73522813087318,
+        "mean": 7.658770082056369,
+        "p50": 6.670717852545019,
+        "p90": 8.035880549065155,
+        "p95": 9.747943447271265,
+        "p99": 39.584195773328915
+      }
+    }
+  },
+  "trials": 2
+}

runs/frontier-code-trace-v0/results/prefill-only-calibration-summary.json

@@ -0,0 +1,539 @@
+{
+  "attention_profile_sha256": "fbcf7e1f95789a6f6d771e24d1fc60958b7daf04eb0db260d27869a19d71d550",
+  "cell_version": "v1",
+  "cells": [
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.0009942595174267852,
+      "drain_tail_s": 3.638989833782034,
+      "offered_load": {
+        "decode_tokens_per_s": 0.0633879781420765,
+        "prefill_tokens_per_s_after_prefix": 884.2945355191257,
+        "prefill_tokens_per_s_raw": 2459.7437158469947,
+        "requests_per_s": 0.0633879781420765
+      },
+      "prefix_cache_hit_ratio": 0.6406200282639031,
+      "requests": 232,
+      "rho": 0.0002,
+      "subcritical_gate": true,
+      "topology": "tp2_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "573cf4b21a3c39789b87eb92a34c8d8c95a8d0d79f769b05719884590f48c639",
+      "ttft_ms": {
+        "count": 232,
+        "max": 22394.384315265255,
+        "mean": 2016.4229573072757,
+        "min": 90.95485497891787,
+        "p50": 876.6052006358223,
+        "p90": 5597.657549404108,
+        "p95": 9955.05948414866,
+        "p99": 15612.433918519466
+      },
+      "waiting_ms": {
+        "count": 232,
+        "max": 13155.607757982125,
+        "mean": 336.5077998372315,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 0.0,
+        "p95": 1592.2631189191934,
+        "p99": 7965.408809250165
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.0005535579625697598,
+      "drain_tail_s": 2.0260221430053207,
+      "offered_load": {
+        "decode_tokens_per_s": 0.0633879781420765,
+        "prefill_tokens_per_s_after_prefix": 880.7972677595628,
+        "prefill_tokens_per_s_raw": 2459.7437158469947,
+        "requests_per_s": 0.0633879781420765
+      },
+      "prefix_cache_hit_ratio": 0.6420421114379927,
+      "requests": 232,
+      "rho": 0.0002,
+      "subcritical_gate": true,
+      "topology": "tp4_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "573cf4b21a3c39789b87eb92a34c8d8c95a8d0d79f769b05719884590f48c639",
+      "ttft_ms": {
+        "count": 232,
+        "max": 12080.695954866542,
+        "mean": 1097.5137410350026,
+        "min": 72.1560867923472,
+        "p50": 516.1309269643652,
+        "p90": 2564.188634116818,
+        "p95": 5585.788369509169,
+        "p99": 8804.50146872767
+      },
+      "waiting_ms": {
+        "count": 232,
+        "max": 7151.845512636555,
+        "mean": 99.8249313518456,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 0.0,
+        "p95": 295.97096758665066,
+        "p99": 2261.809117635907
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.0013382625998707982,
+      "drain_tail_s": 4.898041115527121,
+      "offered_load": {
+        "decode_tokens_per_s": 0.13633879781420766,
+        "prefill_tokens_per_s_after_prefix": 1794.7636612021859,
+        "prefill_tokens_per_s_raw": 5620.918852459016,
+        "requests_per_s": 0.13633879781420766
+      },
+      "prefix_cache_hit_ratio": 0.6808258903220888,
+      "requests": 499,
+      "rho": 0.0004,
+      "subcritical_gate": true,
+      "topology": "tp2_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "b27aec81dd9baa832176679b8e84333720fd70946882fbb079de7b912332feda",
+      "ttft_ms": {
+        "count": 499,
+        "max": 28160.869292443294,
+        "mean": 2470.467680714121,
+        "min": 90.95485497891787,
+        "p50": 877.1610873113787,
+        "p90": 7449.431996229117,
+        "p95": 12179.038589710988,
+        "p99": 18584.71742409069
+      },
+      "waiting_ms": {
+        "count": 499,
+        "max": 25191.932798227754,
+        "mean": 864.2429863694354,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 2060.9470903828583,
+        "p95": 5987.232655719128,
+        "p99": 13173.044246174226
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.0005723542337033552,
+      "drain_tail_s": 2.09481649535428,
+      "offered_load": {
+        "decode_tokens_per_s": 0.13633879781420766,
+        "prefill_tokens_per_s_after_prefix": 1734.9516393442623,
+        "prefill_tokens_per_s_raw": 5620.918852459016,
+        "requests_per_s": 0.13633879781420766
+      },
+      "prefix_cache_hit_ratio": 0.691468838921361,
+      "requests": 499,
+      "rho": 0.0004,
+      "subcritical_gate": true,
+      "topology": "tp4_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "b27aec81dd9baa832176679b8e84333720fd70946882fbb079de7b912332feda",
+      "ttft_ms": {
+        "count": 499,
+        "max": 14638.743702257216,
+        "mean": 1190.143380934895,
+        "min": 72.1560867923472,
+        "p50": 510.84479136716254,
+        "p90": 3181.3298830905296,
+        "p95": 5265.102348068058,
+        "p99": 10414.488385287405
+      },
+      "waiting_ms": {
+        "count": 499,
+        "max": 12819.0035203138,
+        "mean": 255.23631772331117,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 96.2998426639842,
+        "p95": 1728.7002592128508,
+        "p99": 5892.272316403113
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.00023484544575858607,
+      "drain_tail_s": 0.859534331476425,
+      "offered_load": {
+        "decode_tokens_per_s": 0.23743169398907105,
+        "prefill_tokens_per_s_after_prefix": 3120.9784153005467,
+        "prefill_tokens_per_s_raw": 7722.316120218579,
+        "requests_per_s": 0.23743169398907105
+      },
+      "prefix_cache_hit_ratio": 0.5959839894093193,
+      "requests": 869,
+      "rho": 0.0008,
+      "subcritical_gate": true,
+      "topology": "tp2_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "a440006d71f928053da5a88af7e00003169a1f95e05ee8328c5c3ca697ae0fe9",
+      "ttft_ms": {
+        "count": 869,
+        "max": 31384.57081632896,
+        "mean": 3262.922770577521,
+        "min": 80.48812343014333,
+        "p50": 869.0654534193527,
+        "p90": 10573.947661327758,
+        "p95": 15296.650926758037,
+        "p99": 21984.870012304014
+      },
+      "waiting_ms": {
+        "count": 869,
+        "max": 30294.53558294381,
+        "mean": 1770.386177703944,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 6890.822699274853,
+        "p95": 11429.513143558317,
+        "p99": 18054.967406463602
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.00014906636399725652,
+      "drain_tail_s": 0.5455828922299588,
+      "offered_load": {
+        "decode_tokens_per_s": 0.23743169398907105,
+        "prefill_tokens_per_s_after_prefix": 2943.2210382513663,
+        "prefill_tokens_per_s_raw": 7722.316120218579,
+        "requests_per_s": 0.23743169398907105
+      },
+      "prefix_cache_hit_ratio": 0.6190078484508921,
+      "requests": 869,
+      "rho": 0.0008,
+      "subcritical_gate": true,
+      "topology": "tp4_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "a440006d71f928053da5a88af7e00003169a1f95e05ee8328c5c3ca697ae0fe9",
+      "ttft_ms": {
+        "count": 869,
+        "max": 16057.04136311897,
+        "mean": 1275.40149118524,
+        "min": 67.46058986800563,
+        "p50": 428.09781420987747,
+        "p90": 3685.320778708873,
+        "p95": 6286.041945787507,
+        "p99": 10430.684729989898
+      },
+      "waiting_ms": {
+        "count": 869,
+        "max": 15053.141906136261,
+        "mean": 421.3322223628933,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 585.9887748068106,
+        "p95": 3090.558970424443,
+        "p99": 8471.886860828818
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.012967957147314541,
+      "drain_tail_s": 47.46272315917122,
+      "offered_load": {
+        "decode_tokens_per_s": 0.46939890710382515,
+        "prefill_tokens_per_s_after_prefix": 6538.567486338798,
+        "prefill_tokens_per_s_raw": 14510.624316939891,
+        "requests_per_s": 0.46939890710382515
+      },
+      "prefix_cache_hit_ratio": 0.5495261427568396,
+      "requests": 1718,
+      "rho": 0.0016,
+      "subcritical_gate": true,
+      "topology": "tp2_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "60080a3b70c01806782a0abd72c873536aae54fbd9c5ef70d1d5a2ae7a1338ea",
+      "ttft_ms": {
+        "count": 1718,
+        "max": 66578.48546737114,
+        "mean": 8771.952477826575,
+        "min": 80.48812343014333,
+        "p50": 3392.0416080366067,
+        "p90": 24475.218749701922,
+        "p95": 33824.30978066004,
+        "p99": 48636.304823714425
+      },
+      "waiting_ms": {
+        "count": 1718,
+        "max": 65625.6292665521,
+        "mean": 7210.059307774163,
+        "min": 0.0,
+        "p50": 1210.4389057114986,
+        "p90": 21454.74688922785,
+        "p95": 31447.91008182273,
+        "p99": 47140.04440652531
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.0024829272772107624,
+      "drain_tail_s": 9.087513834591391,
+      "offered_load": {
+        "decode_tokens_per_s": 0.46939890710382515,
+        "prefill_tokens_per_s_after_prefix": 6046.31393442623,
+        "prefill_tokens_per_s_raw": 14510.624316939891,
+        "requests_per_s": 0.46939890710382515
+      },
+      "prefix_cache_hit_ratio": 0.5834579374478305,
+      "requests": 1718,
+      "rho": 0.0016,
+      "subcritical_gate": true,
+      "topology": "tp4_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "60080a3b70c01806782a0abd72c873536aae54fbd9c5ef70d1d5a2ae7a1338ea",
+      "ttft_ms": {
+        "count": 1718,
+        "max": 22740.47242973984,
+        "mean": 1982.5181747731513,
+        "min": 67.46058986800563,
+        "p50": 563.4391554845024,
+        "p90": 6378.403293449832,
+        "p95": 8710.966056077563,
+        "p99": 15357.245352912887
+      },
+      "waiting_ms": {
+        "count": 1718,
+        "max": 18888.109661448652,
+        "mean": 1110.9868411863508,
+        "min": 0.0,
+        "p50": 0.0,
+        "p90": 4253.667468357389,
+        "p95": 6886.657874642189,
+        "p99": 13397.793338046433
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.37830325420435085,
+      "drain_tail_s": 1384.5899103879242,
+      "offered_load": {
+        "decode_tokens_per_s": 0.95,
+        "prefill_tokens_per_s_after_prefix": 14302.829234972678,
+        "prefill_tokens_per_s_raw": 31647.095901639343,
+        "requests_per_s": 0.95
+      },
+      "prefix_cache_hit_ratio": 0.548175911436349,
+      "requests": 3477,
+      "rho": 0.0032,
+      "subcritical_gate": false,
+      "topology": "tp2_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "8eeea3f44626e30e6da6ab58950c9a7f52e6cf6eb29c6bd2e35e1c25e4a9c5ce",
+      "ttft_ms": {
+        "count": 3477,
+        "max": 1396505.4377501393,
+        "mean": 713718.2923315652,
+        "min": 523.4377250571841,
+        "p50": 752636.5425455952,
+        "p90": 1219928.0000770227,
+        "p95": 1265657.4963084226,
+        "p99": 1364995.269473642
+      },
+      "waiting_ms": {
+        "count": 3477,
+        "max": 1395552.5815493201,
+        "mean": 711986.6468225399,
+        "min": 0.0,
+        "p50": 751006.927897161,
+        "p90": 1218640.0576069213,
+        "p95": 1264383.90901991,
+        "p99": 1357836.932620136
+      }
+    },
+    {
+      "decode_batch": {
+        "histogram": {},
+        "max": 0,
+        "share_gt_1": 0.0,
+        "share_gt_4": 0.0,
+        "stages": 0
+      },
+      "drain_fraction": 0.0024829272772107624,
+      "drain_tail_s": 9.087513834591391,
+      "offered_load": {
+        "decode_tokens_per_s": 0.95,
+        "prefill_tokens_per_s_after_prefix": 12622.671857923497,
+        "prefill_tokens_per_s_raw": 31647.095901639343,
+        "requests_per_s": 0.95
+      },
+      "prefix_cache_hit_ratio": 0.6012782892549811,
+      "requests": 3477,
+      "rho": 0.0032,
+      "subcritical_gate": true,
+      "topology": "tp4_mns16",
+      "tpot_ms": {
+        "count": 0,
+        "max": null,
+        "mean": null,
+        "min": null,
+        "p50": null,
+        "p90": null,
+        "p95": null,
+        "p99": null
+      },
+      "trace_sha256": "8eeea3f44626e30e6da6ab58950c9a7f52e6cf6eb29c6bd2e35e1c25e4a9c5ce",
+      "ttft_ms": {
+        "count": 3477,
+        "max": 43733.59998241585,
+        "mean": 6397.374766665999,
+        "min": 67.46058986814774,
+        "p50": 3276.4329854612697,
+        "p90": 17833.72132484574,
+        "p95": 23705.880006424282,
+        "p99": 36214.16656225671
+      },
+      "waiting_ms": {
+        "count": 3477,
+        "max": 40748.976762736216,
+        "mean": 5494.564635817035,
+        "min": 0.0,
+        "p50": 1963.6734690996036,
+        "p90": 16680.754893288453,
+        "p95": 22203.49896551297,
+        "p99": 34692.440763376595
+      }
+    }
+  ],
+  "schema": "frontier-code-trace-calibration-summary-v1",
+  "subcritical_rule": "drain tail <= 10% of the 3660s arrival window",
+  "workload_mode": "prefill_only"
+}

runs/frontier-code-trace-v0/summarize_calibration.py

@@ -10,7 +10,9 @@ import re
 from pathlib import Path
 
 
-CELL_RE = re.compile(r"r(?P<rho>[0-9p]+)-tp(?P<tp>[24])-v3")
+CELL_RE = re.compile(
+    r"r(?P<rho>[0-9p]+)-tp(?P<tp>[24])-(?P<version>v[0-9]+)"
+)
 ATTENTION_FLAG = "--random_forrest_execution_time_predictor_config_atten_input_file"
 
 
@@ -18,6 +20,12 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
     parser.add_argument("--sim-root", type=Path, required=True)
     parser.add_argument("--profile-manifest", type=Path, required=True)
+    parser.add_argument(
+        "--workload-mode",
+        choices=("prefill_decode", "prefill_only"),
+        default="prefill_decode",
+    )
+    parser.add_argument("--cell-version", default="v3")
     parser.add_argument("--output", type=Path, required=True)
     return parser.parse_args()
 
@@ -37,7 +45,11 @@ def main() -> None:
     cells = []
     for root in sorted(args.sim_root.iterdir()):
         match = CELL_RE.fullmatch(root.name)
-        if match is None or not (root / "summary.json").is_file():
+        if (
+            match is None
+            or match.group("version") != args.cell_version
+            or not (root / "summary.json").is_file()
+        ):
             continue
         manifest = json.loads((root / "manifest.json").read_text())
         observed_profile_sha = manifest.get("attention_profile_sha256")
@@ -75,6 +87,8 @@ def main() -> None:
         raise ValueError(f"no completed v3 calibration cells below {args.sim_root}")
     payload = {
         "schema": "frontier-code-trace-calibration-summary-v1",
+        "workload_mode": args.workload_mode,
+        "cell_version": args.cell_version,
         "attention_profile_sha256": expected_profile_sha,
         "subcritical_rule": "drain tail <= 10% of the 3660s arrival window",
         "cells": cells,