Audit Frontier prefix-cache trace contract

runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py

@@ -21,6 +21,11 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--port", type=int, required=True)
     parser.add_argument("--requests-file", type=Path, required=True)
     parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument(
+        "--served-model",
+        default=None,
+        help="Optional server-side model alias used only for HTTP routing.",
+    )
     parser.add_argument("--tpot-slo-ms", type=float, default=150.0)
     parser.add_argument("--timeout-seconds", type=float, default=1800.0)
     return parser.parse_args()
@@ -64,6 +69,7 @@ async def request_one(
     scheduled_at: float,
     benchmark_start: float,
     tpot_slo_ms: float,
+    served_model: str | None,
 ) -> dict[str, Any]:
     loop = asyncio.get_running_loop()
     delay = scheduled_at - loop.time()
@@ -81,6 +87,8 @@ async def request_one(
         "success": False,
     }
     body = dict(row["body"])
+    if served_model is not None:
+        body["model"] = served_model
     body.update(
         {
             "temperature": 0,
@@ -178,6 +186,7 @@ async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[d
                     scheduled_at=benchmark_start + float(row["arrived_at"]),
                     benchmark_start=benchmark_start,
                     tpot_slo_ms=args.tpot_slo_ms,
+                    served_model=args.served_model,
                 )
             )
             for row in rows
@@ -219,6 +228,7 @@ def main() -> None:
             "last_arrival_s": arrivals[-1],
             "arrival": "original_trace_timestamp_and_order",
             "input_output_prompt": "exact_source_values",
+            "served_model_alias": args.served_model,
             "ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
             "tpot_slo_ms": args.tpot_slo_ms,
             "target_pass_rate": TARGET_PASS_RATE,

runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py

@@ -84,8 +84,21 @@ def percentile(values: list[float], fraction: float) -> float | None:
     return ordered[math.ceil(fraction * len(ordered)) - 1]
 
 
+def manifest_offered_rate(path: Path) -> tuple[float | None, str | None]:
+    manifest_path = path.with_name("manifest.json")
+    if not manifest_path.is_file():
+        return None, None
+    manifest = json.loads(manifest_path.read_text())
+    if manifest.get("public_csv") != str(path):
+        return None, None
+    rate = manifest.get("global_offered_request_rate")
+    if not isinstance(rate, (int, float)) or not math.isfinite(rate) or rate <= 0:
+        raise ValueError(f"invalid global_offered_request_rate in {manifest_path}")
+    return float(rate), str(manifest_path)
+
+
 def parse_trace(
-    specification: str, *, rate_contract: str = "trace-window"
+    specification: str, *, rate_contract: str = "trace-window", prefix_caching: bool
 ) -> dict[str, Any]:
     if "=" not in specification:
         raise ValueError(f"trace must be LABEL=PATH: {specification}")
@@ -114,7 +127,12 @@ def parse_trace(
     if any(right < left for left, right in zip(arrivals, arrivals[1:])):
         raise ValueError(f"arrival order drift in {path}")
     if rate_contract == "trace-window":
-        offered_request_rate = len(rows) / WINDOW_SECONDS
+        offered_request_rate, rate_manifest = manifest_offered_rate(path)
+        if offered_request_rate is None:
+            offered_request_rate = len(rows) / WINDOW_SECONDS
+            rate_source = "legacy_fixed_600_second_window"
+        else:
+            rate_source = f"manifest:{rate_manifest}"
     elif rate_contract == "uniform-spacing":
         if len(rows) < 2 or arrivals[-1] <= arrivals[0]:
             raise ValueError("uniform-spacing traces require at least two arrivals")
@@ -123,6 +141,7 @@ def parse_trace(
         if any(abs(value - expected_interval) > 1e-9 for value in intervals):
             raise ValueError(f"non-uniform fixed trace: {path}")
         offered_request_rate = 1.0 / expected_interval
+        rate_source = "uniform_spacing"
     else:
         raise ValueError(f"unknown rate contract: {rate_contract}")
     shapes = [
@@ -131,12 +150,22 @@ def parse_trace(
     ]
     if any(prefill <= 0 or decode <= 0 or prefill + decode > 40960 for prefill, decode in shapes):
         raise ValueError(f"out-of-contract shape in {path}")
+    if prefix_caching:
+        for index, (row, (prefill, _)) in enumerate(zip(rows, shapes, strict=True)):
+            block_ids = [value for value in row["block_hash_ids"].split("|") if value]
+            if len(block_ids) != prefill // 16:
+                raise ValueError(
+                    "Frontier prefix-cache trace must expose only complete "
+                    f"16-token blocks: row={index}, ISL={prefill}, "
+                    f"ids={len(block_ids)}, expected={prefill // 16}"
+                )
     return {
         "label": label,
         "path": path,
         "sha256": BASE.sha256(path),
         "requests": len(rows),
         "offered_request_rate": offered_request_rate,
+        "offered_request_rate_source": rate_source,
         "first_arrival_s": arrivals[0],
         "last_arrival_s": arrivals[-1],
         "shapes": shapes,
@@ -236,7 +265,11 @@ def main() -> None:
     if args.allreduce_csv is not None:
         args.allreduce_csv = args.allreduce_csv.resolve()
     traces = [
-        parse_trace(specification, rate_contract=args.rate_contract)
+        parse_trace(
+            specification,
+            rate_contract=args.rate_contract,
+            prefix_caching=args.prefix_caching,
+        )
         for specification in args.trace
     ]
     if len({trace["label"] for trace in traces}) != len(traces):

runs/simulator-tuning-latency-matrix-v0/experiment-card.md

@@ -0,0 +1,228 @@
+# EXP-SIM-TUNING-LATENCY-MATRIX-V0：现有 simulator 能否选择真机低延迟 serving config？
+
+> **状态：** protocol 已冻结；Qwen3.6 release-runtime topology smoke 已通过（2026-07-17）。
+> 没有 Frontier Qwen3.6 contract/profile、trace replay 或 latency-selection 结果；本 card
+> 不授权 600-s matrix launch。
+
+## Claim 与决策
+
+- **研究问题：** 对冻结的 `(H20 hardware, model, serving engine, request trace, config
+  candidate surface)`，现有 simulator 能否为一个明确的 latency objective 选出真机最优、或
+  统计上等价的 serving-engine config？
+- **Parent claim：** “simulator 已经解决 config tuning”只有在完整 candidate surface 上产生
+  正确的真机 latency selection 时才成立；绝对 latency error 或单个吞吐/SLO 结果均不充分。
+- **Hypothesis：** simulator 在 fixed-shape / prefill-only 条件下可能足够，但在 decode、真实
+  arrival 与 prefix/KV state 同时存在时会因 state-conditioned residual 或 coverage failure 而
+  失去 selection fidelity。
+- **Competing hypothesis：** 经冻结、非 evaluation 的 operator/collective calibration 后，
+  simulator 能跨 non-MoE hybrid/MoE 与四种 workload condition 稳定选出真机低延迟 config。
+- **判定：** 每个 `model × workload × metric` 是独立 tuning task，且 config 越低 latency 越好。
+  只有所有预注册且 simulator 声称支持的 task 均通过下面的 coverage 与 selection gates，才能
+  写“该 simulator 已解决其声明范围内的 tuning”；任一失败 cell 是明确 boundary，不被平均值
+  掩盖。
+
+## 固定系统与模型覆盖
+
+| 角色 | 状态 | 运行时契约 | 当前可复用 artifact / 含义 |
+|---|---|---|---|
+| MoE | **可开始准备** | Qwen3-30B-A3B，BF16，community vLLM 0.20.0，dash0 H20；Frontier/Vidur-class fork | `frontier-phase-factorial-v0` 有 `ISL=2048, OSL=1, r=4` 的 request-level real/sim pilot；`frontier-fidelity-envelope-v1` 有 fixed-PD 与 exact Trace-PD materializers。历史 pilot 不是本 card 的最终 600-s evaluation。 |
+| Non-MoE hybrid (not all-attention dense) | **TP1/2/4 smoke passed; profile pending** | `Qwen/Qwen3.6-27B@cea40373b9214dd387123e68841890af30dcd469`，community release `vLLM 0.20.2` / Torch `2.11.0` / Transformers `5.14.1`，dash0 H20；Frontier profile 必须在此 exact stack 采集 | Exact release serves text-only 2048→1 at TP1/2/4. vLLM resolves a 784-token attention page for its GDN/Mamba state, so requested block=16 is not the physical engine state. Qwen3.6 trace, Frontier hybrid model contract, and profile remain missing. |
+
+所有公开结果固定在单机 `dash0`（8×H20）；每个 model 使用相同 engine build 的 real
+server 与 simulator contract。Qwen3.6 bring-up 已确认真实 runtime/page legality；其余最低成本
+gate 是冻结 trace/tokenizer 与完成 Frontier hybrid profile-schema audit。若不能做到，论文结论
+必须限于 MoE，不得写跨模型 generalization。
+
+### Qwen3.6 prerequisite audit (runtime + smoke, 2026-07-17)
+
+| Prerequisite | Evidence | Verdict | Decision |
+|---|---|---|---|
+| Exact model snapshot | Immutable ModelScope snapshot `Qwen/Qwen3.6-27B@cea40373b9214dd387123e68841890af30dcd469` is staged at `/home/admin/cpfs/wjh/models/Qwen/Qwen3.6-27B`; its sorted-file inventory digest is `031cea103726cbf27732b6971e898e210aee77f0c32e4bc703c947e0372349c4`. | **PASS** | Bind every Qwen3.6 trace, real server, and profile manifest to this revision and inventory. |
+| Community vLLM runtime | Clean environment `/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2` contains official wheel `vLLM 0.20.2`, Torch `2.11.0`, and Transformers `5.14.1`; `pip check` passes and CPU-only `get_config(..., trust_remote_code=false)` resolves `Qwen3_5ForConditionalGeneration`. | **PASS (CPU)** | Use exactly this release environment for real serving and profile collection. |
+| Topology compatibility | Fresh TP1/2/4 servers each completed a text-only 2048-prompt/1-completion request with exact usage; all GPUs were released after cleanup. Every topology resolved the attention page from requested 16 to 784 tokens for Mamba/GDN state. | **PASS (GPU smoke)** | This is an engine-configuration legality result, not a latency rank or simulator result. Profile/simulator input must expose the 784-token state/page contract. |
+| Qwen3.6 Frontier model/profile | No local or remote Qwen3.6-named Frontier profile/model directory was found | **MISSING** | New dense profile root is required; Qwen30/MoE profile CSVs are forbidden inputs. |
+| Old dense artifacts | Existing Qwen3.5-27B trace is internal-vLLM/hybrid/speculative | **N/A as baseline** | Keep only as historical context; never feed it to this matrix or profile calibration. |
+
+This audit is not a failed simulator result. The topology gate is closed, but the model contract,
+profile root, and frozen trace are still reproducibility blockers: the Qwen3.6 half is `NOT RUN`,
+not `FAIL`. For a paper claim about a conventional all-attention dense transformer, Qwen3.6 is not
+admissible evidence; it is only a non-MoE hybrid case.
+
+## Workload matrix
+
+每个 model 都运行四个固定 trace。`P` 是 PD disaggregation 的 P-node workload，不是缺失
+decode 的坏数据；它的 `OSL=1` 是设计变量。固定 cases 与 trace cases都使用 open-loop replay，
+600 秒 observation window；`Fixed` 的 request count 为 `ceil(QPS × 600)`。
+
+| ID | trace contract | output contract | prefix/KV contract | 可评 latency objectives |
+|---|---|---|---|---|
+| Fixed-P | 固定 `ISL=2048`、uniform fixed QPS=`4 req/s`，token vector deterministic | `OSL=1` | no shared logical prefix；prefix cache off | mean/p90 TTFT；mean/p90 E2E；TPOT=`N/A` |
+| Fixed-PD | 固定 `ISL=2048`、uniform fixed QPS=`4 req/s` | `OSL=128`，`min=max=128`，`ignore_eos=true` | no shared logical prefix；prefix cache off | mean/p90 TTFT、TPOT、E2E |
+| Trace-P | held-out production trace 的逐 request prompt-token vector、arrival time、request order、session root 与 prefix block relation 原样保留 | **只将 replay output 强制为 `OSL=1`** | source prefix equivalence、warm/cold state、cache policy 原样保留 | mean/p90 TTFT；mean/p90 E2E；TPOT=`N/A` |
+| Trace-PD | held-out production trace 的逐 request input/output lengths、arrival time、order、session root 与 prefix block relation 原样保留 | 原始 `min=max=source OSL`，`ignore_eos=true` | source prefix equivalence、warm/cold state、cache policy 原样保留 | mean/p90 TTFT、TPOT、E2E |
+
+`Trace-P` 应明确称为 **trace-derived prefill-only**：它忠实保留输入、到达、会话与 cache
+state，但不忠实保留原始 output-length distribution；不能被描述成完整 trace-faithful replay。
+`Trace-PD` 才是完整 input/output/arrival/prefix-KV joint replay。
+
+### Trace、correctness 与 cache 的不可变契约
+
+- 每个 trace manifest 列出：request ID、arrival timestamp、input/output token counts、session
+  ID、runtime block identities、trace vector SHA-256、source trace SHA-256、requested and
+  resolved physical page/block size、cache policy、initial cache state、warmup trace digest 与
+  model tokenizer/version。
+- 当 simulator 要模拟 vLLM prefix caching 时，`block_hash_ids` 只能描述可复用的**完整**
+  runtime blocks：adapter 必须导出 `floor(ISL / block_size)` 个 prompt identities，不能将
+  `ceil` 得到的 partial final block 当作 cache hit。该 simulator-facing projection 有独立
+  digest；private real replay 仍保留完整 prompt 与原始 token vector。
+- `Trace-P` 产生单独 manifest：source OSL 和 replay OSL 都写入 digest；只允许 source
+  OSL → 1 这一个变换。`Trace-PD`、real client 与 simulator CSV 的 row-vector digest 必须完全
+  相同。
+- 真实端逐 request 核验 prompt/completion usage 与 manifest 一致；simulator 核验 request ID、
+  input/output shapes、完成数与 trace digest。一项不符即该 cell 的 metric 无效。
+- 每个 `(config, workload, trial)` fresh server；server warmup 用独立 manifest，warmup request
+  不进入指标，也不得改变 evaluation cache 初态。prefix-on trace 以同一明示 initial cache
+  state重放，不能用随机前史替代。
+
+## Candidate-surface 与 baseline 规则
+
+1. 对每个 model 冻结 `C = TP∈{1,2,4} × MNS∈{8,16,32,64}`（12 个 configs），并固定
+   `DP=PP=EP=1`、MBT=8192、chunked prefill on。每个 candidate 还必须冻结 **真实 engine
+   resolved** page/block and state-cache contract；requested CLI `--block-size 16` 不能替代它。
+   Qwen3.6 smoke 在所有 TP 都将其改写为 784 tokens。任何不被某 model/engine 静态支持的
+   config 要在运行前从该 model 的 `C` 整体删除，并记录理由；不能看结果后删除。
+2. `C_real-valid` 是 server launch 成功、所有三个 fresh trials 全部请求完成且 usage/hash 合法的
+   configs。真机 OOM、timeout 或 request error 保留为 real infeasible/safety outcome，而非给它
+   赋无限 latency。
+3. 对每个 simulator，selection coverage 必须涵盖全部 `C_real-valid`。crash、scheduler stall、
+   trace mismatch、缺 request metrics 都是 simulator **coverage failure**，不可当成高 latency 或
+   infeasible 来产生排序。只有完整 coverage 才能形成 tuning verdict。
+4. Frontier、官方 Vidur 与 APEX 分别独立报告版本、patch、profile 与 capability surface。APEX
+   若只支持 DP/PP/TP parallel plan，则它只评 `parallel-plan tuning` 交集；不得用该子问题结果
+   宣称解决 MNS/MBT/cache 等完整 engine tuning。
+
+## Metrics、统计与 selection score
+
+每个完整 request 使用同一 client-boundary 定义：
+
+- `TTFT`：客户端发出请求至收到第一个 generated token；包含 serving queue。
+- `E2E`：客户端发出请求至完成 `[DONE]`。
+- `TPOT`：`(E2E - TTFT) / (completion_tokens - 1)`；只对 `completion_tokens > 1` 的请求定义。
+- 每 trial 的 `mean` 是 request-level arithmetic mean；`p90` 是 nearest-rank
+  `ceil(0.90 × N)`。P-only 的 TPOT 是 JSON `null` / table `N/A`，绝不写 0。
+
+同一 config 的 real 运行三次 fresh-server trial，随机化 config order；每个 trial 都对整个
+trace 计算六项或四项指标。real ranking point estimate 是三个 trial metric 的 median，且报告
+pooled request statistic、每-trial statistic 和 95% hierarchical bootstrap interval（先重采样
+trial，再在 trial 内重采样 request）。simulator 的 seed、determinism 和 repeat policy同样记录。
+
+对每一个 objective，输出：absolute real/sim surface、real bootstrap top set、simulator exact
+top set、top-set hit、sim-chosen config 的 best/worst tie-break regret、Kendall τ-b、non-tied
+pair direction accuracy、coverage 与 error/infeasible table。Latency regret 定义为
+`(real_metric(sim_choice) - real_metric(real_best)) / real_metric(real_best)`。
+
+## 预注册 decision gates
+
+| Gate | Pass | Failure handling |
+|---|---|---|
+| Trace/metric validity | 100% planned requests have matching usage/shape and finite applicable metrics | cell invalid；诊断 harness，不作 performance conclusion |
+| Simulator coverage | 100% of `C_real-valid` output legal request metrics | `FAIL: coverage`；不把 missing cell 排成最差 |
+| Selection | simulator top set 与 bootstrap real top set 有交集 | `FAIL: selection` |
+| Risk | simulator top-set 的 **worst** real tie-break regret `≤5%` | `FAIL: regret`，即使 best tie-break 碰巧为 0 |
+| Global rank diagnostic | τ-b `≥0.8`；若没有非并列 pair 则写 `N/A`，不当通过 | `NEEDS EVIDENCE` / `FAIL` 与 top-set gate 分开报告 |
+
+这里不使用 SLO、capacity 或 throughput 作 selection objective；它们最多作为安全/诊断附录指标。
+P-only 共 4 tasks，PD 共 6 tasks，所以完整 `non-MoE-hybrid/MoE × 4 workloads` matrix 共有 40 个
+独立 latency-selection tasks。
+
+## Calibration 与 information boundary
+
+- 允许 simulator 使用同硬件/同 model/同 runtime 的 offline operator、collective 或 memory
+  profiles，但 profile command、shape grid、cost 和 hash 必须在 evaluation 前冻结。
+- calibration 不得读取 evaluation 的 real request latency、top config 或 E2E scalar；禁止 per-TP
+  或 per-config serving E2E scale。若有 trace-driven calibration，必须只使用按 session root 切分
+  的 calibration fold；Trace-P/PD 使用完全不相交的 held-out root sessions。
+- Fixed traces 与 trace evaluation 都不能同时作为 calibration workload。任何 required simulator
+  patch 在查看该 cell real result 前冻结；patched baseline 与 original release 分开作图。
+
+## Minimal Qwen3.6-27B bring-up and profile plan
+
+This is the only permitted path to add the dense case. It intentionally separates cheap provenance
+work from GPU work, and it does not authorize either stage by itself.
+
+1. **Preflight and topology smoke (complete):** the snapshot/runtime are locked; fresh TP1/2/4
+   servers completed exact-usage text requests. The actual vLLM state/page setting is 784 tokens,
+   not the requested 16. This establishes the profile input contract only.
+2. **Hybrid Frontier contract (not run):** verify Frontier can represent Gated DeltaNet/linear-state,
+   gated attention, FFN, true mixed execution, TP collectives, the 784-token page rule and capacity,
+   without substituting Qwen3.5/Qwen30 fields.
+3. **Minimal profile set (GPU, separately approved):** on the exact Qwen3.6/community-vLLM/dash0
+   stack, collect per-TP Gated-DeltaNet, gated-attention prefill/decode, state/KV and TP2/TP4 all-reduce
+   rows. The shape grid must cover the pre-registered Fixed-P/PD and held-out Trace-P/PD ranges, but
+   must not contain evaluation E2E latency or a fitted time scale. Freeze CSV/schema/command hashes
+   before running any evaluation trace.
+4. **Profile acceptance (CPU):** verify model/runtime identity in every profile manifest; require
+   requested shape coverage, finite values, and no fallback to a MoE or Qwen3.5 profile. Only then may
+   a parent-approved real/simulator latency surface be scheduled.
+
+## 预期产物、复现与成本
+
+- **Figure prototype：** [latency-selection-matrix-schematic.svg](latency-selection-matrix-schematic.svg)
+  （明确标注 `SCHEMATIC — NOT MEASURED DATA`）。最终图替换为每个 task 的 top-set hit / regret /
+  coverage heatmap，并附 per-config latency surfaces。
+- **Raw layout：** `artifacts/<model>/<workload>/<real|simulator>/<config>/trial-*/`；每个目录含
+  command、environment/machine fingerprint、server flags、trace/profile hash、raw request records、
+  stdout/stderr、duration 与 SHA-256 inventory。
+- **Existing reuse audit：** MoE F0 可检验 P-only raw metric reader；现有 F2 可检验 fixed-PD
+  trace materialization；T1 可检验 Trace-PD digest and simulator coverage reporting。它们并不构成
+这个 600-s no-SLO matrix 的完整 real surface。Qwen3.6 requires a new same-stack trace,
+  model contract and profile root; old Qwen3.5 internal-vLLM artifacts are excluded.
+- **Cost estimate, not authorization：** 单 config×trial 假定 `10 min replay + 5 min fresh-server
+  startup/warmup = 15 min`。完整 12-config sweep 的 GPU allocation 为
+  `4 × (TP1 + TP2 + TP4) = 28 GPU`; 因而每个 `model×workload` 的三 trial 约
+  `28 × 0.25 × 3 = 21 H20-GPUh`。8 cells 的 real serving core is **168 H20-GPUh**. The Qwen3.6
+  bring-up/profile set adds at most **6 H20-GPUh** (three topology smokes plus dense/attention/
+  collective rows), giving a **174 H20-GPUh** core estimate and a **190–215 H20-GPUh** planning
+  envelope with retries. Every launch must recompute this from the staged trace duration and actual
+  server preflight; this paragraph is not authorization.
+
+### Required remote launch echo (not executed)
+
+```text
+NOT EXECUTED — awaiting explicit approval after Qwen3.6 model/runtime/profile gates
+host=dash0; remote_root=/home/admin/cpfs/wjh/aituner/aituner
+models=non-moe-hybrid:Qwen/Qwen3.6-27B@<FROZEN_SNAPSHOT>,moe:Qwen3-30B-A3B-BF16
+engine=community-open-source-vLLM@<FROZEN_SOURCE_OR_WHEEL>; simulator=Frontier/Vidur-class,<official baselines if installed>
+workloads=Fixed-P(ISL2048,OSL1,QPS4,600s),Fixed-PD(ISL2048,OSL128,QPS4,600s),
+          Trace-P(heldout exact input/arrival/prefix,OSL1),Trace-PD(heldout exact joint trace)
+surface=TP{1,2,4}×MNS{8,16,32,64}; trials=3; fresh_server=true
+real_paths=runs/simulator-tuning-latency-matrix-v0/artifacts/<model>/<workload>/real/
+expected_cost=174 H20-GPUh core; 190–215 H20-GPUh planning envelope; expected_wall=multiple queued hours
+gate=Qwen3.6 snapshot + community-vLLM fingerprint + resolved-784 hybrid Frontier profile completeness + trace/profile hashes + user/parent approval
+```
+
+## Expected outcomes and stop/pivot rules
+
+- If a simulator passes every supported task, stop diagnosing that simulator and look for a new problem
+  outside its evaluated scope (e.g., unsupported engine knobs or alignment cost).
+- If it fails coverage, repair/liveness is a separate intervention; original capability result remains
+  frozen. First rerun simulator only; do not spend GPU-hours until it can produce a complete surface.
+- If it covers but selects incorrectly, retain the counterexample and run one minimal discriminative
+  mechanism experiment (scheduler state, batch composition, graph/kernel family, or MoE/collective),
+  changing one factor at a time.
+- If Qwen3.6 cannot be profiled on its exact community-vLLM stack, stop the non-MoE-hybrid half rather
+  than treating any internal-vLLM or Qwen3.5 result as a substitute. If the headline requires a pure
+  all-attention dense-transformer case, add that model explicitly rather than relabelling Qwen3.6.
+
+## Benchmark-design audit
+
+| Risk | Pre-registered control |
+|---|---|
+| SLO/capacity result misrepresented as tuning | selection uses only per-request mean/p90 TTFT, TPOT, E2E on the same trace |
+| P-only judged by nonexistent TPOT | OSL=1 and TPOT=`N/A` are explicit workload/metric contracts |
+| Trace simplification hidden | Trace-P is labelled output-normalized; only Trace-PD claims full joint replay |
+| calibration equals evaluation | microprofile/trace calibration folds are disjoint; no E2E correction on evaluation |
+| simulator crash treated as a bad config | coverage gate separates missing metrics from latency ranking |
+| cross-stack or wrong-architecture reuse | only Qwen3.6-27B hybrid profiles collected on its exact community-vLLM stack are admissible; old Qwen3.5/internal-vLLM or Qwen30/MoE artifacts are excluded |
+| selected winning cells only | all 8 cells and all legal configs are frozen before real results |
+| tail/noise omitted | p90, three fresh trials, per-trial values and bootstrap top sets are mandatory |

runs/simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""Derive a per-GPU-normalized real/Frontier Qwen30 trace pair.
+
+Arrival times change as ``t' = t / TP``.  The private JSONL preserves the
+complete prompt identity vector for real replay; the Frontier CSV exports only
+the completed 16-token prompt blocks that vLLM can legally reuse from prefix
+cache. Prompt bodies remain exclusively in the private output.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import math
+from pathlib import Path
+from typing import Any
+
+
+CSV_FIELDS = (
+    "arrived_at",
+    "num_prefill_tokens",
+    "num_decode_tokens",
+    "session_id",
+    "block_hash_ids",
+)
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def update_digest(digest: Any, values: list[Any]) -> None:
+    digest.update(json.dumps(values, separators=(",", ":")).encode())
+    digest.update(b"\n")
+
+
+def full_prompt_block_ids(
+    runtime_block_ids: list[int], input_tokens: int, block_size: int
+) -> list[int]:
+    """Project exact prompt identities to vLLM-cacheable full blocks only."""
+    if input_tokens <= 0 or block_size <= 0:
+        raise ValueError("input_tokens and block_size must be positive")
+    expected_runtime_blocks = math.ceil(input_tokens / block_size)
+    if len(runtime_block_ids) != expected_runtime_blocks:
+        raise ValueError(
+            "runtime block identity count does not match the exact prompt: "
+            f"got {len(runtime_block_ids)}, expected {expected_runtime_blocks}"
+        )
+    return runtime_block_ids[: input_tokens // block_size]
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-public-csv", type=Path, required=True)
+    parser.add_argument("--base-private-jsonl", type=Path, required=True)
+    parser.add_argument("--tp", type=int, required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    parser.add_argument("--base-window-seconds", type=float, default=600.0)
+    parser.add_argument("--runtime-block-size", type=int, default=16)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    if (
+        args.tp <= 0
+        or args.runtime_block_size <= 0
+        or not math.isfinite(args.base_window_seconds)
+        or args.base_window_seconds <= 0
+    ):
+        raise ValueError("TP, runtime block size, and base window must be positive")
+
+    public_rows = list(csv.DictReader(args.base_public_csv.open(newline="")))
+    private_rows = [
+        json.loads(line) for line in args.base_private_jsonl.open() if line.strip()
+    ]
+    if not public_rows or len(public_rows) != len(private_rows):
+        raise ValueError("public/private request counts must match and be non-zero")
+    if set(public_rows[0]) != set(CSV_FIELDS):
+        raise ValueError("unexpected public trace schema")
+
+    normalized_digest = hashlib.sha256()
+    semantic_digest = hashlib.sha256()
+    normalized_private: list[dict[str, Any]] = []
+    normalized_public: list[dict[str, str]] = []
+    original_arrivals: list[float] = []
+    frontier_prefix_digest = hashlib.sha256()
+    partial_prompt_count = 0
+    raw_runtime_block_count = 0
+    frontier_full_block_count = 0
+
+    for index, (public, private) in enumerate(
+        zip(public_rows, private_rows, strict=True)
+    ):
+        original_arrival = float(public["arrived_at"])
+        if not math.isfinite(original_arrival) or original_arrival < 0:
+            raise ValueError(f"invalid arrival at row {index}")
+        if not math.isclose(original_arrival, float(private["arrived_at"]), abs_tol=1e-9):
+            raise ValueError(f"public/private arrival mismatch at row {index}")
+        runtime_block_ids = [int(value) for value in private["runtime_block_ids"]]
+        expected = (
+            int(public["num_prefill_tokens"]),
+            int(public["num_decode_tokens"]),
+            int(public["session_id"]),
+            public["block_hash_ids"],
+        )
+        actual = (
+            int(private["input_length"]),
+            int(private["output_length"]),
+            int(private["session_id"]),
+            "|".join(str(value) for value in runtime_block_ids),
+        )
+        if expected != actual:
+            raise ValueError(f"public/private payload mismatch at row {index}")
+
+        input_tokens = int(private["input_length"])
+        frontier_block_ids = full_prompt_block_ids(
+            runtime_block_ids, input_tokens, args.runtime_block_size
+        )
+        if not frontier_block_ids:
+            raise ValueError(
+                "Frontier prefix-cache replay requires at least one complete "
+                f"runtime block; request {index} has ISL={input_tokens}"
+            )
+        partial_prompt_count += int(input_tokens % args.runtime_block_size != 0)
+        raw_runtime_block_count += len(runtime_block_ids)
+        frontier_full_block_count += len(frontier_block_ids)
+
+        normalized_arrival = original_arrival / args.tp
+        updated = dict(private)
+        updated["arrived_at"] = normalized_arrival
+        normalized_private.append(updated)
+        normalized_public.append(
+            {
+                "arrived_at": f"{normalized_arrival:.12f}",
+                "num_prefill_tokens": public["num_prefill_tokens"],
+                "num_decode_tokens": public["num_decode_tokens"],
+                "session_id": public["session_id"],
+                "block_hash_ids": "|".join(str(value) for value in frontier_block_ids),
+            }
+        )
+        update_digest(
+            semantic_digest,
+            [
+                int(private["source_index"]),
+                int(private["input_length"]),
+                int(private["output_length"]),
+                int(private["session_id"]),
+                runtime_block_ids,
+            ],
+        )
+        update_digest(
+            normalized_digest,
+            [
+                int(private["source_index"]),
+                normalized_arrival,
+                int(private["input_length"]),
+                int(private["output_length"]),
+                int(private["session_id"]),
+                runtime_block_ids,
+            ],
+        )
+        update_digest(
+            frontier_prefix_digest,
+            [
+                int(private["source_index"]),
+                normalized_arrival,
+                input_tokens,
+                frontier_block_ids,
+            ],
+        )
+        original_arrivals.append(original_arrival)
+
+    if any(
+        right < left for left, right in zip(original_arrivals, original_arrivals[1:])
+    ):
+        raise ValueError("base arrival order is not monotonic")
+
+    public_dir = args.output_root / "public"
+    private_dir = args.output_root / "private"
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+    public_path = public_dir / "frontier.csv"
+    private_path = private_dir / "real_requests.jsonl"
+    with public_path.open("w", newline="") as output:
+        writer = csv.DictWriter(output, fieldnames=CSV_FIELDS, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(normalized_public)
+    with private_path.open("w") as output:
+        for row in normalized_private:
+            output.write(json.dumps(row, separators=(",", ":")) + "\n")
+
+    manifest = {
+        "schema": "qwen30-tp-normalized-trace-v2",
+        "privacy": "prompt text exists only under private/ and must not be harvested",
+        "transform": "arrival_seconds_prime = arrival_seconds / tensor_parallel_size",
+        "tensor_parallel_size": args.tp,
+        "requests": len(normalized_private),
+        "base_window_seconds": args.base_window_seconds,
+        "global_offered_request_rate": len(normalized_private)
+        / args.base_window_seconds
+        * args.tp,
+        "per_gpu_offered_request_rate": len(normalized_private)
+        / args.base_window_seconds,
+        "frontier_prefix_block_projection": {
+            "runtime_block_size": args.runtime_block_size,
+            "rule": "full_prompt_blocks_only=floor(input_tokens/runtime_block_size)",
+            "partial_prompt_count": partial_prompt_count,
+            "raw_runtime_block_count": raw_runtime_block_count,
+            "frontier_full_block_count": frontier_full_block_count,
+        },
+        "original_first_arrival_s": original_arrivals[0],
+        "original_last_arrival_s": original_arrivals[-1],
+        "normalized_first_arrival_s": original_arrivals[0] / args.tp,
+        "normalized_last_arrival_s": original_arrivals[-1] / args.tp,
+        "base_public_csv": str(args.base_public_csv.resolve()),
+        "base_public_csv_sha256": sha256(args.base_public_csv),
+        "base_private_jsonl": str(args.base_private_jsonl.resolve()),
+        "base_private_jsonl_sha256": sha256(args.base_private_jsonl),
+        "public_csv": str(public_path.resolve()),
+        "public_csv_sha256": sha256(public_path),
+        "private_jsonl": str(private_path.resolve()),
+        "private_jsonl_sha256": sha256(private_path),
+        "semantic_vector_sha256": semantic_digest.hexdigest(),
+        "normalized_row_vector_sha256": normalized_digest.hexdigest(),
+        "frontier_prefix_vector_sha256": frontier_prefix_digest.hexdigest(),
+    }
+    manifest_path = public_dir / "manifest.json"
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(manifest, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md

@@ -0,0 +1,191 @@
+# Qwen3-30B-A3B Trace-PD `u0p01` latency-selection comparison
+
+> **Status:** the 36-run real surface is valid. The initially reported Frontier
+> surface is **invalidated for simulator fidelity/selection** by a post-hoc
+> prefix-cache trace-contract audit (2026-07-17): its simulator-facing block
+> metadata includes non-cacheable partial final blocks. The raw CPU artifacts
+> are retained for diagnosis, but are not Frontier evidence. This remains the
+> first MoE case of `EXP-SIM-TUNING-LATENCY-MATRIX-V0`, not a claim about the
+> remaining three workload classes.
+
+## Question and decision
+
+Can the frozen Frontier surface select the same low-latency Qwen3-30B-A3B
+configuration as a fresh community-vLLM deployment when both replay exactly
+the same trace-derived prefill+decode request vector? The objective contains
+no SLO or capacity threshold: lower mean/p90 TTFT, TPOT, and E2E is better.
+
+The result is a bounded Trace-PD result only. It cannot establish a claim
+about Fixed-P, Fixed-PD, Trace-P, other trace rates, another model, or a
+modified Frontier.
+
+## Frozen contract
+
+| Item | Value |
+|---|---|
+| Hardware | `dash0`, H20, one replica; fresh vLLM server per config/trial |
+| Model / engine | `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`; community `vLLM 0.20.0+cu129`, BF16 |
+| Config surface | `TP ∈ {1,2,4} × MNS ∈ {8,16,32,64}`; `DP=PP=EP=1`, `MBT=8192`, GPU-memory utilization `0.92`, chunked prefill and prefix caching on |
+| Trace | `trace-exact-v1`, anchor `u0p01`; **129 requests**, original input/output length, order and session are preserved. The first Frontier materialization also copied every 16-token runtime identity, including a final partial block; that is not a legal vLLM prefix-cache-hit representation and invalidates this simulator attempt. For a config of tensor parallelism `TP`, every arrival is transformed as `t′ = t / TP`; the TP1/2/4 traces therefore end at 597.037/298.519/149.259 s. |
+| Offered load | Base rate is `129 / 600 = 0.215 req/s`. TP1/2/4 replay at 0.215/0.430/0.860 req/s globally, respectively, so all three topologies receive 0.215 req/s/GPU. This is a **per-GPU-normalized trace**, not an unchanged-wall-clock trace. |
+| Simulator baseline | The existing same-global-rate T1 surface is excluded: it evaluates TP2/TP4 at lower per-GPU load. A new Frontier surface must use the identical TP-normalized arrival vector for each topology before coverage or latency can be reported. |
+| Trials | Three fresh-server trials per config; independent warmup excluded from the measured trace |
+
+## Validity gates
+
+1. Each real trial must complete all 129 requests, preserve exact prompt and
+   completion-token usage, and have the TP-normalized row-vector digest.
+2. A simulator metric is valid only if its cell has all 129 finite request
+   metrics with matching request ID, prompt/output lengths and transformed
+   arrival vector. A simulator `scheduler_stall` is a coverage result, never
+   an infeasible-latency label.
+3. TTFT, TPOT and E2E are kept in milliseconds. `TPOT` is recorded only for
+   outputs longer than one token (all requests in this Trace-PD anchor
+   qualify).
+4. A failed real run remains an infeasible/safety observation; it is not
+   silently converted to infinite latency or omitted from ranking.
+
+## Analysis and decision rule
+
+For every `config × trial`, retain raw per-request records and compute mean
+and nearest-rank p90 for TTFT, TPOT and E2E. Report pooled 387-request values
+alongside the three per-trial values. A selection-stability bootstrap is
+deferred; it is not needed to evaluate Frontier's coverage gate.
+
+The primary gate is Frontier decision-valid coverage on all 12 cells. If any
+real-valid candidate lacks a legal simulator metric, Frontier receives no
+full-surface selection, top-set overlap, regret, or Kendall-tau score. We
+will still show all valid per-cell latency comparisons and the complete real
+surface, so any coverage failure and its ground truth remain auditable.
+
+The simulator's pre-existing SLO fields do not influence request generation,
+scheduling, or this no-SLO analysis; they are excluded from scoring.
+
+## Post-hoc Frontier trace-contract audit (2026-07-17)
+
+The CPU attempt initially passed only a syntactic trace check (row count,
+IDs, lengths, and SHA). It did **not** validate the semantic invariant needed
+by vLLM prefix caching: only completed 16-token blocks may be cache hits.
+
+- The trace exporter emitted `ceil(ISL / 16)` identities. Thus 122/129 prompts
+  contained one partial final block; the public trace has 36,443 identities,
+  while a full-block Frontier/vLLM cache adapter must expose 36,321.
+- Exact community-vLLM 0.20 source (`vllm.v1.core.kv_cache_manager`) documents
+  that computed prefix blocks "must be full" and bounds hits by
+  `prompt_length - 1`. Frontier's fixture contract only requires a non-empty
+  `block_hash_ids` field and does not validate this full-block invariant.
+- Frontier's prefix manager converts every hit to `len(blocks) * 16`. For the
+  final TP2/MNS16 state, requests 89/111/117 had `(ISL, IDs)` of
+  `(722,46)`, `(678,43)`, and `(706,45)`. If all supplied identities hit, it
+  obtains 736/688/720 computed tokens and hence `num_new_tokens` of
+  -14/-10/-14. The scheduler preserves zero-or-negative-token requests in its
+  waiting queue; no event remains, so the simulator reports
+  `Sequential simulation ended with non-empty scheduler state`.
+
+This proves a trace-adapter/Frontier prefix-cache semantic mismatch, not a
+GPU capacity failure: a targeted CPU hook found no failed KV allocation and
+no PP-admission deferral, only the three non-admitted requests with a full
+8192-token batch budget. The correct next input is a **separate Frontier
+adapter** that exports only `floor(ISL / 16)` complete prompt-block identities;
+it must not modify the private real trace or its input/output/arrival vector.
+
+## Invalid raw Frontier attempt (2026-07-17; retained for diagnosis only)
+
+The TP-normalized surface completed in 156 seconds of CPU-only wall time on
+`dash0`, with Frontier commit `deadc4a321f0baaa534c6ebd17f974123733cdc2`.
+All twelve input traces passed the 129-request, ID/shape, and TP-specific
+trace-SHA checks.  Only two cells reached valid request metrics; the other ten
+ended in `scheduler_stall`.
+
+| Config | Status | mean / p90 TTFT (ms) | mean / p90 TPOT (ms) | mean / p90 E2E (ms) |
+|---|---|---:|---:|---:|
+| TP1/MNS32 | complete | 440032 / 972628 | 145.8 / 151.5 | 936301 / 1645895 |
+| TP1/MNS64 | complete | 174335 / 510962 | 163.6 / 178.9 | 724328 / 1285374 |
+| remaining 10 configs | `scheduler_stall` | N/A | N/A | N/A |
+
+The apparent **2/12 (16.7%)** coverage is not a Frontier coverage result:
+both the ten stalls and the two completed cells consumed the invalid partial
+block metadata. In particular, the two completed cells may receive false
+prefix hits. No latency, ranking, coverage, top-set overlap, regret, or
+Kendall-tau statement about Frontier may be derived from this attempt.
+
+The legacy surface runner writes an offered-rate summary assuming a fixed
+600-second window. That field is not used here because TP2/TP4 have compressed
+arrival clocks. The materialized trace manifests are authoritative: they
+record global 0.215/0.430/0.860 req/s and the matched 0.215 req/s/GPU rate.
+
+## Real result (2026-07-17)
+
+All **36/36** `config × trial` replays completed with exit code zero. Each
+trial passed all validity gates: 129/129 successful requests, exact requested
+and observed input/output token usage, the TP-specific private-trace SHA and
+normalized arrival-vector SHA, and HTTP routing to the explicit alias
+`qwen3-30b-exact-trace`. The cache-populating smoke and earlier failed launch
+attempts are excluded from this table. Values pool 387 requests/config (three
+trials); p90 is nearest-rank.
+
+| Config | mean / p90 TTFT (ms) | mean / p90 TPOT (ms) | mean / p90 E2E (ms) |
+|---|---:|---:|---:|
+| TP1/MNS8 | 88479.9 / 168185.5 | 14.3 / 16.3 | 137996.4 / 222136.0 |
+| TP1/MNS16 | 17378.2 / 42739.8 | 20.6 / 25.3 | 87355.2 / 156177.2 |
+| TP1/MNS32 | 620.5 / 1931.5 | 23.8 / 30.0 | 81051.2 / 162908.6 |
+| TP1/MNS64 | 619.8 / 1956.7 | 24.1 / 30.4 | 82253.7 / 165355.3 |
+| TP2/MNS8 | 95920.1 / 183878.8 | 9.2 / 10.2 | 128296.9 / 215848.3 |
+| TP2/MNS16 | 34443.0 / 77922.4 | 14.1 / 16.1 | 82936.7 / 137726.4 |
+| TP2/MNS32 | 1050.6 / 2521.8 | 18.0 / 21.3 | 62267.5 / 117239.3 |
+| TP2/MNS64 | 375.8 / 1148.6 | 18.2 / 21.6 | 62353.6 / 118392.3 |
+| TP4/MNS8 | 102605.9 / 193359.7 | **6.8 / 7.2** | 126405.4 / 209092.4 |
+| TP4/MNS16 | 35042.6 / 74281.3 | 8.8 / 9.6 | 65779.1 / 110374.3 |
+| TP4/MNS32 | 6296.2 / 19159.9 | 12.2 / 13.8 | 47944.5 / 83825.7 |
+| TP4/MNS64 | **246.0 / 685.5** | 13.2 / 15.4 | **44985.1 / 83763.6** |
+
+Thus real-vLLM chooses TP4/MNS64 for mean/p90 TTFT and E2E, and TP4/MNS8
+for mean/p90 TPOT. These are different single-metric objectives, not a
+claim that one configuration simultaneously optimizes all three.
+
+## Correct conclusion and next gate
+
+This run establishes a **harness finding**, not a Frontier capability finding:
+the initial simulator-facing Trace-PD prefix contract was invalid. The valid
+real surface remains a frozen ground truth, but all raw Frontier numbers above
+are excluded from the research claim. We therefore retract the prior bounded
+counterexample and do not yet know whether Frontier selects the correct config
+on this case.
+
+Before a CPU-only Frontier rerun, the comparison must additionally record
+these remaining alignment gaps:
+
+1. real vLLM uses `FULL_AND_PIECEWISE` CUDA graphs, whereas the raw Frontier
+   command explicitly used `--decode_cuda_graph_mode none`; Frontier exposes
+   `full_decode_only` and `piecewise`, not the identical combined mode;
+2. Frontier used one explicit KV-block count per TP, while real vLLM's graph
+   capture changes the count with MNS (the mismatch is small but measurable);
+3. Frontier skipped CPU-overhead modeling, and its MoE CSV has only
+   `standalone_legacy` gating rows, so the code warned and fell back rather
+   than training the requested `prefill_hot` pseudo-model;
+4. the runner's rate *metadata* incorrectly fixed the TP2/TP4 numerator to a
+   600-second window. Arrival timestamps supplied to the simulator were
+   correct, so this did not alter the raw execution, but the metadata must be
+   fixed before reporting a rerun.
+
+The next admissible Frontier result uses complete-block prefix metadata,
+per-`TP×MNS` real KV capacity, correct TP-normalized rate metadata, and an
+explicit CUDA-graph compatibility decision. Only a complete, semantically
+valid surface may receive the selection metrics. Any remaining mismatch after
+that CPU rerun is then evidence about Frontier's scheduler/profile fidelity,
+not this adapter.
+
+## Cost and output
+
+The measured replay horizons are 597/299/149 s at TP1/2/4. Including five
+minutes of per-launch server start/warmup, 36 launches consume about **13
+H20-GPUh nominally**; applying the launcher's historical 12--35 minute
+fresh-server envelope yields an upper bound of **41 H20-GPUh**. A separate
+CPU-only simulator rerun is mandatory before this GPU stage; no profile
+collection is included.
+
+The remote, experiment-specific output root is
+`/home/admin/cpfs/wjh/aituner/simulator-tuning-latency-q30-tp-normalized-u0p01-20260717`.
+It contains the prompt-free real audit JSON/Markdown, commands, environment,
+GPU/trace/model hashes, raw prompt-free request records, and cache inventory.
+Private prompt text remains on `dash0` and is never copied into the repository.