Track simulator fidelity experiment artifacts

runs/frontier-multicase-sufficiency-v1/analyze_t0_surface.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""Validate and compare frozen T0 Frontier and two-round real surfaces."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import math
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+
+SLOS = ("tpot_40ms", "tpot_120ms", "tpot_150ms", "tpot_180ms")
+RATE_LATTICE = (0.10, 0.20, 0.40, 0.80, 1.20, 1.60, 2.40, 3.20)
+
+
+def sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def rate_key(rate: float) -> str:
+    return f"r{rate:.2f}".replace(".", "p")
+
+
+def find_real_result(roots: list[Path], name: str, round_id: int, rate: float) -> Path:
+    relative = Path(name) / f"round{round_id}/results" / f"{rate_key(rate)}.json"
+    matches = [root / relative for root in roots if (root / relative).is_file()]
+    if len(matches) != 1:
+        raise ValueError(f"expected one real result for {relative}, got {matches}")
+    return matches[0]
+
+
+def capacity(loads: list[dict[str, Any]], slo: str, field: str) -> float | None:
+    values = [float(load["rate"]) for load in loads if bool(load[field][slo]["feasible"])]
+    return max(values) if values else None
+
+
+def real_boundary_status(loads: list[dict[str, Any]], slo: str) -> str:
+    labels = {
+        float(load["rate"]): bool(load["real_conservative"][slo]["feasible"])
+        for load in loads
+    }
+    ordered = [(rate, labels[rate]) for rate in RATE_LATTICE if rate in labels]
+    if any(not left and right for (_, left), (_, right) in zip(ordered, ordered[1:])):
+        return "non_monotonic_requires_full_lattice"
+    if len(labels) == len(RATE_LATTICE):
+        return "complete_lattice"
+    for lower, upper in zip(RATE_LATTICE, RATE_LATTICE[1:]):
+        if labels.get(lower) is True and labels.get(upper) is False:
+            return "adjacent_transition_bracketed"
+    if labels.get(RATE_LATTICE[-1]) is True:
+        return "upper_lattice_reached"
+    if labels.get(RATE_LATTICE[0]) is False and not any(labels.values()):
+        return "lowest_anchor_infeasible"
+    return "unbracketed_requires_expansion"
+
+
+def kendall_tau_b(left: list[float], right: list[float]) -> float | None:
+    concordant = discordant = left_ties = right_ties = 0
+    for i in range(len(left)):
+        for j in range(i + 1, len(left)):
+            x = (left[i] > left[j]) - (left[i] < left[j])
+            y = (right[i] > right[j]) - (right[i] < right[j])
+            if x == 0 and y == 0:
+                continue
+            if x == 0:
+                left_ties += 1
+            elif y == 0:
+                right_ties += 1
+            elif x == y:
+                concordant += 1
+            else:
+                discordant += 1
+    denominator = math.sqrt(
+        (concordant + discordant + left_ties)
+        * (concordant + discordant + right_ties)
+    )
+    return (concordant - discordant) / denominator if denominator else None
+
+
+def sign(left: float, right: float) -> int:
+    return (left > right) - (left < right)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--frontier-freeze", type=Path, required=True)
+    parser.add_argument("--real-plan", type=Path, required=True)
+    parser.add_argument("--real-root", type=Path, action="append", required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    args = parser.parse_args()
+
+    freeze = json.loads(args.frontier_freeze.read_text())
+    plan = json.loads(args.real_plan.read_text())
+    real_roots = [root.resolve() for root in args.real_root]
+    if freeze.get("status") != "frozen_before_real_surface" or len(freeze.get("config_results") or []) != 8:
+        raise ValueError("Frontier freeze is incomplete")
+    if plan.get("frontier_freeze", {}).get("sha256") != sha256(args.frontier_freeze):
+        raise ValueError("real plan does not point to this Frontier freeze")
+    sim_by_name = {item["config"]["name"]: item for item in freeze["config_results"]}
+
+    cells = []
+    for cell in plan["cells"]:
+        config = cell["config"]
+        name = config["name"]
+        sim_loads = {float(load["offered_request_rate"]): load for load in sim_by_name[name]["loads"]}
+        loads = []
+        for rate in cell["rates"]:
+            round_summaries = []
+            files = []
+            for round_id in (1, 2):
+                path = find_real_result(real_roots, name, round_id, float(rate))
+                payload = json.loads(path.read_text())
+                if payload.get("schema") != "qwen235b-t0-rate-anchor-v1":
+                    raise ValueError(f"bad real result schema: {path}")
+                if payload["summary"]["completed"] != 64 or payload["summary"]["failed"] != 0:
+                    raise ValueError(f"incomplete real anchor: {path}")
+                if float(payload["workload"]["offered_request_rate"]) != float(rate):
+                    raise ValueError(f"offered-rate drift: {path}")
+                round_summaries.append(payload["summary"]["slos"])
+                files.append({"path": str(path), "sha256": sha256(path)})
+            conservative = {
+                slo: {
+                    "feasible": all(summary[slo]["feasible"] for summary in round_summaries),
+                    "round_pass_rates": [summary[slo]["pass_rate"] for summary in round_summaries],
+                }
+                for slo in SLOS
+            }
+            loads.append(
+                {
+                    "rate": float(rate),
+                    "real_conservative": conservative,
+                    "sim": sim_loads[float(rate)]["slos"],
+                    "real_files": files,
+                }
+            )
+        cells.append({"config": config, "loads": loads})
+
+    comparisons = {}
+    for slo in SLOS:
+        records = []
+        for cell in cells:
+            real = capacity(cell["loads"], slo, "real_conservative")
+            sim_values = [
+                float(load["offered_request_rate"])
+                for load in sim_by_name[cell["config"]["name"]]["loads"]
+                if bool(load["slos"][slo]["feasible"])
+            ]
+            sim = max(sim_values) if sim_values else None
+            tp = int(cell["config"]["tp"])
+            boundary = real_boundary_status(cell["loads"], slo)
+            records.append(
+                {
+                    "config": cell["config"],
+                    "real_capacity": real,
+                    "sim_capacity": sim,
+                    "real_capacity_per_gpu": real / tp if real is not None else None,
+                    "sim_capacity_per_gpu": sim / tp if sim is not None else None,
+                    "real_boundary_status": boundary,
+                    "expansion_required": boundary in {
+                        "non_monotonic_requires_full_lattice",
+                        "unbracketed_requires_expansion",
+                    },
+                }
+            )
+        rankable = [
+            row
+            for row in records
+            if not row["expansion_required"]
+            and row["real_capacity_per_gpu"] is not None
+            and row["sim_capacity_per_gpu"] is not None
+        ]
+        tau = kendall_tau_b(
+            [row["real_capacity_per_gpu"] for row in rankable],
+            [row["sim_capacity_per_gpu"] for row in rankable],
+        )
+        pairwise = {
+            "all_pairs": 0,
+            "exact_sign_matches": 0,
+            "real_non_tie_pairs": 0,
+            "real_non_tie_direction_matches": 0,
+        }
+        for i, left in enumerate(rankable):
+            for right in rankable[i + 1 :]:
+                real_sign = sign(left["real_capacity_per_gpu"], right["real_capacity_per_gpu"])
+                sim_sign = sign(left["sim_capacity_per_gpu"], right["sim_capacity_per_gpu"])
+                pairwise["all_pairs"] += 1
+                pairwise["exact_sign_matches"] += real_sign == sim_sign
+                if real_sign:
+                    pairwise["real_non_tie_pairs"] += 1
+                    pairwise["real_non_tie_direction_matches"] += real_sign == sim_sign
+        real_best = max((row["real_capacity_per_gpu"] for row in records if row["real_capacity_per_gpu"] is not None), default=None)
+        sim_best = max((row["sim_capacity_per_gpu"] for row in records if row["sim_capacity_per_gpu"] is not None), default=None)
+        sim_top = [row for row in records if sim_best is not None and row["sim_capacity_per_gpu"] == sim_best]
+        real_top = [row for row in records if real_best is not None and row["real_capacity_per_gpu"] == real_best]
+        optimistic_regret = worst_regret = None
+        if real_best is not None and sim_top and all(row["real_capacity_per_gpu"] is not None for row in sim_top):
+            regrets = [(real_best - row["real_capacity_per_gpu"]) / real_best for row in sim_top]
+            optimistic_regret = min(regrets)
+            worst_regret = max(regrets)
+        confusion = {"anchors": 0, "match": 0, "false_feasible": 0, "false_infeasible": 0}
+        for cell in cells:
+            for load in cell["loads"]:
+                real_feasible = bool(load["real_conservative"][slo]["feasible"])
+                sim_feasible = bool(load["sim"][slo]["feasible"])
+                confusion["anchors"] += 1
+                confusion["match"] += real_feasible == sim_feasible
+                confusion["false_feasible"] += sim_feasible and not real_feasible
+                confusion["false_infeasible"] += real_feasible and not sim_feasible
+        comparisons[slo] = {
+            "records": records,
+            "kendall_tau_b": tau,
+            "pairwise": pairwise,
+            "anchor_confusion": confusion,
+            "real_top_set": [row["config"]["name"] for row in real_top],
+            "sim_top_set": [row["config"]["name"] for row in sim_top],
+            "top_set_intersection": sorted(
+                {row["config"]["name"] for row in real_top}
+                & {row["config"]["name"] for row in sim_top}
+            ),
+            "top_set_exact_match": {
+                row["config"]["name"] for row in real_top
+            } == {row["config"]["name"] for row in sim_top},
+            "optimistic_regret": optimistic_regret,
+            "worst_tie_break_regret": worst_regret,
+        }
+
+    run_costs = []
+    for root in real_roots:
+        config_names = [child.name for child in root.iterdir() if child.is_dir() and child.name in sim_by_name]
+        if len(config_names) != 1:
+            raise ValueError(f"expected one config directory in real root {root}, got {config_names}")
+        config = sim_by_name[config_names[0]]["config"]
+        remote_run = root.parents[2] / "remote_run"
+        started = datetime.fromisoformat((remote_run / "started_at").read_text().strip())
+        finished = datetime.fromisoformat((remote_run / "finished_at").read_text().strip())
+        wall_seconds = (finished - started).total_seconds()
+        run_costs.append(
+            {
+                "run_id": root.parents[2].name,
+                "config": config_names[0],
+                "gpu_count": int(config["tp"]),
+                "wall_seconds": wall_seconds,
+                "h20_gpu_hours": wall_seconds * int(config["tp"]) / 3600,
+                "started_at": started.isoformat(),
+                "finished_at": finished.isoformat(),
+            }
+        )
+    fresh_server_anchors = 2 * sum(len(cell["loads"]) for cell in cells)
+    real_execution_cost = {
+        "accepted_fleet_jobs": len(run_costs),
+        "fresh_server_anchors": fresh_server_anchors,
+        "measured_requests": fresh_server_anchors * 64,
+        "warmup_requests": 2
+        * sum(
+            min(32, max(4, math.ceil(float(load["rate"]) * 20)))
+            for cell in cells
+            for load in cell["loads"]
+        ),
+        "accepted_h20_gpu_hours": sum(run["h20_gpu_hours"] for run in run_costs),
+        "campaign_wall_span_seconds": (
+            max(datetime.fromisoformat(run["finished_at"]) for run in run_costs)
+            - min(datetime.fromisoformat(run["started_at"]) for run in run_costs)
+        ).total_seconds(),
+        "runs": run_costs,
+    }
+
+    output = {
+        "schema": "qwen235b-t0-simulator-real-comparison-v1",
+        "frontier_freeze_sha256": sha256(args.frontier_freeze),
+        "real_plan_sha256": sha256(args.real_plan),
+        "real_execution_cost": real_execution_cost,
+        "cells": cells,
+        "comparisons": comparisons,
+    }
+    args.output_root.mkdir(parents=True, exist_ok=True)
+    write_json(args.output_root / "comparison.json", output)
+    with (args.output_root / "capacity.csv").open("w", newline="") as target:
+        writer = csv.DictWriter(
+            target,
+            fieldnames=[
+                "slo",
+                "config",
+                "tp",
+                "mns",
+                "mbt",
+                "real_capacity_per_gpu",
+                "sim_capacity_per_gpu",
+                "real_boundary_status",
+                "expansion_required",
+            ],
+        )
+        writer.writeheader()
+        for slo, comparison in comparisons.items():
+            for row in comparison["records"]:
+                writer.writerow(
+                    {
+                        "slo": slo,
+                        "config": row["config"]["name"],
+                        "tp": row["config"]["tp"],
+                        "mns": row["config"]["mns"],
+                        "mbt": row["config"]["mbt"],
+                        "real_capacity_per_gpu": row["real_capacity_per_gpu"],
+                        "sim_capacity_per_gpu": row["sim_capacity_per_gpu"],
+                        "real_boundary_status": row["real_boundary_status"],
+                        "expansion_required": row["expansion_required"],
+                    }
+                )
+    print(args.output_root / "comparison.json")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/exclusions.json

@@ -0,0 +1,61 @@
+{
+  "schema": "qwen235b-t0-real-exclusions-v1",
+  "policy": "Exclude the complete output directory whenever another process shares any allocated GPU or benchmark endpoint during warmup or a measured anchor.",
+  "excluded_attempts": [
+    {
+      "reason": "Fleet monitor re-probed during model load and oversubscribed both TP4 allocations; the later MNS128 attempts overlapped the MNS64 warmup and start of r0p10.",
+      "remote_quarantine": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/contaminated-queue-race-20260716T045500Z",
+      "run_ids": [
+        "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T045338281933Z",
+        "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T045339503446Z",
+        "qwen235b-t0-real-tp4_mns128_mbt8192-20260716-v1-20260716T045458846499Z",
+        "qwen235b-t0-real-tp4_mns128_mbt16384-20260716-v1-20260716T045500253803Z"
+      ]
+    },
+    {
+      "reason": "A dispatch command still copying the source was mistakenly resubmitted. The duplicate controllers reused the first pair's open ports and issued concurrent warmup requests, so both the original and duplicate outputs were rejected.",
+      "remote_quarantine": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/contaminated-duplicate-dispatch-20260716T0501Z",
+      "run_ids": [
+        "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T050017477963Z",
+        "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T050018748070Z",
+        "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T050123485776Z",
+        "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T050124800483Z"
+      ]
+    },
+    {
+      "reason": "Clean diagnostic run exposed cross-anchor execution-state leakage: TPOT at 1.60 req/s failed after a sustained 0.10 anchor but passed after a 2.40 anchor. The multi-rate-per-server contract does not match Frontier's independent rate runs, so the complete attempt is diagnostic-only.",
+      "remote_quarantine": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/diagnostic-cross-anchor-state-20260716T0504Z",
+      "run_ids": [
+        "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T050405703312Z",
+        "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T050407104293Z"
+      ]
+    }
+  ],
+  "accepted_attempts": [
+    "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T054640909739Z",
+    "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T054642115787Z",
+    "qwen235b-t0-real-tp4_mns64_mbt8192-expansion-r3p20-20260716-v1-20260716T062904775741Z",
+    "qwen235b-t0-real-tp4_mns64_mbt16384-expansion-r3p20-20260716-v1-20260716T062906244819Z",
+    "qwen235b-t0-real-tp4_mns128_mbt8192-20260716-v1-20260716T063826749477Z",
+    "qwen235b-t0-real-tp4_mns128_mbt16384-20260716-v1-20260716T063827805737Z",
+    "qwen235b-t0-real-tp4_mns128_mbt8192-expansion-r3p20-20260716-v1-20260716T072121119478Z",
+    "qwen235b-t0-real-tp4_mns128_mbt16384-expansion-r3p20-20260716-v1-20260716T072122501851Z",
+    "qwen235b-t0-real-tp8_mns64_mbt8192-20260716-v1-20260716T073151443129Z",
+    "qwen235b-t0-real-tp8_mns64_mbt8192-expansion-r2p40-20260716-v1-20260716T081536141326Z",
+    "qwen235b-t0-real-tp8_mns64_mbt8192-expansion-r3p20-20260716-v1-20260716T082603837807Z",
+    "qwen235b-t0-real-tp8_mns64_mbt16384-20260716-v1-20260716T083632620015Z",
+    "qwen235b-t0-real-tp8_mns64_mbt16384-expansion-r2p40-20260716-v1-20260716T092040851202Z",
+    "qwen235b-t0-real-tp8_mns128_mbt8192-20260716-v1-20260716T093245831612Z",
+    "qwen235b-t0-real-tp8_mns128_mbt8192-expansion-r2p40-20260716-v1-20260716T101734683241Z",
+    "qwen235b-t0-real-tp8_mns128_mbt16384-20260716-v1-20260716T102855604708Z",
+    "qwen235b-t0-real-tp8_mns128_mbt16384-expansion-r2p40-20260716-v1-20260716T111437770615Z",
+    "qwen235b-t0-real-tp8_mns128_mbt16384-expansion-r3p20-20260716-v1-20260716T112608807037Z"
+  ],
+  "accepted_warnings": [
+    {
+      "run_id": "qwen235b-t0-real-tp8_mns64_mbt8192-20260716-v1-20260716T073151443129Z",
+      "anchor": "round1/r1p20",
+      "reason": "One rank-7 TCPStore heartbeat warning occurred 1.1 s after the complete 64-request result was written, while all eight workers were terminating after the runner intentionally stopped the fresh server. The application then shut down normally; no request, engine, OOM, or in-window NCCL failure occurred."
+    }
+  ]
+}

runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/real-plan-executed.json

@@ -0,0 +1,299 @@
+{
+  "cells": [
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 64,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns64_mbt8192",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "rates": [
+        0.1,
+        1.6,
+        2.4,
+        3.2
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 64,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns64_mbt16384",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "rates": [
+        0.1,
+        1.6,
+        2.4,
+        3.2
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 128,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns128_mbt8192",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "rates": [
+        0.1,
+        1.6,
+        2.4,
+        3.2
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 128,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns128_mbt16384",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "rates": [
+        0.1,
+        1.6,
+        2.4,
+        3.2
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 64,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns64_mbt8192",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "rates": [
+        0.1,
+        1.2,
+        1.6,
+        2.4,
+        3.2
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 64,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns64_mbt16384",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "rates": [
+        0.1,
+        1.2,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 128,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns128_mbt8192",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "rates": [
+        0.1,
+        1.2,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 128,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns128_mbt16384",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "preregistered_rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "rates": [
+        0.1,
+        1.2,
+        1.6,
+        2.4,
+        3.2
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    }
+  ],
+  "execution_protocol_amendment": {
+    "contract": "fresh server and target-rate warmup for every config-rate-round",
+    "reason": "observed cross-anchor GPU/kernel/batch warm-state leakage",
+    "timing": "after excluded multi-rate-per-server diagnostic and before any accepted real surface cell"
+  },
+  "expansion_artifact_roots": [
+    "artifacts/t0-real-expansion-r2p40-v1",
+    "artifacts/t0-real-expansion-r3p20-v1"
+  ],
+  "expected_total_h20_gpu_hours": 39.98888888888888,
+  "frontier_freeze": {
+    "path": "/home/gahow/phd/aituner/runs/frontier-multicase-sufficiency-v1/artifacts/frontier-t0-surface-v1/frontier_surface_frozen.json",
+    "sha256": "801aa36451c8647f71cc87011144622d2203786e82f189ed1375d964399b106a"
+  },
+  "hard_timeout_hours_per_cell": 2.0,
+  "post_pilot_sensitivities": [
+    "tpot_120ms",
+    "tpot_150ms",
+    "tpot_180ms"
+  ],
+  "preregistered_plan": {
+    "path": "/home/gahow/phd/aituner/runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/real-plan.json",
+    "sha256": "f7acb45d1183a6026dba4bbf9bc2d7578452b815b8de0035ca732512f3d9d47a"
+  },
+  "schema": "qwen235b-t0-real-executed-plan-v1",
+  "selection_slo": "tpot_150ms",
+  "selection_timing": "rate anchors frozen after complete simulator surface and before any accepted real surface cell",
+  "status": "complete",
+  "strict_preregistered_slo": "tpot_40ms"
+}

runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/real-plan-pre-anchor-isolation-amendment.json

@@ -0,0 +1,187 @@
+{
+  "cells": [
+    {
+      "config": {
+        "mbt": 8192,
+        "mns": 64,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns64_mbt8192",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 2.2125,
+      "expected_wall_seconds": 1991.25,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 16384,
+        "mns": 64,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns64_mbt16384",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 2.2125,
+      "expected_wall_seconds": 1991.25,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 8192,
+        "mns": 128,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns128_mbt8192",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 2.2125,
+      "expected_wall_seconds": 1991.25,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 16384,
+        "mns": 128,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns128_mbt16384",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 2.2125,
+      "expected_wall_seconds": 1991.25,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 8192,
+        "mns": 64,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns64_mbt8192",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 4.541666666666667,
+      "expected_wall_seconds": 2043.75,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 16384,
+        "mns": 64,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns64_mbt16384",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 4.541666666666667,
+      "expected_wall_seconds": 2043.75,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 8192,
+        "mns": 128,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns128_mbt8192",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 4.541666666666667,
+      "expected_wall_seconds": 2043.75,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    },
+    {
+      "config": {
+        "mbt": 16384,
+        "mns": 128,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns128_mbt16384",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 4.541666666666667,
+      "expected_wall_seconds": 2043.75,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2
+    }
+  ],
+  "expected_total_h20_gpu_hours": 27.016666666666666,
+  "frontier_freeze": {
+    "path": "/home/gahow/phd/aituner/runs/frontier-multicase-sufficiency-v1/artifacts/frontier-t0-surface-v1/frontier_surface_frozen.json",
+    "sha256": "801aa36451c8647f71cc87011144622d2203786e82f189ed1375d964399b106a"
+  },
+  "hard_timeout_hours_per_cell": 2.0,
+  "post_pilot_sensitivities": [
+    "tpot_120ms",
+    "tpot_150ms",
+    "tpot_180ms"
+  ],
+  "schema": "qwen235b-t0-real-plan-v1",
+  "selection_slo": "tpot_150ms",
+  "selection_timing": "after_complete_simulator_freeze_before_any_real_surface_cell",
+  "strict_preregistered_slo": "tpot_40ms"
+}

runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/real-plan.json

@@ -0,0 +1,240 @@
+{
+  "cells": [
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 64,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns64_mbt8192",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 64,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns64_mbt16384",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 128,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns128_mbt8192",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 128,
+        "moe_ep": 1,
+        "moe_tp": 4,
+        "name": "tp4_mns128_mbt16384",
+        "num_gpu_blocks": 26101,
+        "tp": 4
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 3.284259259259259,
+      "expected_wall_seconds": 2955.833333333333,
+      "rates": [
+        0.1,
+        1.6,
+        2.4
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.60": 32,
+        "2.40": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 64,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns64_mbt8192",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 64,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns64_mbt16384",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 8192,
+        "mns": 128,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns128_mbt8192",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    },
+    {
+      "anchor_isolation": "fresh_server_per_rate_per_round",
+      "config": {
+        "mbt": 16384,
+        "mns": 128,
+        "moe_ep": 8,
+        "moe_tp": 1,
+        "name": "tp8_mns128_mbt16384",
+        "num_gpu_blocks": 62351,
+        "tp": 8
+      },
+      "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+      "expected_h20_gpu_hours": 6.712962962962962,
+      "expected_wall_seconds": 3020.833333333333,
+      "rates": [
+        0.1,
+        1.2,
+        1.6
+      ],
+      "requests_per_anchor": 64,
+      "rounds": 2,
+      "target_rate_warmup_requests": {
+        "0.10": 4,
+        "1.20": 24,
+        "1.60": 32
+      }
+    }
+  ],
+  "execution_protocol_amendment": {
+    "contract": "fresh server and target-rate warmup for every config-rate-round",
+    "reason": "observed cross-anchor GPU/kernel/batch warm-state leakage",
+    "timing": "after excluded multi-rate-per-server diagnostic and before any accepted real surface cell"
+  },
+  "expected_total_h20_gpu_hours": 39.98888888888888,
+  "frontier_freeze": {
+    "path": "/home/gahow/phd/aituner/runs/frontier-multicase-sufficiency-v1/artifacts/frontier-t0-surface-v1/frontier_surface_frozen.json",
+    "sha256": "801aa36451c8647f71cc87011144622d2203786e82f189ed1375d964399b106a"
+  },
+  "hard_timeout_hours_per_cell": 2.0,
+  "post_pilot_sensitivities": [
+    "tpot_120ms",
+    "tpot_150ms",
+    "tpot_180ms"
+  ],
+  "schema": "qwen235b-t0-real-plan-v1",
+  "selection_slo": "tpot_150ms",
+  "selection_timing": "rate anchors frozen after complete simulator surface and before any accepted real surface cell",
+  "strict_preregistered_slo": "tpot_40ms"
+}

runs/frontier-multicase-sufficiency-v1/assemble_frontier_profile_root.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+"""Create an immutable Frontier profile root with measured decode rows."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+MODEL = "Qwen3-235B-A22B-FP8"
+ATTENTION_RELATIVE_PATH = Path("compute/h20") / MODEL / "attention.csv"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-profile-root", type=Path, required=True)
+    parser.add_argument("--decode-attention-csv", type=Path, required=True)
+    parser.add_argument("--true-mixed-attention-csv", type=Path)
+    parser.add_argument("--output-root", type=Path, required=True)
+    return parser.parse_args()
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def read_csv(path: Path) -> tuple[list[str], list[dict[str, str]]]:
+    with path.open(newline="") as source:
+        reader = csv.DictReader(source)
+        if reader.fieldnames is None:
+            raise ValueError(f"missing CSV header: {path}")
+        return reader.fieldnames, list(reader)
+
+
+def is_true(value: str) -> bool:
+    return value.strip().lower() == "true"
+
+
+def profile_hashes(root: Path) -> dict[str, str]:
+    return {
+        str(path.relative_to(root)): sha256(path)
+        for path in sorted(root.rglob("*"))
+        if path.is_file() and path.name != "profile_closure_manifest.json"
+    }
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def main() -> None:
+    args = parse_args()
+    base_root = args.base_profile_root.resolve()
+    decode_csv = args.decode_attention_csv.resolve()
+    output_root = args.output_root.resolve()
+    base_attention = base_root / ATTENTION_RELATIVE_PATH
+
+    if output_root.exists():
+        raise FileExistsError(f"refusing to overwrite profile root: {output_root}")
+    for path in (base_attention, decode_csv):
+        if not path.is_file():
+            raise FileNotFoundError(path)
+
+    base_fields, base_rows = read_csv(base_attention)
+    decode_fields, decode_source_rows = read_csv(decode_csv)
+    if base_fields != decode_fields:
+        raise ValueError("base and decode attention CSV schemas differ")
+    if not base_rows or any(not is_true(row["is_prefill"]) for row in base_rows):
+        raise ValueError("base attention profile must contain only prefill rows")
+    decode_rows = [row for row in decode_source_rows if not is_true(row["is_prefill"])]
+    if not decode_rows:
+        raise ValueError("decode attention profile contains no decode rows")
+    if any(not row["time_stats.attn_decode.median"] for row in decode_rows):
+        raise ValueError("decode attention profile has an empty median")
+
+    dimensions = {
+        (
+            int(row["num_tensor_parallel_workers"]),
+            int(row["batch_size"]),
+            int(row["kv_cache_size"]),
+            row["attention_backend"],
+        )
+        for row in decode_rows
+    }
+    if len(dimensions) != len(decode_rows):
+        raise ValueError("decode attention profile has duplicate coverage coordinates")
+    if any(dimension[-1] != "FLASHINFER" for dimension in dimensions):
+        raise ValueError("decode attention profile is not entirely FlashInfer")
+
+    true_mixed_csv = (
+        args.true_mixed_attention_csv.resolve()
+        if args.true_mixed_attention_csv is not None
+        else None
+    )
+    true_mixed_rows: list[dict[str, str]] = []
+    output_fields = list(base_fields)
+    if true_mixed_csv is not None:
+        if not true_mixed_csv.is_file():
+            raise FileNotFoundError(true_mixed_csv)
+        true_mixed_fields, true_mixed_rows = read_csv(true_mixed_csv)
+        required = {
+            "is_true_mixed_batch",
+            "decode_batch_size",
+            "decode_avg_kv_cache_size",
+            "num_prefill_seqs",
+            "time_stats.attn_decode.median",
+            "time_stats.attn_prefill.median",
+        }
+        missing = required - set(true_mixed_fields)
+        if missing:
+            raise ValueError(f"true-mixed attention CSV lacks columns: {sorted(missing)}")
+        if not true_mixed_rows or any(
+            not is_true(row["is_true_mixed_batch"]) for row in true_mixed_rows
+        ):
+            raise ValueError("true-mixed attention CSV has non-mixed rows")
+        if any(
+            not row["time_stats.attn_decode.median"]
+            or not row["time_stats.attn_prefill.median"]
+            for row in true_mixed_rows
+        ):
+            raise ValueError("true-mixed attention profile has an empty median")
+        if {
+            int(row["num_tensor_parallel_workers"]) for row in true_mixed_rows
+        } != {4, 8}:
+            raise ValueError("true-mixed attention profile must cover TP4 and TP8")
+        output_fields.extend(
+            field for field in true_mixed_fields if field not in output_fields
+        )
+
+    for row in [*base_rows, *decode_rows]:
+        if "is_true_mixed_batch" in output_fields:
+            row["is_true_mixed_batch"] = "False"
+
+    output_root.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(
+        prefix=f".{output_root.name}.", dir=output_root.parent
+    ) as temporary:
+        temporary_root = Path(temporary) / output_root.name
+        shutil.copytree(base_root, temporary_root)
+        merged_attention = temporary_root / ATTENTION_RELATIVE_PATH
+        with merged_attention.open("w", newline="") as output:
+            writer = csv.DictWriter(output, fieldnames=output_fields, lineterminator="\n")
+            writer.writeheader()
+            writer.writerows([*base_rows, *decode_rows, *true_mixed_rows])
+
+        payload = {
+            "schema": "frontier-profile-closure-v1",
+            "model": MODEL,
+            "base_profile_root": str(base_root),
+            "base_attention_sha256": sha256(base_attention),
+            "decode_attention_csv": str(decode_csv),
+            "decode_attention_sha256": sha256(decode_csv),
+            "base_prefill_rows": len(base_rows),
+            "decode_source_rows": len(decode_source_rows),
+            "added_decode_rows": len(decode_rows),
+            "true_mixed_attention_csv": (
+                str(true_mixed_csv) if true_mixed_csv is not None else None
+            ),
+            "true_mixed_attention_sha256": (
+                sha256(true_mixed_csv) if true_mixed_csv is not None else None
+            ),
+            "added_true_mixed_rows": len(true_mixed_rows),
+            "merged_attention_rows": (
+                len(base_rows) + len(decode_rows) + len(true_mixed_rows)
+            ),
+            "decode_dimensions": [list(values) for values in sorted(dimensions)],
+            "output_files_sha256": profile_hashes(temporary_root),
+        }
+        write_json(temporary_root / "profile_closure_manifest.json", payload)
+        temporary_root.rename(output_root)
+
+    print(json.dumps(payload, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/audit_trace_contract.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Audit Qwen235B trace token lengths and source prefix-hash identities."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import platform
+import socket
+import time
+from pathlib import Path
+from typing import Any
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--trace", type=Path, required=True)
+    parser.add_argument("--model", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--max-model-len", type=int, default=40960)
+    parser.add_argument("--source-block-size", type=int, default=64)
+    parser.add_argument("--batch-size", type=int, default=16)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    if args.max_model_len <= 0 or args.source_block_size <= 0 or args.batch_size <= 0:
+        raise ValueError("length and batch-size arguments must be positive")
+
+    import transformers
+    from transformers import AutoTokenizer
+
+    rows = [json.loads(line) for line in args.trace.open() if line.strip()]
+    context_exceeded = [
+        row
+        for row in rows
+        if int(row["input_length"]) + int(row["output_length"]) > args.max_model_len
+    ]
+    zero_output = [row for row in rows if int(row["output_length"]) == 0]
+    eligible = [
+        row
+        for row in rows
+        if int(row["input_length"]) + int(row["output_length"]) <= args.max_model_len
+        and int(row["output_length"]) > 0
+    ]
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    length_mismatch_count = 0
+    hash_count_mismatch_count = 0
+    hash_to_key: dict[str, bytes] = {}
+    key_to_hash: dict[bytes, str] = {}
+    hash_to_key_conflict_count = 0
+    key_to_hash_conflict_count = 0
+    total_tokens = 0
+    full_blocks = 0
+    partial_blocks = 0
+    length_digest = hashlib.sha256()
+    token_digest = hashlib.sha256()
+    started = time.time()
+
+    for start in range(0, len(eligible), args.batch_size):
+        batch = eligible[start : start + args.batch_size]
+        encoded = tokenizer(
+            [row["prompt"] for row in batch],
+            add_special_tokens=False,
+            padding=False,
+            truncation=False,
+        )["input_ids"]
+        for offset, (row, token_ids) in enumerate(zip(batch, encoded, strict=True)):
+            row_index = start + offset
+            actual_length = len(token_ids)
+            expected_length = int(row["input_length"])
+            total_tokens += actual_length
+            length_digest.update(f"{row_index}:{actual_length}\n".encode())
+            if actual_length != expected_length:
+                length_mismatch_count += 1
+
+            source_hashes = row["hash_ids"]
+            expected_hashes = math.ceil(actual_length / args.source_block_size)
+            if len(source_hashes) != expected_hashes:
+                hash_count_mismatch_count += 1
+                continue
+
+            request_token_digest = hashlib.sha256()
+            parent = b"ROOT"
+            for block_index, source_hash in enumerate(source_hashes):
+                begin = block_index * args.source_block_size
+                chunk = token_ids[begin : begin + args.source_block_size]
+                token_payload = b"".join(
+                    int(token_id).to_bytes(4, "little", signed=False)
+                    for token_id in chunk
+                )
+                request_token_digest.update(token_payload)
+                chunk_digest = hashlib.blake2b(token_payload, digest_size=16).digest()
+                key_digest = hashlib.blake2b(
+                    parent + b"\0" + chunk_digest, digest_size=16
+                ).digest()
+                if len(chunk) == args.source_block_size:
+                    full_blocks += 1
+                else:
+                    partial_blocks += 1
+
+                hash_id = str(source_hash)
+                previous_key = hash_to_key.setdefault(hash_id, key_digest)
+                if previous_key != key_digest:
+                    hash_to_key_conflict_count += 1
+                previous_hash = key_to_hash.setdefault(key_digest, hash_id)
+                if previous_hash != hash_id:
+                    key_to_hash_conflict_count += 1
+                parent = hash_id.encode()
+
+            token_digest.update(row_index.to_bytes(4, "little"))
+            token_digest.update(request_token_digest.digest())
+
+    payload: dict[str, Any] = {
+        "schema": "qwen235b-trace-contract-audit-v1",
+        "status": "pass_offline_source_contract"
+        if not any(
+            (
+                length_mismatch_count,
+                hash_count_mismatch_count,
+                hash_to_key_conflict_count,
+                key_to_hash_conflict_count,
+            )
+        )
+        else "fail",
+        "execution": {
+            "host": socket.gethostname(),
+            "device": "cpu_only",
+            "elapsed_seconds": round(time.time() - started, 3),
+            "python_version": platform.python_version(),
+            "tokenizer_class": type(tokenizer).__name__,
+            "transformers_version": transformers.__version__,
+            "model_path": str(args.model.resolve()),
+        },
+        "trace": {
+            "path": str(args.trace.resolve()),
+            "sha256": sha256_file(args.trace),
+            "source_request_count": len(rows),
+            "context_exceeded_count": len(context_exceeded),
+            "zero_output_count": len(zero_output),
+            "exclusion_overlap_count": sum(row in zero_output for row in context_exceeded),
+            "eligible_request_count": len(eligible),
+        },
+        "tokenization": {
+            "total_token_count": total_tokens,
+            "input_length_mismatch_count": length_mismatch_count,
+            "length_order_sha256": length_digest.hexdigest(),
+            "per_request_token_digest_sha256": token_digest.hexdigest(),
+        },
+        "source_hash_contract": {
+            "source_block_size_tokens": args.source_block_size,
+            "hash_count_mismatch_count": hash_count_mismatch_count,
+            "full_block_count": full_blocks,
+            "partial_block_count": partial_blocks,
+            "unique_hash_id_count": len(hash_to_key),
+            "unique_parent_chunk_key_count": len(key_to_hash),
+            "hash_id_to_parent_chunk_conflict_count": hash_to_key_conflict_count,
+            "parent_chunk_to_hash_id_conflict_count": key_to_hash_conflict_count,
+            "key_definition": (
+                "(parent source hash id, BLAKE2b-128 of the tokenizer token-id chunk)"
+            ),
+        },
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    if payload["status"] != "pass_offline_source_contract":
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/fleet.toml

@@ -0,0 +1,24 @@
+version = 1
+
+[paths]
+state_dir = "runs/frontier-multicase-sufficiency-v1/fleet-state"
+artifacts_dir = "runs/frontier-multicase-sufficiency-v1/fleet-artifacts"
+
+[ssh]
+connect_timeout_sec = 10
+
+[scheduler]
+gpu_free_memory_mb = 1024
+gpu_free_utilization_pct = 10
+prefer_pack = true
+
+[sync]
+mode = "scp"
+local_path = "runs/frontier-multicase-sufficiency-v1"
+
+[[hosts]]
+name = "dash0"
+ssh_alias = "dash0"
+enabled = true
+sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1"
+fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-v1"

runs/frontier-multicase-sufficiency-v1/generate_t0_real_jobs.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Render the fleet queue for a frozen T0 real-surface plan."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def quoted(value: object) -> str:
+    return json.dumps(str(value))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--plan", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--config", action="append", dest="configs")
+    parser.add_argument("--rates", nargs="+", type=float)
+    parser.add_argument("--artifact-root", default="artifacts/t0-real-surface-v1")
+    parser.add_argument("--name-suffix", default="")
+    args = parser.parse_args()
+    plan = json.loads(args.plan.read_text())
+    if plan.get("schema") != "qwen235b-t0-real-plan-v1" or len(plan.get("cells") or []) != 8:
+        raise ValueError("invalid or incomplete T0 real plan")
+    lattice = {0.10, 0.20, 0.40, 0.80, 1.20, 1.60, 2.40, 3.20}
+    if args.rates and any(rate not in lattice for rate in args.rates):
+        raise ValueError("rate override must stay on the frozen T0 lattice")
+
+    indexed_cells = list(enumerate(plan["cells"]))
+    if args.configs:
+        requested = set(args.configs)
+        known = {cell["config"]["name"] for _, cell in indexed_cells}
+        if not requested <= known:
+            raise ValueError(f"unknown configs: {sorted(requested - known)}")
+        indexed_cells = [
+            (index, cell)
+            for index, cell in indexed_cells
+            if cell["config"]["name"] in requested
+        ]
+
+    lines = [
+        "# Generated from the frozen T0 real plan; do not edit rates in place.",
+        "version = 1",
+        "",
+    ]
+    for index, cell in indexed_cells:
+        config = cell["config"]
+        suffix = f"-{args.name_suffix}" if args.name_suffix else ""
+        name = f"qwen235b-t0-real-{config['name']}{suffix}-20260716-v1"
+        artifact = f"{args.artifact_root.rstrip('/')}/{config['name']}"
+        rates = args.rates or cell["rates"]
+        lines.extend(
+            [
+                "[[jobs]]",
+                f"name = {quoted(name)}",
+                f"gpus = {int(config['tp'])}",
+                'gpu_model = "H20"',
+                'hosts = ["dash0"]',
+                'command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"',
+                f"artifacts = [{quoted(artifact)}]",
+                "",
+                "[jobs.env]",
+                f"OUTPUT_ROOT = {quoted('/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/' + artifact)}",
+                f"TP = {quoted(config['tp'])}",
+                f"MNS = {quoted(config['mns'])}",
+                f"MBT = {quoted(config['mbt'])}",
+                f"RATES = {quoted(' '.join(f'{rate:.2f}' for rate in rates))}",
+                f"SERVER_PORT = {quoted(18920 + index)}",
+                'VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"',
+                'MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"',
+                "",
+            ]
+        )
+    args.output.write_text("\n".join(lines))
+    print(args.output)
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/jobs.toml

@@ -0,0 +1,43 @@
+# Append-only queue for the simulator-fidelity v1 campaign.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-tp4-smoke-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash run_t0_smoke.sh"
+artifacts = ["artifacts/t0-smoke-20260716"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-smoke-20260716"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6"
+SERVER_PORT = "18910"
+
+[[jobs]]
+name = "qwen235b-decode-attention-profile-20260716-v1"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=30s 900 bash run_decode_attention_profile.sh"
+artifacts = ["artifacts/decode-attention-profile-20260716"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/decode-attention-profile-20260716"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3"
+
+[[jobs]]
+name = "qwen235b-t0-full-attention-profile-20260716-v1"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=30s 900 bash run_t0_full_attention_profile.sh"
+artifacts = ["artifacts/t0-full-attention-profile-20260716"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-full-attention-profile-20260716"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3"

runs/frontier-multicase-sufficiency-v1/jobs_decode_profile.toml

@@ -0,0 +1,16 @@
+# One-shot queue view for the next authorized profile job. The canonical
+# append-only campaign queue remains jobs.toml.
+version = 1
+
+[[jobs]]
+name = "qwen235b-decode-attention-profile-20260716-v1"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=30s 900 bash run_decode_attention_profile.sh"
+artifacts = ["artifacts/decode-attention-profile-20260716"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/decode-attention-profile-20260716"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3"

runs/frontier-multicase-sufficiency-v1/jobs_t0_anchor_isolation_smoke.toml

@@ -0,0 +1,37 @@
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-anchor-isolation-smoke-tp4-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-anchor-isolation-smoke-v1/tp4_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-anchor-isolation-smoke-v1/tp4_mns64_mbt8192"
+TP = "4"
+MNS = "64"
+MBT = "8192"
+RATES = "1.60"
+SERVER_PORT = "18930"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-anchor-isolation-smoke-tp4-mbt16k-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-anchor-isolation-smoke-v1/tp4_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-anchor-isolation-smoke-v1/tp4_mns64_mbt16384"
+TP = "4"
+MNS = "64"
+MBT = "16384"
+RATES = "1.60"
+SERVER_PORT = "18931"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_full_profile.toml

@@ -0,0 +1,15 @@
+# One-shot queue view for the T0 full-coverage attention profile.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-full-attention-profile-20260716-v1"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=30s 900 bash run_t0_full_attention_profile.sh"
+artifacts = ["artifacts/t0-full-attention-profile-20260716"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-full-attention-profile-20260716"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_batch_tp4_mns128.toml

@@ -0,0 +1,38 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns128_mbt8192-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns128_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns128_mbt8192"
+TP = "4"
+MNS = "128"
+MBT = "8192"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18922"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns128_mbt16384-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns128_mbt16384"
+TP = "4"
+MNS = "128"
+MBT = "16384"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18923"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_batch_tp4_mns64.toml

@@ -0,0 +1,38 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns64_mbt8192"
+TP = "4"
+MNS = "64"
+MBT = "8192"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18920"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns64_mbt16384"
+TP = "4"
+MNS = "64"
+MBT = "16384"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18921"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_batch_tp8_mns128_mbt16384.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt16384-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns128_mbt16384"
+TP = "8"
+MNS = "128"
+MBT = "16384"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18927"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_batch_tp8_mns128_mbt8192.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt8192-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns128_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns128_mbt8192"
+TP = "8"
+MNS = "128"
+MBT = "8192"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18926"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_batch_tp8_mns64_mbt16384.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt16384-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns64_mbt16384"
+TP = "8"
+MNS = "64"
+MBT = "16384"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18925"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_batch_tp8_mns64_mbt8192.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt8192-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns64_mbt8192"
+TP = "8"
+MNS = "64"
+MBT = "8192"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18924"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp4_mns128_r3p20.toml

@@ -0,0 +1,38 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns128_mbt8192-expansion-r3p20-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r3p20-v1/tp4_mns128_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r3p20-v1/tp4_mns128_mbt8192"
+TP = "4"
+MNS = "128"
+MBT = "8192"
+RATES = "3.20"
+SERVER_PORT = "18922"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns128_mbt16384-expansion-r3p20-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r3p20-v1/tp4_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r3p20-v1/tp4_mns128_mbt16384"
+TP = "4"
+MNS = "128"
+MBT = "16384"
+RATES = "3.20"
+SERVER_PORT = "18923"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp4_mns64_r3p20.toml

@@ -0,0 +1,38 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns64_mbt8192-expansion-r3p20-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r3p20-v1/tp4_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r3p20-v1/tp4_mns64_mbt8192"
+TP = "4"
+MNS = "64"
+MBT = "8192"
+RATES = "3.20"
+SERVER_PORT = "18920"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns64_mbt16384-expansion-r3p20-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r3p20-v1/tp4_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r3p20-v1/tp4_mns64_mbt16384"
+TP = "4"
+MNS = "64"
+MBT = "16384"
+RATES = "3.20"
+SERVER_PORT = "18921"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp8_mns128_mbt16384_r2p40.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt16384-expansion-r2p40-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r2p40-v1/tp8_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r2p40-v1/tp8_mns128_mbt16384"
+TP = "8"
+MNS = "128"
+MBT = "16384"
+RATES = "2.40"
+SERVER_PORT = "18927"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp8_mns128_mbt16384_r3p20.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt16384-expansion-r3p20-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r3p20-v1/tp8_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r3p20-v1/tp8_mns128_mbt16384"
+TP = "8"
+MNS = "128"
+MBT = "16384"
+RATES = "3.20"
+SERVER_PORT = "18927"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp8_mns128_mbt8192_r2p40.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt8192-expansion-r2p40-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r2p40-v1/tp8_mns128_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r2p40-v1/tp8_mns128_mbt8192"
+TP = "8"
+MNS = "128"
+MBT = "8192"
+RATES = "2.40"
+SERVER_PORT = "18926"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp8_mns64_mbt16384_r2p40.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt16384-expansion-r2p40-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r2p40-v1/tp8_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r2p40-v1/tp8_mns64_mbt16384"
+TP = "8"
+MNS = "64"
+MBT = "16384"
+RATES = "2.40"
+SERVER_PORT = "18925"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp8_mns64_mbt8192_r2p40.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt8192-expansion-r2p40-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r2p40-v1/tp8_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r2p40-v1/tp8_mns64_mbt8192"
+TP = "8"
+MNS = "64"
+MBT = "8192"
+RATES = "2.40"
+SERVER_PORT = "18924"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_expansion_tp8_mns64_mbt8192_r3p20.toml

@@ -0,0 +1,20 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt8192-expansion-r3p20-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-expansion-r3p20-v1/tp8_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-expansion-r3p20-v1/tp8_mns64_mbt8192"
+TP = "8"
+MNS = "64"
+MBT = "8192"
+RATES = "3.20"
+SERVER_PORT = "18924"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/jobs_t0_real_surface.toml

@@ -0,0 +1,146 @@
+# Generated from the frozen T0 real plan; do not edit rates in place.
+version = 1
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns64_mbt8192"
+TP = "4"
+MNS = "64"
+MBT = "8192"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18920"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns64_mbt16384"
+TP = "4"
+MNS = "64"
+MBT = "16384"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18921"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns128_mbt8192-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns128_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns128_mbt8192"
+TP = "4"
+MNS = "128"
+MBT = "8192"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18922"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp4_mns128_mbt16384-20260716-v1"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp4_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp4_mns128_mbt16384"
+TP = "4"
+MNS = "128"
+MBT = "16384"
+RATES = "0.10 1.60 2.40"
+SERVER_PORT = "18923"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt8192-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns64_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns64_mbt8192"
+TP = "8"
+MNS = "64"
+MBT = "8192"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18924"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns64_mbt16384-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns64_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns64_mbt16384"
+TP = "8"
+MNS = "64"
+MBT = "16384"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18925"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt8192-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns128_mbt8192"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns128_mbt8192"
+TP = "8"
+MNS = "128"
+MBT = "8192"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18926"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+
+[[jobs]]
+name = "qwen235b-t0-real-tp8_mns128_mbt16384-20260716-v1"
+gpus = 8
+gpu_model = "H20"
+hosts = ["dash0"]
+command = "timeout --signal=TERM --kill-after=60s 7200 bash run_t0_real_config.sh"
+artifacts = ["artifacts/t0-real-surface-v1/tp8_mns128_mbt16384"]
+
+[jobs.env]
+OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/t0-real-surface-v1/tp8_mns128_mbt16384"
+TP = "8"
+MNS = "128"
+MBT = "16384"
+RATES = "0.10 1.20 1.60"
+SERVER_PORT = "18927"
+VENV_ROOT = "/tmp/wjh-frontier-vllm0102-smoke/.venv"
+MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"

runs/frontier-multicase-sufficiency-v1/prepare_t0_real_plan.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Select predeclared simulator-lattice anchors for blind real confirmation."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+from typing import Any
+
+
+RATES = (0.10, 0.20, 0.40, 0.80, 1.20, 1.60, 2.40, 3.20)
+CONFIG_NAMES = {
+    f"tp{tp}_mns{mns}_mbt{mbt}"
+    for tp in (4, 8)
+    for mns in (64, 128)
+    for mbt in (8192, 16384)
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--frontier-freeze", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--selection-slo", default="tpot_150ms")
+    return parser.parse_args()
+
+
+def sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def select_rates(loads: list[dict[str, Any]], slo: str) -> list[float]:
+    ordered = sorted(loads, key=lambda item: float(item["offered_request_rate"]))
+    rates = [float(item["offered_request_rate"]) for item in ordered]
+    feasible = [bool(item["slos"][slo]["feasible"]) for item in ordered]
+    selected = {rates[0]}
+    for index in range(len(rates) - 1):
+        if feasible[index] != feasible[index + 1]:
+            selected.update((rates[index], rates[index + 1]))
+    if len(selected) == 1:
+        if all(feasible):
+            selected.update(rates[-2:])
+        elif not any(feasible):
+            selected.update(rates[:2])
+    return sorted(selected)
+
+
+def warmup_requests(rate: float) -> int:
+    return min(32, max(4, math.ceil(rate * 20.0)))
+
+
+def main() -> None:
+    args = parse_args()
+    freeze_path = args.frontier_freeze.resolve()
+    freeze = json.loads(freeze_path.read_text())
+    if freeze.get("schema") != "frontier-qwen235b-t0-surface-v1":
+        raise ValueError("unexpected Frontier freeze schema")
+    if freeze.get("status") != "frozen_before_real_surface":
+        raise ValueError("Frontier surface is not frozen")
+    results = freeze.get("config_results") or []
+    if len(results) != 8 or any(len(item.get("loads") or []) != 8 for item in results):
+        raise ValueError("Frontier surface is incomplete")
+    names = {item.get("config", {}).get("name") for item in results}
+    if names != CONFIG_NAMES:
+        raise ValueError(f"Frontier config set mismatch: {names}")
+    for item in results:
+        rates = tuple(sorted(float(load["offered_request_rate"]) for load in item["loads"]))
+        if rates != RATES:
+            raise ValueError(f"Frontier rate lattice mismatch for {item['config']['name']}: {rates}")
+
+    cells = []
+    total_expected_seconds = 0.0
+    for item in results:
+        config = item["config"]
+        rates = select_rates(item["loads"], args.selection_slo)
+        # Every anchor gets an independent server in both rounds.  The estimate
+        # includes server startup, target-rate warmup and conservative drain
+        # allowances for both the discarded and measured request streams.
+        expected_seconds = 2 * sum(
+            120.0
+            + (warmup_requests(rate) - 1) / rate
+            + 60.0
+            + 63.0 / rate
+            + 60.0
+            for rate in rates
+        )
+        total_expected_seconds += expected_seconds * int(config["tp"])
+        cells.append(
+            {
+                "config": config,
+                "rates": rates,
+                "rounds": 2,
+                "requests_per_anchor": 64,
+                "anchor_isolation": "fresh_server_per_rate_per_round",
+                "target_rate_warmup_requests": {
+                    f"{rate:.2f}": warmup_requests(rate) for rate in rates
+                },
+                "expected_wall_seconds": expected_seconds,
+                "expected_h20_gpu_hours": expected_seconds * int(config["tp"]) / 3600.0,
+                "expansion_rule": "if real labels do not bracket a transition, expand to the next frozen lattice anchor and repeat both directions",
+            }
+        )
+    payload = {
+        "schema": "qwen235b-t0-real-plan-v1",
+        "frontier_freeze": {"path": str(freeze_path), "sha256": sha256(freeze_path)},
+        "selection_slo": args.selection_slo,
+        "selection_timing": "rate anchors frozen after complete simulator surface and before any accepted real surface cell",
+        "execution_protocol_amendment": {
+            "timing": "after excluded multi-rate-per-server diagnostic and before any accepted real surface cell",
+            "reason": "observed cross-anchor GPU/kernel/batch warm-state leakage",
+            "contract": "fresh server and target-rate warmup for every config-rate-round",
+        },
+        "strict_preregistered_slo": "tpot_40ms",
+        "post_pilot_sensitivities": ["tpot_120ms", "tpot_150ms", "tpot_180ms"],
+        "cells": cells,
+        "expected_total_h20_gpu_hours": total_expected_seconds / 3600.0,
+        "hard_timeout_hours_per_cell": 2.0,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps({"cells": len(cells), "expected_total_h20_gpu_hours": payload["expected_total_h20_gpu_hours"]}, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/protocol.md

@@ -0,0 +1,123 @@
+# Qwen235B simulator fidelity 预注册协议 v1
+
+状态：`IN PROGRESS`。更新日期：2026-07-16。T0 的 simulator 与 real 8-config surface 已完成；T1 和 T2 尚未运行。pilot 单独记录在 [t0-smoke-report.md](t0-smoke-report.md)，最终 T0 rank evaluation 记录在 [comparison.json](results/t0-final/comparison.json) 和根目录 [simulator-fidelity.md](../../simulator-fidelity.md)。
+
+## Research question 与成功标准
+
+对同一 workload、SLO 和候选 config surface，Frontier 是否能找到真机上低 regret 的 config，而不要求绝对 latency 或 capacity 完全一致？
+
+主目标统一为：
+
+```text
+capacity(c) = max { 被真实测试的 offered req/s | 至少 95% requests 满足全部 SLO }
+score(c) = capacity(c) / config 实际占用的 GPU 数
+```
+
+每个 case 都报告 real/sim capacity、top set、worst tie-break regret、Kendall tau-b、informative-pair direction、anchor-level SLO confusion，以及为达到该 fidelity 使用的 profile、patch 和真机 calibration 成本。只有同时满足下列条件，才能说 Frontier 对该 compatibility envelope 是足够的 config ranker：
+
+- worst selected-config regret 不超过 5%；
+- tie-aware rank correlation 不低于 0.8，且有足够的 non-tied pairs；
+- ground-truth capacity bracket 不足以反转最优决策；
+- 不使用被评测 workload/config 的端到端测量做 per-action calibration。
+
+## 冻结的平台与软件边界
+
+| 项目 | 设置 |
+|---|---|
+| machine | 仅 `dash0`，8×NVIDIA H20；不得调度到 `dash1` |
+| remote repo | `/home/admin/cpfs/wjh/aituner/aituner`；实验使用独立 clean worktree/clone，不修改当前 dirty checkout |
+| model | `/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8` |
+| serving | community vLLM `0.10.2` isolated environment，eager，BF16 KV；包版本与二进制 hash 入 manifest |
+| simulator | Frontier upstream commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + 明示、hash 后的 FP8/MoE/trace patches |
+| trace | `thinking_w20260327_1000.jsonl`，SHA256 `f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2` |
+| source window | 600 s，15,479 requests，natural offered rate 25.798 req/s |
+
+每个 real trial 使用 fresh server。第一轮 config 顺序由固定 seed 随机化，第二轮反序；同一 config 的 offered-load anchors 从低到高和从高到低各跑一次。任何 OOM、engine crash、timeout 和右删失 cell 都作为结果保留，不能静默丢弃。
+
+## Trace fidelity contract
+
+source row 同时包含 exact prompt、arrival timestamp、`input_length`、`output_length`、session/turn/parent、`sampling_u`，以及 block-size=64 的 `hash_ids`。主 trace case 不设置 output override，也不按 input/output length 选择“好跑”的 cohort。
+
+community model 的最大 context 是 40,960 tokens。预先冻结的 eligible universe 如下：
+
+| universe | requests | input mean/p50/p95 | output mean/p50/p95 | total mean/p50/p95 |
+|---|---:|---:|---:|---:|
+| source | 15,479 | 3,660.0 / 1,491 / 19,610.6 | 3,924.6 / 3,435 / 8,945.1 | 7,584.6 / 5,317 / 25,102.1 |
+| eligible | 15,401 | 3,575.0 / 1,490 / 18,887 | 3,823.3 / 3,417 / 8,768 | 7,398.2 / 5,305 / 23,697 |
+
+唯一排除项是 72 个 `input_length + output_length > 40960` rows，以及 6 个 `output_length = 0` rows；二者无重叠。eligible 比例为 99.50%。
+
+capacity search 使用 `sampling_u <= u` 调整 offered rate。`sampling_u` 在同一 session 内共享，因此这是与长度无关的 session-coherent thinning，而不是 workload filtering。对每个 anchor，real 与 sim 必须具有相同的 source row IDs、arrival/order hash、input/output vector hash、session vector hash 和 prefix vector hash。每个被选 request 仍使用原始 arrival、prompt、input、output 和 hash。
+
+第一个 correctness gate 已完成：在 `dash0` 上使用 community model 的 `Qwen2TokenizerFast` 全量处理 15,401 个 prompts、55,057,919 tokens，实际 token length 与 trace `input_length` 的 mismatch 为 0；在 852,407 个完整和 15,131 个末尾 partial source blocks 上，`hash_id ↔ (parent_hash_id, 64-token chunk)` 均无冲突。结果和 digest 记录在 [trace-contract-audit.json](trace-contract-audit.json)。这说明 exact prompt/source hash 可以表达原 trace 的 prefix-equivalence relation。
+
+在启动 GPU 前还需完成第二个 correctness gate：
+
+1. real 与 Frontier 的 KV block size 都冻结为 16。source `hash_ids` 的 block size 是 64，因此应从 exact prompt token IDs 生成 block-16 content/parent identities，并验证每四个完整 block-16 对应的 64-token source equality/reuse relation 没有改变。两侧逐 request 比较 computed/hit/allocated block counts。
+
+## T0：fixed-shape sanity case
+
+T0 是最简单的机制隔离基线，不声称代表 production trace。
+
+| 项目 | 设置 |
+|---|---|
+| requests | deterministic 生成；ISL=2,048、OSL=128 |
+| arrivals | uniform QPS；相同 request IDs/order；每个 anchor 至少 64 个 completed requests |
+| prefix | off；不同 token content，保证没有共享完整 KV blocks |
+| SLO | TTFT `<= 1000 ms + 1000×input_tokens/8000`（本 case 为 1,256 ms），TPOT `<= 40 ms`，joint pass rate ≥0.95 |
+| configs | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}`；沿用已闭合的 TP4/TP8 MoE mapping |
+
+先在 simulator 中用宽 anchor lattice 找到每个 config 的边界，再冻结 simulator output 和 hash；真机只运行相同 anchors。T0 回答的是：在没有 length variance、prefix reuse 和 burstiness 时，profile composition 与 scheduler 是否已经能保持 config rank。它不能用于回答真实 trace fidelity。
+
+首次真机 smoke 之后、完整 surface 之前预先记录如下 amendment：TP4 的 zero-queue TPOT 已经是 136.2 ms，因此 40 ms 主 SLO 对至少 TP4 没有非零 capacity。40 ms 仍保留为 preregistered primary，不因结果不可行而删除；同时固定报告 TPOT 120/150/180 ms sensitivity。150 ms 是 post-pilot 的 decision-bearing sensitivity，因为它略高于已观测 TP4 idle floor，又仍可能在 batch/queue 增大时产生 capacity knee；它不能冒充 blind primary result。
+
+完整 simulator lattice 固定为 system offered rate `{0.10,0.20,0.40,0.80,1.20,1.60,2.40,3.20}` req/s，每个点 64 requests。所有 8×8 simulator cells 必须在任何 real surface cell 之前冻结。真机对每个 config 先测 simulator 在 150 ms sensitivity 下的相邻 transition anchors，并始终加入最低 0.10 req/s anchor；若没有 transition，则测边界方向的两个 anchors。若这些 real labels 没有 bracket，按冻结 lattice 向外扩展，且 simulator 结果不得重算。每个被测 anchor 做两个 fresh-server rounds，rate 顺序正向/反向，所有 strict/sensitivity SLO 均在同一 raw requests 上离线打分。真机 anchor 只有在两个 round 都达到 95% joint pass rate 时才保守地记为 feasible；两个 round 标签不一致时同时报告原始 pass rates，并把该点记为重复性不确定而不是择优取样。
+
+第一次多-rate 真机 run 后、接受任何 real surface cell 之前增加一项 execution amendment。原 runner 每个 round 只启动一次 server，正向 `0.10→1.60→2.40` 时 1.60 的前 24 个 requests 出现 114--234 ms 的递减 TPOT transient；反向先运行 2.40 后，同一 1.60 anchor 从第一个 request 起稳定在 113--128 ms。两个独立 TP4 config 同时复现，说明前一 rate 的 GPU/kernel/batch warm state 泄漏到后一 rate，而 Frontier 的各 rate 是独立 simulator run。该诊断 run 整体排除，不作为 ground truth。修正后的每个 `(config, rate, round)` 使用独立 fresh server，并在测量前按同一 target rate 丢弃 `min(32, max(4, ceil(rate×20)))` 个 exact-shape warmup requests；随后从空 scheduler queue 开始记录 64 requests。正/逆序只用于复测和控制机器时间漂移，不再共享 execution state。这个 amendment 是观察 diagnostic transient 后作出的，必须在结果中明示，不能写成原始 blind protocol。
+
+steady-QPS profile closure 不能只含 pure decode。完整 profile 预先覆盖 TP `{4,8}`、decode batch `{1,2,4,8,16,32,64,96,128}`、KV 2,048--2,175，并加入 prefill-chunk=2,048 的 true mixed prefill+decode rows。最终 attention root 含 726 个既有 prefill rows、162 个 standard decode rows 和 216 个 true-mixed rows；没有使用 T0 端到端 latency 做 calibration。
+
+## T1：trace-faithful mixed case
+
+T1 直接使用上述 15,401-row eligible universe：不设置 completion override，不改变入选 request 的 input/output，不做 length-stratified sampling，prefix caching 在 real 与 sim 两侧同时打开。
+
+| 项目 | 设置 |
+|---|---|
+| selection | source `sampling_u <= u`；同 session 一起入选；real/sim 共用 frozen anchor files |
+| arrival | 原 timestamp、600 s window、原 burst/order；`replay_time_scale=1.0` |
+| output | `min_tokens=max_tokens=output_length`；记录实际 usage，必须与 trace 相等 |
+| prefix | exact prompt token blocks；两侧相同 block size、capacity、cache policy 与 sticky session routing |
+| SLO | 与 T0 相同的 TTFT 规则和 TPOT 40 ms；另报告 TPOT 20/50 ms sensitivity，不用 sensitivity 改选主结论 |
+| configs | 与 T0 相同的 8 cells，先隔离 workload state，再扩展 DP/EP topology |
+
+由于 natural QPS 远高于该模型 capacity，直接全量同时到达只会让所有 config 都 infeasible，不能产生排序信息。这里 `sampling_u` 是 trace 原生的负载抽样维度；它只改变入选 session 数量，不改变入选请求的 joint distribution 和字段。每个边界至少用两个预先冻结的 session-hash folds 或两个独立 trace windows 重复，避免一个低-u 小样本偶然决定 ranking；若暂时只有一个 window，结论标为 single-window evidence。
+
+## T2：Qwen235B strict decode-only case
+
+T2 不用“短 prefill + 长 output”冒充 decode-only。它要求一个可检查的 initial-KV contract：request 到达 scheduler 时，与 input token IDs 对应的 KV blocks 已经 resident，request 状态为 prefill complete，首次被调度的 token 是第一个 decode token。
+
+两侧 contract 必须共同记录：initial KV token/block count、block identities、replica placement、填充/传输开始结束时间、首次 scheduler admission time。若真机 connector 在 GPU 上同步填充 dummy KV，这项带宽和同步成本会干扰并发 decode；Frontier 必须显式建模这段 admission cost，或真机把它移到计时区间之前。不能只忽略 TTFT，就假设 connector 没有影响 TPOT。
+
+T2 分两步，避免把 execution mechanisms 混成一个不可解释的 gap：
+
+| 层次 | Workload / mechanisms | Config surface | 目的 |
+|---|---|---|---|
+| T2a controlled decode | fixed ISL=2,048、OSL=512；BF16 KV；eager；prefix/speculation/CUDA graph off | `TP∈{4,8} × MNS∈{64,128} × MBT∈{256,384}`，DP=1 | 验证 initial-KV、decode attention/batching 与 TP rank |
+| T2b topology stress | trace exact input/output；EP8；仍先关闭 EAGLE3 与 decode graph | `(TP4,DP2,EP8)` 与 `(TP2,DP4,EP8)` × MNS `{64,128}` × MBT `{256,384}` | 激活 replica count、expert communication、KV residency 的耦合 |
+
+只有 T2a/T2b 对齐后，才按单变量顺序加入 FP8 KV、EAGLE3、DeepEP/NVSHMEM 和 `FULL_DECODE_ONLY` CUDA graph；每加入一项都重新冻结 simulator ranking 并测真机。现有 internal-vLLM decode surface 同时打开了这些机制，而且 8 个 config 的 capacity brackets 全部可能包含最优值，只作为 historical real-only evidence，不作为本协议 ground truth。
+
+## Run order 与停止条件
+
+1. 完成 tokenizer、row vector、prefix-block correctness audit；不使用 GPU。tokenizer/source-hash 离线部分已完成，block-16 runtime-counter parity 待完成。
+2. 在 dash0 做一次 community vLLM TP4 model-load + T0 one-request smoke。已完成：server 与 exact 2,048/128-token requests 成功，但最低负载 TPOT 为 136.2 ms，40 ms primary SLO 不可行；对应 Frontier run 暴露 decode attention profile 未闭合。
+3. 完成 T0 的 simulator 8-cell surface并冻结 SHA256；随后运行 T0 real anchors。已完成：64/64 simulator cells 冻结，8/8 real config boundaries 在 150 ms decision-bearing SLO 下闭合。
+4. 实现并单测 trace block translation，完成 T1 simulator freeze，再运行 T1 real anchors。
+5. 实现 community-vLLM 0.10.2 与 Frontier 的 initial-KV contract；一请求和并发请求状态机 smoke 通过后，运行 T2a。
+6. 只有 profile closure 包含 EP8 all-to-all 且 T2a 可解释时，运行 T2b。
+
+每层遇到 mismatch，先按 request/stage 分解 queue wait、prefill、decode step、collective、MoE 和 KV admission residual。只有证据定位到某一 composition invariant 失效，才修改 simulator；不通过端到端 scalar 把 ranking 调到正确。
+
+## 首次 GPU launch gate
+
+首次 GPU smoke、full attention closure 与完整 T0 sweep 均已于 2026-07-16 在 `dash0` 完成，GPU 已释放。接受的 T0 ground truth 为 68 个 fresh-server anchors，消耗 36.26 H20-GPU-hours；污染和 cross-anchor-state diagnostic attempts 已隔离。任何 T1/T2 GPU launch 都必须重新 echo resolved workload、artifact paths、GPU 预算与预计时长。

runs/frontier-multicase-sufficiency-v1/results/t0-final/capacity.csv

@@ -0,0 +1,33 @@
+slo,config,tp,mns,mbt,real_capacity_per_gpu,sim_capacity_per_gpu,real_boundary_status,expansion_required
+tpot_40ms,tp4_mns64_mbt8192,4,64,8192,,,lowest_anchor_infeasible,False
+tpot_40ms,tp4_mns64_mbt16384,4,64,16384,,,lowest_anchor_infeasible,False
+tpot_40ms,tp4_mns128_mbt8192,4,128,8192,,,lowest_anchor_infeasible,False
+tpot_40ms,tp4_mns128_mbt16384,4,128,16384,,,lowest_anchor_infeasible,False
+tpot_40ms,tp8_mns64_mbt8192,8,64,8192,,,lowest_anchor_infeasible,False
+tpot_40ms,tp8_mns64_mbt16384,8,64,16384,,,lowest_anchor_infeasible,False
+tpot_40ms,tp8_mns128_mbt8192,8,128,8192,,,lowest_anchor_infeasible,False
+tpot_40ms,tp8_mns128_mbt16384,8,128,16384,,,lowest_anchor_infeasible,False
+tpot_120ms,tp4_mns64_mbt8192,4,64,8192,0.025,0.2,unbracketed_requires_expansion,True
+tpot_120ms,tp4_mns64_mbt16384,4,64,16384,0.025,0.2,unbracketed_requires_expansion,True
+tpot_120ms,tp4_mns128_mbt8192,4,128,8192,0.025,0.2,unbracketed_requires_expansion,True
+tpot_120ms,tp4_mns128_mbt16384,4,128,16384,0.025,0.2,unbracketed_requires_expansion,True
+tpot_120ms,tp8_mns64_mbt8192,8,64,8192,0.0125,0.05,unbracketed_requires_expansion,True
+tpot_120ms,tp8_mns64_mbt16384,8,64,16384,0.0125,0.05,unbracketed_requires_expansion,True
+tpot_120ms,tp8_mns128_mbt8192,8,128,8192,0.0125,0.05,unbracketed_requires_expansion,True
+tpot_120ms,tp8_mns128_mbt16384,8,128,16384,0.0125,0.05,unbracketed_requires_expansion,True
+tpot_150ms,tp4_mns64_mbt8192,4,64,8192,0.6,0.4,adjacent_transition_bracketed,False
+tpot_150ms,tp4_mns64_mbt16384,4,64,16384,0.6,0.4,adjacent_transition_bracketed,False
+tpot_150ms,tp4_mns128_mbt8192,4,128,8192,0.6,0.4,adjacent_transition_bracketed,False
+tpot_150ms,tp4_mns128_mbt16384,4,128,16384,0.6,0.4,adjacent_transition_bracketed,False
+tpot_150ms,tp8_mns64_mbt8192,8,64,8192,0.3,0.15,adjacent_transition_bracketed,False
+tpot_150ms,tp8_mns64_mbt16384,8,64,16384,0.2,0.15,adjacent_transition_bracketed,False
+tpot_150ms,tp8_mns128_mbt8192,8,128,8192,0.2,0.15,adjacent_transition_bracketed,False
+tpot_150ms,tp8_mns128_mbt16384,8,128,16384,0.3,0.15,adjacent_transition_bracketed,False
+tpot_180ms,tp4_mns64_mbt8192,4,64,8192,0.8,0.6,upper_lattice_reached,False
+tpot_180ms,tp4_mns64_mbt16384,4,64,16384,0.8,0.6,upper_lattice_reached,False
+tpot_180ms,tp4_mns128_mbt8192,4,128,8192,0.8,0.6,upper_lattice_reached,False
+tpot_180ms,tp4_mns128_mbt16384,4,128,16384,0.8,0.6,upper_lattice_reached,False
+tpot_180ms,tp8_mns64_mbt8192,8,64,8192,0.4,0.2,upper_lattice_reached,False
+tpot_180ms,tp8_mns64_mbt16384,8,64,16384,0.3,0.2,unbracketed_requires_expansion,True
+tpot_180ms,tp8_mns128_mbt8192,8,128,8192,0.3,0.2,unbracketed_requires_expansion,True
+tpot_180ms,tp8_mns128_mbt16384,8,128,16384,0.4,0.2,upper_lattice_reached,False

runs/frontier-multicase-sufficiency-v1/run_decode_attention_profile.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:-$(pwd)/artifacts/decode-attention-profile-20260716}"
+FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh-frontier-vllm0102-smoke/.venv}"
+PROFILE_ROOT="${OUTPUT_ROOT}/profiles"
+LOG_DIR="${OUTPUT_ROOT}/logs"
+PROVENANCE_DIR="${OUTPUT_ROOT}/provenance"
+MODEL="Qwen3-235B-A22B-FP8"
+
+mkdir -p "${PROFILE_ROOT}" "${LOG_DIR}" "${PROVENANCE_DIR}"
+exec > >(tee -a "${LOG_DIR}/profile.log") 2>&1
+
+if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
+  echo "ERROR: CUDA_VISIBLE_DEVICES must contain the fleet-allocated GPU" >&2
+  exit 1
+fi
+IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}"
+if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
+  echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
+  exit 1
+fi
+
+echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} operator=FlashInfer_attention phase=decode TP_workers=4 batch_sizes=1,2 kv_sizes=2048,2176 block=16 measurement=CUDA_EVENT output=${OUTPUT_ROOT} expected_wall=5-10m hard_wall=900s hard_gpu_cap=0.25_H20h"
+date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
+nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader
+
+test -x "${VENV_ROOT}/bin/python"
+test -f "${FRONTIER_ROOT}/pyproject.toml"
+test -f "${FRONTIER_ROOT}/data/config/models/${MODEL}.json"
+sha256sum run_decode_attention_profile.sh > "${PROVENANCE_DIR}/source.sha256"
+
+export PYTHONPATH="${FRONTIER_ROOT}"
+export TOKENIZERS_PARALLELISM=false
+export TORCH_CUDA_ARCH_LIST=9.0
+
+cd "${FRONTIER_ROOT}"
+timeout --signal=TERM --kill-after=30s 600 \
+  "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
+  --disable_ray \
+  --models "${MODEL}" \
+  --num_gpus 1 \
+  --max_model_len 40960 \
+  --max_seq_len 2176 \
+  --min_batch_size 1 \
+  --max_batch_size 2 \
+  --batch_size_list 1 2 \
+  --decode_kv_cache_size_list 2048 2176 \
+  --num_tensor_parallel_workers 4 \
+  --max_pipeline_parallel_size 1 \
+  --attention_backend FLASHINFER \
+  --block_size 16 \
+  --profile_only_decode \
+  --device h20 \
+  --profile_method cuda_event \
+  --output_dir "${PROFILE_ROOT}" \
+  --yes
+
+ATTENTION_CSV="${PROFILE_ROOT}/compute/h20/${MODEL}/attention.csv"
+test -s "${ATTENTION_CSV}"
+"${VENV_ROOT}/bin/python" - "${ATTENTION_CSV}" \
+  > "${PROVENANCE_DIR}/coverage.json" <<'PY'
+import json
+import sys
+
+import pandas as pd
+
+path = sys.argv[1]
+frame = pd.read_csv(path)
+decode = frame[frame["is_prefill"] == False]  # noqa: E712
+payload = {
+    "path": path,
+    "row_count": len(frame),
+    "decode_row_count": len(decode),
+    "batch_sizes": sorted(int(value) for value in decode["batch_size"].unique()),
+    "kv_cache_sizes": sorted(int(value) for value in decode["kv_cache_size"].unique()),
+    "attn_decode_median_non_null": int(
+        decode["time_stats.attn_decode.median"].notna().sum()
+    ),
+}
+print(json.dumps(payload, indent=2, sort_keys=True))
+if payload["decode_row_count"] < 4 or payload["attn_decode_median_non_null"] < 4:
+    raise SystemExit(1)
+PY
+
+sha256sum \
+  "${ATTENTION_CSV}" \
+  "${PROVENANCE_DIR}/coverage.json" \
+  "${PROVENANCE_DIR}/source.sha256" \
+  > "${PROVENANCE_DIR}/artifacts.sha256"
+nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader
+date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
+echo "DECODE_ATTENTION_PROFILE_COMPLETE"

runs/frontier-multicase-sufficiency-v1/run_frontier_t0_smoke.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+"""Run Frontier on the exact single/concurrency-2 T0 smoke workload."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import os
+import subprocess
+import time
+from pathlib import Path
+from typing import Any
+
+
+MODEL = "Qwen3-235B-A22B-FP8"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--frontier-source", type=Path, required=True)
+    parser.add_argument("--profile-root", type=Path, required=True)
+    parser.add_argument("--python", type=Path, default=Path("/usr/bin/python3.12"))
+    parser.add_argument("--output-root", type=Path, required=True)
+    return parser.parse_args()
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def write_trace(path: Path, request_count: int) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="") as output:
+        writer = csv.DictWriter(
+            output,
+            fieldnames=[
+                "arrived_at",
+                "num_prefill_tokens",
+                "num_decode_tokens",
+                "slo_ttft_ms",
+            ],
+        )
+        writer.writeheader()
+        for _ in range(request_count):
+            writer.writerow(
+                {
+                    "arrived_at": 0.0,
+                    "num_prefill_tokens": 2048,
+                    "num_decode_tokens": 128,
+                    "slo_ttft_ms": 1256.0,
+                }
+            )
+
+
+def build_command(
+    *,
+    args: argparse.Namespace,
+    trace: Path,
+    run_dir: Path,
+    run_id: str,
+) -> list[str]:
+    compute_root = args.profile_root / "compute/h20" / MODEL
+    network = args.profile_root / "network/h20_nccl/all_reduce.csv"
+    return [
+        str(args.python),
+        "-m",
+        "frontier.main",
+        "--simulation_mode",
+        "offline",
+        "--offline_use_generated_request_arrivals",
+        "--sys_arch",
+        "co-location",
+        "--cluster_config_num_replicas",
+        "1",
+        "--replica_config_model_name",
+        MODEL,
+        "--replica_config_attn_tensor_parallel_size",
+        "4",
+        "--replica_config_attn_data_parallel_size",
+        "1",
+        "--replica_config_moe_tensor_parallel_size",
+        "4",
+        "--replica_config_moe_expert_parallel_size",
+        "1",
+        "--replica_config_total_expert_num",
+        "128",
+        "--replica_config_router_topk",
+        "8",
+        "--replica_config_moe_routing_mode",
+        "simulation",
+        "--replica_config_moe_routing_seed",
+        "42",
+        "--replica_config_num_pipeline_stages",
+        "1",
+        "--replica_config_device",
+        "h20",
+        "--replica_config_network_device",
+        "h20_dgx",
+        "--cc_backend_config_type",
+        "vidur",
+        "--vidur_cc_backend_config_profiling_data_dir",
+        str(args.profile_root),
+        "--vidur_cc_backend_config_cache_dir",
+        str(run_dir / "cache/collectives"),
+        "--vidur_cc_backend_config_all_reduce_input_file",
+        str(network),
+        "--replica_scheduler_config_type",
+        "vllm_v1",
+        "--decode_cuda_graph_mode",
+        "none",
+        "--vllm_v1_scheduler_config_batch_size_cap",
+        "64",
+        "--vllm_v1_scheduler_config_block_size",
+        "16",
+        "--vllm_v1_scheduler_config_num_blocks",
+        "26101",
+        "--vllm_v1_scheduler_config_num_blocks_mode",
+        "explicit",
+        "--vllm_v1_scheduler_config_max_tokens_in_batch",
+        "8192",
+        "--vllm_v1_scheduler_config_enable_chunked_prefill",
+        "--no-vllm_v1_scheduler_config_enable_prefix_caching",
+        "--request_generator_config_type",
+        "trace_replay",
+        "--trace_request_generator_config_trace_file",
+        str(trace),
+        "--trace_request_generator_config_time_scale_factor",
+        "1",
+        "--trace_request_generator_config_prefill_scale_factor",
+        "1",
+        "--trace_request_generator_config_decode_scale_factor",
+        "1",
+        "--trace_request_generator_config_max_tokens",
+        "40960",
+        "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode",
+        "--random_forrest_execution_time_predictor_config_linear_op_input_file",
+        str(compute_root / "linear_op.csv"),
+        "--random_forrest_execution_time_predictor_config_atten_input_file",
+        str(compute_root / "attention.csv"),
+        "--random_forrest_execution_time_predictor_config_moe_input_file",
+        str(compute_root / "moe.csv"),
+        "--random_forrest_execution_time_predictor_config_all_reduce_input_file",
+        str(network),
+        "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size",
+        "16384",
+        "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request",
+        "40960",
+        "--random_forrest_execution_time_predictor_config_prediction_max_batch_size",
+        "128",
+        "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling",
+        "--metrics_config_cache_dir",
+        str(run_dir / "cache/execution"),
+        "--metrics_config_output_dir",
+        str(run_dir / "metrics"),
+        "--metrics_config_run_id",
+        run_id,
+        "--metrics_config_write_metrics",
+        "--metrics_config_store_request_metrics",
+        "--no-metrics_config_store_plots",
+        "--no-metrics_config_enable_chrome_trace",
+        "--no-metrics_config_write_json_trace",
+    ]
+
+
+def find_request_metrics(run_dir: Path) -> Path:
+    matches = list((run_dir / "metrics").rglob("request_metrics.csv"))
+    if len(matches) != 1:
+        raise RuntimeError(f"expected one request_metrics.csv, found {matches}")
+    return matches[0]
+
+
+def score(metrics_path: Path) -> dict[str, Any]:
+    with metrics_path.open(newline="") as source:
+        rows = list(csv.DictReader(source))
+    requests = []
+    for row in rows:
+        ttft_ms = float(row["ttft"])
+        e2e_ms = float(row["request_e2e_time"])
+        decode_tokens = int(float(row["request_num_decode_tokens"]))
+        aligned_tpot_ms = (
+            (e2e_ms - ttft_ms) / (decode_tokens - 1) if decode_tokens > 1 else 0.0
+        )
+        requests.append(
+            {
+                "request_id": int(row["Request Id"]),
+                "prompt_tokens": int(float(row["request_num_prefill_tokens"])),
+                "completion_tokens": decode_tokens,
+                "ttft_ms": ttft_ms,
+                "e2e_ms": e2e_ms,
+                "tpot_ms_aligned": aligned_tpot_ms,
+                "frontier_decode_e2e_time_per_token_ms": float(
+                    row["decode_e2e_time_per_token"]
+                ),
+                "joint_slo_pass": ttft_ms <= 1256.0 and aligned_tpot_ms <= 40.0,
+            }
+        )
+    return {
+        "request_count": len(requests),
+        "joint_slo_pass_count": sum(row["joint_slo_pass"] for row in requests),
+        "requests": requests,
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    source = args.frontier_source.resolve()
+    profiles = args.profile_root.resolve()
+    output = args.output_root.resolve()
+    required = [
+        source / "pyproject.toml",
+        profiles / f"compute/h20/{MODEL}/linear_op.csv",
+        profiles / f"compute/h20/{MODEL}/attention.csv",
+        profiles / f"compute/h20/{MODEL}/moe.csv",
+        profiles / "network/h20_nccl/all_reduce.csv",
+    ]
+    missing = [str(path) for path in required if not path.is_file()]
+    if missing:
+        raise FileNotFoundError(missing)
+
+    results: dict[str, Any] = {}
+    for label, count in (("single", 1), ("concurrency2", 2)):
+        run_dir = output / label
+        trace = run_dir / "trace.csv"
+        write_trace(trace, count)
+        command = build_command(
+            args=args,
+            trace=trace,
+            run_dir=run_dir,
+            run_id=f"t0_{label}",
+        )
+        write_json(run_dir / "command.json", command)
+        env = dict(os.environ)
+        env["PYTHONPATH"] = str(source)
+        started = time.time()
+        with (run_dir / "stdout.log").open("w") as stdout:
+            completed = subprocess.run(
+                command,
+                cwd=source,
+                env=env,
+                stdout=stdout,
+                stderr=subprocess.STDOUT,
+                check=False,
+                text=True,
+                timeout=300,
+            )
+        if completed.returncode != 0:
+            raise RuntimeError(
+                f"Frontier {label} failed with {completed.returncode}; "
+                f"see {run_dir / 'stdout.log'}"
+            )
+        metrics = find_request_metrics(run_dir)
+        results[label] = {
+            "elapsed_seconds": time.time() - started,
+            "trace_sha256": sha256(trace),
+            "request_metrics_path": str(metrics),
+            "request_metrics_sha256": sha256(metrics),
+            **score(metrics),
+        }
+
+    profile_files = required[1:]
+    payload = {
+        "schema": "frontier-qwen235b-t0-smoke-v1",
+        "contract": {
+            "topology": "TP4/DP1/MoE-TP4/EP1",
+            "mns": 64,
+            "mbt": 8192,
+            "block_size": 16,
+            "num_gpu_blocks": 26101,
+            "prefix_caching": False,
+            "input_tokens": 2048,
+            "output_tokens": 128,
+            "arrivals": "all at t=0",
+        },
+        "frontier": {
+            "source": str(source),
+            "declared_upstream_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
+            "python_and_config_tree_sha256": "172fc7ae19c40e67030208ff488d0d5d90764888ed76a7a543be6037ba62dc11",
+        },
+        "profiles": {str(path): sha256(path) for path in profile_files},
+        "results": results,
+    }
+    write_json(output / "summary.json", payload)
+    print(json.dumps(payload["results"], indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/run_frontier_t0_surface.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""Freeze the full fixed-shape T0 Frontier response surface."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import math
+import os
+import subprocess
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+
+MODEL = "Qwen3-235B-A22B-FP8"
+FRONTIER_DECLARED_UPSTREAM_COMMIT = "d9cfeb6d8791fbf2f295dd9744c56a666171776e"
+FRONTIER_PYTHON_CONFIG_TREE_SHA256 = "172fc7ae19c40e67030208ff488d0d5d90764888ed76a7a543be6037ba62dc11"
+RATES = (0.10, 0.20, 0.40, 0.80, 1.20, 1.60, 2.40, 3.20)
+TPOT_SLOS_MS = (40.0, 120.0, 150.0, 180.0)
+TTFT_SLO_MS = 1256.0
+TARGET_PASS_RATE = 0.95
+
+
+@dataclass(frozen=True)
+class Config:
+    tp: int
+    mns: int
+    mbt: int
+    moe_tp: int
+    moe_ep: int
+    num_gpu_blocks: int
+
+    @property
+    def name(self) -> str:
+        return f"tp{self.tp}_mns{self.mns}_mbt{self.mbt}"
+
+
+GRID = tuple(
+    Config(
+        tp=tp,
+        mns=mns,
+        mbt=mbt,
+        moe_tp=4 if tp == 4 else 1,
+        moe_ep=1 if tp == 4 else 8,
+        num_gpu_blocks=26101 if tp == 4 else 62351,
+    )
+    for tp in (4, 8)
+    for mns in (64, 128)
+    for mbt in (8192, 16384)
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--frontier-source", type=Path, required=True)
+    parser.add_argument("--profile-root", type=Path, required=True)
+    parser.add_argument("--python", type=Path, required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    parser.add_argument("--requests", type=int, default=64)
+    parser.add_argument("--rate", type=float, action="append")
+    parser.add_argument("--config", action="append")
+    parser.add_argument("--subprocess-timeout-seconds", type=int, default=1800)
+    return parser.parse_args()
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    os.replace(temporary, path)
+
+
+def rate_key(rate: float) -> str:
+    return f"r{rate:.2f}".replace(".", "p")
+
+
+def write_trace(path: Path, *, request_count: int, rate: float) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="") as output:
+        writer = csv.DictWriter(
+            output,
+            fieldnames=[
+                "arrived_at",
+                "num_prefill_tokens",
+                "num_decode_tokens",
+                "slo_ttft_ms",
+            ],
+        )
+        writer.writeheader()
+        for request_id in range(request_count):
+            writer.writerow(
+                {
+                    "arrived_at": f"{request_id / rate:.12f}",
+                    "num_prefill_tokens": 2048,
+                    "num_decode_tokens": 128,
+                    "slo_ttft_ms": TTFT_SLO_MS,
+                }
+            )
+
+
+def profile_paths(root: Path) -> dict[str, Path]:
+    compute = root / "compute/h20" / MODEL
+    paths = {
+        "linear": compute / "linear_op.csv",
+        "attention": compute / "attention.csv",
+        "moe": compute / "moe.csv",
+        "all_reduce": root / "network/h20_nccl/all_reduce.csv",
+        "closure_manifest": root / "profile_closure_manifest.json",
+    }
+    missing = [str(path) for path in paths.values() if not path.is_file()]
+    if missing:
+        raise FileNotFoundError(missing)
+    return paths
+
+
+def validate_attention_coverage(path: Path) -> dict[str, Any]:
+    with path.open(newline="") as source:
+        rows = list(csv.DictReader(source))
+    standard_decode = [
+        row
+        for row in rows
+        if row["is_prefill"].lower() == "false"
+        and row.get("is_true_mixed_batch", "").lower() != "true"
+    ]
+    true_mixed = [
+        row for row in rows if row.get("is_true_mixed_batch", "").lower() == "true"
+    ]
+    standard_by_tp = {
+        str(tp): sum(int(row["num_tensor_parallel_workers"]) == tp for row in standard_decode)
+        for tp in (4, 8)
+    }
+    mixed_by_tp = {
+        str(tp): sum(int(row["num_tensor_parallel_workers"]) == tp for row in true_mixed)
+        for tp in (4, 8)
+    }
+    if standard_by_tp != {"4": 81, "8": 81}:
+        raise ValueError(f"standard decode coverage mismatch: {standard_by_tp}")
+    if mixed_by_tp != {"4": 108, "8": 108}:
+        raise ValueError(f"true-mixed coverage mismatch: {mixed_by_tp}")
+    return {
+        "rows": len(rows),
+        "standard_decode_by_tp": standard_by_tp,
+        "true_mixed_by_tp": mixed_by_tp,
+    }
+
+
+def build_command(
+    args: argparse.Namespace,
+    paths: dict[str, Path],
+    config: Config,
+    trace: Path,
+    run_dir: Path,
+) -> list[str]:
+    cache = args.output_root / "cache" / f"tp{config.tp}"
+    return [
+        str(args.python), "-m", "frontier.main",
+        "--simulation_mode", "offline",
+        "--offline_use_generated_request_arrivals",
+        "--sys_arch", "co-location",
+        "--cluster_config_num_replicas", "1",
+        "--replica_config_model_name", MODEL,
+        "--replica_config_attn_tensor_parallel_size", str(config.tp),
+        "--replica_config_attn_data_parallel_size", "1",
+        "--replica_config_moe_tensor_parallel_size", str(config.moe_tp),
+        "--replica_config_moe_expert_parallel_size", str(config.moe_ep),
+        "--replica_config_total_expert_num", "128",
+        "--replica_config_router_topk", "8",
+        "--replica_config_moe_routing_mode", "simulation",
+        "--replica_config_moe_routing_seed", "42",
+        "--replica_config_num_pipeline_stages", "1",
+        "--replica_config_device", "h20",
+        "--replica_config_network_device", "h20_dgx",
+        "--cc_backend_config_type", "vidur",
+        "--vidur_cc_backend_config_profiling_data_dir", str(args.profile_root),
+        "--vidur_cc_backend_config_cache_dir", str(cache / "collectives"),
+        "--vidur_cc_backend_config_all_reduce_input_file", str(paths["all_reduce"]),
+        "--replica_scheduler_config_type", "vllm_v1",
+        "--decode_cuda_graph_mode", "none",
+        "--vllm_v1_scheduler_config_batch_size_cap", str(config.mns),
+        "--vllm_v1_scheduler_config_block_size", "16",
+        "--vllm_v1_scheduler_config_num_blocks", str(config.num_gpu_blocks),
+        "--vllm_v1_scheduler_config_num_blocks_mode", "explicit",
+        "--vllm_v1_scheduler_config_max_tokens_in_batch", str(config.mbt),
+        "--vllm_v1_scheduler_config_enable_chunked_prefill",
+        "--no-vllm_v1_scheduler_config_enable_prefix_caching",
+        "--request_generator_config_type", "trace_replay",
+        "--trace_request_generator_config_trace_file", str(trace),
+        "--trace_request_generator_config_time_scale_factor", "1",
+        "--trace_request_generator_config_prefill_scale_factor", "1",
+        "--trace_request_generator_config_decode_scale_factor", "1",
+        "--trace_request_generator_config_max_tokens", "40960",
+        "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode",
+        "--random_forrest_execution_time_predictor_config_linear_op_input_file", str(paths["linear"]),
+        "--random_forrest_execution_time_predictor_config_atten_input_file", str(paths["attention"]),
+        "--random_forrest_execution_time_predictor_config_moe_input_file", str(paths["moe"]),
+        "--random_forrest_execution_time_predictor_config_all_reduce_input_file", str(paths["all_reduce"]),
+        "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size", "16384",
+        "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request", "40960",
+        "--random_forrest_execution_time_predictor_config_prediction_max_batch_size", "128",
+        "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling",
+        "--metrics_config_cache_dir", str(cache / "execution"),
+        "--metrics_config_output_dir", str(run_dir / "metrics"),
+        "--metrics_config_run_id", f"{config.name}_{run_dir.name}",
+        "--metrics_config_write_metrics",
+        "--metrics_config_store_request_metrics",
+        "--no-metrics_config_store_plots",
+        "--no-metrics_config_enable_chrome_trace",
+        "--no-metrics_config_write_json_trace",
+    ]
+
+
+def find_metrics(run_dir: Path) -> Path:
+    matches = list((run_dir / "metrics").rglob("request_metrics.csv"))
+    if len(matches) != 1:
+        raise RuntimeError(f"expected one request_metrics.csv, got {matches}")
+    return matches[0]
+
+
+def score(metrics: Path, expected_requests: int) -> dict[str, Any]:
+    with metrics.open(newline="") as source:
+        rows = list(csv.DictReader(source))
+    if len(rows) != expected_requests:
+        raise ValueError(f"request count mismatch: {len(rows)} != {expected_requests}")
+    requests = []
+    for row in rows:
+        prompt = int(float(row["request_num_prefill_tokens"]))
+        completion = int(float(row["request_num_decode_tokens"]))
+        ttft = float(row["ttft"])
+        e2e = float(row["request_e2e_time"])
+        tpot = (e2e - ttft) / (completion - 1)
+        if prompt != 2048 or completion != 128:
+            raise ValueError("request shape drift")
+        if not all(math.isfinite(value) and value >= 0 for value in (ttft, e2e, tpot)):
+            raise ValueError("non-finite or negative latency")
+        requests.append({"request_id": int(row["Request Id"]), "ttft_ms": ttft, "tpot_ms": tpot, "e2e_ms": e2e})
+    slos = {}
+    for limit in TPOT_SLOS_MS:
+        passed = sum(row["ttft_ms"] <= TTFT_SLO_MS and row["tpot_ms"] <= limit for row in requests)
+        pass_rate = passed / len(requests)
+        slos[f"tpot_{int(limit)}ms"] = {
+            "passed": passed,
+            "pass_rate": pass_rate,
+            "feasible": pass_rate >= TARGET_PASS_RATE,
+        }
+    return {"requests": requests, "slos": slos}
+
+
+def main() -> None:
+    args = parse_args()
+    args.frontier_source = args.frontier_source.resolve()
+    args.profile_root = args.profile_root.resolve()
+    # A venv interpreter is commonly a symlink to the system executable.
+    # Keep the venv path so Python discovers that environment's site-packages.
+    args.python = args.python.absolute()
+    args.output_root = args.output_root.resolve()
+    if args.requests < 2:
+        raise ValueError("requests must be at least two")
+    rates = tuple(args.rate or RATES)
+    if any(rate <= 0 for rate in rates) or len(set(rates)) != len(rates):
+        raise ValueError("rates must be positive and unique")
+    selected = list(GRID)
+    if args.config:
+        names = set(args.config)
+        selected = [config for config in GRID if config.name in names]
+        if {config.name for config in selected} != names:
+            raise ValueError(f"unknown configs: {names - {config.name for config in selected}}")
+
+    paths = profile_paths(args.profile_root)
+    coverage = validate_attention_coverage(paths["attention"])
+    args.output_root.mkdir(parents=True, exist_ok=True)
+    traces = {}
+    for rate in rates:
+        path = args.output_root / "traces" / f"{rate_key(rate)}.csv"
+        write_trace(path, request_count=args.requests, rate=rate)
+        traces[rate] = path
+
+    config_results = []
+    environment = dict(os.environ)
+    environment.update(
+        {
+            "PYTHONPATH": str(args.frontier_source),
+            "WANDB_DISABLED": "true",
+            "VIDUR_DISABLE_WANDB": "1",
+            # The best-effort source emits a per-layer OP-TRACE at INFO.  It is
+            # diagnostic only and can produce hundreds of MiB per T0 cell.
+            "FRONTIER_LOG_LEVEL": "WARNING",
+        }
+    )
+    for config in selected:
+        loads = []
+        for rate in rates:
+            run_dir = args.output_root / "runs" / config.name / rate_key(rate)
+            result_path = run_dir / "result.json"
+            if result_path.is_file():
+                result = json.loads(result_path.read_text())
+                if result.get("status") == "completed":
+                    loads.append(result)
+                    continue
+            run_dir.mkdir(parents=True, exist_ok=True)
+            command = build_command(args, paths, config, traces[rate], run_dir)
+            write_json(run_dir / "command.json", command)
+            started = time.time()
+            with (run_dir / "stdout.log").open("w") as output:
+                completed = subprocess.run(
+                    command,
+                    cwd=args.frontier_source,
+                    env=environment,
+                    stdout=output,
+                    stderr=subprocess.STDOUT,
+                    timeout=args.subprocess_timeout_seconds,
+                    check=False,
+                    text=True,
+                )
+            if completed.returncode != 0:
+                raise RuntimeError(f"Frontier failed: {config.name} rate={rate}, rc={completed.returncode}")
+            metrics = find_metrics(run_dir)
+            result = {
+                "status": "completed",
+                "config": asdict(config) | {"name": config.name},
+                "offered_request_rate": rate,
+                "request_rate_per_gpu": rate / config.tp,
+                "elapsed_seconds": time.time() - started,
+                "trace_sha256": sha256(traces[rate]),
+                "request_metrics_sha256": sha256(metrics),
+                **score(metrics, args.requests),
+            }
+            write_json(result_path, result)
+            loads.append(result)
+            print(json.dumps({"config": config.name, "rate": rate, "elapsed_seconds": result["elapsed_seconds"], "slos": result["slos"]}, sort_keys=True), flush=True)
+        config_results.append({"config": asdict(config) | {"name": config.name}, "loads": loads})
+
+    rankings = {}
+    for slo in (f"tpot_{int(value)}ms" for value in TPOT_SLOS_MS):
+        records = []
+        for item in config_results:
+            feasible = [load["offered_request_rate"] for load in item["loads"] if load["slos"][slo]["feasible"]]
+            capacity = max(feasible) if feasible else None
+            records.append({
+                "config": item["config"],
+                "maximum_tested_feasible_request_rate": capacity,
+                "maximum_tested_feasible_request_rate_per_gpu": capacity / item["config"]["tp"] if capacity is not None else None,
+                "lower_censored": capacity is None,
+                "upper_censored": capacity == max(rates) if capacity is not None else False,
+            })
+        records.sort(key=lambda row: (-(row["maximum_tested_feasible_request_rate_per_gpu"] if row["maximum_tested_feasible_request_rate_per_gpu"] is not None else -1), row["config"]["name"]))
+        rankings[slo] = records
+
+    is_complete_preregistered_surface = (
+        selected == list(GRID) and rates == RATES and args.requests == 64
+    )
+    manifest = {
+        "schema": "frontier-qwen235b-t0-surface-v1",
+        "status": (
+            "frozen_before_real_surface"
+            if is_complete_preregistered_surface
+            else "partial_surface_not_decision_bearing"
+        ),
+        "contract": {"requests_per_anchor": args.requests, "rates": rates, "input_tokens": 2048, "output_tokens": 128, "ttft_slo_ms": TTFT_SLO_MS, "tpot_slos_ms": TPOT_SLOS_MS, "target_pass_rate": TARGET_PASS_RATE, "prefix_caching": False},
+        "frontier": {
+            "source": str(args.frontier_source),
+            "declared_upstream_commit": FRONTIER_DECLARED_UPSTREAM_COMMIT,
+            "python_and_config_tree_sha256": FRONTIER_PYTHON_CONFIG_TREE_SHA256,
+            "fingerprint_source": "prefill-grid-v3 frozen run manifest for the same immutable source snapshot",
+        },
+        "runner": {"path": str(Path(__file__).resolve()), "sha256": sha256(Path(__file__).resolve())},
+        "profiles": {"root": str(args.profile_root), "coverage": coverage, "files_sha256": {name: sha256(path) for name, path in paths.items()}},
+        "config_results": config_results,
+        "rankings": rankings,
+    }
+    write_json(args.output_root / "frontier_surface_frozen.json", manifest)
+    print(args.output_root / "frontier_surface_frozen.json")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/run_t0_full_attention_profile.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:-$(pwd)/artifacts/t0-full-attention-profile-20260716}"
+FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh-frontier-vllm0102-smoke/.venv}"
+PROFILE_ROOT="${OUTPUT_ROOT}/profiles"
+LOG_DIR="${OUTPUT_ROOT}/logs"
+PROVENANCE_DIR="${OUTPUT_ROOT}/provenance"
+MODEL="Qwen3-235B-A22B-FP8"
+
+mkdir -p "${PROFILE_ROOT}" "${LOG_DIR}" "${PROVENANCE_DIR}"
+exec > >(tee -a "${LOG_DIR}/profile.log") 2>&1
+
+if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
+  echo "ERROR: CUDA_VISIBLE_DEVICES must contain the fleet-allocated GPU" >&2
+  exit 1
+fi
+IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}"
+if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
+  echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
+  exit 1
+fi
+
+echo "FULL_PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} operator=FlashInfer_attention phases=standard_decode,true_mixed TP_workers=4,8 batch_sizes=1,2,4,8,16,32,64,96,128 kv_sizes=2048:2175 true_mixed_prefill_chunk=2048 block=16 measurement=CUDA_EVENT output=${OUTPUT_ROOT} expected_wall=5-10m hard_wall=900s hard_gpu_cap=0.25_H20h"
+date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
+nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader
+
+test -x "${VENV_ROOT}/bin/python"
+test -f "${FRONTIER_ROOT}/pyproject.toml"
+test -f "${FRONTIER_ROOT}/data/config/models/${MODEL}.json"
+sha256sum run_t0_full_attention_profile.sh > "${PROVENANCE_DIR}/source.sha256"
+
+export PYTHONPATH="${FRONTIER_ROOT}"
+export TOKENIZERS_PARALLELISM=false
+export TORCH_CUDA_ARCH_LIST=9.0
+
+cd "${FRONTIER_ROOT}"
+timeout --signal=TERM --kill-after=30s 780 \
+  "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
+  --disable_ray \
+  --models "${MODEL}" \
+  --num_gpus 1 \
+  --max_model_len 40960 \
+  --max_seq_len 2176 \
+  --min_batch_size 1 \
+  --max_batch_size 128 \
+  --batch_size_list 1 2 4 8 16 32 64 96 128 \
+  --decode_kv_cache_size_list 2048 2064 2080 2096 2112 2128 2144 2160 2175 \
+  --num_tensor_parallel_workers 4 8 \
+  --max_pipeline_parallel_size 1 \
+  --attention_backend FLASHINFER \
+  --block_size 16 \
+  --enable_true_mixed \
+  --true_mixed_prefill_batch_sizes 1 2 4 7 \
+  --true_mixed_prefill_chunk_sizes 2048 \
+  --true_mixed_decode_batch_sizes 1 2 4 8 16 32 64 96 124 127 \
+  --true_mixed_decode_kv_cache_sizes 2048 2112 2175 \
+  --true_mixed_prefill_kv_cache_size 0 \
+  --device h20 \
+  --profile_method cuda_event \
+  --output_dir "${PROFILE_ROOT}" \
+  --yes
+
+MODEL_PROFILE_DIR="${PROFILE_ROOT}/compute/h20/${MODEL}"
+STANDARD_CSV="${MODEL_PROFILE_DIR}/attention.csv"
+TRUE_MIXED_CSV="${MODEL_PROFILE_DIR}/attention_true_mixed.csv"
+COMBINED_CSV="${MODEL_PROFILE_DIR}/attention_combined.csv"
+test -s "${STANDARD_CSV}"
+test -s "${TRUE_MIXED_CSV}"
+test -s "${COMBINED_CSV}"
+
+"${VENV_ROOT}/bin/python" - "${STANDARD_CSV}" "${TRUE_MIXED_CSV}" \
+  > "${PROVENANCE_DIR}/coverage.json" <<'PY'
+import json
+import sys
+
+import pandas as pd
+
+standard_path, true_mixed_path = sys.argv[1:]
+standard = pd.read_csv(standard_path)
+true_mixed = pd.read_csv(true_mixed_path)
+decode = standard[standard["is_prefill"] == False]  # noqa: E712
+payload = {
+    "standard_path": standard_path,
+    "standard_rows": len(standard),
+    "decode_rows": len(decode),
+    "decode_rows_by_tp": {
+        str(int(key)): int(value)
+        for key, value in decode.groupby("num_tensor_parallel_workers").size().items()
+    },
+    "decode_batch_sizes": sorted(int(value) for value in decode["batch_size"].unique()),
+    "decode_kv_cache_sizes": sorted(
+        int(value) for value in decode["kv_cache_size"].unique()
+    ),
+    "decode_median_non_null": int(
+        decode["time_stats.attn_decode.median"].notna().sum()
+    ),
+    "true_mixed_path": true_mixed_path,
+    "true_mixed_rows": len(true_mixed),
+    "true_mixed_rows_by_tp": {
+        str(int(key)): int(value)
+        for key, value in true_mixed.groupby("num_tensor_parallel_workers").size().items()
+    },
+    "true_mixed_decode_median_non_null": int(
+        true_mixed["time_stats.attn_decode.median"].notna().sum()
+    ),
+    "true_mixed_prefill_median_non_null": int(
+        true_mixed["time_stats.attn_prefill.median"].notna().sum()
+    ),
+}
+print(json.dumps(payload, indent=2, sort_keys=True))
+expected_batch_sizes = [1, 2, 4, 8, 16, 32, 64, 96, 128]
+expected_kv_sizes = [2048, 2064, 2080, 2096, 2112, 2128, 2144, 2160, 2175]
+if payload["decode_rows_by_tp"] != {"4": 81, "8": 81}:
+    raise SystemExit(1)
+if payload["decode_batch_sizes"] != expected_batch_sizes:
+    raise SystemExit(1)
+if payload["decode_kv_cache_sizes"] != expected_kv_sizes:
+    raise SystemExit(1)
+if payload["decode_median_non_null"] != 162:
+    raise SystemExit(1)
+if set(payload["true_mixed_rows_by_tp"]) != {"4", "8"}:
+    raise SystemExit(1)
+if payload["true_mixed_rows"] < 100:
+    raise SystemExit(1)
+if payload["true_mixed_decode_median_non_null"] != payload["true_mixed_rows"]:
+    raise SystemExit(1)
+if payload["true_mixed_prefill_median_non_null"] != payload["true_mixed_rows"]:
+    raise SystemExit(1)
+PY
+
+sha256sum \
+  "${STANDARD_CSV}" \
+  "${TRUE_MIXED_CSV}" \
+  "${COMBINED_CSV}" \
+  "${PROVENANCE_DIR}/coverage.json" \
+  "${PROVENANCE_DIR}/source.sha256" \
+  > "${PROVENANCE_DIR}/artifacts.sha256"
+nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader
+date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
+echo "T0_FULL_ATTENTION_PROFILE_COMPLETE"

runs/frontier-multicase-sufficiency-v1/run_t0_real_config.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+TP="${TP:?TP is required}"
+MNS="${MNS:?MNS is required}"
+MBT="${MBT:?MBT is required}"
+RATES="${RATES:?RATES is required}"
+SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh-frontier-vllm0102-smoke/.venv}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
+SERVED_MODEL="qwen3-235b-t0-surface"
+SERVER_PID=""
+
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
+exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1
+
+cleanup() {
+  if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+    kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
+    for _ in $(seq 1 30); do
+      kill -0 "${SERVER_PID}" 2>/dev/null || break
+      sleep 1
+    done
+    kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
+  fi
+  SERVER_PID=""
+}
+trap cleanup EXIT INT TERM
+
+IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:-}"
+if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then
+  echo "ERROR: expected ${TP} allocated GPUs, got ${CUDA_VISIBLE_DEVICES:-unset}" >&2
+  exit 1
+fi
+read -r -a RATE_ARRAY <<< "${RATES}"
+if [[ "${#RATE_ARRAY[@]}" -lt 1 ]]; then
+  echo "ERROR: at least one frozen rate is required" >&2
+  exit 1
+fi
+
+echo "REAL_CONFIG_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} config=TP${TP}_MNS${MNS}_MBT${MBT} rates=${RATES// /,} repeats=2 requests_per_anchor=64 isolation=fresh_server_per_anchor target_warmup=min32_max4_ceil_rate_x20 trace=fixed_ISL2048_OSL128 prefix=off runtime=community_vllm_0.10.2 execution=eager kv=BF16 output=${OUTPUT_ROOT}"
+date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
+sha256sum run_t0_real_config.sh t0_rate_client.py > "${OUTPUT_ROOT}/provenance/source.sha256"
+"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${MBT}" "${RATES}" \
+  > "${OUTPUT_ROOT}/provenance/contract.json" <<'PY'
+import importlib.metadata as metadata
+import json
+import platform
+import sys
+
+tp, mns, mbt, rates = sys.argv[1:]
+print(json.dumps({
+    "python": platform.python_version(),
+    "torch": metadata.version("torch"),
+    "transformers": metadata.version("transformers"),
+    "vllm": metadata.version("vllm"),
+    "config": {"tp": int(tp), "mns": int(mns), "mbt": int(mbt)},
+    "rates": [float(value) for value in rates.split()],
+    "rounds": 2,
+    "requests_per_anchor": 64,
+    "anchor_isolation": "fresh_server_per_rate_per_round",
+    "target_rate_warmup_requests": "min(32, max(4, ceil(rate * 20)))",
+    "ttft_slo_ms": 1256.0,
+    "tpot_slos_ms": [40.0, 120.0, 150.0, 180.0],
+}, indent=2, sort_keys=True))
+PY
+nvidia-smi --query-gpu=index,name,uuid,driver_version --format=csv,noheader \
+  > "${OUTPUT_ROOT}/provenance/gpus.csv"
+sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256"
+
+export TOKENIZERS_PARALLELISM=false
+export VLLM_USE_V1=1
+export VLLM_ATTENTION_BACKEND=FLASHINFER
+export TORCH_CUDA_ARCH_LIST=9.0
+
+EXTRA_FLAGS=()
+NUM_BLOCKS=26101
+if [[ "${TP}" -eq 8 ]]; then
+  EXTRA_FLAGS+=(--enable-expert-parallel)
+  NUM_BLOCKS=62351
+fi
+
+for ROUND in 1 2; do
+  ROUND_ROOT="${OUTPUT_ROOT}/round${ROUND}"
+  mkdir -p "${ROUND_ROOT}/logs" "${ROUND_ROOT}/results"
+  ORDERED_RATES=("${RATE_ARRAY[@]}")
+  if [[ "${ROUND}" -eq 2 ]]; then
+    ORDERED_RATES=()
+    for ((index=${#RATE_ARRAY[@]}-1; index>=0; index--)); do ORDERED_RATES+=("${RATE_ARRAY[index]}"); done
+  fi
+  for RATE in "${ORDERED_RATES[@]}"; do
+    KEY="$(printf 'r%.2f' "${RATE}" | tr '.' 'p')"
+    SERVER_LOG="${ROUND_ROOT}/logs/server_${KEY}.log"
+    setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
+      --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \
+      --tensor-parallel-size "${TP}" --disable-custom-all-reduce --quantization fp8 \
+      --gpu-memory-utilization 0.80 --num-gpu-blocks-override "${NUM_BLOCKS}" \
+      --kv-cache-dtype auto --max-model-len 40960 --max-num-batched-tokens "${MBT}" \
+      --max-num-seqs "${MNS}" --no-enable-prefix-caching --enable-chunked-prefill \
+      --enforce-eager --disable-log-requests "${EXTRA_FLAGS[@]}" \
+      > "${SERVER_LOG}" 2>&1 &
+    SERVER_PID=$!
+    READY=0
+    for _ in $(seq 1 180); do
+      if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \
+        > "${ROUND_ROOT}/results/models_${KEY}.json" 2>/dev/null; then
+        READY=1
+        break
+      fi
+      if ! kill -0 "${SERVER_PID}" 2>/dev/null; then tail -200 "${SERVER_LOG}"; exit 1; fi
+      sleep 5
+    done
+    if [[ "${READY}" -ne 1 ]]; then tail -200 "${SERVER_LOG}"; exit 1; fi
+
+    WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY'
+import math
+import sys
+print(min(32, max(4, math.ceil(float(sys.argv[1]) * 20.0))))
+PY
+)"
+    "${VENV_ROOT}/bin/python" t0_rate_client.py --port "${SERVER_PORT}" \
+      --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
+      --requests "${WARMUP_REQUESTS}" \
+      --output "${ROUND_ROOT}/results/warmup_${KEY}.json"
+    "${VENV_ROOT}/bin/python" t0_rate_client.py --port "${SERVER_PORT}" \
+      --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
+      --requests 64 --output "${ROUND_ROOT}/results/${KEY}.json"
+    cleanup
+  done
+done
+
+find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
+  | sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256"
+date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
+echo "T0_REAL_CONFIG_COMPLETE"

runs/frontier-multicase-sufficiency-v1/run_t0_smoke.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:-$(pwd)/artifacts/t0-smoke-20260716}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh-frontier-vllm0102-smoke/.venv}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
+FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6}"
+SERVER_PORT="${SERVER_PORT:-18910}"
+SERVED_MODEL="qwen3-235b-t0-smoke"
+LOG_DIR="${OUTPUT_ROOT}/logs"
+RESULT_DIR="${OUTPUT_ROOT}/results"
+PROVENANCE_DIR="${OUTPUT_ROOT}/provenance"
+SERVER_PID=""
+
+mkdir -p "${LOG_DIR}" "${RESULT_DIR}" "${PROVENANCE_DIR}"
+exec > >(tee -a "${LOG_DIR}/smoke.log") 2>&1
+
+cleanup() {
+  if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+    kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
+    for _ in $(seq 1 30); do
+      if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
+        break
+      fi
+      sleep 1
+    done
+    kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT INT TERM
+
+if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
+  echo "ERROR: CUDA_VISIBLE_DEVICES must contain the four fleet-allocated GPUs" >&2
+  exit 1
+fi
+IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}"
+if [[ "${#GPU_IDS[@]}" -ne 4 ]]; then
+  echo "ERROR: expected exactly four GPUs, got ${CUDA_VISIBLE_DEVICES}" >&2
+  exit 1
+fi
+
+echo "LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} trace=fixed_ISL2048_OSL128 prefix=off qps=single_then_concurrency2 runtime=community_vllm_0.10.2 topology=TP4_DP1 execution=eager kv=BF16 spec=off cuda_graph=off output=${OUTPUT_ROOT} expected_wall=20-30m hard_wall=1800s hard_gpu_cap=2_H20h"
+date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
+nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader
+
+test -x "${VENV_ROOT}/bin/vllm"
+test -x "${VENV_ROOT}/bin/python"
+test -f "${MODEL_ROOT}/config.json"
+test -f "${FRONTIER_ROOT}/pyproject.toml"
+test -f "$(pwd)/t0_smoke_client.py"
+
+sha256sum run_t0_smoke.sh t0_smoke_client.py > "${PROVENANCE_DIR}/source.sha256"
+"${VENV_ROOT}/bin/python" - <<'PY' > "${PROVENANCE_DIR}/environment.json"
+import importlib.metadata as metadata
+import json
+import platform
+
+print(json.dumps({
+    "python": platform.python_version(),
+    "torch": metadata.version("torch"),
+    "transformers": metadata.version("transformers"),
+    "vllm": metadata.version("vllm"),
+}, indent=2, sort_keys=True))
+PY
+
+export TOKENIZERS_PARALLELISM=false
+export VLLM_USE_V1=1
+export VLLM_ATTENTION_BACKEND=FLASHINFER
+export TORCH_CUDA_ARCH_LIST=9.0
+
+echo "STAGE server_start"
+setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
+  --host 127.0.0.1 \
+  --port "${SERVER_PORT}" \
+  --served-model-name "${SERVED_MODEL}" \
+  --tensor-parallel-size 4 \
+  --disable-custom-all-reduce \
+  --quantization fp8 \
+  --gpu-memory-utilization 0.80 \
+  --kv-cache-dtype auto \
+  --max-model-len 40960 \
+  --max-num-batched-tokens 8192 \
+  --max-num-seqs 64 \
+  --no-enable-prefix-caching \
+  --enable-chunked-prefill \
+  --enforce-eager \
+  --disable-log-requests \
+  > "${LOG_DIR}/server.log" 2>&1 &
+SERVER_PID=$!
+
+READY=0
+for _ in $(seq 1 180); do
+  if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \
+    > "${RESULT_DIR}/models.json" 2>/dev/null; then
+    READY=1
+    break
+  fi
+  if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
+    echo "ERROR: vLLM server exited before readiness" >&2
+    tail -200 "${LOG_DIR}/server.log" >&2 || true
+    exit 1
+  fi
+  sleep 5
+done
+if [[ "${READY}" -ne 1 ]]; then
+  echo "ERROR: vLLM server did not become ready within 900 seconds" >&2
+  tail -200 "${LOG_DIR}/server.log" >&2 || true
+  exit 1
+fi
+
+echo "STAGE fixed_shape_single"
+"${VENV_ROOT}/bin/python" t0_smoke_client.py \
+  --port "${SERVER_PORT}" \
+  --served-model "${SERVED_MODEL}" \
+  --model-path "${MODEL_ROOT}" \
+  --input-tokens 2048 \
+  --output-tokens 128 \
+  --concurrency 1 \
+  --requests 1 \
+  --output "${RESULT_DIR}/single.json"
+
+echo "STAGE fixed_shape_concurrency2"
+"${VENV_ROOT}/bin/python" t0_smoke_client.py \
+  --port "${SERVER_PORT}" \
+  --served-model "${SERVED_MODEL}" \
+  --model-path "${MODEL_ROOT}" \
+  --input-tokens 2048 \
+  --output-tokens 128 \
+  --concurrency 2 \
+  --requests 2 \
+  --output "${RESULT_DIR}/concurrency2.json"
+
+cleanup
+SERVER_PID=""
+
+sha256sum \
+  "${PROVENANCE_DIR}/environment.json" \
+  "${PROVENANCE_DIR}/source.sha256" \
+  "${RESULT_DIR}/models.json" \
+  "${RESULT_DIR}/single.json" \
+  "${RESULT_DIR}/concurrency2.json" \
+  > "${PROVENANCE_DIR}/artifacts.sha256"
+nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader
+date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
+echo "T0_SMOKE_COMPLETE"

runs/frontier-multicase-sufficiency-v1/t0-smoke-report.md

@@ -0,0 +1,89 @@
+# Qwen235B T0 fixed-shape smoke report
+
+日期：2026-07-16。状态：real 与 profile-closed Frontier smoke complete；不是 config-rank evaluation。
+
+## Setup
+
+| 项目 | 设置 |
+|---|---|
+| host | `dash0`，仅 GPU 0--3；GPU 4--7 始终空闲 |
+| model | `Qwen/Qwen3-235B-A22B-FP8` |
+| runtime | community vLLM 0.10.2、torch 2.8.0、Transformers 4.55.2 |
+| config | TP4/DP1、MNS64、MBT8192、block16、26,101 KV blocks/GPU |
+| execution | FlashInfer、eager、BF16 KV、prefix/spec/CUDA graph off |
+| workload | exact prompt token IDs；ISL=2,048、OSL=128；single 和两个同时到达 requests |
+| SLO | TTFT ≤1,256 ms、TPOT ≤40 ms，joint pass |
+| GPU cost | 02:33:48--02:36:01 UTC，133 s × 4 H20 = 0.148 H20-hours |
+
+vLLM 报告每卡 model weights 约55.13 GiB、KV capacity 417,616 tokens，以及 `num_gpu_blocks=26101`。这与此前输入 Frontier 的 TP4 KV capacity 一致。
+
+## Real result
+
+| load | request | usage | TTFT | TPOT | E2E | joint pass |
+|---|---:|---|---:|---:|---:|---:|
+| single | 0 | 2,048+128 | 586.23 ms | 136.18 ms | 17,881.30 ms | no |
+| concurrency=2 | 0 | 2,048+128 | 186.09 ms | 124.87 ms | 16,045.24 ms | no |
+| concurrency=2 | 1 | 2,048+128 | 434.29 ms | 123.78 ms | 16,154.03 ms | no |
+
+所有 response 的 usage completion tokens 和逐 chunk token IDs 都是128；不是 EOS 提前退出或 client length-hint 误差。artifact SHA256 全部通过 harvest 后复验。
+
+单请求已经违反40 ms TPOT，说明该 SLO 对 community/eager controlled stack 在最低负载下不可行。concurrency=2 的 batching 将平均 TPOT 改善到124.33 ms，但仍不足以通过。这是 pilot finding，不应用于事后挑选一个恰好产生所需 ranking 的阈值。
+
+## Frontier result
+
+Frontier 使用冻结的 best-effort source：
+
+```text
+upstream commit: d9cfeb6d8791fbf2f295dd9744c56a666171776e
+source/config tree SHA256: 172fc7ae19c40e67030208ff488d0d5d90764888ed76a7a543be6037ba62dc11
+profile root: profiles-best-effort-final-v2
+TP4/DP1/MoE-TP4/EP1, MNS64, MBT8192, block16, 26101 blocks
+```
+
+simulator 正确加载1个请求，执行2,048-token prefill，并提交第一个 decode token。下一轮成为 pure-decode batch 时失败：
+
+```text
+Skipping eager attn_decode training: no standard decode rows
+ValueError: attention decode prediction cache not found for cluster monolithic
+```
+
+profile audit 确认 final attention CSV 有726 rows，但 `is_prefill` 只有 `True`。另外两个已有 Qwen235 attention files 也只有 prefill rows。由于没有产生完整 request metrics，本文不报告 simulator TTFT/TPOT，也不把 crash 当成 SLO failure。使用 dummy/fallback latency 会把缺失 measurement coverage 隐藏成虚假的 simulator prediction。
+
+## Profile closure 与 rerun
+
+在 `dash0` 单张 H20 上，用同一 vLLM 0.10.2/FlashInfer stack 测量 TP4、batch `{1,2}`、KV `{2048,2176}` 的4个 CUDA-event decode-attention points；4/4 rows 的 `attn_decode.median` 非空。它们与原726个 prefill rows 按完全相同的55-column schema 合并到新的 immutable profile root，未覆盖原 profile，也未加入 dummy 或 E2E calibration。
+
+同一 Frontier smoke 随后精确完成1/2个请求：
+
+| load | request | TTFT | TPOT | E2E |
+|---|---:|---:|---:|---:|
+| single | 0 | 267.67 ms | 88.27 ms | 11,478.45 ms |
+| concurrency=2 | 0 | 470.66 ms | 90.54 ms | 11,969.73 ms |
+| concurrency=2 | 1 | 470.66 ms | 90.54 ms | 11,969.73 ms |
+
+请求数、ISL/OSL 与 real contract 完全一致，所有 latency 均有限且非负，representation gate 因而通过。absolute TPOT error 并非固定比例：single 低估约35%，concurrency=2 低估约27%。这组数据只能证明 simulator 现在能表达该 state path，不能证明 config rank 正确。
+
+steady-QPS 下还会出现 prefill+decode true-mixed batches，因此 full surface 没有沿用4-row smoke profile。后续完整 closure 覆盖 TP4/TP8 各81个 standard-decode 与108个 true-mixed points；合并后的 attention root 共1,104 rows，并已确认 `attn_decode_in_mixed` 从每个 TP 的108个真实 samples 训练。
+
+## Interpretation
+
+Change：首次把原 prefill-only compatibility envelope用于有真实 decode tokens 的最简单 fixed-shape case。
+
+Expected effect：如果 profiles 和 execution model 已闭合，Frontier 应至少产生同一 config 的 TTFT/TPOT，之后才能讨论绝对 gap 或 rank。
+
+Verification：real exact-token streaming run；Frontier 同 config/blocks/profile run；profile CSV 与完整 traceback 审计。
+
+Result：real serving 成功；原 Frontier compatibility envelope 在 pure-decode attention 处断裂；补齐最小 measurement coverage 后 smoke 通过，但出现27%--35%的 TPOT absolute error。因此此前的 prefill rank-match 仍不能支持“Frontier 已足够解决 Qwen235 mixed config selection”的外推，必须看完整 config response surface。
+
+Remaining risk：完整 profile 已加入 mixed-attention coverage，但 MoE decode routing context、EP8 communication、batch-dependent TPOT 和 simulator/real action ranking 仍需由 full surface 检验；smoke 只关闭第一个必要缺口，并未证明它是唯一缺口。
+
+## Evidence
+
+- Real single request：[single.json](fleet-artifacts/qwen235b-t0-tp4-smoke-20260716-v1-20260716T023346844896Z/artifacts/artifacts/t0-smoke-20260716/results/single.json)
+- Real concurrency=2：[concurrency2.json](fleet-artifacts/qwen235b-t0-tp4-smoke-20260716-v1-20260716T023346844896Z/artifacts/artifacts/t0-smoke-20260716/results/concurrency2.json)
+- Server log：[server.log](fleet-artifacts/qwen235b-t0-tp4-smoke-20260716-v1-20260716T023346844896Z/artifacts/artifacts/t0-smoke-20260716/logs/server.log)
+- Frontier command：[command.json](frontier-smoke-failure/single/command.json)
+- Frontier traceback：[stdout.log](frontier-smoke-failure/single/stdout.log)
+- Frontier trace：[trace.csv](frontier-smoke-failure/single/trace.csv)
+- Minimal decode profile coverage：[coverage.json](fleet-artifacts/qwen235b-decode-attention-profile-20260716-v1-20260716T030929269383Z/artifacts/artifacts/decode-attention-profile-20260716/provenance/coverage.json)
+- Full attention profile coverage：[coverage.json](fleet-artifacts/qwen235b-t0-full-attention-profile-20260716-v1-20260716T032536186518Z/artifacts/artifacts/t0-full-attention-profile-20260716/provenance/coverage.json)

runs/frontier-multicase-sufficiency-v1/t0_rate_client.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""Open-loop fixed-shape completion workload for one T0 offered-load anchor."""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import hashlib
+import http.client
+import json
+import math
+import statistics
+import time
+from pathlib import Path
+from typing import Any
+
+
+TPOT_SLOS_MS = (40.0, 120.0, 150.0, 180.0)
+TTFT_SLO_MS = 1256.0
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, required=True)
+    parser.add_argument("--served-model", required=True)
+    parser.add_argument("--model-path", type=Path, required=True)
+    parser.add_argument("--rate", type=float, required=True)
+    parser.add_argument("--requests", type=int, default=64)
+    parser.add_argument("--input-tokens", type=int, default=2048)
+    parser.add_argument("--output-tokens", type=int, default=128)
+    parser.add_argument("--timeout-seconds", type=float, default=900.0)
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def percentile(values: list[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, max(0, math.ceil(fraction * len(ordered)) - 1))
+    return ordered[index]
+
+
+def run_request(
+    *,
+    request_index: int,
+    scheduled_at: float,
+    benchmark_start: float,
+    args: argparse.Namespace,
+    prompt_ids: list[int],
+) -> dict[str, Any]:
+    delay = scheduled_at - time.perf_counter()
+    if delay > 0:
+        time.sleep(delay)
+    admitted = time.perf_counter()
+    record: dict[str, Any] = {
+        "request_index": request_index,
+        "scheduled_s": scheduled_at - benchmark_start,
+        "admitted_s": admitted - benchmark_start,
+        "admission_lag_ms": (admitted - scheduled_at) * 1000.0,
+        "success": False,
+    }
+    connection = http.client.HTTPConnection(args.host, args.port, timeout=args.timeout_seconds)
+    body = {
+        "model": args.served_model,
+        "prompt": prompt_ids,
+        "min_tokens": args.output_tokens,
+        "max_tokens": args.output_tokens,
+        "ignore_eos": True,
+        "temperature": 0,
+        "stream": True,
+        "stream_options": {"include_usage": True},
+        "return_token_ids": True,
+    }
+    try:
+        started = time.perf_counter()
+        connection.request(
+            "POST",
+            "/v1/completions",
+            body=json.dumps(body, separators=(",", ":")).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        response = connection.getresponse()
+        if response.status != 200:
+            raise RuntimeError(f"HTTP {response.status}: {response.read().decode(errors='replace')}")
+        first_token_at = None
+        last_token_at = None
+        streamed_tokens = 0
+        usage = None
+        while True:
+            raw = response.readline()
+            if not raw:
+                break
+            line = raw.decode(errors="replace").strip()
+            if not line.startswith("data:"):
+                continue
+            data = line[5:].strip()
+            if data == "[DONE]":
+                break
+            payload = json.loads(data)
+            if payload.get("usage"):
+                usage = payload["usage"]
+            emitted = 0
+            for choice in payload.get("choices") or []:
+                token_ids = choice.get("token_ids") or []
+                emitted += len(token_ids) if token_ids else int(bool(choice.get("text")))
+            if emitted:
+                now = time.perf_counter()
+                first_token_at = first_token_at or now
+                last_token_at = now
+                streamed_tokens += emitted
+        finished = time.perf_counter()
+        if first_token_at is None or last_token_at is None or usage is None:
+            raise RuntimeError("missing streaming tokens or usage")
+        prompt_tokens = int(usage["prompt_tokens"])
+        completion_tokens = int(usage["completion_tokens"])
+        if prompt_tokens != args.input_tokens or completion_tokens != args.output_tokens:
+            raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}")
+        ttft = (first_token_at - started) * 1000.0
+        tpot = (last_token_at - first_token_at) * 1000.0 / (completion_tokens - 1)
+        record.update(
+            {
+                "success": True,
+                "prompt_tokens": prompt_tokens,
+                "completion_tokens": completion_tokens,
+                "streamed_token_count": streamed_tokens,
+                "ttft_ms": ttft,
+                "tpot_ms": tpot,
+                "e2e_ms": (finished - started) * 1000.0,
+            }
+        )
+    except Exception as error:  # Preserve failed requests as SLO failures.
+        record["error"] = f"{type(error).__name__}: {error}"
+    finally:
+        connection.close()
+    return record
+
+
+def main() -> None:
+    args = parse_args()
+    if args.rate <= 0 or args.requests <= 0:
+        raise ValueError("rate and requests must be positive")
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
+    excluded = set(tokenizer.all_special_ids)
+    candidates = [token_id for token_id in range(tokenizer.vocab_size) if token_id not in excluded]
+    if len(candidates) < args.requests + 1:
+        raise RuntimeError("tokenizer has too few non-special token IDs")
+    base_id = candidates[0]
+    prompts = [
+        [candidates[index + 1], *([base_id] * (args.input_tokens - 1))]
+        for index in range(args.requests)
+    ]
+    prompt_hash = hashlib.sha256(
+        "\n".join(",".join(map(str, prompt)) for prompt in prompts).encode()
+    ).hexdigest()
+
+    benchmark_start = time.perf_counter() + 2.0
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.requests) as pool:
+        futures = [
+            pool.submit(
+                run_request,
+                request_index=index,
+                scheduled_at=benchmark_start + index / args.rate,
+                benchmark_start=benchmark_start,
+                args=args,
+                prompt_ids=prompts[index],
+            )
+            for index in range(args.requests)
+        ]
+        requests = [future.result() for future in futures]
+    requests.sort(key=lambda row: int(row["request_index"]))
+    completed = [row for row in requests if row["success"]]
+    slos = {}
+    for limit in TPOT_SLOS_MS:
+        passed = sum(
+            row["success"]
+            and float(row["ttft_ms"]) <= TTFT_SLO_MS
+            and float(row["tpot_ms"]) <= limit
+            for row in requests
+        )
+        slos[f"tpot_{int(limit)}ms"] = {
+            "passed": passed,
+            "pass_rate": passed / len(requests),
+            "feasible": passed / len(requests) >= 0.95,
+        }
+    ttfts = [float(row["ttft_ms"]) for row in completed]
+    tpots = [float(row["tpot_ms"]) for row in completed]
+    payload = {
+        "schema": "qwen235b-t0-rate-anchor-v1",
+        "workload": {
+            "offered_request_rate": args.rate,
+            "request_count": args.requests,
+            "input_tokens": args.input_tokens,
+            "output_tokens": args.output_tokens,
+            "prefix_caching": False,
+            "arrival": "open_loop_uniform",
+            "last_scheduled_arrival_s": (args.requests - 1) / args.rate,
+            "prompt_vector_sha256": prompt_hash,
+        },
+        "summary": {
+            "completed": len(completed),
+            "failed": len(requests) - len(completed),
+            "ttft_p50_ms": percentile(ttfts, 0.50),
+            "ttft_p95_ms": percentile(ttfts, 0.95),
+            "tpot_p50_ms": percentile(tpots, 0.50),
+            "tpot_p95_ms": percentile(tpots, 0.95),
+            "admission_lag_max_ms": max(float(row["admission_lag_ms"]) for row in requests),
+            "slos": slos,
+        },
+        "requests": requests,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(payload["summary"], sort_keys=True), flush=True)
+    if len(completed) != args.requests:
+        raise SystemExit(2)
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/t0_smoke_client.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""Issue exact fixed-shape completion requests and record streaming latency."""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import http.client
+import json
+import statistics
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, required=True)
+    parser.add_argument("--served-model", required=True)
+    parser.add_argument("--model-path", type=Path, required=True)
+    parser.add_argument("--input-tokens", type=int, default=2048)
+    parser.add_argument("--output-tokens", type=int, default=128)
+    parser.add_argument("--concurrency", type=int, required=True)
+    parser.add_argument("--requests", type=int, required=True)
+    parser.add_argument("--timeout-seconds", type=float, default=600.0)
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def run_request(
+    *,
+    request_index: int,
+    args: argparse.Namespace,
+    prompt_token_id: int,
+    start_barrier: threading.Barrier,
+) -> dict[str, Any]:
+    body = {
+        "model": args.served_model,
+        "prompt": [prompt_token_id] * args.input_tokens,
+        "min_tokens": args.output_tokens,
+        "max_tokens": args.output_tokens,
+        "ignore_eos": True,
+        "temperature": 0,
+        "stream": True,
+        "stream_options": {"include_usage": True},
+        "return_token_ids": True,
+    }
+    encoded = json.dumps(body, separators=(",", ":")).encode()
+    connection = http.client.HTTPConnection(
+        args.host, args.port, timeout=args.timeout_seconds
+    )
+    start_barrier.wait()
+    started = time.perf_counter()
+    connection.request(
+        "POST",
+        "/v1/completions",
+        body=encoded,
+        headers={"Content-Type": "application/json"},
+    )
+    response = connection.getresponse()
+    if response.status != 200:
+        detail = response.read().decode(errors="replace")
+        raise RuntimeError(f"request {request_index} failed: HTTP {response.status}: {detail}")
+
+    first_token_at: float | None = None
+    last_token_at: float | None = None
+    streamed_token_count = 0
+    usage: dict[str, Any] | None = None
+    while True:
+        raw = response.readline()
+        if not raw:
+            break
+        line = raw.decode(errors="replace").strip()
+        if not line.startswith("data:"):
+            continue
+        data = line[5:].strip()
+        if data == "[DONE]":
+            break
+        payload = json.loads(data)
+        if payload.get("usage"):
+            usage = payload["usage"]
+        emitted = 0
+        for choice in payload.get("choices") or []:
+            token_ids = choice.get("token_ids") or []
+            if token_ids:
+                emitted += len(token_ids)
+            elif choice.get("text"):
+                emitted += 1
+        if emitted:
+            now = time.perf_counter()
+            if first_token_at is None:
+                first_token_at = now
+            last_token_at = now
+            streamed_token_count += emitted
+    finished = time.perf_counter()
+    connection.close()
+
+    if first_token_at is None or last_token_at is None or usage is None:
+        raise RuntimeError(
+            f"request {request_index} missing streaming token or usage metadata"
+        )
+    prompt_tokens = int(usage["prompt_tokens"])
+    completion_tokens = int(usage["completion_tokens"])
+    if prompt_tokens != args.input_tokens or completion_tokens != args.output_tokens:
+        raise RuntimeError(
+            f"request {request_index} usage mismatch: prompt={prompt_tokens}, "
+            f"completion={completion_tokens}"
+        )
+
+    ttft_ms = (first_token_at - started) * 1000.0
+    tpot_ms = (
+        (last_token_at - first_token_at) * 1000.0 / (completion_tokens - 1)
+        if completion_tokens > 1
+        else 0.0
+    )
+    return {
+        "request_index": request_index,
+        "prompt_tokens": prompt_tokens,
+        "completion_tokens": completion_tokens,
+        "streamed_token_count": streamed_token_count,
+        "ttft_ms": ttft_ms,
+        "tpot_ms": tpot_ms,
+        "e2e_ms": (finished - started) * 1000.0,
+        "ttft_slo_ms": 1000.0 + args.input_tokens / 8.0,
+        "tpot_slo_ms": 40.0,
+        "joint_slo_pass": ttft_ms <= 1000.0 + args.input_tokens / 8.0
+        and tpot_ms <= 40.0,
+    }
+
+
+def percentile(values: list[float], fraction: float) -> float:
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, max(0, round((len(ordered) - 1) * fraction)))
+    return ordered[index]
+
+
+def main() -> None:
+    args = parse_args()
+    if args.concurrency <= 0 or args.requests < args.concurrency:
+        raise ValueError("requests must be at least concurrency, and both must be positive")
+    if args.input_tokens <= 0 or args.output_tokens <= 0:
+        raise ValueError("token lengths must be positive")
+
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
+    candidate_ids = tokenizer.encode(" hello", add_special_tokens=False)
+    if not candidate_ids:
+        raise RuntimeError("tokenizer returned no prompt token id")
+    prompt_token_id = int(candidate_ids[0])
+
+    results: list[dict[str, Any]] = []
+    for batch_start in range(0, args.requests, args.concurrency):
+        batch_count = min(args.concurrency, args.requests - batch_start)
+        barrier = threading.Barrier(batch_count)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=batch_count) as pool:
+            futures = [
+                pool.submit(
+                    run_request,
+                    request_index=batch_start + offset,
+                    args=args,
+                    prompt_token_id=prompt_token_id,
+                    start_barrier=barrier,
+                )
+                for offset in range(batch_count)
+            ]
+            results.extend(future.result() for future in futures)
+
+    ttfts = [float(row["ttft_ms"]) for row in results]
+    tpots = [float(row["tpot_ms"]) for row in results]
+    payload = {
+        "schema": "qwen235b-t0-smoke-v1",
+        "workload": {
+            "input_tokens": args.input_tokens,
+            "output_tokens": args.output_tokens,
+            "uniform_qps": None,
+            "prefix_caching": False,
+            "concurrency": args.concurrency,
+            "request_count": args.requests,
+            "prompt_token_id": prompt_token_id,
+        },
+        "summary": {
+            "completed_requests": len(results),
+            "joint_slo_pass_count": sum(bool(row["joint_slo_pass"]) for row in results),
+            "ttft_mean_ms": statistics.fmean(ttfts),
+            "ttft_p95_ms": percentile(ttfts, 0.95),
+            "tpot_mean_ms": statistics.fmean(tpots),
+            "tpot_p95_ms": percentile(tpots, 0.95),
+        },
+        "requests": sorted(results, key=lambda row: int(row["request_index"])),
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(payload["summary"], sort_keys=True), flush=True)
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-multicase-sufficiency-v1/trace-contract-audit.json

@@ -0,0 +1,41 @@
+{
+  "schema": "qwen235b-trace-contract-audit-v1",
+  "status": "pass_offline_source_contract",
+  "recorded_at": "2026-07-16",
+  "execution": {
+    "host": "dash0",
+    "device": "cpu_only",
+    "elapsed_seconds": 45.643,
+    "tokenizer_class": "Qwen2TokenizerFast",
+    "transformers_version": "4.55.2",
+    "model_path": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8"
+  },
+  "trace": {
+    "path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/thinking_w20260327_1000.jsonl",
+    "sha256": "f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2",
+    "source_request_count": 15479,
+    "eligible_request_count": 15401,
+    "eligibility": "input_length + output_length <= 40960 and output_length > 0"
+  },
+  "tokenization": {
+    "eligible_request_count": 15401,
+    "total_token_count": 55057919,
+    "input_length_mismatch_count": 0,
+    "length_order_sha256": "8bb1c9b7278b261fe1be695ff16dd0c4735d63bd0b1830b612953961fefc370a",
+    "per_request_token_digest_sha256": "b15b01c00042813c76701ffcc75fc0facde3c01806e167afdc69b5b5861e2381"
+  },
+  "source_hash_contract": {
+    "source_block_size_tokens": 64,
+    "full_block_count": 852407,
+    "partial_block_count": 15131,
+    "unique_hash_id_count": 509437,
+    "unique_parent_chunk_key_count": 509437,
+    "hash_id_to_parent_chunk_conflict_count": 0,
+    "parent_chunk_to_hash_id_conflict_count": 0,
+    "key_definition": "(parent source hash id, BLAKE2b-128 of the tokenizer token-id chunk)"
+  },
+  "interpretation": {
+    "established": "The exact prompts reproduce input_length, and source hash ids preserve the tokenizer-visible parent/chunk prefix-equivalence relation over the eligible universe.",
+    "not_yet_established": "Runtime parity of vLLM and Frontier block-size-16 computed, hit, allocated, eviction, and placement counters."
+  }
+}