Wait for Qwen235 GPU memory cleanup

runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Compare pooled real and Frontier Qwen235 latency-selection surfaces."""
+
+from __future__ import annotations
+
+import argparse
+import itertools
+import json
+import math
+from pathlib import Path
+
+
+CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128")
+CASES = ("fixed-pd", "fixed-po", "trace-pd", "trace-po")
+
+
+def percentile(values: list[float], fraction: float):
+    if not values:
+        return None
+    return sorted(values)[math.ceil(len(values) * fraction) - 1]
+
+
+def summarize(values: list[float]):
+    if not values:
+        return {"mean": None, "p90": None}
+    return {"mean": sum(values) / len(values), "p90": percentile(values, 0.9)}
+
+
+def real_surface(root: Path, case: str):
+    surface = {}
+    for config in CONFIGS:
+        requests = []
+        trials = []
+        for trial in (1, 2, 3):
+            path = root / "real" / case / "real" / config / f"trial{trial}" / "results" / "result.json"
+            payload = json.loads(path.read_text())
+            summary = payload["summary"]
+            if summary["failed"] or summary["completed"] != len(payload["requests"]):
+                raise ValueError(f"invalid real result: {path}")
+            if any(not row["success"] for row in payload["requests"]):
+                raise ValueError(f"failed request: {path}")
+            requests.extend(payload["requests"])
+            trials.append(str(path.resolve()))
+        ttft = summarize([float(row["ttft_ms"]) for row in requests])
+        tpot = summarize([float(row["tpot_ms"]) for row in requests if row["tpot_ms"] is not None])
+        e2e = summarize([float(row["e2e_ms"]) for row in requests])
+        surface[config] = {
+            "ttft_mean_ms": ttft["mean"], "ttft_p90_ms": ttft["p90"],
+            "tpot_mean_ms": tpot["mean"], "tpot_p90_ms": tpot["p90"],
+            "e2e_mean_ms": e2e["mean"], "e2e_p90_ms": e2e["p90"],
+            "request_samples": len(requests), "trials": trials,
+        }
+    return surface
+
+
+def sim_surface(root: Path, case: str):
+    payload = json.loads((root / "sim" / case / "frontier_surface.json").read_text())
+    surface = {}
+    for result in payload["results"]:
+        if result["status"] != "completed":
+            continue
+        config = result["config"]["name"]
+        if config in surface:
+            raise ValueError(f"duplicate simulator config: {config}")
+        surface[config] = {key: value for key, value in result["metrics"].items() if key.endswith("_ms")}
+    if set(surface) != set(CONFIGS):
+        raise ValueError(f"simulator coverage failure for {case}: {sorted(surface)}")
+    return surface
+
+
+def compare_metric(real: dict, sim: dict, metric: str):
+    applicable = [config for config in CONFIGS if real[config][metric] is not None and sim[config][metric] is not None]
+    real_winner = min(applicable, key=lambda config: (real[config][metric], config))
+    sim_winner = min(applicable, key=lambda config: (sim[config][metric], config))
+    regret = real[sim_winner][metric] / real[real_winner][metric] - 1
+    informative = agreement = 0
+    reversals = []
+    for left, right in itertools.combinations(applicable, 2):
+        real_direction = (real[left][metric] > real[right][metric]) - (real[left][metric] < real[right][metric])
+        sim_direction = (sim[left][metric] > sim[right][metric]) - (sim[left][metric] < sim[right][metric])
+        if real_direction and sim_direction:
+            informative += 1
+            agreement += int(real_direction == sim_direction)
+            if real_direction != sim_direction:
+                reversals.append([left, right])
+    return {
+        "real_winner": real_winner,
+        "sim_winner": sim_winner,
+        "winner_match": real_winner == sim_winner,
+        "selected_real_regret": regret,
+        "informative_pairs": informative,
+        "pair_direction_agreement": agreement / informative if informative else None,
+        "reversals": reversals,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--campaign-root", type=Path, required=True)
+    parser.add_argument("--json-output", type=Path, required=True)
+    parser.add_argument("--markdown-output", type=Path, required=True)
+    args = parser.parse_args()
+    cases = {}
+    lines = ["# Qwen235 vLLM 0.20 Frontier vs real", "", "| case | metric | Frontier winner | real winner | match | regret | pair agreement |", "|---|---|---|---|---:|---:|---:|"]
+    for case in CASES:
+        real = real_surface(args.campaign_root, case)
+        sim = sim_surface(args.campaign_root, case)
+        metrics = ["ttft_mean_ms", "ttft_p90_ms", "e2e_mean_ms", "e2e_p90_ms"]
+        if case.endswith("pd"):
+            metrics[2:2] = ["tpot_mean_ms", "tpot_p90_ms"]
+        comparisons = {}
+        for metric in metrics:
+            item = compare_metric(real, sim, metric)
+            comparisons[metric] = item
+            lines.append(
+                f"| {case} | {metric} | {item['sim_winner']} | {item['real_winner']} | "
+                f"{'yes' if item['winner_match'] else 'no'} | {item['selected_real_regret']:.1%} | "
+                f"{item['pair_direction_agreement']:.1%} |"
+            )
+        cases[case] = {"real": real, "sim": sim, "comparison": comparisons}
+    payload = {"schema": "qwen235-v020-simulator-real-comparison-v1", "cases": cases}
+    args.json_output.parent.mkdir(parents=True, exist_ok=True)
+    args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    args.markdown_output.write_text("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/assemble_qwen235_v020_profiles.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Assemble immutable flat profiles for the Qwen235 Frontier runner."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+
+MODEL = "Qwen3-235B-A22B"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    for name in ("cuda_common", "cuda_moe_tp4", "cuda_moe_ep8", "kernel_common", "kernel_moe_tp4", "kernel_moe_ep8"):
+        parser.add_argument(f"--{name.replace('_', '-')}", type=Path, required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    return parser.parse_args()
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    value.update(path.read_bytes())
+    return value.hexdigest()
+
+
+def model_file(root: Path, name: str) -> Path:
+    path = root / "compute" / "h20" / MODEL / name
+    if not path.is_file():
+        raise FileNotFoundError(path)
+    return path
+
+
+def merge_csv(inputs: list[Path], output: Path) -> None:
+    fields = None
+    rows = []
+    for path in inputs:
+        with path.open(newline="") as source:
+            reader = csv.DictReader(source)
+            if fields is None:
+                fields = reader.fieldnames
+            elif reader.fieldnames != fields:
+                raise ValueError(f"CSV schema mismatch: {path}")
+            rows.extend(reader)
+    if not fields or not rows:
+        raise ValueError("cannot merge empty profile CSV")
+    with output.open("w", newline="") as target:
+        writer = csv.DictWriter(target, fieldnames=fields, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def main() -> None:
+    args = parse_args()
+    output = args.output_root.resolve()
+    if output.exists():
+        raise FileExistsError(output)
+    output.mkdir(parents=True)
+    sources = {
+        "linear_op.csv": model_file(args.cuda_common, "linear_op.csv"),
+        "attention.csv": model_file(args.cuda_common, "attention.csv"),
+        "linear_op_kernel_only.csv": model_file(args.kernel_common, "linear_op_kernel_only.csv"),
+        "attention_kernel_only.csv": model_file(args.kernel_common, "attention_kernel_only.csv"),
+    }
+    for name, source in sources.items():
+        shutil.copy2(source, output / name)
+    merge_csv(
+        [model_file(args.cuda_moe_tp4, "moe.csv"), model_file(args.cuda_moe_ep8, "moe.csv")],
+        output / "moe.csv",
+    )
+    merge_csv(
+        [model_file(args.kernel_moe_tp4, "moe_kernel_only.csv"), model_file(args.kernel_moe_ep8, "moe_kernel_only.csv")],
+        output / "moe_kernel_only.csv",
+    )
+    outputs = {path.name: digest(path) for path in sorted(output.glob("*.csv"))}
+    manifest = {
+        "schema": "qwen235-v020-frontier-profile-v1",
+        "model": MODEL,
+        "measurement_families": ["CUDA_EVENT", "KERNEL_ONLY"],
+        "moe_runtime_paths": {"tp4_ep1": "TRITON", "tp8_ep8": "FLASHINFER_CUTLASS"},
+        "inputs": {name: str(getattr(args, name).resolve()) for name in ("cuda_common", "cuda_moe_tp4", "cuda_moe_ep8", "kernel_common", "kernel_moe_tp4", "kernel_moe_ep8")},
+        "outputs": outputs,
+    }
+    (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(manifest, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Extract simulator-visible graph/KV metadata from Qwen235 server starts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--case-root", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    configs = {}
+    for config in CONFIGS:
+        log = args.case_root / "real" / config / "trial1" / "logs" / "server.log"
+        text = log.read_text(errors="replace")
+        token_matches = re.findall(r"GPU KV cache size:\s*([0-9,]+) tokens", text)
+        capture_matches = re.findall(r"cudagraph_capture_sizes': \[([^]]+)\]", text)
+        if not token_matches or not capture_matches:
+            raise ValueError(f"missing runtime metadata in {log}")
+        tokens = int(token_matches[-1].replace(",", ""))
+        if tokens % 16:
+            raise ValueError(f"KV token count is not block aligned: {tokens}")
+        capture = [int(value.strip()) for value in capture_matches[-1].split(",")]
+        configs[config] = {
+            "num_gpu_blocks": tokens // 16,
+            "capture_sizes": capture,
+            "source_log": str(log.resolve()),
+        }
+    payload = {"schema": "qwen235-v020-runtime-contract-v1", "configs": configs}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(payload, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md

@@ -0,0 +1,89 @@
+# EXP-SIMFID-Q30-FIXED-PD-PRESSURE: calibrate a nontrivial fixed-shape workload
+
+> Status: complete (2026-07-19). This is a real-only workload calibration; its
+> measurements are not part of a Frontier-versus-vLLM winner comparison.
+
+## Question
+
+The former Fixed-PD workload (`2048 -> 128`, 0.215 request/s/GPU) was nearly
+single-request at TP4: its real mean E2E was 604 ms at a TP4 cluster rate of
+0.86 request/s, or about 0.52 in-flight requests by Little's law. It therefore
+does not test the decode batching regime of Trace-PD.
+
+Can a no-prefix, uniform `4096 -> 256` workload be assigned a fixed offered
+rate that reaches the healthy Trace-PD operating point without using a
+simulator result for calibration?
+
+## Controlled calibration
+
+The reference is the real Trace-PD `TP4/MNS64` cell, chosen before this probe
+because it is the real TTFT/E2E winner on that surface:
+
+| Reference metric | Target |
+|---|---:|
+| mean TTFT | 245.95 ms |
+| mean TPOT | 13.18 ms |
+| mean E2E | 44.99 s |
+| offered rate | 0.215 request/s/GPU; 0.86 request/s at TP4 |
+| in-flight proxy | 38.69 requests |
+
+The probe fixes Qwen3-30B-A3B BF16, community vLLM 0.20.0, H20, TP4, MNS64,
+MBT=8192, chunked prefill, and prefix caching off. An incomplete one-trial
+range-finding pilot showed global 4 rps stable and global 8 rps already badly
+overloaded; it is not used in the decision. The formal probe therefore compares
+per-GPU rates `{1, 1.125, 1.25, 1.5}` (global TP4 rates `{4, 4.5, 5, 6}`
+request/s). Every rate has 257 exact `4096 -> 256` requests in each of three
+fresh-server trials; rate orders are rotated across trials. The launcher
+requires a validated vLLM 0.20 FlashInfer kernel cache; this avoids including
+one-off custom-kernel JIT in server startup and does not alter request latency
+measurement.
+
+## Decision rule
+
+Pool the three trials for each rate. Among rates with every request completed,
+choose the rate minimizing the Euclidean distance of the two *relative* errors
+for mean TTFT and mean TPOT from the above targets. Report p90 TTFT/TPOT/E2E,
+trial variation, and `global_rate * mean_E2E` as an in-flight proxy, but do not
+turn any of them into an SLO. The selected rate becomes a frozen workload
+contract. The subsequent 12-cell real/simulator surface must be fresh and is
+not allowed to reuse this calibration data.
+
+## Interpretation boundary
+
+This selects a workload regime, not a simulator parameter and not a serving
+capacity point. It may make Fixed-PD more comparable to Trace-PD in batching
+pressure, but it deliberately continues to exclude trace-shaped arrivals and
+prefix reuse; those are separately evaluated workload dimensions.
+
+## Result
+
+All 12 planned cells completed (`257` requests × `3` fresh-server trials per
+rate, zero client failures). The raw artifact is
+`dash0:/home/admin/cpfs/wjh/aituner/qwen30-fixed-pd-pressure-20260719-r5`;
+it records the vLLM version, model/config checksums, H20 inventory, FlashInfer
+workspace, and an artifact checksum manifest. Pooled results are:
+
+| Global / per-GPU rps | TTFT mean / p90 (ms) | TPOT mean / p90 (ms) | E2E mean / p90 (ms) | In-flight proxy |
+|---|---:|---:|---:|---:|
+| 4 / 1 | 121.53 / 125.12 | 10.75 / 11.83 | 2862.42 / 3138.92 | 11.45 |
+| 4.5 / 1.125 | 122.76 / 126.29 | 11.76 / 12.55 | 3122.32 / 3322.26 | 14.05 |
+| 5 / 1.25 | 125.09 / 129.66 | 15.16 / 16.58 | 3989.73 / 4354.22 | 19.95 |
+| 6 / 1.5 | 129.27 / 134.69 | 21.33 / 23.03 | 5569.14 / 6005.12 | 33.41 |
+
+The predeclared two-metric rule picks `4.5` global rps (`1.125` rps/GPU), but
+that is only the *least mismatched* tested point: TTFT is still 50.1% below
+the Trace-PD target (122.76 vs 245.95 ms), while TPOT is 10.7% below it (11.76
+vs 13.18 ms). Raising load to 5 global rps moves TPOT past target (+15.0%)
+while TTFT barely changes (125.09 ms); at 6 global rps TPOT is +61.9% while
+TTFT remains 47.4% below target. Trial-level mean standard deviations are at
+most 0.78 ms for TTFT and 0.34 ms for TPOT, so this is not trial-order noise.
+
+**Conclusion:** `4096 -> 256` with uniform no-prefix arrivals cannot be called
+a pressure-matched Fixed-PD control simply by setting RPS. It has much shorter
+request residence time (3.12 s versus 44.99 s) and lower concurrency (14.05
+versus 38.69 by the same Little-law proxy) at its closest point. For the
+paper's fixed-versus-trace comparison, retain this case only as a deliberately
+different fixed-shape workload; do not interpret any simulator gap as being
+caused solely by trace fidelity. A future pressure-matched fixed control must
+also increase output length (or otherwise preserve residence time), then repeat
+this real-only calibration before reopening the Frontier-versus-vLLM surface.

runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md

@@ -0,0 +1,49 @@
+# EXP-SIMFID-Q30-FIXED-PRESSURE-SURFACE: high-pressure Fixed-PD / Fixed-PO
+
+> Status: approved and launching (2026-07-19).
+
+## Question
+
+When uniform Fixed-PD is moved from the near-singleton workload to
+`4096 -> 256` at `1.125` request/s/GPU, does Frontier still choose a different
+latency-optimal configuration from real community vLLM 0.20? Does removing
+decode (`4096 -> 1`) change that selection boundary?
+
+## Setup and decision rule
+
+- Qwen3-30B-A3B BF16 on dash0 H20; community vLLM 0.20.0.
+- Fixed-PD `4096 -> 256` and Fixed-PO `4096 -> 1`; 257 uniform,
+  prefix-disjoint requests; prefix caching off.
+- Rate is frozen at 1.125 request/s/GPU, so global rates are 1.125, 2.25, and
+  4.5 request/s for TP1, TP2, and TP4.
+- Surface: `TP in {1,2,4}` x `MNS in {8,16,32,64}`, MBT=8192. Real
+  measurements use three fresh-server trials with rotated order.
+- Frontier is `deadc4a`, `piecewise`, the frozen CUDA-event profile plus the
+  graph-compatible KERNEL_ONLY profile and measured collectives used by the
+  prior Trace-PD comparison. No latency calibration is fitted to this case.
+- Compare exact winners, selected-config real regret, and pairwise order for
+  mean/p90 TTFT, E2E, and TPOT where decode exists. SLO is not scored.
+
+The pressure-probe measurements selected the workload but are excluded from
+the real evaluation pool. A failed or incomplete Frontier cell is a coverage
+failure, not a high-latency observation.
+
+## Expected output and decision
+
+The final table has one row per case/objective with Frontier winner, real
+winner, regret, and pairwise agreement. If the old Fixed-PD reversal persists,
+the failure is not an artifact of singleton load. If it disappears, simulator
+fidelity has a load-regime boundary that must be localized. Fixed-PO isolates
+whether decode-state composition is necessary for either outcome.
+
+The previously reviewed matrix schematic remains the figure prototype:
+`../simulator-tuning-latency-matrix-v0/latency-selection-matrix-schematic.svg`.
+
+## Provenance
+
+Remote output root:
+`dash0:/home/admin/cpfs/wjh/aituner/qwen30-fixed-pressure-surface-20260719-r1`.
+The campaign records source/profile/model hashes, runtime versions, GPU
+inventory, per-cell commands, raw request records, and an artifact checksum
+manifest.
+

runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""Run Frontier on the four-cell Qwen235 vLLM 0.20 latency surface."""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+HERE = Path(__file__).resolve().parent
+Q30_RUNNER = HERE / "run_frontier_qwen30_exact_trace_surface.py"
+
+
+def load_q30():
+    spec = importlib.util.spec_from_file_location("q30_exact_surface", Q30_RUNNER)
+    if spec is None or spec.loader is None:
+        raise ImportError(Q30_RUNNER)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+Q30 = load_q30()
+BASE = Q30.BASE
+MODEL = "Qwen3-235B-A22B"
+
+
+@dataclass(frozen=True)
+class Config:
+    tp: int
+    mns: int
+    moe_tp: int
+    moe_ep: int
+
+    @property
+    def name(self) -> str:
+        return f"tp{self.tp}_ep{self.moe_ep}_mns{self.mns}"
+
+
+GRID = tuple(
+    Config(tp, mns, 4 if tp == 4 else 1, 1 if tp == 4 else 8)
+    for tp in (4, 8)
+    for mns in (64, 128)
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--frontier-source", type=Path, required=True)
+    parser.add_argument("--replayserve-root", type=Path, required=True)
+    parser.add_argument("--profile-root", type=Path, required=True)
+    parser.add_argument("--python-deps", type=Path, required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    parser.add_argument("--runtime-contract", type=Path, required=True)
+    parser.add_argument("--trace-tp", action="append", required=True, help="TP=PATH")
+    parser.add_argument("--config", action="append")
+    parser.add_argument("--prefix-caching", action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--allreduce-csv", type=Path, required=True)
+    parser.add_argument("--timeout-seconds", type=float, default=7200)
+    parser.add_argument("--resume", action="store_true")
+    return parser.parse_args()
+
+
+def profile_paths(root: Path) -> dict[str, Path]:
+    paths = {
+        "linear": root / "linear_op.csv",
+        "attention": root / "attention.csv",
+        "moe": root / "moe.csv",
+        "linear_kernel": root / "linear_op_kernel_only.csv",
+        "attention_kernel": root / "attention_kernel_only.csv",
+        "moe_kernel": root / "moe_kernel_only.csv",
+        "manifest": root / "manifest.json",
+    }
+    missing = [str(path) for path in paths.values() if not path.is_file()]
+    if missing:
+        raise FileNotFoundError(missing)
+    manifest = json.loads(paths["manifest"].read_text())
+    for name, path in paths.items():
+        if name == "manifest":
+            continue
+        expected = manifest.get("outputs", {}).get(path.name)
+        if expected != BASE.sha256(path):
+            raise ValueError(f"profile hash mismatch: {path}")
+    return paths
+
+
+def knobs(config: Config, paths: dict[str, Path], contract: dict, cache: Path, prefix: bool):
+    resolved = contract[config.name]
+    return {
+        "simulation_mode": "online",
+        "sys_arch": "co-location",
+        "num_replicas": 1,
+        "cluster_scheduler": "sticky_round_robin",
+        "model_name": MODEL,
+        "device": "h20",
+        "network_device": "h20_dgx",
+        "attn_tensor_parallel_size": config.tp,
+        "attn_data_parallel_size": 1,
+        "moe_tensor_parallel_size": config.moe_tp,
+        "moe_expert_parallel_size": config.moe_ep,
+        "num_pipeline_stages": 1,
+        "replica_scheduler": "vllm_v1",
+        "decode_cuda_graph_mode": "piecewise",
+        "batch_size_cap": config.mns,
+        "max_tokens_in_batch": 8192,
+        "long_prefill_token_threshold": 0,
+        "block_size": 16,
+        "num_blocks_mode": "explicit",
+        "num_blocks": int(resolved["num_gpu_blocks"]),
+        "gpu_memory_utilization": 0.80,
+        "non_kv_cache_overhead_bytes": 0,
+        "trace_max_tokens": 40960,
+        "cache_dir": str(cache / config.name),
+        "enable_prefix_caching": prefix,
+        "enable_dummy_mode": False,
+        "linear_op_input_file": str(paths["linear"]),
+        "atten_input_file": str(paths["attention"]),
+        "moe_input_file": str(paths["moe"]),
+        "linear_op_kernel_only_input_file": str(paths["linear_kernel"]),
+        "atten_kernel_only_input_file": str(paths["attention_kernel"]),
+        "moe_kernel_only_input_file": str(paths["moe_kernel"]),
+        "prediction_max_prefill_chunk_size": 8192,
+        "prediction_max_tokens_per_request": 40960,
+        "prediction_max_batch_size": max(int(v) for v in resolved["capture_sizes"]),
+        "no_cache": True,
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    for name in ("frontier_source", "replayserve_root", "profile_root", "python_deps", "output_root", "runtime_contract", "allreduce_csv"):
+        setattr(args, name, getattr(args, name).resolve())
+    paths = profile_paths(args.profile_root)
+    contract = json.loads(args.runtime_contract.read_text())["configs"]
+    trace_by_tp = {}
+    for specification in args.trace_tp:
+        raw_tp, separator, path = specification.partition("=")
+        if not separator:
+            raise ValueError(f"trace-tp must be TP=PATH: {specification}")
+        tp = int(raw_tp)
+        trace_by_tp[tp] = Q30.parse_trace(
+            f"eval={path}", rate_contract="trace-window", prefix_caching=args.prefix_caching
+        )
+    if set(trace_by_tp) != {4, 8}:
+        raise ValueError("trace-tp must provide exactly TP4 and TP8")
+    selected = list(GRID)
+    if args.config:
+        wanted = set(args.config)
+        selected = [config for config in GRID if config.name in wanted]
+        if {config.name for config in selected} != wanted:
+            raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}")
+
+    builder = BASE.load_module(
+        "qwen235_frontier_builder", args.replayserve_root / "tools/run_frontier_sweep.py"
+    )
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "PYTHONPATH": ":".join([str(args.python_deps), str(args.frontier_source)]),
+            "CUDA_VISIBLE_DEVICES": "",
+            "NVIDIA_VISIBLE_DEVICES": "void",
+            "WANDB_DISABLED": "true",
+            "VIDUR_DISABLE_WANDB": "1",
+            "FRONTIER_LOG_LEVEL": "WARNING",
+            "PYTHONDONTWRITEBYTECODE": "1",
+        }
+    )
+    results = []
+    for config in selected:
+        config_knobs = knobs(config, paths, contract, args.output_root / "cache", args.prefix_caching)
+        for trace in (trace_by_tp[config.tp],):
+            run_dir = args.output_root / "runs" / config.name / trace["label"]
+            result_path = run_dir / "result.json"
+            if args.resume and result_path.is_file():
+                results.append(json.loads(result_path.read_text()))
+                continue
+            run_dir.mkdir(parents=True, exist_ok=True)
+            command = builder.build_frontier_command(
+                python_bin="/usr/bin/python3",
+                trace_file=trace["path"],
+                metrics_root=run_dir / "metrics",
+                run_id=f"qwen235_v020_{config.name}_{trace['label']}",
+                knobs=config_knobs,
+            )
+            command.extend(["--cudagraph_capture_sizes", *(str(v) for v in contract[config.name]["capture_sizes"])])
+            command = BASE.configure_cc_command(
+                command,
+                backend="vidur",
+                allreduce_csv=args.allreduce_csv,
+                cache=args.output_root / "cc-cache",
+            )
+            BASE.write_json(run_dir / "command.json", command)
+            started = time.time()
+            with (run_dir / "stdout.log").open("w") as stdout, (run_dir / "stderr.log").open("w") as stderr:
+                completed = subprocess.run(
+                    command,
+                    cwd=args.frontier_source,
+                    env=environment,
+                    stdout=stdout,
+                    stderr=stderr,
+                    timeout=args.timeout_seconds,
+                    check=False,
+                )
+            result = {
+                "config": asdict(config) | {"name": config.name},
+                "trace": Q30.trace_manifest_entry(trace),
+                "elapsed_seconds": time.time() - started,
+                "returncode": completed.returncode,
+            }
+            if completed.returncode == 0:
+                metrics = Q30.find_request_metrics(run_dir)
+                result.update(status="completed", metrics=Q30.score(metrics, trace["shapes"]), request_metrics_sha256=BASE.sha256(metrics))
+            else:
+                stderr_text = (run_dir / "stderr.log").read_text(errors="replace")
+                result.update(status="failed", failure_class=Q30.classify_frontier_failure(stderr_text))
+            BASE.write_json(result_path, result)
+            results.append(result)
+            print(json.dumps({"config": config.name, "trace": trace["label"], "status": result["status"]}, sort_keys=True), flush=True)
+
+    manifest = {
+        "schema": "qwen235-v020-frontier-latency-surface-v1",
+        "frontier_commit": subprocess.check_output(["git", "-C", str(args.frontier_source), "rev-parse", "HEAD"], text=True).strip(),
+        "profiles": {name: BASE.sha256(path) for name, path in paths.items()},
+        "runtime_contract_sha256": BASE.sha256(args.runtime_contract),
+        "prefix_caching": args.prefix_caching,
+        "results": results,
+    }
+    BASE.write_json(args.output_root / "frontier_surface.json", manifest)
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+Q30_ROOT="${Q30_ROOT:?Q30_ROOT is required}"
+Q30_SESSION="${Q30_SESSION:-q30_fixed_pressure_20260719}"
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+
+# FlashInfer's TensorRT-LLM MoE runtime invokes `nvcc` by name when it
+# autotunes a previously unseen FP8 grouped-GEMM shape.
+export PATH="/usr/local/cuda/bin:${PATH}"
+export FRONTIER_VLLM_MODEL_TYPE=qwen3_moe
+export VLLM_KV_CACHE_LAYOUT=NHD
+command -v nvcc >/dev/null
+
+mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}"
+exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1
+echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}"
+date -u +SUPERVISOR_START_UTC=%Y-%m-%dT%H:%M:%SZ
+
+while tmux has-session -t "${Q30_SESSION}" 2>/dev/null; do
+  printf 'WAIT_Q30_UTC=%s results=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    "$(find "${Q30_ROOT}/real" -path '*/results/result.json' 2>/dev/null | wc -l)"
+  sleep 30
+done
+grep -q 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' "${Q30_ROOT}/controller.log" || {
+  echo 'ERROR: Q30 session ended without completion marker; refusing Q235 launch' >&2
+  exit 1
+}
+nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 > 16 {exit 1}'
+
+mkdir -p "${OUTPUT_ROOT}/profile-smoke"
+env CUDA_VISIBLE_DEVICES=0 PYTHONPATH="${FRONTIER_SOURCE}" \
+  "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/smoke_qwen235_v020_frontier_moe.py" \
+    --frontier-source "${FRONTIER_SOURCE}" \
+    --output "${OUTPUT_ROOT}/profile-smoke/moe.json"
+
+OUTPUT_ROOT="${PROFILE_ROOT}" FRONTIER_SOURCE="${FRONTIER_SOURCE}" \
+  VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \
+  bash "${RUNNER_DIR}/run_qwen235_v020_profiles.sh"
+
+CAMPAIGN_ROOT="${OUTPUT_ROOT}" PROFILE_ROOT="${PROFILE_ROOT}/frozen" \
+  FRONTIER_SOURCE="${FRONTIER_SOURCE}" VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \
+  bash "${RUNNER_DIR}/run_qwen235_v020_campaign.sh"
+date -u +SUPERVISOR_END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_AFTER_Q30_COMPLETE

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}"
+PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
+PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}"
+ALLREDUCE_CSV="${ALLREDUCE_CSV:-${RUNNER_DIR}/profiles/measured-allreduce.csv}"
+BASE_PUBLIC=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv
+BASE_PRIVATE=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl
+FIXED_PER_GPU_RATE="${FIXED_PER_GPU_RATE:-0.2}"
+REQUESTS=129
+
+mkdir -p "${CAMPAIGN_ROOT}"/{provenance,traces,real,sim,analysis}
+exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1
+echo "Q235_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 simulator=Frontier-5b953f5-piecewise cases={Fixed-PD:4096x256@0.2rps/gpu,Fixed-PO:4096x1@0.2rps/gpu,Trace-PD:u0p01-original-OSL,Trace-PO:u0p01-OSL1} requests=129 transform=t_prime=t/TP configs={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored expected_wall=10-30h expected_cost=90-220_H20-GPUh profile=${PROFILE_ROOT} output=${CAMPAIGN_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+sha256sum "${BASH_SOURCE[0]}" "${BASE_PUBLIC}" "${BASE_PRIVATE}" "${MODEL_ROOT}/config.json" \
+  "${PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" > "${CAMPAIGN_ROOT}/provenance/input.sha256"
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit"
+git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit"
+
+NORMALIZER="${RUNNER_DIR}/../simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py"
+MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py"
+for tp in 4 8; do
+  "${VENV_ROOT}/bin/python" "${NORMALIZER}" --base-public-csv "${BASE_PUBLIC}" \
+    --base-private-jsonl "${BASE_PRIVATE}" --tp "${tp}" \
+    --output-root "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}"
+  "${VENV_ROOT}/bin/python" "${MATERIALIZER}" trace \
+    --base-public "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/public/frontier.csv" \
+    --base-private "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/private/real_requests.jsonl" \
+    --output-tokens 1 --tp "${tp}" --output-root "${CAMPAIGN_ROOT}/traces/trace-po/tp${tp}"
+  for spec in fixed-pd:256 fixed-po:1; do
+    case_name="${spec%%:*}"; osl="${spec##*:}"
+    "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed --model "${MODEL_ROOT}" \
+      --input-tokens 4096 --output-tokens "${osl}" --requests "${REQUESTS}" \
+      --per-gpu-rate "${FIXED_PER_GPU_RATE}" --tp "${tp}" \
+      --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}"
+  done
+done
+
+run_real() {
+  local case_name="$1" prefix="$2" port="$3"
+  CASE_NAME="${case_name}" PREFIX_CACHING="${prefix}" \
+    TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \
+    RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \
+    BASE_PORT="${port}" bash "${RUNNER_DIR}/run_qwen235_v020_real_surface.sh"
+}
+run_real fixed-pd false 9300
+run_real fixed-po false 9400
+run_real trace-pd true 9500
+run_real trace-po true 9600
+
+"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \
+  --case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \
+  --output "${CAMPAIGN_ROOT}/provenance/runtime-contract.json"
+
+run_sim() {
+  local case_name="$1" prefix_flag="$2"
+  "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/run_frontier_qwen235_v020_surface.py" \
+    --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \
+    --profile-root "${PROFILE_ROOT}" --python-deps "${PYTHON_DEPS}" \
+    --output-root "${CAMPAIGN_ROOT}/sim/${case_name}" \
+    --runtime-contract "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" \
+    --trace-tp "4=${CAMPAIGN_ROOT}/traces/${case_name}/tp4/public/frontier.csv" \
+    --trace-tp "8=${CAMPAIGN_ROOT}/traces/${case_name}/tp8/public/frontier.csv" \
+    "${prefix_flag}" --allreduce-csv "${ALLREDUCE_CSV}" --resume
+}
+run_sim fixed-pd --no-prefix-caching
+run_sim fixed-po --no-prefix-caching
+run_sim trace-pd --prefix-caching
+run_sim trace-po --prefix-caching
+
+"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/analyze_qwen235_v020_campaign.py" \
+  --campaign-root "${CAMPAIGN_ROOT}" \
+  --json-output "${CAMPAIGN_ROOT}/analysis/comparison.json" \
+  --markdown-output "${CAMPAIGN_ROOT}/analysis/comparison.md"
+find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
+  | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256"
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_V020_CAMPAIGN_COMPLETE

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+MODEL=Qwen3-235B-A22B
+TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 512 1024 2048 4096 8192)
+BATCHES=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256)
+KV=(128 1024 2048 4096 8192 16384 32768 40960)
+
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
+exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1
+
+echo "Q235_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} attention_tp={4,8} moe_paths={TP4/EP1:Triton,TP1/EP8:FlashInfer-CUTLASS} token_points=${#TOKENS[@]} batch_points=${#BATCHES[@]} kv_points=${#KV[@]} parallel_gpus=6 expected_wall=20-75m expected_cost=2-7_H20-GPUh output=${OUTPUT_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \
+  | awk '$2 > 16 {exit 1}'
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
+"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version"
+# Frontier's model-config loader resolves data/config/models from the current
+# source checkout, so all profiling entrypoints must run from this directory.
+cd "${FRONTIER_SOURCE}"
+
+common_profile() {
+  local gpu="$1" method="$2" root="$3"
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \
+      --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" \
+      --profile_method "${method}" --yes
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 40960 \
+      --max_batch_size 256 --batch_size_list "${BATCHES[@]}" \
+      --decode_kv_cache_size_list "${KV[@]}" \
+      --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
+      --profile_method "${method}" --yes
+}
+
+moe_profile() {
+  local gpu="$1" method="$2" tp="$3" ep="$4" root="$5"
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \
+      --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
+      --num_samples_per_distribution 1 --profile_method "${method}" --yes
+}
+
+declare -a pids=()
+common_profile 0 cuda_event "${OUTPUT_ROOT}/cuda-common" > "${OUTPUT_ROOT}/logs/cuda-common.log" 2>&1 & pids+=("$!")
+common_profile 1 record_function "${OUTPUT_ROOT}/kernel-common" > "${OUTPUT_ROOT}/logs/kernel-common.log" 2>&1 & pids+=("$!")
+moe_profile 2 cuda_event 4 1 "${OUTPUT_ROOT}/cuda-moe-tp4" > "${OUTPUT_ROOT}/logs/cuda-moe-tp4.log" 2>&1 & pids+=("$!")
+moe_profile 3 cuda_event 1 8 "${OUTPUT_ROOT}/cuda-moe-ep8" > "${OUTPUT_ROOT}/logs/cuda-moe-ep8.log" 2>&1 & pids+=("$!")
+moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_ROOT}/logs/kernel-moe-tp4.log" 2>&1 & pids+=("$!")
+moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!")
+failed=0
+for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
+[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
+
+"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \
+  --cuda-common "${OUTPUT_ROOT}/cuda-common" \
+  --cuda-moe-tp4 "${OUTPUT_ROOT}/cuda-moe-tp4" --cuda-moe-ep8 "${OUTPUT_ROOT}/cuda-moe-ep8" \
+  --kernel-common "${OUTPUT_ROOT}/kernel-common" \
+  --kernel-moe-tp4 "${OUTPUT_ROOT}/kernel-moe-tp4" --kernel-moe-ep8 "${OUTPUT_ROOT}/kernel-moe-ep8" \
+  --output-root "${OUTPUT_ROOT}/frozen"
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_V020_PROFILES_COMPLETE

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+# Three fresh-server trials for the four-cell Qwen235 vLLM 0.20 surface.
+
+set -euo pipefail
+
+OUT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+TRACE_ROOT="${TRACE_ROOT:?TRACE_ROOT is required}"
+CASE_NAME="${CASE_NAME:?CASE_NAME is required}"
+PREFIX_CACHING="${PREFIX_CACHING:?PREFIX_CACHING is required}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+RUNNER="${RUNNER_DIR}/run_qwen30_exact_trace_real_anchor.sh"
+CLIENT="${RUNNER_DIR}/qwen30_exact_trace_client.py"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
+SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}"
+IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
+IDLE_GPU_SETTLE_ATTEMPTS="${IDLE_GPU_SETTLE_ATTEMPTS:-120}"
+RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}"
+PORT="${BASE_PORT:-9300}"
+REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")"
+
+export PATH="/usr/local/cuda/bin:${PATH}"
+command -v nvcc >/dev/null
+
+[[ "$(wc -l < "${TRACE_ROOT}/tp8/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || {
+  echo 'ERROR: TP-specific request counts differ' >&2
+  exit 1
+}
+case "${CASE_NAME}" in fixed-pd|fixed-po|trace-pd|trace-po) ;; *) exit 2 ;; esac
+case "${PREFIX_CACHING}" in true|false) ;; *) exit 2 ;; esac
+
+mkdir -p "${OUT}/provenance"
+declare -a WAVE_PIDS=()
+
+has_valid_result() {
+  local result="$1"
+  [[ -s "${result}" ]] || return 1
+  "${VENV_ROOT}/bin/python" - "${result}" "${REQUEST_COUNT}" <<'PY'
+import json, sys
+s = json.load(open(sys.argv[1])).get("summary", {})
+raise SystemExit(0 if s.get("completed") == int(sys.argv[2]) and s.get("failed") == 0 else 1)
+PY
+}
+
+preflight_gpus() {
+  local attempt
+  nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \
+    | tee -a "${OUT}/controller.log"
+  for ((attempt = 1; attempt <= IDLE_GPU_SETTLE_ATTEMPTS; attempt++)); do
+    if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
+      | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then
+      return 0
+    fi
+    sleep 1
+  done
+  echo "ERROR: GPU memory did not settle below ${IDLE_GPU_MEMORY_TOLERANCE_MIB} MiB after ${IDLE_GPU_SETTLE_ATTEMPTS}s" >&2
+  return 1
+}
+
+assert_no_server() {
+  ! pgrep -fa 'qwen3-235b-v020-eval' > "${OUT}/provenance/unexpected_server_processes.txt"
+}
+
+wait_for_wave() {
+  local failed=0 pid
+  for pid in "${WAVE_PIDS[@]}"; do wait "${pid}" || failed=1; done
+  WAVE_PIDS=()
+  [[ "${failed}" -eq 0 ]]
+}
+
+launch_config() {
+  local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1
+  if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi
+  local config="tp${tp}_ep${ep_size}_mns${mns}"
+  local run_out="${OUT}/real/${config}/trial${trial}"
+  local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}"
+  PORT=$((PORT + 1))
+  if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then
+    printf 'CONFIG_REUSED_VALID case=%s trial=%s config=%s\n' "${CASE_NAME}" "${trial}" "${config}"
+    return 0
+  fi
+  mkdir -p "${run_out}"
+  (
+    cd "${RUNNER_DIR}"
+    set +e
+    env HOME=/tmp/wjh XDG_CACHE_HOME=/tmp/wjh/.cache \
+      VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm CUDA_VISIBLE_DEVICES="${gpus}" \
+      TP="${tp}" MNS="${mns}" TRACE_LABEL="${CASE_NAME}/tp${tp}-normalized" \
+      PREFIX_CACHING="${PREFIX_CACHING}" ENABLE_EXPERT_PARALLEL="${ep}" \
+      MODEL_QUANTIZATION=fp8 DISABLE_CUSTOM_ALL_REDUCE=true GPU_MEMORY_UTILIZATION=0.80 \
+      SERVED_MODEL=qwen3-235b-v020-eval SERVER_PORT="${port}" OUTPUT_ROOT="${run_out}" \
+      REQUESTS_FILE="${requests}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \
+      SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS}" CLIENT_TIMEOUT_SECONDS=6000 \
+      EXACT_TRACE_CLIENT="${CLIENT}" \
+      timeout --signal=TERM --kill-after=60s 7800 bash "${RUNNER}" \
+      > "${run_out}/launcher.stdout.log" 2> "${run_out}/launcher.stderr.log"
+    status="$?"
+    printf '%s\n' "${status}" > "${run_out}/launcher.exit_code"
+    exit "${status}"
+  ) &
+  WAVE_PIDS+=("$!")
+}
+
+run_trial() {
+  local trial="$1" order="$2"
+  IFS=',' read -r -a mnss <<< "${order}"
+  preflight_gpus && assert_no_server
+  printf 'WAVE_START case=%s trial=%s tp=4 mns=%s,%s\n' \
+    "${CASE_NAME}" "${trial}" "${mnss[0]}" "${mnss[1]}" | tee -a "${OUT}/controller.log"
+  launch_config "${trial}" 4 "${mnss[0]}" '0,1,2,3'
+  launch_config "${trial}" 4 "${mnss[1]}" '4,5,6,7'
+  wait_for_wave && preflight_gpus && assert_no_server
+  for mns in "${mnss[@]}"; do
+    printf 'WAVE_START case=%s trial=%s tp=8 mns=%s\n' \
+      "${CASE_NAME}" "${trial}" "${mns}" | tee -a "${OUT}/controller.log"
+    launch_config "${trial}" 8 "${mns}" '0,1,2,3,4,5,6,7'
+    wait_for_wave && preflight_gpus && assert_no_server
+  done
+}
+
+{
+  printf '%s\n' "Q235_REAL_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 checkpoint=FP8 cases=${CASE_NAME} trace=${TRACE_ROOT}/tp{4,8} requests=${REQUEST_COUNT} transform=t_prime=t/TP prefix=${PREFIX_CACHING} surface={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored output=${OUT}/real"
+  date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+  sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${MODEL_ROOT}/config.json" \
+    > "${OUT}/provenance/real-input.sha256"
+  run_trial 1 '64,128'
+  run_trial 2 '128,64'
+  run_trial 3 '64,128'
+  date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+  printf '%s\n' 'Q235_REAL_SURFACE_COMPLETE'
+} 2>&1 | tee -a "${OUT}/controller.log"

runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh

@@ -9,11 +9,16 @@ MNS="${MNS:?MNS is required}"
 TRACE_LABEL="${TRACE_LABEL:?TRACE_LABEL is required}"
 SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
 PREFIX_CACHING="${PREFIX_CACHING:-true}"
+ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:-false}"
+MODEL_QUANTIZATION="${MODEL_QUANTIZATION:-}"
+DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-false}"
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}"
 VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
 FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}"
 SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-120}"
-SERVED_MODEL="qwen3-30b-exact-trace"
+CLIENT_TIMEOUT_SECONDS="${CLIENT_TIMEOUT_SECONDS:-1800}"
+SERVED_MODEL="${SERVED_MODEL:-qwen3-30b-exact-trace}"
 EXACT_TRACE_CLIENT="${EXACT_TRACE_CLIENT:-qwen30_exact_trace_client.py}"
 SERVER_PID=""
 
@@ -53,8 +58,23 @@ case "${PREFIX_CACHING}" in
     ;;
 esac
 
+case "${ENABLE_EXPERT_PARALLEL}" in
+  true) EP_FLAG=(--enable-expert-parallel) ;;
+  false) EP_FLAG=() ;;
+  *) echo "ERROR: ENABLE_EXPERT_PARALLEL must be true or false" >&2; exit 1 ;;
+esac
+case "${DISABLE_CUSTOM_ALL_REDUCE}" in
+  true) CUSTOM_AR_FLAG=(--disable-custom-all-reduce) ;;
+  false) CUSTOM_AR_FLAG=() ;;
+  *) echo "ERROR: DISABLE_CUSTOM_ALL_REDUCE must be true or false" >&2; exit 1 ;;
+esac
+QUANT_FLAG=()
+if [[ -n "${MODEL_QUANTIZATION}" ]]; then
+  QUANT_FLAG=(--quantization "${MODEL_QUANTIZATION}")
+fi
+
 REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")"
-echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) hard_wall=3600s"
+echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}"
 date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
 sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \
   ../frontier-phase-factorial-v0/qwen30_prefill_client.py \
@@ -74,9 +94,10 @@ ulimit -n 65536
 
 setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
   --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \
-  --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \
+  --tensor-parallel-size "${TP}" --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
   --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
   "${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \
+  "${CUSTOM_AR_FLAG[@]}" "${QUANT_FLAG[@]}" "${EP_FLAG[@]}" \
   > "${OUTPUT_ROOT}/logs/server.log" 2>&1 &
 SERVER_PID=$!
 READY=0
@@ -110,7 +131,7 @@ fi
   --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \
   --served-model "${SERVED_MODEL}" \
   --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \
-  --timeout-seconds 1800
+  --timeout-seconds "${CLIENT_TIMEOUT_SECONDS}"
 
 cleanup
 find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \

runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh

@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+
+# Real-only, pressure-matching probe for the next Fixed-PD workload. This
+# intentionally profiles one anchor, then freezes the workload before any
+# Frontier-vs-real selection comparison.
+
+set -euo pipefail
+
+OUT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+CLIENT="${CLIENT:-${RUNNER_DIR}/../frontier-phase-factorial-v0/qwen30_prefill_client.py}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
+# Reuse the validated vLLM 0.20 H20 kernel cache. A per-output workspace
+# starts costly FlashInfer MoE JIT compilation and changes startup behavior
+# without changing the serving configuration being profiled.
+FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:?FLASHINFER_WORKSPACE_BASE is required}"
+GPU_IDS="${GPU_IDS:-0,1,2,3}"
+TP="${TP:-4}"
+MNS="${MNS:-64}"
+REQUESTS="${REQUESTS:-257}"
+GLOBAL_RATES="${GLOBAL_RATES:-4 4.5 5 6}"
+SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-180}"
+PORT="${PORT:-8930}"
+SERVED_MODEL="qwen3-30b-fixed-pd-pressure"
+SERVER_PID=""
+
+[[ "${TP}" == "4" ]] || { echo 'ERROR: this calibrated probe is TP4-only' >&2; exit 1; }
+[[ "${MNS}" == "64" ]] || { echo 'ERROR: this calibrated probe is MNS64-only' >&2; exit 1; }
+[[ "${REQUESTS}" =~ ^[1-9][0-9]*$ ]] || { echo 'ERROR: REQUESTS must be positive' >&2; exit 1; }
+[[ -f "${CLIENT}" ]] || { echo "ERROR: client missing: ${CLIENT}" >&2; exit 1; }
+[[ -f "${MODEL_ROOT}/config.json" ]] || { echo "ERROR: model missing: ${MODEL_ROOT}" >&2; exit 1; }
+[[ -d "${FLASHINFER_WORKSPACE_BASE}" ]] || { echo "ERROR: FlashInfer workspace missing: ${FLASHINFER_WORKSPACE_BASE}" >&2; exit 1; }
+read -r -a RATE_VALUES <<< "${GLOBAL_RATES}"
+[[ "${#RATE_VALUES[@]}" -eq 4 ]] || { echo 'ERROR: GLOBAL_RATES must contain exactly four rates' >&2; exit 1; }
+for rate in "${RATE_VALUES[@]}"; do
+  awk -v value="${rate}" 'BEGIN {exit !(value > 0)}' || { echo "ERROR: invalid rate: ${rate}" >&2; exit 1; }
+done
+
+mkdir -p "${OUT}/provenance" "${OUT}/trials" "${FLASHINFER_WORKSPACE_BASE}"
+exec > >(tee -a "${OUT}/controller.log") 2>&1
+
+# Match the validated Qwen30 real-surface runner's file-descriptor budget for
+# vLLM's multiprocessing shared-memory transport. The failed r1 probe had the
+# default remote limit (1024) and stalled before KV-cache creation; r2 tests
+# whether this runner-contract difference is causal.
+ulimit -n 65536
+
+cleanup_server() {
+  if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+    kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
+    for _ in $(seq 1 30); do
+      kill -0 "${SERVER_PID}" 2>/dev/null || break
+      sleep 1
+    done
+    kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
+  fi
+  SERVER_PID=""
+}
+trap cleanup_server EXIT
+trap 'cleanup_server; exit 130' INT
+trap 'cleanup_server; exit 143' TERM
+
+assert_idle() {
+  nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader
+  nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
+    | awk '$1 > 16 {exit 1}'
+}
+
+wait_ready() {
+  local target="$1"
+  for _ in $(seq 1 "${SERVER_READY_ATTEMPTS}"); do
+    if curl -fsS --max-time 2 "http://127.0.0.1:${PORT}/v1/models" > "${target}/models.json" 2>/dev/null; then
+      return 0
+    fi
+    if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
+      tail -200 "${target}/server.log" >&2 || true
+      return 1
+    fi
+    sleep 3
+  done
+  echo "ERROR: vLLM did not become ready in $((SERVER_READY_ATTEMPTS * 3)) seconds" >&2
+  return 1
+}
+
+start_server() {
+  local target="$1"
+  export TOKENIZERS_PARALLELISM=false
+  export VLLM_USE_V1=1
+  export TORCH_CUDA_ARCH_LIST=9.0
+  export HF_HUB_OFFLINE=1
+  export TRANSFORMERS_OFFLINE=1
+  export FLASHINFER_WORKSPACE_BASE
+  export HOME=/tmp/wjh
+  export XDG_CACHE_HOME=/tmp/wjh/.cache
+  export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm
+  export CUDA_VISIBLE_DEVICES="${GPU_IDS}"
+  setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
+    --host 127.0.0.1 --port "${PORT}" --served-model-name "${SERVED_MODEL}" \
+    --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \
+    --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
+    --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \
+    > "${target}/server.log" 2>&1 &
+  SERVER_PID=$!
+  wait_ready "${target}"
+}
+
+run_client() {
+  local target="$1" rate="$2"
+  timeout --signal=TERM --kill-after=60s 1800 \
+    "${VENV_ROOT}/bin/python" "${CLIENT}" \
+      --port "${PORT}" --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" \
+      --rate "${rate}" --requests "${REQUESTS}" --input-tokens 4096 --output-tokens 256 \
+      --timeout-seconds 1200 --output "${target}/result.json"
+}
+
+warmup_server() {
+  local target="$1"
+  timeout --signal=TERM --kill-after=60s 600 \
+    "${VENV_ROOT}/bin/python" "${CLIENT}" \
+      --port "${PORT}" --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" \
+      --rate 1 --requests 4 --input-tokens 512 --output-tokens 1 \
+      --timeout-seconds 300 --output "${target}/result.json"
+}
+
+analyze() {
+  "${VENV_ROOT}/bin/python" - "${OUT}" "${GLOBAL_RATES}" <<'PY'
+import json
+import math
+import statistics
+import sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+rates = [float(value) for value in sys.argv[2].split()]
+target = {"ttft_ms": 245.9527667526406, "tpot_ms": 13.178025610291787}
+
+def p90(values):
+    return sorted(values)[math.ceil(0.9 * len(values)) - 1]
+
+rows = []
+for rate in rates:
+    label = f"r{rate:g}"
+    trial_means = {"ttft_ms": [], "tpot_ms": [], "e2e_ms": []}
+    pooled = {key: [] for key in trial_means}
+    for trial in range(1, 4):
+        path = root / "trials" / f"trial{trial}" / label / "result.json"
+        payload = json.loads(path.read_text())
+        workload = payload["workload"]
+        if (float(workload["offered_request_rate"]) != rate or workload["request_count"] != 257
+                or workload["input_tokens"] != 4096 or workload["output_tokens"] != 256
+                or workload["prefix_caching"] is not False):
+            raise ValueError(f"workload drift: {path}")
+        requests = payload["requests"]
+        if len(requests) != 257 or any(not request["success"] for request in requests):
+            raise ValueError(f"incomplete client result: {path}")
+        for key in pooled:
+            values = [float(request[key]) for request in requests]
+            pooled[key].extend(values)
+            trial_means[key].append(statistics.mean(values))
+    row = {
+        "global_rate": rate,
+        "per_gpu_rate": rate / 4.0,
+        "requests_per_trial": 257,
+        "trials": 3,
+        "metrics": {
+            key: {
+                "pooled_mean_ms": statistics.mean(values),
+                "pooled_p90_ms": p90(values),
+                "trial_mean_stdev_ms": statistics.stdev(trial_means[key]),
+            }
+            for key, values in pooled.items()
+        },
+    }
+    row["inflight_proxy"] = rate * row["metrics"]["e2e_ms"]["pooled_mean_ms"] / 1000.0
+    row["relative_distance"] = math.sqrt(sum(
+        ((row["metrics"][key]["pooled_mean_ms"] - target[key]) / target[key]) ** 2
+        for key in target
+    ))
+    rows.append(row)
+
+winner = min(rows, key=lambda row: (row["relative_distance"], row["global_rate"]))
+payload = {
+    "schema": "qwen30-fixed-pd-pressure-probe-v1",
+    "target_trace_pd_tp4_mns64": target,
+    "decision_rule": "minimum Euclidean distance of relative mean TTFT and TPOT errors",
+    "rates": rows,
+    "recommended_global_rate": winner["global_rate"],
+    "recommended_per_gpu_rate": winner["per_gpu_rate"],
+}
+(root / "pressure-analysis.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+lines = [
+    "# Fixed-PD pressure probe",
+    "",
+    "| Global / per-GPU rps | TTFT mean / p90 (ms) | TPOT mean / p90 (ms) | E2E mean / p90 (ms) | In-flight proxy | Relative distance |",
+    "|---|---:|---:|---:|---:|---:|",
+]
+for row in rows:
+    metric = row["metrics"]
+    lines.append(
+        f"| {row['global_rate']:g} / {row['per_gpu_rate']:g} | "
+        f"{metric['ttft_ms']['pooled_mean_ms']:.2f} / {metric['ttft_ms']['pooled_p90_ms']:.2f} | "
+        f"{metric['tpot_ms']['pooled_mean_ms']:.2f} / {metric['tpot_ms']['pooled_p90_ms']:.2f} | "
+        f"{metric['e2e_ms']['pooled_mean_ms']:.2f} / {metric['e2e_ms']['pooled_p90_ms']:.2f} | "
+        f"{row['inflight_proxy']:.2f} | {row['relative_distance']:.3f} |"
+    )
+lines.extend([
+    "",
+    f"Recommended frozen rate: **{winner['global_rate']:g} global rps / {winner['per_gpu_rate']:g} rps per GPU**.",
+    "Selection uses only pooled mean TTFT and TPOT; p90 and in-flight proxy are audit outputs.",
+])
+(root / "pressure-analysis.md").write_text("\n".join(lines) + "\n")
+print(json.dumps(payload, sort_keys=True))
+PY
+}
+
+{
+  echo "FIXED_PD_PRESSURE_PROBE_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} engine=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 gpus=${GPU_IDS} prefix=false shape=4096_to_256 requests_per_rate=${REQUESTS} global_rates={${GLOBAL_RATES}} rate_contract=global_rate_divided_by_TP flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} trials=3 fresh_server=true metric_target=TracePD_TP4_MNS64_meanTTFT245.95ms_meanTPOT13.18ms expected_wall=12-20m expected_cost=0.8-1.4_H20-GPUh output=${OUT}"
+  date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+  assert_idle
+  sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256"
+  "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version"
+  "${VENV_ROOT}/bin/python" -c 'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' > "${OUT}/provenance/runtime.versions"
+  ulimit -n > "${OUT}/provenance/open-file-limit"
+  readlink -f "${FLASHINFER_WORKSPACE_BASE}" > "${OUT}/provenance/flashinfer-workspace"
+  nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.before.csv"
+
+  declare -a ORDERS=(
+    "${RATE_VALUES[*]}"
+    "${RATE_VALUES[3]} ${RATE_VALUES[2]} ${RATE_VALUES[1]} ${RATE_VALUES[0]}"
+    "${RATE_VALUES[1]} ${RATE_VALUES[3]} ${RATE_VALUES[0]} ${RATE_VALUES[2]}"
+  )
+  for trial in 1 2 3; do
+    trial_root="${OUT}/trials/trial${trial}"
+    mkdir -p "${trial_root}"
+    echo "TRIAL_START trial=${trial} order=${ORDERS[$((trial - 1))]}"
+    start_server "${trial_root}"
+    warmup_server "${trial_root}/warmup"
+    for rate in ${ORDERS[$((trial - 1))]}; do
+      rate_root="${trial_root}/r${rate}"
+      mkdir -p "${rate_root}"
+      echo "RATE_START trial=${trial} global_rate=${rate} per_gpu_rate=$(awk -v value="${rate}" 'BEGIN {printf "%.3f", value / 4}')"
+      run_client "${rate_root}" "${rate}"
+      echo "RATE_COMPLETE trial=${trial} global_rate=${rate}"
+    done
+    cleanup_server
+    assert_idle
+    echo "TRIAL_COMPLETE trial=${trial}"
+  done
+  analyze
+  find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256"
+  nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.after.csv"
+  date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+  echo 'FIXED_PD_PRESSURE_PROBE_COMPLETE'
+} >> "${OUT}/controller.log" 2>&1

runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+# Frozen Qwen30 Fixed-PD/Fixed-PO Frontier-versus-real campaign. The pressure
+# calibration artifacts are deliberately not reused as evaluation results.
+
+set -euo pipefail
+
+CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}"
+REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}"
+PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}"
+PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}"
+KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}"
+ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}"
+FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:?FLASHINFER_WORKSPACE_BASE is required}"
+REQUESTS="${REQUESTS:-257}"
+PER_GPU_RATE="${PER_GPU_RATE:-1.125}"
+
+[[ "${REQUESTS}" == "257" ]] || { echo 'ERROR: frozen campaign requires REQUESTS=257' >&2; exit 1; }
+[[ "${PER_GPU_RATE}" == "1.125" ]] || { echo 'ERROR: frozen campaign requires PER_GPU_RATE=1.125' >&2; exit 1; }
+
+MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py"
+REAL_RUNNER="${RUNNER_DIR}/run_qwen30_latency_case_real_surface.sh"
+SIM_RUNNER="${RUNNER_DIR}/run_frontier_qwen30_exact_trace_surface.py"
+REAL_AUDITOR="${RUNNER_DIR}/audit_qwen30_latency_case.py"
+COMPARATOR="${RUNNER_DIR}/analyze_qwen30_latency_case.py"
+
+mkdir -p "${CAMPAIGN_ROOT}/provenance" "${CAMPAIGN_ROOT}/traces" \
+  "${CAMPAIGN_ROOT}/real" "${CAMPAIGN_ROOT}/sim" "${CAMPAIGN_ROOT}/analysis"
+exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1
+
+printf '%s\n' "Q30_FIXED_PRESSURE_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 cases={fixed-pd:4096_to_256,fixed-po:4096_to_1} prefix=false requests=${REQUESTS} per_gpu_rate=${PER_GPU_RATE} global_rates={TP1:1.125,TP2:2.25,TP4:4.5} surface_per_case=TP{1,2,4}xMNS{8,16,32,64} real_trials=3 fresh_server=true simulator=Frontier-deadc4a_piecewise_graph-kernel-only metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} expected_wall=2-5h expected_cost=12-24_H20-GPUh output=${CAMPAIGN_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+
+sha256sum "${BASH_SOURCE[0]}" "${MATERIALIZER}" "${REAL_RUNNER}" \
+  "${SIM_RUNNER}" "${REAL_AUDITOR}" "${COMPARATOR}" \
+  "${MODEL_ROOT}/config.json" "${PROFILE_ROOT}/manifest.json" \
+  "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" \
+  > "${CAMPAIGN_ROOT}/provenance/input.sha256"
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit"
+git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit"
+"${VENV_ROOT}/bin/vllm" --version > "${CAMPAIGN_ROOT}/provenance/vllm.version"
+nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \
+  > "${CAMPAIGN_ROOT}/provenance/gpus.before.csv"
+
+prepare_case() {
+  local case_name="$1" output_tokens="$2"
+  local tp
+  for tp in 1 2 4; do
+    "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed \
+      --model "${MODEL_ROOT}" --input-tokens 4096 --output-tokens "${output_tokens}" \
+      --requests "${REQUESTS}" --per-gpu-rate "${PER_GPU_RATE}" --tp "${tp}" \
+      --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}"
+  done
+}
+
+run_real_case() {
+  local case_name="$1" base_port="$2"
+  CASE_NAME="${case_name}" PREFIX_CACHING=false \
+    TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" \
+    OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \
+    RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \
+    FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_WORKSPACE_BASE}" \
+    BASE_PORT="${base_port}" RESUME_VALID_CELLS=true \
+    bash "${REAL_RUNNER}"
+}
+
+run_sim_tp() {
+  local case_name="$1" tp="$2"
+  local sim_root="${CAMPAIGN_ROOT}/sim/${case_name}"
+  local -a configs=()
+  local mns
+  for mns in 8 16 32 64; do
+    configs+=(--config "tp${tp}_mns${mns}")
+  done
+  /usr/bin/python3 "${SIM_RUNNER}" \
+    --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \
+    --profile-root "${PROFILE_ROOT}" --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \
+    --python-deps "${PYTHON_DEPS}" --output-root "${sim_root}" \
+    --trace "tp${tp}=${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}/public/frontier.csv" \
+    "${configs[@]}" --rate-contract uniform-spacing --no-prefix-caching \
+    --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \
+    --timeout-seconds 3600 --predictor-training-job-threads 4 \
+    --decode-cuda-graph-mode piecewise --align-real-graph-runtime \
+    --fresh-predictor-cache --resume --continue-on-failure
+}
+
+run_sim_case() {
+  local case_name="$1" failed=0 pid
+  local -a pids=()
+  mkdir -p "${CAMPAIGN_ROOT}/sim/${case_name}"
+  for tp in 1 2 4; do
+    run_sim_tp "${case_name}" "${tp}" \
+      > "${CAMPAIGN_ROOT}/sim/${case_name}/launcher-tp${tp}.log" 2>&1 &
+    pids+=("$!")
+  done
+  for pid in "${pids[@]}"; do
+    wait "${pid}" || failed=1
+  done
+  [[ "${failed}" -eq 0 ]] || return 1
+}
+
+analyze_case() {
+  local case_name="$1"
+  local analysis_root="${CAMPAIGN_ROOT}/analysis/${case_name}"
+  mkdir -p "${analysis_root}"
+  "${VENV_ROOT}/bin/python" "${REAL_AUDITOR}" \
+    --case-root "${CAMPAIGN_ROOT}/real/${case_name}" \
+    --traces-root "${CAMPAIGN_ROOT}/traces/${case_name}" \
+    --json-output "${analysis_root}/real-audit.json" \
+    --markdown-output "${analysis_root}/real-audit.md"
+  "${VENV_ROOT}/bin/python" "${COMPARATOR}" \
+    --sim-root "${CAMPAIGN_ROOT}/sim/${case_name}" \
+    --real-audit "${analysis_root}/real-audit.json" \
+    --json-output "${analysis_root}/comparison.json" \
+    --markdown-output "${analysis_root}/comparison.md"
+}
+
+prepare_case fixed-pd 256
+prepare_case fixed-po 1
+
+run_real_case fixed-pd 9000
+run_real_case fixed-po 9100
+
+run_sim_case fixed-pd
+analyze_case fixed-pd
+run_sim_case fixed-po
+analyze_case fixed-po
+
+nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \
+  > "${CAMPAIGN_ROOT}/provenance/gpus.after.csv"
+find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
+  | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256"
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+printf '%s\n' 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE'

runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh

@@ -21,6 +21,14 @@ SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}"
 IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
 RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-false}"
 PORT="${BASE_PORT:-8300}"
+REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp1/private/real_requests.jsonl")"
+
+for tp in 2 4; do
+  [[ "$(wc -l < "${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || {
+    echo "ERROR: TP-specific request counts differ" >&2
+    exit 1
+  }
+done
 
 case "${PREFIX_CACHING}" in
   true|false) ;;
@@ -186,7 +194,7 @@ run_tp_wave() {
 }
 
 {
-  printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real"
+  printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=${REQUEST_COUNT} transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real"
   date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
   mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}"
   sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \

runs/frontier-fidelity-envelope-v1/smoke_qwen235_v020_frontier_moe.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""One-cell gate for the two Qwen235 vLLM 0.20 MoE runtime backends."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import torch
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--frontier-source", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def routing(tokens: int, experts: int = 128, topk: int = 8):
+    ids = torch.arange(tokens * topk, device="cuda", dtype=torch.int64)
+    ids = (ids % experts).view(tokens, topk)
+    weights = torch.full((tokens, topk), 1.0 / topk, device="cuda")
+    return weights, ids
+
+
+def main() -> None:
+    args = parse_args()
+    sys.path.insert(0, str(args.frontier_source.resolve()))
+    from frontier.profiling.moe.moe_vllm_kernel import profile_fused_moe_kernel
+
+    weights, ids = routing(8)
+    cells = []
+    cells.append(
+        {
+            "name": "tp4_ep1_triton",
+            "stats": profile_fused_moe_kernel(
+                num_tokens=8,
+                num_experts=128,
+                hidden_dim=4096,
+                expert_hidden_dim=1536,
+                top_k=8,
+                topk_weights=weights,
+                topk_ids=ids,
+                tensor_parallel_size=4,
+                use_fp8=True,
+                block_shape=[128, 128],
+                warmup_steps=1,
+                active_steps=2,
+            ),
+        }
+    )
+    expert_map = torch.full((128,), -1, device="cuda", dtype=torch.int32)
+    expert_map[:16] = torch.arange(16, device="cuda", dtype=torch.int32)
+    cells.append(
+        {
+            "name": "tp1_ep8_flashinfer_cutlass",
+            "stats": profile_fused_moe_kernel(
+                num_tokens=8,
+                num_experts=16,
+                hidden_dim=4096,
+                expert_hidden_dim=1536,
+                top_k=8,
+                topk_weights=weights,
+                topk_ids=ids,
+                tensor_parallel_size=1,
+                use_fp8=True,
+                block_shape=[128, 128],
+                warmup_steps=1,
+                active_steps=2,
+                global_num_experts=128,
+                expert_map=expert_map,
+            ),
+        }
+    )
+    payload = {"schema": "qwen235-v020-frontier-moe-smoke-v1", "cells": cells}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(payload, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()