Add decode batch-grid stability experiment

runs/frontier-decode-batch-grid-v0/.gitignore

@@ -0,0 +1,3 @@
+fleet-artifacts/
+fleet-state/
+remote-outputs/

runs/frontier-decode-batch-grid-v0/analyze_decode_trace.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Summarize graph-on vLLM Torch traces by decode component."""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import json
+import math
+import statistics
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+COMPONENT_NAMES = (
+    "attention",
+    "linear_norm_rope",
+    "router",
+    "moe",
+    "collective",
+    "output_head",
+    "other",
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--trace-root", type=Path, required=True)
+    parser.add_argument("--label", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def classify(name: str, occurrences: int, steps: int) -> str:
+    lower = name.lower()
+    if any(token in lower for token in ("nccl", "allreduce", "all_reduce")):
+        return "collective"
+    if "topkgating" in lower or "fused_topk" in lower:
+        return "router"
+    if any(
+        token in lower
+        for token in (
+            "fused_moe",
+            "moefcgemm",
+            "tensorrt_llm::kernels::cutlass_kernels",
+            "groupproblemshape",
+            "memcpy32_post",
+        )
+    ):
+        return "moe"
+    if any(
+        token in lower
+        for token in (
+            "flashattn",
+            "flashattnfwd",
+            "reshape_and_cache",
+            "prepare_varlen_num_blocks",
+        )
+    ):
+        return "attention"
+    if "nvjet" in lower and occurrences <= steps * 2:
+        return "output_head"
+    if any(
+        token in lower
+        for token in (
+            "nvjet",
+            "cublaslt",
+            "rms_norm",
+            "rsqrt",
+            "triton_red_fused_2",
+            "triton_poi_fused_3",
+            "triton_red_fused_0",
+            "triton_poi_fused_1",
+        )
+    ):
+        return "linear_norm_rope"
+    return "other"
+
+
+def stats(values: list[float]) -> dict[str, float | int]:
+    ordered = sorted(values)
+    return {
+        "n": len(values),
+        "mean_ms": statistics.fmean(values),
+        "population_std_ms": statistics.pstdev(values),
+        "p50_ms": statistics.median(ordered),
+        "p95_ms": ordered[math.ceil(0.95 * len(ordered)) - 1],
+    }
+
+
+def load_events(path: Path) -> list[dict]:
+    opener = gzip.open if path.suffix == ".gz" else open
+    with opener(path, "rt") as source:
+        return json.load(source)["traceEvents"]
+
+
+def analyze_rank(path: Path) -> dict:
+    events = load_events(path)
+    kernels = [event for event in events if event.get("cat") == "kernel"]
+    all_windows = sorted(
+        (
+            event
+            for event in events
+            if event.get("cat") == "gpu_user_annotation"
+            and str(event.get("name", "")).startswith("execute_")
+        ),
+        key=lambda event: float(event["ts"]),
+    )
+    if not all_windows:
+        raise ValueError(f"{path}: no GPU execute annotations")
+    window_names = Counter(str(window["name"]) for window in all_windows)
+    selected_name = window_names.most_common(1)[0][0]
+    windows = [
+        window for window in all_windows if str(window["name"]) == selected_name
+    ]
+
+    selected: list[dict] = []
+    step_kernels: list[list[dict]] = []
+    for window in windows:
+        start = float(window["ts"])
+        end = start + float(window["dur"])
+        current = [
+            kernel for kernel in kernels if start <= float(kernel["ts"]) < end
+        ]
+        selected.extend(current)
+        step_kernels.append(current)
+
+    occurrences = Counter(str(kernel["name"]) for kernel in selected)
+    component_steps: dict[str, list[float]] = defaultdict(list)
+    busy_steps: list[float] = []
+    wall_steps = [float(window["dur"]) / 1000.0 for window in windows]
+    for current in step_kernels:
+        per_component: dict[str, float] = defaultdict(float)
+        for kernel in current:
+            name = str(kernel["name"])
+            per_component[classify(name, occurrences[name], len(windows))] += (
+                float(kernel["dur"]) / 1000.0
+            )
+        for name in COMPONENT_NAMES:
+            component_steps[name].append(per_component[name])
+        busy_steps.append(sum(per_component.values()))
+
+    return {
+        "trace": str(path),
+        "selected_execute_annotation": selected_name,
+        "execute_annotation_histogram": dict(sorted(window_names.items())),
+        "steps": len(windows),
+        "execute_wall": stats(wall_steps),
+        "gpu_kernel_busy": stats(busy_steps),
+        "non_kernel_gap": stats(
+            [wall - busy for wall, busy in zip(wall_steps, busy_steps)]
+        ),
+        "components": {
+            name: stats(values) for name, values in component_steps.items()
+        },
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    traces = sorted(args.trace_root.rglob("*.pt.trace.json*"))
+    if not traces:
+        raise SystemExit(f"no traces under {args.trace_root}")
+    ranks = [analyze_rank(path) for path in traces]
+    payload = {
+        "schema": "frontier-decode-batch-trace.v1",
+        "label": args.label,
+        "contract": {
+            "timing": "CUDA graph-on GPU execute annotations",
+            "component_time": "sum of CUDA kernel durations inside execute range",
+            "tp_aggregation": "per-rank; slowest rank mean approximates critical path",
+        },
+        "ranks": ranks,
+        "rank_summary": {
+            "ranks": len(ranks),
+            "slowest_rank_execute_mean_ms": max(
+                rank["execute_wall"]["mean_ms"] for rank in ranks
+            ),
+            "slowest_rank_kernel_busy_mean_ms": max(
+                rank["gpu_kernel_busy"]["mean_ms"] for rank in ranks
+            ),
+            "component_rank_mean_ms": {
+                name: statistics.fmean(
+                    rank["components"][name]["mean_ms"] for rank in ranks
+                )
+                for name in COMPONENT_NAMES
+            },
+        },
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-decode-batch-grid-v0/experiment-card.md

@@ -0,0 +1,68 @@
+# 实验 EXP-DECODE-BATCH-GRID：BC-8 是否可由稳定 whole-layer b2--b8 curve 修复
+
+> **状态：** entry gate PASS，待 TP4/b2 GPU smoke
+>
+> Parent campaign：[`../frontier-simulator-gap-campaign-v0/README.md`](../frontier-simulator-gap-campaign-v0/README.md)
+
+## Claim 与决策
+
+- **Parent claim：** 最新 1h trace 中 TP4 的 TPOT/E2E 仍稳定正偏约 10--23%；
+  历史 BC-8 knee-right 真机排序为 `TP2<TP4<TP1`，sim 为
+  `TP2<TP1<TP4`。这是当前最明显的 decode/state residual。
+- **目的：** 判断 BC-8 是 b2--b8 whole-layer service curve 可工程修复，
+  还是必须进入 event-level admission/batch-formation 建模。
+- **Competing hypotheses：**
+  - H1：现有 b2 service sample 不稳定或 b>4 常数外推，导致 TP4 residence
+    被高估；稳定 whole-layer curve 可恢复排序。
+  - H2：给定 batch 的 whole-layer curve 已准，错误来自 simulator 与 vLLM
+    batch formation/event semantics 不同。
+- **事前预测：** 若 H1 成立，TP4/b2 repeat CV≤10%，其 median 比当前进入
+  BC-8 的 curve 低至少 0.5 ms，注入后 `TP4<TP1`；若 H2 成立，
+  same-state residual <0.5 ms，但 BC-8 排序仍错。
+- **判定规则：**
+  - 先做 TP4/b2 两个 fresh-process repeats；CV>10% 时追加第三次，取
+    median，不允许从单次 favorable sample 选值。
+  - smoke 稳定后补 TP2/TP4 × b{2,4,6,8}；b>4 不再使用 b4 常数外推。
+  - whole-layer 注入只替换 pure-decode service curve，不改变 prefill。
+  - BC-8 排序仍错则停止 profile 修补，升级 event-level state telemetry。
+
+## Setup
+
+- **自变量：** TP `{2,4}`；pure-decode batch `{2,4,6,8}`；fresh-process
+  repeat `{1,2}`，不稳定 cell 追加 repeat 3。
+- **控制变量：** Qwen3-30B-A3B BF16；H20；vLLM 0.20；context=2048；
+  output=128；graph-on；MNS=16；no prefix caching；同模型/runtime/cache。
+- **Hardware：** dash1--dash4；每轮调度前要求目标主机 8×H20
+  idle/healthy。每主机同时只启动一个 engine，避免 CPU/JIT contention。
+- **Metrics：** critical-rank execute wall；MoE/attention/collective/other
+  kernels；repeat median/CV；sim same-state residual；BC-8 TPOT 完整排序。
+- **Baselines：**
+  - 旧 serving grid：TP4/b2 execute=`5.044 ms`，单 process 内
+    population std=`1.561 ms`，不足以进入判决链。
+  - BC-8：real `4.305<4.449<4.876 ms`；sim
+    `5.12<5.49<5.86 ms`（TP2/TP4/TP1 对应完整排序见结果审计）。
+
+## 预期产物与 review
+
+- **预期数据：** 每 cell trace、request result、runtime/GPU provenance；
+  `results/grid.json`；BC-8 counterfactual 与 verdict。
+- **Figure prototype：** `figure-prototype.png`。左图画 b2 repeat instability
+  与目标 CI，右图画 BC-8 real/sim/corrected 排序。
+- **人工 review：** campaign 已批准按第三优先级推进。
+- **Review 意见：** 先 TP4/b2 smoke；不在稳定性 gate 前铺完整网格。
+
+## 复现信息
+
+- **Frontier baseline：** `deadc4a321f0baaa534c6ebd17f974123733cdc2`；
+  joint serving-path curves + structured-attention analysis branch。
+- **Remote：** 复用 dash1 的 clean detached experiment worktree；canonical
+  checkout 的用户 dirty changes 不动。
+- **Known limits：** 固定 2048→128 state-matched workload；本实验只支持
+  decode curve 与 BC-8，不外推到 chat prefill shape 或其它模型。
+
+## 结果
+
+- **Entry gate：** PASS。TP4 1h trace TPOT mean 正偏
+  `+10.6%--+23.0%`，E2E mean `+9.8%--+19.2%`；BC-8 完整排序仍错。
+- **观察事实：** 待 GPU。
+- **Decision：** 待 GPU。

runs/frontier-decode-batch-grid-v0/fleet.toml

@@ -0,0 +1,46 @@
+version = 1
+
+[paths]
+state_dir = "runs/frontier-decode-batch-grid-v0/fleet-state"
+artifacts_dir = "runs/frontier-decode-batch-grid-v0/fleet-artifacts"
+
+[ssh]
+connect_timeout_sec = 10
+
+[scheduler]
+gpu_free_memory_mb = 1024
+gpu_free_utilization_pct = 10
+prefer_pack = true
+
+# Code is synchronized by Git. The mandatory scp pass uses an empty directory.
+[sync]
+mode = "scp"
+local_path = "/tmp/frontier-decode-batch-grid-v0-empty-sync"
+
+[[hosts]]
+name = "dash1"
+ssh_alias = "dash1"
+enabled = true
+sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0"
+fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash1"
+
+[[hosts]]
+name = "dash2"
+ssh_alias = "dash2"
+enabled = true
+sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0"
+fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash2"
+
+[[hosts]]
+name = "dash3"
+ssh_alias = "dash3"
+enabled = true
+sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0"
+fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash3"
+
+[[hosts]]
+name = "dash4"
+ssh_alias = "dash4"
+enabled = true
+sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0"
+fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash4"

runs/frontier-decode-batch-grid-v0/jobs-smoke-r1.toml

@@ -0,0 +1,16 @@
+version = 1
+
+[[jobs]]
+name = "decode-batch-tp4-b2-r1-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "2"
+SERVER_PORT = "9831"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"

runs/frontier-decode-batch-grid-v0/jobs-smoke-r2.toml

@@ -0,0 +1,16 @@
+version = 1
+
+[[jobs]]
+name = "decode-batch-tp4-b2-r2-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "2"
+SERVER_PORT = "9832"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"

runs/frontier-decode-batch-grid-v0/materialize_decode_batch.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Create simultaneous, prefix-disjoint requests for a fixed decode batch."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=Path, required=True)
+    parser.add_argument("--batch", type=int, required=True)
+    parser.add_argument("--input-tokens", type=int, default=2048)
+    parser.add_argument("--output-tokens", type=int, default=128)
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    if min(args.batch, args.input_tokens, args.output_tokens) <= 0:
+        raise ValueError("batch and token counts must be positive")
+    if args.input_tokens + args.output_tokens > 40960:
+        raise ValueError("request exceeds the server max model length")
+
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    special = set(tokenizer.all_special_ids)
+    candidates = [
+        token for token in range(tokenizer.vocab_size) if token not in special
+    ]
+    if len(candidates) < args.batch + 1:
+        raise ValueError("tokenizer has too few non-special token IDs")
+
+    base = candidates[0]
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("w") as output:
+        for index in range(args.batch):
+            row = {
+                "source_index": index,
+                "arrived_at": 0.0,
+                "input_length": args.input_tokens,
+                "output_length": args.output_tokens,
+                "session_id": index,
+                "runtime_block_ids": [],
+                "body": {
+                    "prompt": [
+                        candidates[index + 1],
+                        *([base] * (args.input_tokens - 1)),
+                    ],
+                    "min_tokens": args.output_tokens,
+                    "max_tokens": args.output_tokens,
+                    "ignore_eos": True,
+                },
+            }
+            output.write(json.dumps(row, separators=(",", ":")) + "\n")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-decode-batch-grid-v0/plot_figure_prototype.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Render the preregistered EXP-DECODE-BATCH-GRID schematic."""
+
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+ROOT = Path(__file__).resolve().parent
+
+
+def main() -> None:
+    fig, axes = plt.subplots(1, 2, figsize=(9.6, 4.0))
+
+    ax = axes[0]
+    repeats = np.array([3.7, 5.0, 6.4])
+    ax.scatter([1, 2, 3], repeats, color="#d95f02", s=45, label="old/possible repeats")
+    ax.axhspan(4.1, 4.6, color="#1b9e77", alpha=0.18, label="stable target band")
+    ax.axhline(np.median(repeats), color="#7570b3", ls="--", label="median")
+    ax.set(xlabel="TP4/b2 fresh-process repeat", ylabel="execute wall (ms)")
+    ax.set_title("A. Stability gate before curve fit")
+    ax.set_xticks([1, 2, 3])
+    ax.legend(frameon=False, fontsize=8)
+
+    ax = axes[1]
+    labels = ["TP2", "TP4", "TP1"]
+    real = [4.305, 4.449, 4.876]
+    sim = [5.12, 5.86, 5.49]
+    corrected = [4.7, 4.9, 5.3]
+    x = np.arange(3)
+    width = 0.25
+    ax.bar(x - width, real, width, label="real", color="#1b9e77")
+    ax.bar(x, sim, width, label="current sim", color="#d95f02")
+    ax.bar(x + width, corrected, width, label="target corrected", color="#7570b3")
+    ax.set_xticks(x, labels)
+    ax.set_ylabel("BC-8 TPOT (ms)")
+    ax.set_title("B. Complete-order recovery gate")
+    ax.legend(frameon=False, fontsize=8)
+
+    fig.suptitle("EXP-DECODE-BATCH-GRID preregistered figure prototype")
+    fig.tight_layout()
+    fig.savefig(ROOT / "figure-prototype.png", dpi=180)
+    fig.savefig(ROOT / "figure-prototype.svg")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-decode-batch-grid-v0/run_decode_profile.sh

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+TP="${TP:?TP is required}"
+PROFILE_BATCH="${PROFILE_BATCH:?PROFILE_BATCH is required}"
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
+VENV_ROOT="${VENV_ROOT:-/home/admin/cpfs/wjh/venvs/vllm-0.20.0-cu129-workload-regime-v2}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}"
+SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-900}"
+ACTIVE_ITERATIONS="${ACTIVE_ITERATIONS:-16}"
+SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_ROOT}/../.." && pwd)"
+MATERIALIZER="${SCRIPT_ROOT}/materialize_decode_batch.py"
+CLIENT="${PROJECT_ROOT}/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py"
+SERVER_PID=""
+
+if [[ ! "${PROFILE_BATCH}" =~ ^(2|4|6|8)$ ]]; then
+  echo "ERROR: PROFILE_BATCH must be 2, 4, 6, or 8" >&2
+  exit 1
+fi
+IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?GPU allocation is required}"
+if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then
+  echo "ERROR: TP=${TP}, but CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" >&2
+  exit 1
+fi
+
+mkdir -p "${OUTPUT_ROOT}"
+OUTPUT_ROOT="$(cd "${OUTPUT_ROOT}" && pwd)"
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" \
+  "${OUTPUT_ROOT}/requests" "${OUTPUT_ROOT}/results" \
+  "${OUTPUT_ROOT}/trace-staging" "${OUTPUT_ROOT}/traces/profile"
+exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1
+
+cleanup() {
+  if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+    kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
+    for _ in $(seq 1 30); do
+      kill -0 "${SERVER_PID}" 2>/dev/null || break
+      sleep 1
+    done
+    kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
+  fi
+  SERVER_PID=""
+}
+trap cleanup EXIT INT TERM
+
+sha256sum "${BASH_SOURCE[0]}" "${MATERIALIZER}" "${CLIENT}" \
+  "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/inputs.sha256"
+"${VENV_ROOT}/bin/python" -c \
+  'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' \
+  > "${OUTPUT_ROOT}/provenance/runtime.versions"
+nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total \
+  --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.before.csv"
+ps -eo user,pid,ppid,etimes,pcpu,pmem,args --sort=pid \
+  > "${OUTPUT_ROOT}/provenance/processes.before.txt"
+uptime > "${OUTPUT_ROOT}/provenance/uptime.before.txt"
+env | sort > "${OUTPUT_ROOT}/provenance/environment.txt"
+
+PROFILE_CONFIG="$("${VENV_ROOT}/bin/python" - "${OUTPUT_ROOT}/trace-staging" \
+  "${ACTIVE_ITERATIONS}" <<'PY'
+import json
+import sys
+
+print(json.dumps({
+    "profiler": "torch",
+    "torch_profiler_dir": sys.argv[1],
+    "torch_profiler_with_stack": False,
+    "torch_profiler_record_shapes": True,
+    "torch_profiler_use_gzip": True,
+    "ignore_frontend": True,
+    "wait_iterations": 0,
+    "warmup_iterations": 2,
+    "active_iterations": int(sys.argv[2]),
+}, separators=(",", ":")))
+PY
+)"
+printf '%s\n' "${PROFILE_CONFIG}" \
+  > "${OUTPUT_ROOT}/provenance/profiler-config.json"
+
+export TOKENIZERS_PARALLELISM=false
+export VLLM_USE_V1=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUTPUT_ROOT}/flashinfer-workspace}"
+mkdir -p "${FLASHINFER_WORKSPACE_BASE}"
+ulimit -n 65536
+
+"${VENV_ROOT}/bin/python" "${MATERIALIZER}" --model "${MODEL_ROOT}" \
+  --batch "${PROFILE_BATCH}" --input-tokens 2048 --output-tokens 128 \
+  --output "${OUTPUT_ROOT}/requests/b${PROFILE_BATCH}.jsonl"
+
+printf 'LAUNCH host=%s tp=%s batch=%s gpus=%s output=%s\n' \
+  "$(hostname)" "${TP}" "${PROFILE_BATCH}" "${CUDA_VISIBLE_DEVICES}" \
+  "${OUTPUT_ROOT}"
+setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
+  --host 127.0.0.1 --port "${SERVER_PORT}" \
+  --served-model-name qwen30-decode-batch-profile \
+  --tensor-parallel-size "${TP}" \
+  --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
+  --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs 16 \
+  --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \
+  --enable-logging-iteration-details --profiler-config "${PROFILE_CONFIG}" \
+  > "${OUTPUT_ROOT}/logs/server.log" 2>&1 &
+SERVER_PID=$!
+
+READY=0
+for _ in $(seq 1 "${SERVER_READY_ATTEMPTS}"); do
+  if curl -fsS --max-time 2 \
+    "http://127.0.0.1:${SERVER_PORT}/v1/models" \
+    > "${OUTPUT_ROOT}/results/models.json" 2>/dev/null; then
+    READY=1
+    break
+  fi
+  if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
+    tail -200 "${OUTPUT_ROOT}/logs/server.log"
+    exit 1
+  fi
+  sleep 3
+done
+if [[ "${READY}" -ne 1 ]]; then
+  echo "ERROR: server readiness timeout" >&2
+  tail -200 "${OUTPUT_ROOT}/logs/server.log"
+  exit 1
+fi
+
+run_client() {
+  local label="$1"
+  "${VENV_ROOT}/bin/python" "${CLIENT}" \
+    --port "${SERVER_PORT}" \
+    --requests-file "${OUTPUT_ROOT}/requests/b${PROFILE_BATCH}.jsonl" \
+    --served-model qwen30-decode-batch-profile \
+    --output "${OUTPUT_ROOT}/results/${label}.json" \
+    --tpot-slo-ms 150 --timeout-seconds 1800
+}
+
+# Exercise the same graph and scheduler path twice before profiling.
+run_client warmup-1
+run_client warmup-2
+
+curl -fsS -X POST "http://127.0.0.1:${SERVER_PORT}/start_profile" \
+  > "${OUTPUT_ROOT}/logs/start-profile.txt"
+run_client profile
+
+deadline=$((SECONDS + 120))
+while (( SECONDS < deadline )); do
+  trace_count="$(find "${OUTPUT_ROOT}/trace-staging" -maxdepth 1 -type f \
+    -name '*.pt.trace.json*' | wc -l)"
+  if (( trace_count >= TP )); then
+    break
+  fi
+  sleep 2
+done
+trace_count="$(find "${OUTPUT_ROOT}/trace-staging" -maxdepth 1 -type f \
+  -name '*.pt.trace.json*' | wc -l)"
+if (( trace_count < TP )); then
+  echo "ERROR: expected ${TP} rank traces, found ${trace_count}" >&2
+  exit 1
+fi
+find "${OUTPUT_ROOT}/trace-staging" -maxdepth 1 -type f \
+  -name '*.pt.trace.json*' -exec mv -t "${OUTPUT_ROOT}/traces/profile" {} +
+curl -fsS -X POST "http://127.0.0.1:${SERVER_PORT}/stop_profile" \
+  > "${OUTPUT_ROOT}/logs/stop-profile.txt"
+
+cleanup
+nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total \
+  --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.after.csv"
+find "${OUTPUT_ROOT}" -type f \
+  ! -path '*/provenance/artifacts.sha256' -print0 \
+  | sort -z | xargs -0 sha256sum \
+  > "${OUTPUT_ROOT}/provenance/artifacts.sha256"
+echo DECODE_BATCH_PROFILE_COMPLETE