diff --git a/runs/frontier-decode-batch-grid-v0/.gitignore b/runs/frontier-decode-batch-grid-v0/.gitignore new file mode 100644 index 0000000..011358f --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/.gitignore @@ -0,0 +1,3 @@ +fleet-artifacts/ +fleet-state/ +remote-outputs/ diff --git a/runs/frontier-decode-batch-grid-v0/analyze_decode_trace.py b/runs/frontier-decode-batch-grid-v0/analyze_decode_trace.py new file mode 100755 index 0000000..2ccea1f --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/analyze_decode_trace.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Summarize graph-on vLLM Torch traces by decode component.""" + +from __future__ import annotations + +import argparse +import gzip +import json +import math +import statistics +from collections import Counter, defaultdict +from pathlib import Path + + +COMPONENT_NAMES = ( + "attention", + "linear_norm_rope", + "router", + "moe", + "collective", + "output_head", + "other", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--trace-root", type=Path, required=True) + parser.add_argument("--label", required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def classify(name: str, occurrences: int, steps: int) -> str: + lower = name.lower() + if any(token in lower for token in ("nccl", "allreduce", "all_reduce")): + return "collective" + if "topkgating" in lower or "fused_topk" in lower: + return "router" + if any( + token in lower + for token in ( + "fused_moe", + "moefcgemm", + "tensorrt_llm::kernels::cutlass_kernels", + "groupproblemshape", + "memcpy32_post", + ) + ): + return "moe" + if any( + token in lower + for token in ( + "flashattn", + "flashattnfwd", + "reshape_and_cache", + "prepare_varlen_num_blocks", + ) + ): + return "attention" + if "nvjet" in lower and occurrences <= steps * 2: + return "output_head" + if any( + token in lower + for token in ( + "nvjet", + "cublaslt", + "rms_norm", + "rsqrt", + "triton_red_fused_2", + "triton_poi_fused_3", + "triton_red_fused_0", + "triton_poi_fused_1", + ) + ): + return "linear_norm_rope" + return "other" + + +def stats(values: list[float]) -> dict[str, float | int]: + ordered = sorted(values) + return { + "n": len(values), + "mean_ms": statistics.fmean(values), + "population_std_ms": statistics.pstdev(values), + "p50_ms": statistics.median(ordered), + "p95_ms": ordered[math.ceil(0.95 * len(ordered)) - 1], + } + + +def load_events(path: Path) -> list[dict]: + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rt") as source: + return json.load(source)["traceEvents"] + + +def analyze_rank(path: Path) -> dict: + events = load_events(path) + kernels = [event for event in events if event.get("cat") == "kernel"] + all_windows = sorted( + ( + event + for event in events + if event.get("cat") == "gpu_user_annotation" + and str(event.get("name", "")).startswith("execute_") + ), + key=lambda event: float(event["ts"]), + ) + if not all_windows: + raise ValueError(f"{path}: no GPU execute annotations") + window_names = Counter(str(window["name"]) for window in all_windows) + selected_name = window_names.most_common(1)[0][0] + windows = [ + window for window in all_windows if str(window["name"]) == selected_name + ] + + selected: list[dict] = [] + step_kernels: list[list[dict]] = [] + for window in windows: + start = float(window["ts"]) + end = start + float(window["dur"]) + current = [ + kernel for kernel in kernels if start <= float(kernel["ts"]) < end + ] + selected.extend(current) + step_kernels.append(current) + + occurrences = Counter(str(kernel["name"]) for kernel in selected) + component_steps: dict[str, list[float]] = defaultdict(list) + busy_steps: list[float] = [] + wall_steps = [float(window["dur"]) / 1000.0 for window in windows] + for current in step_kernels: + per_component: dict[str, float] = defaultdict(float) + for kernel in current: + name = str(kernel["name"]) + per_component[classify(name, occurrences[name], len(windows))] += ( + float(kernel["dur"]) / 1000.0 + ) + for name in COMPONENT_NAMES: + component_steps[name].append(per_component[name]) + busy_steps.append(sum(per_component.values())) + + return { + "trace": str(path), + "selected_execute_annotation": selected_name, + "execute_annotation_histogram": dict(sorted(window_names.items())), + "steps": len(windows), + "execute_wall": stats(wall_steps), + "gpu_kernel_busy": stats(busy_steps), + "non_kernel_gap": stats( + [wall - busy for wall, busy in zip(wall_steps, busy_steps)] + ), + "components": { + name: stats(values) for name, values in component_steps.items() + }, + } + + +def main() -> None: + args = parse_args() + traces = sorted(args.trace_root.rglob("*.pt.trace.json*")) + if not traces: + raise SystemExit(f"no traces under {args.trace_root}") + ranks = [analyze_rank(path) for path in traces] + payload = { + "schema": "frontier-decode-batch-trace.v1", + "label": args.label, + "contract": { + "timing": "CUDA graph-on GPU execute annotations", + "component_time": "sum of CUDA kernel durations inside execute range", + "tp_aggregation": "per-rank; slowest rank mean approximates critical path", + }, + "ranks": ranks, + "rank_summary": { + "ranks": len(ranks), + "slowest_rank_execute_mean_ms": max( + rank["execute_wall"]["mean_ms"] for rank in ranks + ), + "slowest_rank_kernel_busy_mean_ms": max( + rank["gpu_kernel_busy"]["mean_ms"] for rank in ranks + ), + "component_rank_mean_ms": { + name: statistics.fmean( + rank["components"][name]["mean_ms"] for rank in ranks + ) + for name in COMPONENT_NAMES + }, + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-decode-batch-grid-v0/experiment-card.md b/runs/frontier-decode-batch-grid-v0/experiment-card.md new file mode 100644 index 0000000..541c135 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/experiment-card.md @@ -0,0 +1,68 @@ +# 实验 EXP-DECODE-BATCH-GRID：BC-8 是否可由稳定 whole-layer b2--b8 curve 修复 + +> **状态：** entry gate PASS，待 TP4/b2 GPU smoke +> +> Parent campaign：[`../frontier-simulator-gap-campaign-v0/README.md`](../frontier-simulator-gap-campaign-v0/README.md) + +## Claim 与决策 + +- **Parent claim：** 最新 1h trace 中 TP4 的 TPOT/E2E 仍稳定正偏约 10--23%； + 历史 BC-8 knee-right 真机排序为 `TP24 常数外推，导致 TP4 residence + 被高估；稳定 whole-layer curve 可恢复排序。 + - H2：给定 batch 的 whole-layer curve 已准，错误来自 simulator 与 vLLM + batch formation/event semantics 不同。 +- **事前预测：** 若 H1 成立，TP4/b2 repeat CV≤10%，其 median 比当前进入 + BC-8 的 curve 低至少 0.5 ms，注入后 `TP410% 时追加第三次，取 + median，不允许从单次 favorable sample 选值。 + - smoke 稳定后补 TP2/TP4 × b{2,4,6,8}；b>4 不再使用 b4 常数外推。 + - whole-layer 注入只替换 pure-decode service curve，不改变 prefill。 + - BC-8 排序仍错则停止 profile 修补，升级 event-level state telemetry。 + +## Setup + +- **自变量：** TP `{2,4}`；pure-decode batch `{2,4,6,8}`；fresh-process + repeat `{1,2}`，不稳定 cell 追加 repeat 3。 +- **控制变量：** Qwen3-30B-A3B BF16；H20；vLLM 0.20；context=2048； + output=128；graph-on；MNS=16；no prefix caching；同模型/runtime/cache。 +- **Hardware：** dash1--dash4；每轮调度前要求目标主机 8×H20 + idle/healthy。每主机同时只启动一个 engine，避免 CPU/JIT contention。 +- **Metrics：** critical-rank execute wall；MoE/attention/collective/other + kernels；repeat median/CV；sim same-state residual；BC-8 TPOT 完整排序。 +- **Baselines：** + - 旧 serving grid：TP4/b2 execute=`5.044 ms`，单 process 内 + population std=`1.561 ms`，不足以进入判决链。 + - BC-8：real `4.305<4.449<4.876 ms`；sim + `5.12<5.49<5.86 ms`（TP2/TP4/TP1 对应完整排序见结果审计）。 + +## 预期产物与 review + +- **预期数据：** 每 cell trace、request result、runtime/GPU provenance； + `results/grid.json`；BC-8 counterfactual 与 verdict。 +- **Figure prototype：** `figure-prototype.png`。左图画 b2 repeat instability + 与目标 CI，右图画 BC-8 real/sim/corrected 排序。 +- **人工 review：** campaign 已批准按第三优先级推进。 +- **Review 意见：** 先 TP4/b2 smoke；不在稳定性 gate 前铺完整网格。 + +## 复现信息 + +- **Frontier baseline：** `deadc4a321f0baaa534c6ebd17f974123733cdc2`； + joint serving-path curves + structured-attention analysis branch。 +- **Remote：** 复用 dash1 的 clean detached experiment worktree；canonical + checkout 的用户 dirty changes 不动。 +- **Known limits：** 固定 2048→128 state-matched workload；本实验只支持 + decode curve 与 BC-8，不外推到 chat prefill shape 或其它模型。 + +## 结果 + +- **Entry gate：** PASS。TP4 1h trace TPOT mean 正偏 + `+10.6%--+23.0%`，E2E mean `+9.8%--+19.2%`；BC-8 完整排序仍错。 +- **观察事实：** 待 GPU。 +- **Decision：** 待 GPU。 diff --git a/runs/frontier-decode-batch-grid-v0/figure-prototype.png b/runs/frontier-decode-batch-grid-v0/figure-prototype.png new file mode 100644 index 0000000..b1db93b Binary files /dev/null and b/runs/frontier-decode-batch-grid-v0/figure-prototype.png differ diff --git a/runs/frontier-decode-batch-grid-v0/figure-prototype.svg b/runs/frontier-decode-batch-grid-v0/figure-prototype.svg new file mode 100644 index 0000000..cce3f63 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/figure-prototype.svg @@ -0,0 +1,1870 @@ + + + + + + + + 2026-07-23T16:49:49.913314 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/runs/frontier-decode-batch-grid-v0/fleet.toml b/runs/frontier-decode-batch-grid-v0/fleet.toml new file mode 100644 index 0000000..6d34c27 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/fleet.toml @@ -0,0 +1,46 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-decode-batch-grid-v0/fleet-state" +artifacts_dir = "runs/frontier-decode-batch-grid-v0/fleet-artifacts" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 1024 +gpu_free_utilization_pct = 10 +prefer_pack = true + +# Code is synchronized by Git. The mandatory scp pass uses an empty directory. +[sync] +mode = "scp" +local_path = "/tmp/frontier-decode-batch-grid-v0-empty-sync" + +[[hosts]] +name = "dash1" +ssh_alias = "dash1" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash1" + +[[hosts]] +name = "dash2" +ssh_alias = "dash2" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash2" + +[[hosts]] +name = "dash3" +ssh_alias = "dash3" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash3" + +[[hosts]] +name = "dash4" +ssh_alias = "dash4" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-decode-batch-grid-v0" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-decode-batch-grid-v0/dash4" diff --git a/runs/frontier-decode-batch-grid-v0/jobs-smoke-r1.toml b/runs/frontier-decode-batch-grid-v0/jobs-smoke-r1.toml new file mode 100644 index 0000000..d531d0b --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/jobs-smoke-r1.toml @@ -0,0 +1,16 @@ +version = 1 + +[[jobs]] +name = "decode-batch-tp4-b2-r1-20260723" +gpus = 4 +gpu_model = "H20" +hosts = ["dash1"] +command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh" +artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1"] + +[jobs.env] +TP = "4" +PROFILE_BATCH = "2" +SERVER_PORT = "9831" +OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1" +FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4" diff --git a/runs/frontier-decode-batch-grid-v0/jobs-smoke-r2.toml b/runs/frontier-decode-batch-grid-v0/jobs-smoke-r2.toml new file mode 100644 index 0000000..55aa1d4 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/jobs-smoke-r2.toml @@ -0,0 +1,16 @@ +version = 1 + +[[jobs]] +name = "decode-batch-tp4-b2-r2-20260723" +gpus = 4 +gpu_model = "H20" +hosts = ["dash1"] +command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh" +artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2"] + +[jobs.env] +TP = "4" +PROFILE_BATCH = "2" +SERVER_PORT = "9832" +OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2" +FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4" diff --git a/runs/frontier-decode-batch-grid-v0/materialize_decode_batch.py b/runs/frontier-decode-batch-grid-v0/materialize_decode_batch.py new file mode 100755 index 0000000..64ba032 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/materialize_decode_batch.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Create simultaneous, prefix-disjoint requests for a fixed decode batch.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--batch", type=int, required=True) + parser.add_argument("--input-tokens", type=int, default=2048) + parser.add_argument("--output-tokens", type=int, default=128) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if min(args.batch, args.input_tokens, args.output_tokens) <= 0: + raise ValueError("batch and token counts must be positive") + if args.input_tokens + args.output_tokens > 40960: + raise ValueError("request exceeds the server max model length") + + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) + special = set(tokenizer.all_special_ids) + candidates = [ + token for token in range(tokenizer.vocab_size) if token not in special + ] + if len(candidates) < args.batch + 1: + raise ValueError("tokenizer has too few non-special token IDs") + + base = candidates[0] + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w") as output: + for index in range(args.batch): + row = { + "source_index": index, + "arrived_at": 0.0, + "input_length": args.input_tokens, + "output_length": args.output_tokens, + "session_id": index, + "runtime_block_ids": [], + "body": { + "prompt": [ + candidates[index + 1], + *([base] * (args.input_tokens - 1)), + ], + "min_tokens": args.output_tokens, + "max_tokens": args.output_tokens, + "ignore_eos": True, + }, + } + output.write(json.dumps(row, separators=(",", ":")) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-decode-batch-grid-v0/plot_figure_prototype.py b/runs/frontier-decode-batch-grid-v0/plot_figure_prototype.py new file mode 100644 index 0000000..3d0e727 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/plot_figure_prototype.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Render the preregistered EXP-DECODE-BATCH-GRID schematic.""" + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +ROOT = Path(__file__).resolve().parent + + +def main() -> None: + fig, axes = plt.subplots(1, 2, figsize=(9.6, 4.0)) + + ax = axes[0] + repeats = np.array([3.7, 5.0, 6.4]) + ax.scatter([1, 2, 3], repeats, color="#d95f02", s=45, label="old/possible repeats") + ax.axhspan(4.1, 4.6, color="#1b9e77", alpha=0.18, label="stable target band") + ax.axhline(np.median(repeats), color="#7570b3", ls="--", label="median") + ax.set(xlabel="TP4/b2 fresh-process repeat", ylabel="execute wall (ms)") + ax.set_title("A. Stability gate before curve fit") + ax.set_xticks([1, 2, 3]) + ax.legend(frameon=False, fontsize=8) + + ax = axes[1] + labels = ["TP2", "TP4", "TP1"] + real = [4.305, 4.449, 4.876] + sim = [5.12, 5.86, 5.49] + corrected = [4.7, 4.9, 5.3] + x = np.arange(3) + width = 0.25 + ax.bar(x - width, real, width, label="real", color="#1b9e77") + ax.bar(x, sim, width, label="current sim", color="#d95f02") + ax.bar(x + width, corrected, width, label="target corrected", color="#7570b3") + ax.set_xticks(x, labels) + ax.set_ylabel("BC-8 TPOT (ms)") + ax.set_title("B. Complete-order recovery gate") + ax.legend(frameon=False, fontsize=8) + + fig.suptitle("EXP-DECODE-BATCH-GRID preregistered figure prototype") + fig.tight_layout() + fig.savefig(ROOT / "figure-prototype.png", dpi=180) + fig.savefig(ROOT / "figure-prototype.svg") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-decode-batch-grid-v0/run_decode_profile.sh b/runs/frontier-decode-batch-grid-v0/run_decode_profile.sh new file mode 100755 index 0000000..079eaf0 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/run_decode_profile.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP is required}" +PROFILE_BATCH="${PROFILE_BATCH:?PROFILE_BATCH is required}" +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" +VENV_ROOT="${VENV_ROOT:-/home/admin/cpfs/wjh/venvs/vllm-0.20.0-cu129-workload-regime-v2}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}" +SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-900}" +ACTIVE_ITERATIONS="${ACTIVE_ITERATIONS:-16}" +SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_ROOT}/../.." && pwd)" +MATERIALIZER="${SCRIPT_ROOT}/materialize_decode_batch.py" +CLIENT="${PROJECT_ROOT}/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py" +SERVER_PID="" + +if [[ ! "${PROFILE_BATCH}" =~ ^(2|4|6|8)$ ]]; then + echo "ERROR: PROFILE_BATCH must be 2, 4, 6, or 8" >&2 + exit 1 +fi +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?GPU allocation is required}" +if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then + echo "ERROR: TP=${TP}, but CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_ROOT}" +OUTPUT_ROOT="$(cd "${OUTPUT_ROOT}" && pwd)" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" \ + "${OUTPUT_ROOT}/requests" "${OUTPUT_ROOT}/results" \ + "${OUTPUT_ROOT}/trace-staging" "${OUTPUT_ROOT}/traces/profile" +exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1 + +cleanup() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID="" +} +trap cleanup EXIT INT TERM + +sha256sum "${BASH_SOURCE[0]}" "${MATERIALIZER}" "${CLIENT}" \ + "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/inputs.sha256" +"${VENV_ROOT}/bin/python" -c \ + 'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' \ + > "${OUTPUT_ROOT}/provenance/runtime.versions" +nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.before.csv" +ps -eo user,pid,ppid,etimes,pcpu,pmem,args --sort=pid \ + > "${OUTPUT_ROOT}/provenance/processes.before.txt" +uptime > "${OUTPUT_ROOT}/provenance/uptime.before.txt" +env | sort > "${OUTPUT_ROOT}/provenance/environment.txt" + +PROFILE_CONFIG="$("${VENV_ROOT}/bin/python" - "${OUTPUT_ROOT}/trace-staging" \ + "${ACTIVE_ITERATIONS}" <<'PY' +import json +import sys + +print(json.dumps({ + "profiler": "torch", + "torch_profiler_dir": sys.argv[1], + "torch_profiler_with_stack": False, + "torch_profiler_record_shapes": True, + "torch_profiler_use_gzip": True, + "ignore_frontend": True, + "wait_iterations": 0, + "warmup_iterations": 2, + "active_iterations": int(sys.argv[2]), +}, separators=(",", ":"))) +PY +)" +printf '%s\n' "${PROFILE_CONFIG}" \ + > "${OUTPUT_ROOT}/provenance/profiler-config.json" + +export TOKENIZERS_PARALLELISM=false +export VLLM_USE_V1=1 +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 +export FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUTPUT_ROOT}/flashinfer-workspace}" +mkdir -p "${FLASHINFER_WORKSPACE_BASE}" +ulimit -n 65536 + +"${VENV_ROOT}/bin/python" "${MATERIALIZER}" --model "${MODEL_ROOT}" \ + --batch "${PROFILE_BATCH}" --input-tokens 2048 --output-tokens 128 \ + --output "${OUTPUT_ROOT}/requests/b${PROFILE_BATCH}.jsonl" + +printf 'LAUNCH host=%s tp=%s batch=%s gpus=%s output=%s\n' \ + "$(hostname)" "${TP}" "${PROFILE_BATCH}" "${CUDA_VISIBLE_DEVICES}" \ + "${OUTPUT_ROOT}" +setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${SERVER_PORT}" \ + --served-model-name qwen30-decode-batch-profile \ + --tensor-parallel-size "${TP}" \ + --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \ + --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs 16 \ + --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + --enable-logging-iteration-details --profiler-config "${PROFILE_CONFIG}" \ + > "${OUTPUT_ROOT}/logs/server.log" 2>&1 & +SERVER_PID=$! + +READY=0 +for _ in $(seq 1 "${SERVER_READY_ATTEMPTS}"); do + if curl -fsS --max-time 2 \ + "http://127.0.0.1:${SERVER_PORT}/v1/models" \ + > "${OUTPUT_ROOT}/results/models.json" 2>/dev/null; then + READY=1 + break + fi + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + tail -200 "${OUTPUT_ROOT}/logs/server.log" + exit 1 + fi + sleep 3 +done +if [[ "${READY}" -ne 1 ]]; then + echo "ERROR: server readiness timeout" >&2 + tail -200 "${OUTPUT_ROOT}/logs/server.log" + exit 1 +fi + +run_client() { + local label="$1" + "${VENV_ROOT}/bin/python" "${CLIENT}" \ + --port "${SERVER_PORT}" \ + --requests-file "${OUTPUT_ROOT}/requests/b${PROFILE_BATCH}.jsonl" \ + --served-model qwen30-decode-batch-profile \ + --output "${OUTPUT_ROOT}/results/${label}.json" \ + --tpot-slo-ms 150 --timeout-seconds 1800 +} + +# Exercise the same graph and scheduler path twice before profiling. +run_client warmup-1 +run_client warmup-2 + +curl -fsS -X POST "http://127.0.0.1:${SERVER_PORT}/start_profile" \ + > "${OUTPUT_ROOT}/logs/start-profile.txt" +run_client profile + +deadline=$((SECONDS + 120)) +while (( SECONDS < deadline )); do + trace_count="$(find "${OUTPUT_ROOT}/trace-staging" -maxdepth 1 -type f \ + -name '*.pt.trace.json*' | wc -l)" + if (( trace_count >= TP )); then + break + fi + sleep 2 +done +trace_count="$(find "${OUTPUT_ROOT}/trace-staging" -maxdepth 1 -type f \ + -name '*.pt.trace.json*' | wc -l)" +if (( trace_count < TP )); then + echo "ERROR: expected ${TP} rank traces, found ${trace_count}" >&2 + exit 1 +fi +find "${OUTPUT_ROOT}/trace-staging" -maxdepth 1 -type f \ + -name '*.pt.trace.json*' -exec mv -t "${OUTPUT_ROOT}/traces/profile" {} + +curl -fsS -X POST "http://127.0.0.1:${SERVER_PORT}/stop_profile" \ + > "${OUTPUT_ROOT}/logs/stop-profile.txt" + +cleanup +nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.after.csv" +find "${OUTPUT_ROOT}" -type f \ + ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum \ + > "${OUTPUT_ROOT}/provenance/artifacts.sha256" +echo DECODE_BATCH_PROFILE_COMPLETE