diff --git a/runs/frontier-fidelity-envelope-v1/.gitignore b/runs/frontier-fidelity-envelope-v1/.gitignore new file mode 100644 index 0000000..008f888 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +a1-native-smoke/ +a2-measured-fix-smoke/ +simulator-a1/ +simulator-a2/ +simulator-a3/ +fleet-state/ +fleet-artifacts/ diff --git a/runs/frontier-fidelity-envelope-v1/experiment-card.md b/runs/frontier-fidelity-envelope-v1/experiment-card.md new file mode 100644 index 0000000..c1563db --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/experiment-card.md @@ -0,0 +1,76 @@ +# EXP-SIMFID-ENVELOPE-V1:Frontier best-effort fidelity envelope + +> **状态:** 已批准,准备执行(2026-07-17)。用户要求先把 simulator 现有能力跑到最好,并同时覆盖 fixed input/output 与真实 trace replay。 + +## Claim 与可证伪假设 + +- **研究问题:** 在不使用被评测 config/workload 的 serving E2E calibration 时,Frontier 的 measured operator/collective profiles 与 scheduler state abstraction,是否足以找到真机上的低-regret config? +- **H-CC:** Qwen30 prefill-only 的 TP 排序错误主要来自默认 analytical all-reduce;注入同机、同 TP 的 measured collective 后,已知 `2048/1` surface 的 regret 降至不超过 5%,Kendall tau-b 升至至少 0.8。 +- **H-BATCH:** 若 H-CC 不足,错误主要来自 pure-prefill attention 只有 batch=1 profile,而 Frontier 在多请求 batch 上使用没有 coverage 的 `attn_prefill_mixed` 外推;增加 MBT 可达的真实 batch composition 后可恢复排序。 +- **H-STATE:** 若 measured collective 与 batch-composition profile 都不能恢复排序,则缺失量位于 isolated operators 之外的 scheduler-state-conditioned step composition;继续增加静态 kernel rows不是有效修复。 +- **成功门槛:** worst selected-config regret `<=5%`、tie-aware Kendall tau-b `>=0.8`、真机 capacity bracket 不足以反转 top decision,并且没有使用同一 surface 的 E2E scalar calibration。 + +## Simulator ablation(先用已有 ground truth,零新增 GPU 成本) + +| variant | compute profile | collective | 目的 | +|---|---|---|---| +| A0 | vLLM 0.20 frozen profile-v2 | Frontier analytical | 已冻结 baseline | +| A1 | 同 A0 | Frontier 原生 Vidur + measured TP2/TP4 CSV | 检查原生 profile consumption;大 payload fallback 保留并计数 | +| A2 | 同 A0 | measured Vidur,cache miss 直接调用已训练 estimator | 最小 correctness fix;消除 `>100k elements` 静默 analytical fallback | +| A3 | 增加 pure-prefill batch-composition rows | 同 A2 | 检验 batch-composition coverage 是否是剩余误差来源 | + +A1/A2/A3 都重新运行完整 `TP∈{1,2,4} × MNS∈{8,16,32,64}` surface;TP1 无 all-reduce。所有 simulator variants 在查看新增真机 case 前冻结。A2 是单独标注、带单测的 compatibility patch,不与 Frontier upstream 原生能力混写。 + +## Workload matrix + +| ID | workload | arrival / prefix | phase role | +|---|---|---|---| +| F0 | fixed `ISL=2048, OSL=1` | uniform QPS;distinct prefixes | 已有 real ground truth,选择 A0--A3 | +| F1 | fixed `ISL=512, OSL=1` | uniform QPS;distinct prefixes | short-prefill、多请求 batch composition | +| F2 | fixed `ISL=2048, OSL=128` | uniform QPS;distinct prefixes | true prefill+decode mixed serving | +| T1 | `thinking_w20260327_1000` eligible trace | 原 timestamp/order;exact prompt/output/session/hash;prefix on | production joint distribution 与 cache/scheduler feedback | + +T1 只排除已经审计的 72 个超 40,960 context rows 和 6 个 zero-output rows,eligible universe 为 `15,401/15,479`。负载轴只使用 trace 已有、同 session 共享的 `sampling_u`;入选 request 的 arrival、input/output、prompt、hash 和相对次序不变。真实 runtime 设置 `min_tokens=max_tokens=output_length` 且 `ignore_eos=true`,逐请求核对 usage。 + +## 固定系统与 config surface + +| 项目 | 冻结设置 | +|---|---| +| machine | 仅 `dash0`,8×NVIDIA H20 | +| model/runtime | `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`;community vLLM 0.20.0+cu129;BF16 weight/activation/KV | +| simulator | Frontier `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + manifest 中列出的现有 compatibility patches;A2 patch 独立 hash | +| configs | `TP∈{1,2,4} × MNS∈{8,16,32,64}`;DP=PP=EP=1;MBT=8192;block=16 | +| runtime | chunked prefill on;fixed cases prefix off;T1 prefix on;fresh server per `(config, load, round)` | +| score | `capacity(c)=max tested offered req/s with joint SLO pass rate >=0.95`;primary `capacity/actual TP GPUs` | +| SLO | TTFT `<=1000ms + 1000×ISL/8000`;mixed case同时要求 TPOT `<=150ms`;另报告 50/100/180ms sensitivity,不用 sensitivity 改写 primary | + +F0 沿用已经冻结的 rate lattice与两个 fresh-server rounds。F1/F2 先由冻结 simulator 给出 boundary,再加入共同 per-GPU guard anchors,避免只测 simulator 预测附近而漏掉真实最优。每个 boundary anchor 两个 fresh-server rounds,二者都 pass 才算 feasible。 + +T1 保持原 600 秒 arrival window。先在 simulator 上冻结 `sampling_u` bracket;真机只运行 topological guard set `{TP1,TP2,TP4} × {MNS8,MNS32,MNS64}`,若 top set 或 bracket 仍可能被未测 MNS 反转再补 MNS16。每个入选 source-row vector在 real/sim 两侧必须有相同 digest;至少两个 session-hash folds,若本轮只完成一个 window则明确标为 single-window evidence。 + +## 诊断与停止规则 + +1. A1 必须报告每次 collective prediction 的 measured-model hit 与 analytical fallback 次数;不能只看最终 rank。 +2. A2 对超过 100k elements 的 payload 必须由单测证明走 estimator;A2 若不改变任何 TP2/TP4 step,立即停止并检查 CLI/config 注入,不进入 GPU。 +3. A3 profile 只覆盖 MBT=8192 可达的 pure-prefill composition:F0 为 `1/2/4 × q2048`,F1 为 `1/2/4/8/16 × q512`,TP1/2/4 分别实测;不做无边界的 profile sweep。 +4. 若 A3 在 F0 仍不能达到 fidelity gate,先采集 `TP1@8, TP2@16, TP4@32` 的 per-step batch/queue/component residual;禁止用 per-TP E2E scale把答案拟合正确。 +5. 只有 best-effort simulator 在 F0 通过或形成可解释、可定位的失败后,才运行 F1/F2/T1 真机;任一 case 的结论不外推到其它 workload。 + +## 预期成本与产物 + +- simulator A1--A3:CPU only,约 1--3 小时总 CPU wall,0 GPU-hour。 +- attention composition profile:3 张 H20 并行,预计 5--10 分钟,`<0.5 H20-GPU-hour`。 +- F1/F2 real boundary:预计合计 12--24 H20-GPU-hours,smoke 后再锁定。 +- T1 real boundary:600 秒原 arrival window使单 anchor较贵;预计 30--60 H20-GPU-hours,必须在 simulator bracket 和一配置 smoke 后重新 echo 精确预算。 +- 产物:variant/profile manifests、full surfaces、anchor-level request metrics、rank/regret/confusion tables、profile-consumption counters,以及 fixed-vs-trace fidelity summary figure。 + +## Benchmark design audit + +| 风险 | 处理 | +|---|---| +| selective benchmarking | 预先冻结 F0/F1/F2/T1,不因结果删除失败 case | +| calibration=evaluation | 禁止使用同一 surface 的 serving E2E scalar;microprofile GPU 成本单独报告 | +| trace filtering | 只做 context/zero-output correctness exclusion和 session-coherent thinning,不按长度筛选 | +| simulator-guided real sampling | 使用共同 guard anchors;未闭合 bracket 不能宣布 top match | +| absolute-vs-rank metric | 同时报绝对 capacity/latency、rank、regret、tau-b、pair direction和 SLO confusion | +| hidden fallback | A1/A2 强制计数 measured-model hit/fallback,并写入 frozen manifest | diff --git a/runs/frontier-fidelity-envelope-v1/fleet.toml b/runs/frontier-fidelity-envelope-v1/fleet.toml new file mode 100644 index 0000000..612e4f7 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/fleet.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-fidelity-envelope-v1/fleet-state" +artifacts_dir = "runs/frontier-fidelity-envelope-v1/fleet-artifacts" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 1024 +gpu_free_utilization_pct = 10 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-fidelity-envelope-v1/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/fidelity-envelope-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1" diff --git a/runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml b/runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml new file mode 100644 index 0000000..7e2aa3b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml @@ -0,0 +1,43 @@ +version = 1 + +[[jobs]] +name = "qwen30-attention-composition-tp1-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_composition.sh" +artifacts = ["artifacts/attention-composition-tp1-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-composition-tp1-v1" + +[[jobs]] +name = "qwen30-attention-composition-tp2-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_composition.sh" +artifacts = ["artifacts/attention-composition-tp2-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-composition-tp2-v1" + +[[jobs]] +name = "qwen30-attention-composition-tp4-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_composition.sh" +artifacts = ["artifacts/attention-composition-tp4-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "4" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-composition-tp4-v1" diff --git a/runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py b/runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py new file mode 100644 index 0000000..bb57bc0 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Convert frozen vLLM collective measurements to Frontier Vidur CC CSV.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from pathlib import Path + + +FIELDS = ( + "time_stats.all_reduce.min", + "time_stats.all_reduce.max", + "time_stats.all_reduce.mean", + "time_stats.all_reduce.median", + "time_stats.all_reduce.std", + "rank", + "num_workers", + "size", + "collective", + "devices_per_node", + "max_devices_per_node", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def convert(input_path: Path, output_path: Path) -> dict[str, object]: + payload = json.loads(input_path.read_text()) + if payload.get("schema_version") != "qwen30_vllm020_allreduce_frozen.v1": + raise ValueError(f"unexpected input schema: {payload.get('schema_version')!r}") + + rows = [] + seen = set() + for source in payload["rows"]: + tp = int(source["tensor_parallel_size"]) + tokens = int(source["num_tokens"]) + key = (tp, tokens) + if key in seen: + raise ValueError(f"duplicate collective row: {key}") + seen.add(key) + if tp not in (2, 4): + raise ValueError(f"unsupported TP: {tp}") + expected_bytes = tokens * int(source["hidden_dim"]) * 2 + if int(source["payload_bytes"]) != expected_bytes: + raise ValueError(f"payload mismatch for {key}") + + # Frontier Vidur consumes only the median target. The raw profiler kept + # per-rank distributions but not aligned per-repeat critical-path + # samples, so do not invent critical-path min/mean/max/std. Repeating + # the observed critical-path median in the unused fields keeps the CSV + # schema explicit without changing the trained target. + median = float(source["critical_path_median_ms"]) + rows.append( + { + "time_stats.all_reduce.min": median, + "time_stats.all_reduce.max": median, + "time_stats.all_reduce.mean": median, + "time_stats.all_reduce.median": median, + "time_stats.all_reduce.std": 0.0, + "rank": 0, + "num_workers": tp, + "size": expected_bytes, + "collective": "all_reduce", + "devices_per_node": tp, + "max_devices_per_node": 8, + } + ) + + expected = {(tp, tokens) for tp in (2, 4) for tokens in (1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192)} + if seen != expected: + raise ValueError(f"collective coverage mismatch: missing={expected - seen}, extra={seen - expected}") + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(sorted(rows, key=lambda row: (row["num_workers"], row["size"]))) + + return { + "schema": "frontier-vidur-allreduce-materialization-v1", + "source": str(input_path.resolve()), + "source_sha256": sha256(input_path), + "output": str(output_path.resolve()), + "output_sha256": sha256(output_path), + "rows": len(rows), + "tp_coverage": [2, 4], + "target": "time_stats.all_reduce.median", + "unused_stat_policy": "repeat critical_path_median; std=0", + "payload_contract": "size=num_tokens*hidden_dim*2_bytes", + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args() + manifest = convert(args.input, args.output) + args.manifest.parent.mkdir(parents=True, exist_ok=True) + args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/mock-fidelity-envelope.png b/runs/frontier-fidelity-envelope-v1/mock-fidelity-envelope.png new file mode 100644 index 0000000..0e63a4e Binary files /dev/null and b/runs/frontier-fidelity-envelope-v1/mock-fidelity-envelope.png differ diff --git a/runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch b/runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch new file mode 100644 index 0000000..56eac1f --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch @@ -0,0 +1,84 @@ +diff --git a/frontier/cc_backend/backends/vidur_cc_backend.py b/frontier/cc_backend/backends/vidur_cc_backend.py +index ca1983a..0c57f05 100644 +--- a/frontier/cc_backend/backends/vidur_cc_backend.py ++++ b/frontier/cc_backend/backends/vidur_cc_backend.py +@@ -882,2 +882,21 @@ class VidurCCBackend(BaseCCBackend): +- # Fallback to analytical if not in cache +- logger.debug(f"num_tokens={num_tokens} not in cache, using analytical fallback") ++ # The precomputed lookup is capped at 100k elements, while realistic ++ # TP payloads are commonly much larger. A cache miss does not mean the ++ # measured-data model is unavailable: predict on demand and memoize the ++ # exact payload instead of silently switching model families. ++ with self._cache_lock: ++ model = self._models.get("all_reduce") ++ if model is not None: ++ features = pd.DataFrame({"num_tokens": [num_tokens]}) ++ result = float(model.predict(features)[0]) ++ with self._cache_lock: ++ self._predictions["all_reduce"][(num_tokens,)] = result ++ logger.debug( ++ f"predict_allreduce: data_size={data_size_bytes}, num_tokens={num_tokens}, " ++ f"result={result:.6f} ms (ML model, on-demand cache miss)" ++ ) ++ return max(0.0, result) ++ ++ logger.debug( ++ f"num_tokens={num_tokens} not in cache and model unavailable, " ++ "using analytical fallback" ++ ) +diff --git a/tests/unit/test_vidur_cc_large_payload.py b/tests/unit/test_vidur_cc_large_payload.py +new file mode 100644 +index 0000000..7e87aa7 +--- /dev/null ++++ b/tests/unit/test_vidur_cc_large_payload.py +@@ -0,0 +1,50 @@ ++from __future__ import annotations ++ ++import threading ++import unittest ++ ++import numpy as np ++ ++from frontier.cc_backend.backends.vidur_cc_backend import VidurCCBackend ++ ++ ++class RecordingModel: ++ def __init__(self, value: float) -> None: ++ self.value = value ++ self.features = [] ++ ++ def predict(self, features): ++ self.features.append(features.copy()) ++ return np.array([self.value]) ++ ++ ++class VidurCCLargePayloadTest(unittest.TestCase): ++ def test_cache_miss_uses_measured_model_and_memoizes(self) -> None: ++ backend = object.__new__(VidurCCBackend) ++ backend._cache_lock = threading.RLock() ++ backend._num_devices = 2 ++ backend._predictions = {"all_reduce": {(100000,): 0.1}} ++ model = RecordingModel(0.321) ++ backend._models = {"all_reduce": model} ++ backend._analytical_fallback_allreduce = lambda *_: self.fail( ++ "analytical fallback must not run when the measured model exists" ++ ) ++ ++ data_size_bytes = 2048 * 2048 * 2 ++ expected_elements = data_size_bytes // 2 ++ first = backend.predict_allreduce(data_size_bytes, num_devices=2) ++ second = backend.predict_allreduce(data_size_bytes, num_devices=2) ++ ++ self.assertEqual(first, 0.321) ++ self.assertEqual(second, 0.321) ++ self.assertEqual(len(model.features), 1) ++ self.assertEqual( ++ int(model.features[0].iloc[0]["num_tokens"]), expected_elements ++ ) ++ self.assertEqual( ++ backend._predictions["all_reduce"][(expected_elements,)], 0.321 ++ ) ++ ++ ++if __name__ == "__main__": ++ unittest.main() diff --git a/runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py b/runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py new file mode 100644 index 0000000..8d21765 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Render the preregistered fidelity-envelope figure prototype.""" + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + + +OUT = Path(__file__).with_name("mock-fidelity-envelope.png") + + +def main() -> None: + workloads = ["F0\n2048/1", "F1\n512/1", "F2\n2048/128", "T1\nexact trace"] + variants = ["A0 analytical", "A1 native measured", "A2 measured+fix", "A3 +batch profile"] + # Prototype values are deliberately marked as MOCK and encode possible, + # distinguishable outcomes only. They are never read by result analysis. + regret = np.array( + [ + [12.5, 14.0, 18.0, 25.0], + [11.0, 13.0, 16.0, 22.0], + [7.5, 10.0, 12.0, 18.0], + [2.0, 4.0, 7.0, 12.0], + ] + ) + + fig, (ax0, ax1) = plt.subplots( + 1, 2, figsize=(11.5, 4.5), gridspec_kw={"width_ratios": [1.35, 1.0]} + ) + x = np.arange(len(workloads)) + width = 0.19 + colors = ["#7f7f7f", "#4c78a8", "#f58518", "#54a24b"] + for index, (variant, color) in enumerate(zip(variants, colors)): + ax0.bar( + x + (index - 1.5) * width, + regret[index], + width, + label=variant, + color=color, + ) + ax0.axhline(5.0, color="#d62728", linestyle="--", linewidth=1.5, label="5% gate") + ax0.set_xticks(x, workloads) + ax0.set_ylabel("Worst selected-config regret (%)") + ax0.set_title("A. Rank fidelity across workload complexity") + ax0.legend(fontsize=8, ncol=2, frameon=False) + ax0.grid(axis="y", alpha=0.25) + + consumption = np.array( + [ + [0, 0, 0], + [35, 65, 0], + [100, 0, 0], + [100, 0, 100], + ] + ) + bottom = np.zeros(len(variants)) + labels = ["measured collective hit", "analytical fallback", "batch-profile coverage"] + stack_colors = ["#4c78a8", "#e45756", "#54a24b"] + for values, label, color in zip(consumption.T, labels, stack_colors): + ax1.barh(variants, values, left=bottom, label=label, color=color) + bottom += values + ax1.set_xlim(0, 200) + ax1.set_xlabel("Coverage counters (normalized; separate axes by mechanism)") + ax1.set_title("B. Profile consumption, not just final rank") + ax1.grid(axis="x", alpha=0.25) + ax1.legend(fontsize=8, frameon=False, loc="lower right") + + fig.suptitle("MOCK / preregistered layout — values are not experimental results", fontsize=12) + fig.tight_layout() + fig.savefig(OUT, dpi=180, bbox_inches="tight") + print(OUT) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv new file mode 100644 index 0000000..85fedab --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv @@ -0,0 +1,25 @@ +time_stats.all_reduce.min,time_stats.all_reduce.max,time_stats.all_reduce.mean,time_stats.all_reduce.median,time_stats.all_reduce.std,rank,num_workers,size,collective,devices_per_node,max_devices_per_node +0.08288000151515007,0.08288000151515007,0.08288000151515007,0.08288000151515007,0.0,0,2,4096,all_reduce,2,8 +0.0793600007891655,0.0793600007891655,0.0793600007891655,0.0793600007891655,0.0,0,2,32768,all_reduce,2,8 +0.0713919997215271,0.0713919997215271,0.0713919997215271,0.0713919997215271,0.0,0,2,65536,all_reduce,2,8 +0.08056000247597694,0.08056000247597694,0.08056000247597694,0.08056000247597694,0.0,0,2,131072,all_reduce,2,8 +0.0865279994904995,0.0865279994904995,0.0865279994904995,0.0865279994904995,0.0,0,2,262144,all_reduce,2,8 +0.07135999947786331,0.07135999947786331,0.07135999947786331,0.07135999947786331,0.0,0,2,524288,all_reduce,2,8 +0.07321599870920181,0.07321599870920181,0.07321599870920181,0.07321599870920181,0.0,0,2,1048576,all_reduce,2,8 +0.09025600180029869,0.09025600180029869,0.09025600180029869,0.09025600180029869,0.0,0,2,2097152,all_reduce,2,8 +0.08083200082182884,0.08083200082182884,0.08083200082182884,0.08083200082182884,0.0,0,2,4194304,all_reduce,2,8 +0.10891199856996536,0.10891199856996536,0.10891199856996536,0.10891199856996536,0.0,0,2,8388608,all_reduce,2,8 +0.1703840047121048,0.1703840047121048,0.1703840047121048,0.1703840047121048,0.0,0,2,16777216,all_reduce,2,8 +0.25539200007915497,0.25539200007915497,0.25539200007915497,0.25539200007915497,0.0,0,2,33554432,all_reduce,2,8 +0.1021759994328022,0.1021759994328022,0.1021759994328022,0.1021759994328022,0.0,0,4,4096,all_reduce,4,8 +0.12694399803876877,0.12694399803876877,0.12694399803876877,0.12694399803876877,0.0,0,4,32768,all_reduce,4,8 +0.09161599725484848,0.09161599725484848,0.09161599725484848,0.09161599725484848,0.0,0,4,65536,all_reduce,4,8 +0.08580800145864487,0.08580800145864487,0.08580800145864487,0.08580800145864487,0.0,0,4,131072,all_reduce,4,8 +0.09867199882864952,0.09867199882864952,0.09867199882864952,0.09867199882864952,0.0,0,4,262144,all_reduce,4,8 +0.09646400064229965,0.09646400064229965,0.09646400064229965,0.09646400064229965,0.0,0,4,524288,all_reduce,4,8 +0.08377600088715553,0.08377600088715553,0.08377600088715553,0.08377600088715553,0.0,0,4,1048576,all_reduce,4,8 +0.1128000020980835,0.1128000020980835,0.1128000020980835,0.1128000020980835,0.0,0,4,2097152,all_reduce,4,8 +0.08755199983716011,0.08755199983716011,0.08755199983716011,0.08755199983716011,0.0,0,4,4194304,all_reduce,4,8 +0.12361599877476692,0.12361599877476692,0.12361599877476692,0.12361599877476692,0.0,0,4,8388608,all_reduce,4,8 +0.20030399411916733,0.20030399411916733,0.20030399411916733,0.20030399411916733,0.0,0,4,16777216,all_reduce,4,8 +0.2924960106611252,0.2924960106611252,0.2924960106611252,0.2924960106611252,0.0,0,4,33554432,all_reduce,4,8 diff --git a/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json new file mode 100644 index 0000000..6ea2791 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json @@ -0,0 +1,15 @@ +{ + "output": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv", + "output_sha256": "9d693fd406616b599e57bcde66c980c7fc2831b3acf37d3eb633cec80ea0070d", + "payload_contract": "size=num_tokens*hidden_dim*2_bytes", + "rows": 24, + "schema": "frontier-vidur-allreduce-materialization-v1", + "source": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/allreduce.json", + "source_sha256": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "target": "time_stats.all_reduce.median", + "tp_coverage": [ + 2, + 4 + ], + "unused_stat_policy": "repeat critical_path_median; std=0" +} diff --git a/runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md b/runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md new file mode 100644 index 0000000..bf2fe4b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md @@ -0,0 +1,2 @@ +This directory is the gpu-fleet synchronization marker. Experiment code is +synchronized to dash0 through the project Git branch before dispatch. diff --git a/runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh b/runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh new file mode 100644 index 0000000..03afe43 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP must be set to 1, 2, or 4}" +case "${TP}" in + 1|2|4) ;; + *) echo "ERROR: invalid TP=${TP}" >&2; exit 1 ;; +esac + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +CAMPAIGN_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROFILE_SCRIPT="${CAMPAIGN_ROOT}/../frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py" +LOG_DIR="${OUTPUT_ROOT}/logs" +PROVENANCE_DIR="${OUTPUT_ROOT}/provenance" +BATCH_SPECS=(2q512 4q512 8q512 16q512 2q2k 4q2k) + +mkdir -p "${LOG_DIR}" "${PROVENANCE_DIR}" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${LOG_DIR}/composition.log") 2>&1 + +if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then + echo "ERROR: CUDA_VISIBLE_DEVICES must contain the fleet-allocated GPU" >&2 + exit 1 +fi +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp=${TP} batch_specs=${BATCH_SPECS[*]} profile_script=${PROFILE_SCRIPT} output=${OUTPUT_ROOT} expected_wall=3-8m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test -x "${VENV_ROOT}/bin/python" +test -f "${VLLM_SOURCE}/benchmarks/attention_benchmarks/runner.py" +test -f "${MODEL}/config.json" +test -f "${PROFILE_SCRIPT}" + +git -C "${CAMPAIGN_ROOT}/../.." rev-parse HEAD > "${PROVENANCE_DIR}/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${PROVENANCE_DIR}/vllm-source.commit" +sha256sum "${PROFILE_SCRIPT}" "${BASH_SOURCE[0]}" > "${PROVENANCE_DIR}/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" > "${PROVENANCE_DIR}/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total --format=csv,noheader > "${PROVENANCE_DIR}/gpus.csv" +printf '%s\n' "${BATCH_SPECS[@]}" > "${PROVENANCE_DIR}/batch-specs.txt" + +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" "${PROFILE_SCRIPT}" \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/flashattn-composition-tp${TP}.json" \ + --tp "${TP}" \ + --batch-specs "${BATCH_SPECS[@]}" \ + --warmup-iters 5 \ + --repeats 10 \ + --profile-kv-update + +test -s "${OUTPUT_ROOT}/raw/flashattn-composition-tp${TP}.json" +sha256sum "${OUTPUT_ROOT}/raw/flashattn-composition-tp${TP}.json" "${PROVENANCE_DIR}"/* > "${OUTPUT_ROOT}/artifacts.sha256" +nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FLASHATTN_COMPOSITION_COMPLETE tp=${TP} cases=${#BATCH_SPECS[@]}" diff --git a/runs/frontier-fidelity-envelope-v1/smoke-report.md b/runs/frontier-fidelity-envelope-v1/smoke-report.md new file mode 100644 index 0000000..f55c31b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/smoke-report.md @@ -0,0 +1,15 @@ +# Frontier measured-collective smoke + +日期:2026-07-17。设备:local CPU(Frontier simulation only)。Frontier commit:`d9cfeb6d8791fbf2f295dd9744c56a666171776e`,沿用既有 dirty compatibility patch set;A2 额外 patch SHA256 为 `35cc6be846589faf8cb5fa3ce5fdfe0aee8f086ba7dbb5dbcdc677148f19a3c8`。 + +固定 cell:Qwen3-30B-A3B BF16 profile-v2,`TP2/MNS8/MBT8192`,`ISL=2048/OSL=1`,64 requests,8 req/s,prefix off,TTFT SLO 1256 ms。 + +| variant | CC path for 2048-token payload | TTFT p50/p95/max (ms) | pass rate | +|---|---|---:|---:| +| A0 analytical | analytical | 122.2406276 | 1.0 | +| A1 native Vidur | `4,194,304 > 100,000` elements,lookup miss 后 analytical fallback | 122.2406276 | 1.0 | +| A2 measured + direct miss | measured random-forest estimator;exact payload memoized | 120.8898909 | 1.0 | + +A1 与 A0 的所有 TTFT 数值完全一致,验证 measured CSV 虽成功加载和训练,但没有参与该 payload 的 prediction。A2 model 对该 payload 的预测为 `0.09877793 ms`;同一个 TP2/2048 row 的 measured critical-path median 是 `0.10891200 ms`。A2 相对 A1 的 E2E delta 为 `-1.3507367 ms`(`-1.105%`),与每层多次 collective 的累计量级一致,因此通过“execution path 必须变化”的 smoke gate。 + +这个 smoke 只证明 profile consumption;不证明 ranking 已恢复。下一步必须运行完整 A2 TP×MNS surface,并与已冻结 real capacity比较。 diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py new file mode 100644 index 0000000..3f5a79c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import csv +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parent + + +def load(name: str): + path = ROOT / name + spec = importlib.util.spec_from_file_location(path.stem, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class FidelityEnvelopeTest(unittest.TestCase): + def test_materialize_allreduce(self) -> None: + module = load("materialize_frontier_allreduce.py") + rows = [] + for tp in (2, 4): + for tokens in (1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192): + rows.append( + { + "tensor_parallel_size": tp, + "num_tokens": tokens, + "hidden_dim": 2048, + "payload_bytes": tokens * 2048 * 2, + "critical_path_median_ms": tp + tokens / 1000, + } + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "allreduce.json" + source.write_text( + json.dumps( + { + "schema_version": "qwen30_vllm020_allreduce_frozen.v1", + "rows": rows, + } + ) + ) + output = root / "all_reduce.csv" + manifest = module.convert(source, output) + self.assertEqual(manifest["rows"], 24) + with output.open(newline="") as handle: + converted = list(csv.DictReader(handle)) + self.assertEqual(converted[0]["num_workers"], "2") + self.assertEqual(converted[0]["size"], "4096") + self.assertEqual(converted[-1]["num_workers"], "4") + self.assertEqual(converted[-1]["size"], str(8192 * 2048 * 2)) + self.assertEqual( + converted[-1]["time_stats.all_reduce.median"], + str(4 + 8192 / 1000), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py index 10249d7..ecd0e74 100644 --- a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py +++ b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py @@ -48,6 +48,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--requests", type=int, default=64) parser.add_argument("--rate", type=float, action="append") parser.add_argument("--config", action="append") + parser.add_argument( + "--cc-backend", choices=("analytical", "vidur"), default="analytical" + ) + parser.add_argument("--allreduce-csv", type=Path) parser.add_argument("--timeout-seconds", type=float, default=900.0) parser.add_argument("--resume", action="store_true") return parser.parse_args() @@ -182,6 +186,37 @@ def knobs(config: Config, paths: dict[str, Path], cache: Path) -> dict[str, Any] } +def configure_cc_command( + command: list[str], *, backend: str, allreduce_csv: Path | None, cache: Path +) -> list[str]: + configured = list(command) + option = "--cc_backend_config_type" + try: + index = configured.index(option) + except ValueError as error: + raise ValueError(f"Frontier command is missing {option}") from error + configured[index + 1] = backend + if backend == "analytical": + if allreduce_csv is not None: + raise ValueError("--allreduce-csv requires --cc-backend vidur") + return configured + if allreduce_csv is None: + raise ValueError("--cc-backend vidur requires --allreduce-csv") + configured.extend( + [ + "--vidur_cc_backend_config_all_reduce_input_file", + str(allreduce_csv), + "--vidur_cc_backend_config_cache_dir", + str(cache), + "--vidur_cc_backend_config_k_fold_cv_splits", + "6", + "--vidur_cc_backend_config_num_training_job_threads", + "1", + ] + ) + return configured + + def find_metrics(run_dir: Path) -> tuple[Path, Path]: systems = list((run_dir / "metrics").rglob("system_metrics.json")) requests = list((run_dir / "metrics").rglob("request_metrics.csv")) @@ -231,6 +266,10 @@ def main() -> None: args.profile_root = args.profile_root.resolve() args.python_deps = args.python_deps.resolve() args.output_root = args.output_root.resolve() + if args.allreduce_csv is not None: + args.allreduce_csv = args.allreduce_csv.resolve() + if not args.allreduce_csv.is_file(): + raise FileNotFoundError(args.allreduce_csv) rates = tuple(args.rate or RATES) selected = list(GRID) if args.config: @@ -274,6 +313,12 @@ def main() -> None: run_id=f"qwen30_prefill_{config.name}_r{rate:g}", knobs=config_knobs, ) + command = configure_cc_command( + command, + backend=args.cc_backend, + allreduce_csv=args.allreduce_csv, + cache=args.output_root / "cc-cache", + ) write_json(run_dir / "command.json", command) environment = os.environ.copy() pythonpath = [str(args.python_deps), str(args.frontier_source)] @@ -389,6 +434,15 @@ def main() -> None: "coverage": coverage, "sha256": {name: sha256(path) for name, path in paths.items()}, }, + "collective": { + "backend": args.cc_backend, + "allreduce_csv": ( + str(args.allreduce_csv) if args.allreduce_csv is not None else None + ), + "allreduce_csv_sha256": ( + sha256(args.allreduce_csv) if args.allreduce_csv is not None else None + ), + }, "config_results": config_results, "capacity": capacities, } diff --git a/runs/frontier-phase-factorial-v0/test_phase_factorial.py b/runs/frontier-phase-factorial-v0/test_phase_factorial.py index 4172926..0ceb170 100644 --- a/runs/frontier-phase-factorial-v0/test_phase_factorial.py +++ b/runs/frontier-phase-factorial-v0/test_phase_factorial.py @@ -43,3 +43,20 @@ def test_kendall_tau_b() -> None: analysis = load("analyze_qwen30_prefill_fidelity.py") assert analysis.kendall_tau_b([1, 2, 3], [1, 2, 3])["kendall_tau_b"] == 1 assert analysis.kendall_tau_b([1, 2, 3], [3, 2, 1])["kendall_tau_b"] == -1 + + +def test_configure_cc_command(tmp_path: Path) -> None: + surface = load("run_frontier_qwen30_prefill_surface.py") + base = ["python", "--cc_backend_config_type", "analytical", "--other", "x"] + analytical = surface.configure_cc_command( + base, backend="analytical", allreduce_csv=None, cache=tmp_path + ) + assert analytical == base + profile = tmp_path / "all_reduce.csv" + profile.write_text("header\n") + vidur = surface.configure_cc_command( + base, backend="vidur", allreduce_csv=profile, cache=tmp_path / "cache" + ) + assert vidur[2] == "vidur" + assert "--vidur_cc_backend_config_all_reduce_input_file" in vidur + assert str(profile) in vidur diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py index 0d420e0..16f1b68 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -114,7 +114,7 @@ MOE_METADATA = ( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--linear", type=Path, required=True) - parser.add_argument("--attention", type=Path, nargs=3, required=True) + parser.add_argument("--attention", type=Path, nargs="+", required=True) parser.add_argument("--moe", type=Path, required=True) parser.add_argument("--router", type=Path, required=True) parser.add_argument("--allreduce", type=Path, nargs=2, required=True)