Evaluate Qwen30 prefill simulator fidelity
This commit is contained in:
BIN
docs/assets/simulator-fidelity/qwen30-prefill-ranking.png
Normal file
BIN
docs/assets/simulator-fidelity/qwen30-prefill-ranking.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 106 KiB |
@@ -0,0 +1,462 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the frozen Frontier surface with conservative real capacities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
RUN_PATTERN = re.compile(r"qwen30-prefill-real-tp(?P<tp>\d+)-mns(?P<mns>\d+)-")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--fleet-artifacts", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--simulator-manifest", type=Path, action="append", required=True
|
||||
)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def sign(value: float) -> int:
|
||||
return (value > 0) - (value < 0)
|
||||
|
||||
|
||||
def kendall_tau_b(real: list[float], simulated: list[float]) -> dict[str, Any]:
|
||||
if len(real) != len(simulated):
|
||||
raise ValueError("ranking vectors have different lengths")
|
||||
concordant = discordant = real_only_ties = simulated_only_ties = both_ties = 0
|
||||
for left in range(len(real)):
|
||||
for right in range(left + 1, len(real)):
|
||||
real_sign = sign(real[left] - real[right])
|
||||
sim_sign = sign(simulated[left] - simulated[right])
|
||||
if real_sign == 0 and sim_sign == 0:
|
||||
both_ties += 1
|
||||
elif real_sign == 0:
|
||||
real_only_ties += 1
|
||||
elif sim_sign == 0:
|
||||
simulated_only_ties += 1
|
||||
elif real_sign == sim_sign:
|
||||
concordant += 1
|
||||
else:
|
||||
discordant += 1
|
||||
denominator = math.sqrt(
|
||||
(concordant + discordant + real_only_ties)
|
||||
* (concordant + discordant + simulated_only_ties)
|
||||
)
|
||||
tau = (concordant - discordant) / denominator if denominator else None
|
||||
return {
|
||||
"kendall_tau_b": tau,
|
||||
"concordant": concordant,
|
||||
"discordant": discordant,
|
||||
"real_only_ties": real_only_ties,
|
||||
"simulator_only_ties": simulated_only_ties,
|
||||
"both_ties": both_ties,
|
||||
}
|
||||
|
||||
|
||||
def result_files_by_anchor(root: Path) -> dict[tuple[str, str], Path]:
|
||||
selected: dict[tuple[str, str], Path] = {}
|
||||
digests: dict[tuple[str, str], str] = {}
|
||||
for path in root.glob("artifacts/**/round*/results/r*.json"):
|
||||
if path.name.startswith("warmup_"):
|
||||
continue
|
||||
round_name = path.parent.parent.name
|
||||
key = (round_name, path.name)
|
||||
digest = sha256(path)
|
||||
if key in digests and digests[key] != digest:
|
||||
raise RuntimeError(
|
||||
f"conflicting duplicate real anchor {key} under {root}"
|
||||
)
|
||||
digests[key] = digest
|
||||
if key not in selected or len(path.parts) < len(selected[key].parts):
|
||||
selected[key] = path
|
||||
return selected
|
||||
|
||||
|
||||
def find_real_runs(root: Path) -> dict[str, Path]:
|
||||
candidates: dict[str, list[Path]] = defaultdict(list)
|
||||
for path in root.iterdir():
|
||||
if not path.is_dir():
|
||||
continue
|
||||
match = RUN_PATTERN.search(path.name)
|
||||
if not match:
|
||||
continue
|
||||
name = f"tp{int(match.group('tp'))}_mns{int(match.group('mns'))}"
|
||||
candidates[name].append(path)
|
||||
selected: dict[str, Path] = {}
|
||||
for name, paths in candidates.items():
|
||||
complete = [
|
||||
path
|
||||
for path in paths
|
||||
if (path / "remote_run" / "exit_code").is_file()
|
||||
and (path / "remote_run" / "exit_code").read_text().strip() == "0"
|
||||
]
|
||||
measured_counts = {
|
||||
path: len(result_files_by_anchor(path))
|
||||
for path in complete
|
||||
}
|
||||
if not measured_counts:
|
||||
raise RuntimeError(f"no successful run for {name}")
|
||||
maximum = max(measured_counts.values())
|
||||
richest = [path for path, count in measured_counts.items() if count == maximum]
|
||||
if len(richest) != 1:
|
||||
raise RuntimeError(f"ambiguous richest successful run for {name}: {richest}")
|
||||
selected[name] = richest[0]
|
||||
if len(selected) != 12:
|
||||
raise RuntimeError(f"expected 12 real configs, got {sorted(selected)}")
|
||||
return selected
|
||||
|
||||
|
||||
def campaign_resources(root: Path) -> dict[str, Any]:
|
||||
runs = []
|
||||
gpu_hours = 0.0
|
||||
for path in sorted(root.iterdir()):
|
||||
match = RUN_PATTERN.search(path.name)
|
||||
exit_code = path / "remote_run" / "exit_code"
|
||||
started_at = path / "remote_run" / "started_at"
|
||||
finished_at = path / "remote_run" / "finished_at"
|
||||
if (
|
||||
not path.is_dir()
|
||||
or not match
|
||||
or not exit_code.is_file()
|
||||
or exit_code.read_text().strip() != "0"
|
||||
or not started_at.is_file()
|
||||
or not finished_at.is_file()
|
||||
):
|
||||
continue
|
||||
started = datetime.fromisoformat(started_at.read_text().strip())
|
||||
finished = datetime.fromisoformat(finished_at.read_text().strip())
|
||||
duration_seconds = (finished - started).total_seconds()
|
||||
tp = int(match.group("tp"))
|
||||
run_gpu_hours = duration_seconds * tp / 3600.0
|
||||
gpu_hours += run_gpu_hours
|
||||
runs.append(
|
||||
{
|
||||
"run": path.name,
|
||||
"tp": tp,
|
||||
"duration_seconds": duration_seconds,
|
||||
"gpu_hours": run_gpu_hours,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"successful_fleet_jobs": len(runs),
|
||||
"gpu_hours": gpu_hours,
|
||||
"runs": runs,
|
||||
}
|
||||
|
||||
|
||||
def parse_real_config(name: str, run_root: Path) -> dict[str, Any]:
|
||||
result_files = sorted(result_files_by_anchor(run_root).values())
|
||||
if len(result_files) not in {10, 16}:
|
||||
raise RuntimeError(f"expected 10 base or 16 refined anchors for {name}, got {len(result_files)}")
|
||||
by_rate: dict[float, list[dict[str, Any]]] = defaultdict(list)
|
||||
for path in result_files:
|
||||
payload = load_json(path)
|
||||
rate = float(payload["workload"]["offered_request_rate"])
|
||||
by_rate[rate].append(
|
||||
{
|
||||
"path": str(path.resolve()),
|
||||
"sha256": sha256(path),
|
||||
"summary": payload["summary"],
|
||||
}
|
||||
)
|
||||
tp = int(name.split("_")[0][2:])
|
||||
base_rates = [4.0, 8.0, 16.0, 32.0, 64.0]
|
||||
refined_rates = sorted({*base_rates, *(tp * value for value in (5.0, 6.0, 7.0))})
|
||||
if tuple(sorted(by_rate)) not in {tuple(base_rates), tuple(refined_rates)}:
|
||||
raise RuntimeError(f"unexpected rate grid for {name}: {sorted(by_rate)}")
|
||||
anchors = []
|
||||
for rate, rounds in sorted(by_rate.items()):
|
||||
if len(rounds) != 2:
|
||||
raise RuntimeError(f"expected two rounds for {name}@{rate}, got {len(rounds)}")
|
||||
round_feasible = [bool(row["summary"]["slo"]["feasible"]) for row in rounds]
|
||||
anchors.append(
|
||||
{
|
||||
"rate": rate,
|
||||
"rounds": rounds,
|
||||
"conservative_feasible": all(round_feasible),
|
||||
"round_feasible": round_feasible,
|
||||
"round_ttft_p95_ms": [
|
||||
float(row["summary"]["ttft_p95_ms"]) for row in rounds
|
||||
],
|
||||
}
|
||||
)
|
||||
feasible = [row["rate"] for row in anchors if row["conservative_feasible"]]
|
||||
capacity = max(feasible, default=0.0)
|
||||
return {
|
||||
"name": name,
|
||||
"tp": tp,
|
||||
"mns": int(name.split("_mns")[1]),
|
||||
"anchors": anchors,
|
||||
"capacity": capacity,
|
||||
"capacity_per_gpu": capacity / tp,
|
||||
"source_run": str(run_root.resolve()),
|
||||
}
|
||||
|
||||
|
||||
def parse_simulator(manifests: list[Path]) -> tuple[dict[str, Any], list[dict[str, str]]]:
|
||||
configs: dict[str, Any] = {}
|
||||
sources = []
|
||||
for path in manifests:
|
||||
payload = load_json(path)
|
||||
if payload["status"] not in {"complete", "partial_not_decision_bearing"}:
|
||||
raise RuntimeError(f"simulator manifest has invalid status: {path}")
|
||||
sources.append({"path": str(path.resolve()), "sha256": sha256(path)})
|
||||
result_by_name = {
|
||||
result["config"]["name"]: result for result in payload["config_results"]
|
||||
}
|
||||
for capacity in payload["capacity"]:
|
||||
name = capacity["config"]["name"]
|
||||
result = result_by_name.get(name)
|
||||
if result is None:
|
||||
raise RuntimeError(f"missing simulator config result {name}")
|
||||
entry = configs.setdefault(
|
||||
name,
|
||||
{
|
||||
"name": name,
|
||||
"tp": int(capacity["config"]["tp"]),
|
||||
"mns": int(capacity["config"]["mns"]),
|
||||
"anchor_by_rate": {},
|
||||
},
|
||||
)
|
||||
for load in result["loads"]:
|
||||
rate = float(load["offered_request_rate"])
|
||||
if rate in entry["anchor_by_rate"]:
|
||||
raise RuntimeError(f"duplicate simulator anchor {name}@{rate}")
|
||||
entry["anchor_by_rate"][rate] = {
|
||||
"rate": rate,
|
||||
"feasible": bool(load["score"]["feasible"]),
|
||||
"pass_rate": float(load["score"]["pass_rate"]),
|
||||
"ttft_p95_ms": float(load["score"]["ttft_p95_ms"]),
|
||||
}
|
||||
if len(configs) != 12:
|
||||
raise RuntimeError(f"expected 12 simulator configs, got {sorted(configs)}")
|
||||
for entry in configs.values():
|
||||
entry["anchors"] = [
|
||||
entry["anchor_by_rate"][rate] for rate in sorted(entry["anchor_by_rate"])
|
||||
]
|
||||
del entry["anchor_by_rate"]
|
||||
feasible = [row["rate"] for row in entry["anchors"] if row["feasible"]]
|
||||
entry["capacity"] = max(feasible, default=0.0)
|
||||
entry["capacity_per_gpu"] = entry["capacity"] / entry["tp"]
|
||||
return configs, sources
|
||||
|
||||
|
||||
def compare(real: dict[str, Any], simulated: dict[str, Any]) -> dict[str, Any]:
|
||||
names = sorted(real, key=lambda name: (real[name]["tp"], real[name]["mns"]))
|
||||
if set(names) != set(simulated):
|
||||
raise RuntimeError("real and simulator config sets differ")
|
||||
real_scores = [real[name]["capacity_per_gpu"] for name in names]
|
||||
sim_scores = [simulated[name]["capacity_per_gpu"] for name in names]
|
||||
real_best = max(real_scores)
|
||||
sim_best = max(sim_scores)
|
||||
real_top = [name for name in names if real[name]["capacity_per_gpu"] == real_best]
|
||||
sim_top = [
|
||||
name for name in names if simulated[name]["capacity_per_gpu"] == sim_best
|
||||
]
|
||||
worst_sim_choice = min(real[name]["capacity_per_gpu"] for name in sim_top)
|
||||
best_sim_choice = max(real[name]["capacity_per_gpu"] for name in sim_top)
|
||||
tau = kendall_tau_b(real_scores, sim_scores)
|
||||
|
||||
pairwise = {"all": {"comparable": 0, "correct": 0}, "within_tp": {}}
|
||||
for left in range(len(names)):
|
||||
for right in range(left + 1, len(names)):
|
||||
real_sign = sign(real_scores[left] - real_scores[right])
|
||||
sim_sign = sign(sim_scores[left] - sim_scores[right])
|
||||
if real_sign:
|
||||
pairwise["all"]["comparable"] += 1
|
||||
pairwise["all"]["correct"] += int(real_sign == sim_sign)
|
||||
if real[names[left]]["tp"] == real[names[right]]["tp"] and real_sign:
|
||||
key = f"tp{real[names[left]]['tp']}"
|
||||
bucket = pairwise["within_tp"].setdefault(
|
||||
key, {"comparable": 0, "correct": 0}
|
||||
)
|
||||
bucket["comparable"] += 1
|
||||
bucket["correct"] += int(real_sign == sim_sign)
|
||||
for bucket in [pairwise["all"], *pairwise["within_tp"].values()]:
|
||||
bucket["accuracy"] = (
|
||||
bucket["correct"] / bucket["comparable"]
|
||||
if bucket["comparable"]
|
||||
else None
|
||||
)
|
||||
|
||||
confusion = {"real_pass_sim_pass": 0, "real_pass_sim_fail": 0,
|
||||
"real_fail_sim_pass": 0, "real_fail_sim_fail": 0}
|
||||
for name in names:
|
||||
real_anchors = {row["rate"]: row for row in real[name]["anchors"]}
|
||||
sim_anchors = {row["rate"]: row for row in simulated[name]["anchors"]}
|
||||
if set(real_anchors) != set(sim_anchors):
|
||||
raise RuntimeError(f"anchor grids differ for {name}")
|
||||
for rate in real_anchors:
|
||||
real_pass = real_anchors[rate]["conservative_feasible"]
|
||||
sim_pass = sim_anchors[rate]["feasible"]
|
||||
key = f"real_{'pass' if real_pass else 'fail'}_sim_{'pass' if sim_pass else 'fail'}"
|
||||
confusion[key] += 1
|
||||
return {
|
||||
"config_order": names,
|
||||
"real_top_set": real_top,
|
||||
"simulator_top_set": sim_top,
|
||||
"top_set_exact_match": real_top == sim_top,
|
||||
"top_set_overlap": sorted(set(real_top) & set(sim_top)),
|
||||
"top1_regret_best": (real_best - best_sim_choice) / real_best,
|
||||
"top1_regret_worst": (real_best - worst_sim_choice) / real_best,
|
||||
"real_best_capacity_per_gpu": real_best,
|
||||
"simulator_best_capacity_per_gpu": sim_best,
|
||||
"kendall": tau,
|
||||
"pairwise_non_tied": pairwise,
|
||||
"anchor_confusion": confusion,
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("w", newline="") as handle:
|
||||
writer = csv.DictWriter(
|
||||
handle, fieldnames=list(rows[0]), lineterminator="\n"
|
||||
)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def plot(path: Path, rows: list[dict[str, Any]], metrics: dict[str, Any]) -> None:
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
labels = [f"TP{row['tp']}\nMNS{row['mns']}" for row in rows]
|
||||
x = np.arange(len(rows))
|
||||
width = 0.36
|
||||
figure, axes = plt.subplots(
|
||||
1, 2, figsize=(13.5, 5.0), gridspec_kw={"width_ratios": [3.3, 1.0]}
|
||||
)
|
||||
axis = axes[0]
|
||||
axis.bar(x - width / 2, [row["real"] for row in rows], width, label="Real vLLM")
|
||||
axis.bar(
|
||||
x + width / 2,
|
||||
[row["simulator"] for row in rows],
|
||||
width,
|
||||
label="Frontier profile-only",
|
||||
)
|
||||
axis.set_xticks(x, labels, fontsize=8)
|
||||
axis.set_ylabel("Max tested SLO-feasible request rate / GPU")
|
||||
axis.set_title("Qwen3-30B-A3B prefill-only: config ranking")
|
||||
axis.grid(axis="y", alpha=0.25)
|
||||
axis.legend(frameon=False, ncols=2)
|
||||
for separator in (3.5, 7.5):
|
||||
axis.axvline(separator, color="0.75", linewidth=0.8)
|
||||
tau = metrics["kendall"]["kendall_tau_b"]
|
||||
annotation = f"worst regret={metrics['top1_regret_worst'] * 100:.1f}%"
|
||||
annotation += (
|
||||
f"\nKendall τ-b={tau:.3f}"
|
||||
if tau is not None
|
||||
else "\nKendall τ-b=undefined"
|
||||
)
|
||||
axis.text(
|
||||
0.01,
|
||||
0.98,
|
||||
annotation,
|
||||
transform=axis.transAxes,
|
||||
va="top",
|
||||
fontsize=9,
|
||||
bbox={"facecolor": "white", "edgecolor": "0.8", "alpha": 0.9},
|
||||
)
|
||||
|
||||
confusion = metrics["anchor_confusion"]
|
||||
matrix = np.array(
|
||||
[
|
||||
[confusion["real_pass_sim_pass"], confusion["real_pass_sim_fail"]],
|
||||
[confusion["real_fail_sim_pass"], confusion["real_fail_sim_fail"]],
|
||||
]
|
||||
)
|
||||
image = axes[1].imshow(matrix, cmap="Blues", vmin=0)
|
||||
axes[1].set_xticks([0, 1], ["Sim pass", "Sim fail"])
|
||||
axes[1].set_yticks([0, 1], ["Real pass", "Real fail"])
|
||||
axes[1].set_title(f"{int(matrix.sum())} anchor decisions")
|
||||
for row in range(2):
|
||||
for column in range(2):
|
||||
axes[1].text(column, row, int(matrix[row, column]), ha="center", va="center")
|
||||
figure.colorbar(image, ax=axes[1], fraction=0.047, pad=0.04)
|
||||
figure.suptitle("ISL=2048, OSL=1, TTFT≤1256 ms, 95% pass gate; two real rounds")
|
||||
figure.tight_layout()
|
||||
figure.savefig(path, dpi=180, bbox_inches="tight")
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
real_runs = find_real_runs(args.fleet_artifacts.resolve())
|
||||
real = {name: parse_real_config(name, path) for name, path in real_runs.items()}
|
||||
simulated, simulator_sources = parse_simulator(
|
||||
[path.resolve() for path in args.simulator_manifest]
|
||||
)
|
||||
metrics = compare(real, simulated)
|
||||
rows = [
|
||||
{
|
||||
"config": name,
|
||||
"tp": real[name]["tp"],
|
||||
"mns": real[name]["mns"],
|
||||
"real": real[name]["capacity_per_gpu"],
|
||||
"simulator": simulated[name]["capacity_per_gpu"],
|
||||
}
|
||||
for name in metrics["config_order"]
|
||||
]
|
||||
resources = campaign_resources(args.fleet_artifacts.resolve())
|
||||
resources["fresh_server_anchors"] = sum(
|
||||
len(config["anchors"]) * 2 for config in real.values()
|
||||
)
|
||||
resources["measured_requests"] = resources["fresh_server_anchors"] * 64
|
||||
resources["warmup_requests"] = sum(
|
||||
min(32, max(4, math.ceil(anchor["rate"] * 2.0))) * 2
|
||||
for config in real.values()
|
||||
for anchor in config["anchors"]
|
||||
)
|
||||
args.output_root.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"schema": "qwen30-prefill-fidelity-comparison-v1",
|
||||
"objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu",
|
||||
"contract": {
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"input_tokens": 2048,
|
||||
"output_tokens": 1,
|
||||
"prefix_caching": False,
|
||||
"ttft_slo_ms": 1256.0,
|
||||
"target_pass_rate": 0.95,
|
||||
"real_anchor_merge": "both_fresh_server_rounds_must_pass",
|
||||
},
|
||||
"metrics": metrics,
|
||||
"real_campaign_resources": resources,
|
||||
"real": real,
|
||||
"simulator": simulated,
|
||||
"simulator_sources": simulator_sources,
|
||||
}
|
||||
(args.output_root / "comparison.json").write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
write_csv(args.output_root / "capacity.csv", rows)
|
||||
plot(args.output_root / "qwen30-prefill-ranking.png", rows, metrics)
|
||||
print(json.dumps(metrics, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,6 @@
|
||||
# 实验 EXP-SIMFID-PHASE-FACTORIAL：prefill-only 是否是 simulator ranking 的容易区间？
|
||||
|
||||
> **状态：** 已批准，运行中（用户于 2026-07-17 明确要求先完成 235B mixed 与 30B prefill-only）
|
||||
> **状态：** 已完成（2026-07-17）
|
||||
|
||||
## Claim 与决策
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
- **30B system context：** community vLLM 0.20.0+cu129，BF16 weights/activation/KV，TP∈{1,2,4}，MNS∈{8,16,32,64}，MBT=8192，chunked prefill on，prefix off；real 保留 runtime 默认 CUDA graph，Frontier profile-only 不做 E2E calibration。
|
||||
- **30B workload：** fixed ISL=2048、OSL=1，64 个不同 token-chain prompts，uniform open-loop QPS；fresh server per `(config, rate, round)`，target-rate warmup 与 measured requests 分离。
|
||||
- **30B SLO：** TTFT≤1256 ms，至少 61/64 requests 通过；primary score 为最大共同 tested feasible req/s / 实际 TP GPUs。
|
||||
- **Boundary refinement rule：** base grid `{4,8,16,32,64}` 先定位每个 TP 的 pass→fail 区间；若除以 TP 后的离散容量产生无法区分的 top tie，则在查看最终 ranking 前追加共同 per-GPU lattice `5/6/7 req/s/GPU`，即 TP1 测 `{5,6,7}`、TP2 测 `{10,12,14}`、TP4 测 `{20,24,28}`。refinement 不替换或删除 base anchors。
|
||||
- **235B baseline：** 已冻结 `ISL=2048, OSL=128`、8 configs、68 个 fresh-server anchors；primary sensitivity TTFT≤1256 ms、TPOT≤150 ms。
|
||||
- **Baselines：** real community vLLM；Frontier same-stack profile-only；historical Qwen30 mixed profile-only；historical frozen per-TP calibration 只作为 upper bound，不参与本 case 拟合。
|
||||
- **Metrics：** top set、worst tie-break regret、Kendall τ-b、exact/non-tied pair direction、anchor confusion、absolute capacity、TTFT p50/p95、real trial variance与GPU-hour。
|
||||
@@ -35,7 +36,7 @@
|
||||
| Selective benchmarking | PASS for initial screen | 同时报已有 235B mixed success和内部 pairwise failure；后续 expansion 由预注册 verdict 触发 |
|
||||
| Simplified workload | NEEDS EVIDENCE | fixed-shape 只用于 phase isolation，不外推 trace-faithful mixed |
|
||||
| Calibration=evaluation | PASS | 新 case 不用 serving E2E 数据拟合 scale |
|
||||
| Missing significance | NEEDS EVIDENCE until run | boundary anchors做独立 fresh-server repeat，保留 disagreement |
|
||||
| Missing significance | PASS for ranking screen | 96 个 config-rate cells 均做两个独立 fresh-server rounds；两轮都 pass 才算 feasible |
|
||||
| Relative-only result | PASS by design | 同时报 req/s/GPU、TTFT distribution、rank/regret |
|
||||
|
||||
## 复现信息
|
||||
@@ -43,12 +44,13 @@
|
||||
- **Code：** AITuner branch `codex/fidelity-prefix-pilot-20260714`；Frontier upstream `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + frozen known patches。
|
||||
- **Environment：** 只使用 dash0 8×H20；Qwen30 venv `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；model `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`。
|
||||
- **产物路径：** local/remote `runs/frontier-phase-factorial-v0/`；raw GPU artifacts 由 fleet harvest，condensed JSON/CSV 进入结果目录。
|
||||
- **已知 deviation：** 235B 为 FP8/vLLM0.10.2/FlashInfer eager，30B 为 BF16/vLLM0.20/FA3/default CUDA graph；因此跨模型只检验 hypothesis consistency，causal phase claim 最终仍需 same-model phase pair。
|
||||
- **已知 deviation：** 235B 为 FP8/vLLM0.10.2/FlashInfer eager，30B 为 BF16/vLLM0.20/FA3/default CUDA graph；因此跨模型只检验 hypothesis consistency，causal phase claim 最终仍需 same-model phase pair。初始 continuous fleet monitor 未在 fresh-server 间隙保留 controller-level GPU reservation，产生重叠 launch；该 attempt 整体移动到 `invalid-overlap-*`，不进入统计。后续一次试图在“其余 4 卡”上并行 refinement 时，探针再次命中 TP4 fresh-server 空窗，把 4 个 TP1 jobs 放到了同一 GPU set。这四个 jobs 尚未产生 measured result；但当时 TP4/MNS64 已产生的单个 anchor 也按污染处理。两者都整体移到 `invalid-overlap-20260716T1750Z`，TP4/MNS64 从空目录重跑。此后的 barrier waves 不在运行中追加 job；但 Wave 3 的 harvest monitor 在未及时返回 launch 状态时已发射 MNS16/32，紧接的 monitor retry 又在它们的启动空窗发射 MNS64。三者都未产生 measured result，整体移到 `invalid-overlap-20260716T1836Z`。最终 TP4 waves 使用只含本波 jobs 的独立 queue state，不再依赖 pending-job 探针调度。
|
||||
|
||||
## 结果
|
||||
|
||||
- **观察事实：** 235B fixed-shape mixed 已完成：real/sim TP4 top set exact match，worst regret=0、τ-b=0.8944；但 20 个 real non-tie pairs 只保持 16 个，10/34 anchors false-infeasible，TP8 MNS×MBT interaction 被漏掉。30B prefill-only 待运行。
|
||||
- **异常：** 无。
|
||||
- **Interpretation 与剩余 alternatives：** 强版本“mixed 必然失败”已被 235B top-set result 削弱；仍可能存在 phase-dependent error magnitude，由 topology margin 掩盖。
|
||||
- **Claim update：** unchanged，等待 30B prefill-only。
|
||||
- **下一步：** freeze 30B simulator surface → guided real anchors → joint verdict；只有判定需要时扩展 decode-heavy 235B 或 trace-shaped 30B prefill。
|
||||
- **观察事实：** 235B fixed-shape mixed 中 real/sim 的四个 TP4 top configs 完全一致，worst regret=0、τ-b=0.8944；但 20 个 real non-tie pairs 只保持 16 个，10/34 anchors false-infeasible，TP8 MNS×MBT interaction 被漏掉。30B prefill-only 中真机 capacity/GPU 为 TP1=7、TP2=7、TP4=8，Frontier 为 TP1=8、TP2=8、TP4=6；real top set 是四个 TP4 configs，simulator top set 是全部八个 TP1/TP2 configs，无交集。worst regret=12.5%、τ-b=-1.0，32 个 real non-tie pairs 中 0 个同序。96 个 anchor labels 中有 8 个 false-feasible 和 8 个 false-infeasible。
|
||||
- **实验成本：** 接受 24 个 fleet jobs、192 个 fresh-server anchors、12,288 个 measured requests 和 4,512 个 warmups，消耗 12.0744 H20-GPU-hours。
|
||||
- **异常与排除：** fleet controller 在 fresh-server 空窗期没有保留 GPU reservation，产生了三批重叠 launch。污染 attempt 不进入 accepted artifact root，未产生 measured result 的重叠 jobs 也不被计数；同一波中已产生的 TP4/MNS64 单 anchor 同样按污染丢弃，从空目录重跑。最终 TP4 refinement 使用彼此独立的 queue states。analyzer 按 `(round, filename)` 去重相同 harvest copy，如果 hash 冲突则直接报错。
|
||||
- **Interpretation 与剩余 alternatives：** `H-phase` 的强形式（prefill-only 是 fidelity 充分条件）被否证；`H-margin` 与数据更一致。235B 的真机 TP4/TP8 最优 margin 为 2×，足以掩盖内部 residual；30B 的 8-vs-7 margin 被 TP-dependent saturation residual 穿过。但这是跨 stack comparison，不能把差异因果归结为 model size。
|
||||
- **Claim update：** “prefill-only 容易，decode/mixed 困难”的强假设被否证。新的可证伪命题是：config-ranking fidelity 取决于 scheduler-state-conditioned action residual 是否大于 real decision margin。
|
||||
- **下一步：** 不继续扩展跨模型 phase cases。在 Qwen30 prefill-only 上依次做 measured-collective injection、batch-composition-conditioned pure-prefill attention/step profile、TP1@8/TP2@16/TP4@32 的 scheduler batch/queue/per-step trace 对齐，最后再测 routing/graph。
|
||||
|
||||
24
runs/frontier-phase-factorial-v0/fleet-base-rerun.toml
Normal file
24
runs/frontier-phase-factorial-v0/fleet-base-rerun.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
version = 1
|
||||
|
||||
[paths]
|
||||
state_dir = "runs/frontier-phase-factorial-v0/fleet-state-base-rerun"
|
||||
artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive"
|
||||
|
||||
[ssh]
|
||||
connect_timeout_sec = 10
|
||||
|
||||
[scheduler]
|
||||
gpu_free_memory_mb = 95000
|
||||
gpu_free_utilization_pct = 5
|
||||
prefer_pack = true
|
||||
|
||||
[sync]
|
||||
mode = "scp"
|
||||
local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker"
|
||||
|
||||
[[hosts]]
|
||||
name = "dash0"
|
||||
ssh_alias = "dash0"
|
||||
enabled = true
|
||||
sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker"
|
||||
fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0"
|
||||
24
runs/frontier-phase-factorial-v0/fleet-exclusive.toml
Normal file
24
runs/frontier-phase-factorial-v0/fleet-exclusive.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
version = 1
|
||||
|
||||
[paths]
|
||||
state_dir = "runs/frontier-phase-factorial-v0/fleet-state-exclusive"
|
||||
artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive"
|
||||
|
||||
[ssh]
|
||||
connect_timeout_sec = 10
|
||||
|
||||
[scheduler]
|
||||
gpu_free_memory_mb = 95000
|
||||
gpu_free_utilization_pct = 5
|
||||
prefer_pack = true
|
||||
|
||||
[sync]
|
||||
mode = "scp"
|
||||
local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker"
|
||||
|
||||
[[hosts]]
|
||||
name = "dash0"
|
||||
ssh_alias = "dash0"
|
||||
enabled = true
|
||||
sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker"
|
||||
fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0"
|
||||
24
runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave3.toml
Normal file
24
runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave3.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
version = 1
|
||||
|
||||
[paths]
|
||||
state_dir = "runs/frontier-phase-factorial-v0/fleet-state-refine-tp4-wave3"
|
||||
artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive"
|
||||
|
||||
[ssh]
|
||||
connect_timeout_sec = 10
|
||||
|
||||
[scheduler]
|
||||
gpu_free_memory_mb = 95000
|
||||
gpu_free_utilization_pct = 5
|
||||
prefer_pack = true
|
||||
|
||||
[sync]
|
||||
mode = "scp"
|
||||
local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker"
|
||||
|
||||
[[hosts]]
|
||||
name = "dash0"
|
||||
ssh_alias = "dash0"
|
||||
enabled = true
|
||||
sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker"
|
||||
fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0"
|
||||
24
runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave4.toml
Normal file
24
runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave4.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
version = 1
|
||||
|
||||
[paths]
|
||||
state_dir = "runs/frontier-phase-factorial-v0/fleet-state-refine-tp4-wave4"
|
||||
artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive"
|
||||
|
||||
[ssh]
|
||||
connect_timeout_sec = 10
|
||||
|
||||
[scheduler]
|
||||
gpu_free_memory_mb = 95000
|
||||
gpu_free_utilization_pct = 5
|
||||
prefer_pack = true
|
||||
|
||||
[sync]
|
||||
mode = "scp"
|
||||
local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker"
|
||||
|
||||
[[hosts]]
|
||||
name = "dash0"
|
||||
ssh_alias = "dash0"
|
||||
enabled = true
|
||||
sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker"
|
||||
fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0"
|
||||
24
runs/frontier-phase-factorial-v0/fleet-refine.toml
Normal file
24
runs/frontier-phase-factorial-v0/fleet-refine.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
version = 1
|
||||
|
||||
[paths]
|
||||
state_dir = "runs/frontier-phase-factorial-v0/fleet-state-refine"
|
||||
artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive"
|
||||
|
||||
[ssh]
|
||||
connect_timeout_sec = 10
|
||||
|
||||
[scheduler]
|
||||
gpu_free_memory_mb = 95000
|
||||
gpu_free_utilization_pct = 5
|
||||
prefer_pack = true
|
||||
|
||||
[sync]
|
||||
mode = "scp"
|
||||
local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker"
|
||||
|
||||
[[hosts]]
|
||||
name = "dash0"
|
||||
ssh_alias = "dash0"
|
||||
enabled = true
|
||||
sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker"
|
||||
fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0"
|
||||
21
runs/frontier-phase-factorial-v0/jobs_base_rerun.toml
Normal file
21
runs/frontier-phase-factorial-v0/jobs_base_rerun.toml
Normal file
@@ -0,0 +1,21 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns64-20260717-v2b-exclusive"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns64-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "64"
|
||||
RATES = "4 8 16 32 64"
|
||||
SERVER_PORT = "8731"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
@@ -1,7 +1,7 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns8-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp1-mns8-20260717-v2-exclusive"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -21,7 +21,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns16-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp1-mns16-20260717-v2-exclusive"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -41,7 +41,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns32-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp1-mns32-20260717-v2-exclusive"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -61,7 +61,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns64-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp1-mns64-20260717-v2-exclusive"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -81,7 +81,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns8-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp2-mns8-20260717-v2-exclusive"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -101,7 +101,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns16-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp2-mns16-20260717-v2-exclusive"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -121,7 +121,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns32-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp2-mns32-20260717-v2-exclusive"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -141,7 +141,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns64-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp2-mns64-20260717-v2-exclusive"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -161,7 +161,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns8-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp4-mns8-20260717-v2-exclusive"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -181,7 +181,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns16-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp4-mns16-20260717-v2-exclusive"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -201,7 +201,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns32-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp4-mns32-20260717-v2-exclusive"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
@@ -221,7 +221,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns64-20260717-v1"
|
||||
name = "qwen30-prefill-real-tp4-mns64-20260717-v2-exclusive"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
|
||||
240
runs/frontier-phase-factorial-v0/jobs_refine.toml
Normal file
240
runs/frontier-phase-factorial-v0/jobs_refine.toml
Normal file
@@ -0,0 +1,240 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns8-20260717-v3-refine"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp1-mns8-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "1"
|
||||
MNS = "8"
|
||||
RATES = "5 6 7"
|
||||
SERVER_PORT = "8720"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns8-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns16-20260717-v3-refine"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp1-mns16-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "1"
|
||||
MNS = "16"
|
||||
RATES = "5 6 7"
|
||||
SERVER_PORT = "8721"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns16-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns32-20260717-v3-refine"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp1-mns32-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "1"
|
||||
MNS = "32"
|
||||
RATES = "5 6 7"
|
||||
SERVER_PORT = "8722"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns32-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp1-mns64-20260717-v3-refine"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp1-mns64-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "1"
|
||||
MNS = "64"
|
||||
RATES = "5 6 7"
|
||||
SERVER_PORT = "8723"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns64-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns8-20260717-v3-refine"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp2-mns8-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "2"
|
||||
MNS = "8"
|
||||
RATES = "10 12 14"
|
||||
SERVER_PORT = "8724"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns8-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns16-20260717-v3-refine"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp2-mns16-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "2"
|
||||
MNS = "16"
|
||||
RATES = "10 12 14"
|
||||
SERVER_PORT = "8725"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns16-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns32-20260717-v3-refine"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp2-mns32-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "2"
|
||||
MNS = "32"
|
||||
RATES = "10 12 14"
|
||||
SERVER_PORT = "8726"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns32-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp2-mns64-20260717-v3-refine"
|
||||
gpus = 2
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp2-mns64-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "2"
|
||||
MNS = "64"
|
||||
RATES = "10 12 14"
|
||||
SERVER_PORT = "8727"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns64-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns8-20260717-v3-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns8-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "8"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8728"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns8-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns16-20260717-v3-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns16-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "16"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8729"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns16-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns32-20260717-v3-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns32-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "32"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8730"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns32-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns64-20260717-v3-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns64-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "64"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8731"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
41
runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave3.toml
Normal file
41
runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave3.toml
Normal file
@@ -0,0 +1,41 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns16-20260717-v4-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns16-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "16"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8729"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns16-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns32-20260717-v4-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns32-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "32"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8730"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns32-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
21
runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave4.toml
Normal file
21
runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave4.toml
Normal file
@@ -0,0 +1,21 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "qwen30-prefill-real-tp4-mns64-20260717-v4-refine"
|
||||
gpus = 4
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash0"]
|
||||
command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh"
|
||||
artifacts = ["artifacts/real-tp4-mns64-v1"]
|
||||
|
||||
[jobs.env]
|
||||
HOME = "/tmp/wjh"
|
||||
XDG_CACHE_HOME = "/tmp/wjh/.cache"
|
||||
VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm"
|
||||
TP = "4"
|
||||
MNS = "64"
|
||||
RATES = "20 24 28"
|
||||
SERVER_PORT = "8731"
|
||||
OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1"
|
||||
VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1"
|
||||
MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
|
||||
13
runs/frontier-phase-factorial-v0/results/final/capacity.csv
Normal file
13
runs/frontier-phase-factorial-v0/results/final/capacity.csv
Normal file
@@ -0,0 +1,13 @@
|
||||
config,tp,mns,real,simulator
|
||||
tp1_mns8,1,8,7.0,8.0
|
||||
tp1_mns16,1,16,7.0,8.0
|
||||
tp1_mns32,1,32,7.0,8.0
|
||||
tp1_mns64,1,64,7.0,8.0
|
||||
tp2_mns8,2,8,7.0,8.0
|
||||
tp2_mns16,2,16,7.0,8.0
|
||||
tp2_mns32,2,32,7.0,8.0
|
||||
tp2_mns64,2,64,7.0,8.0
|
||||
tp4_mns8,4,8,8.0,6.0
|
||||
tp4_mns16,4,16,8.0,6.0
|
||||
tp4_mns32,4,32,8.0,6.0
|
||||
tp4_mns64,4,64,8.0,6.0
|
||||
|
5860
runs/frontier-phase-factorial-v0/results/final/comparison.json
Normal file
5860
runs/frontier-phase-factorial-v0/results/final/comparison.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,461 @@
|
||||
{
|
||||
"capacity": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp1_mns16",
|
||||
"tp": 1
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 7.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp1_mns32",
|
||||
"tp": 1
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 7.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp1_mns64",
|
||||
"tp": 1
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 7.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp1_mns8",
|
||||
"tp": 1
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 7.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
}
|
||||
],
|
||||
"config_results": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp1_mns8",
|
||||
"tp": 1
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp1_mns8",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.505136728286743,
|
||||
"offered_request_rate": 5.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.011124187030024,
|
||||
"ttft_max_ms": 171.58529929215405,
|
||||
"ttft_p50_ms": 171.58529929215405,
|
||||
"ttft_p95_ms": 171.58529929215405
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp1_mns8",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.398303508758545,
|
||||
"offered_request_rate": 6.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.915042512446176,
|
||||
"ttft_max_ms": 432.43322440414465,
|
||||
"ttft_p50_ms": 282.618152646152,
|
||||
"ttft_p95_ms": 333.9001759332376
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp1_mns8",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.264978170394897,
|
||||
"offered_request_rate": 7.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 6.89299458621687,
|
||||
"ttft_max_ms": 432.97081304433016,
|
||||
"ttft_p50_ms": 297.34516049744997,
|
||||
"ttft_p95_ms": 427.64609771465524
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp1_mns16",
|
||||
"tp": 1
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp1_mns16",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.103909015655518,
|
||||
"offered_request_rate": 5.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.011124187030024,
|
||||
"ttft_max_ms": 171.58529929215405,
|
||||
"ttft_p50_ms": 171.58529929215405,
|
||||
"ttft_p95_ms": 171.58529929215405
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp1_mns16",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.05237078666687,
|
||||
"offered_request_rate": 6.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.915042512446176,
|
||||
"ttft_max_ms": 432.43322440414465,
|
||||
"ttft_p50_ms": 282.618152646152,
|
||||
"ttft_p95_ms": 333.9001759332376
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp1_mns16",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.048887729644775,
|
||||
"offered_request_rate": 7.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 6.89299458621687,
|
||||
"ttft_max_ms": 432.97081304433016,
|
||||
"ttft_p50_ms": 297.34516049744997,
|
||||
"ttft_p95_ms": 427.64609771465524
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp1_mns32",
|
||||
"tp": 1
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp1_mns32",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.058743476867676,
|
||||
"offered_request_rate": 5.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.011124187030024,
|
||||
"ttft_max_ms": 171.58529929215405,
|
||||
"ttft_p50_ms": 171.58529929215405,
|
||||
"ttft_p95_ms": 171.58529929215405
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp1_mns32",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.106712818145752,
|
||||
"offered_request_rate": 6.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.915042512446176,
|
||||
"ttft_max_ms": 432.43322440414465,
|
||||
"ttft_p50_ms": 282.618152646152,
|
||||
"ttft_p95_ms": 333.9001759332376
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp1_mns32",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 10.1020348072052,
|
||||
"offered_request_rate": 7.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 6.89299458621687,
|
||||
"ttft_max_ms": 432.97081304433016,
|
||||
"ttft_p50_ms": 297.34516049744997,
|
||||
"ttft_p95_ms": 427.64609771465524
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp1_mns64",
|
||||
"tp": 1
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp1_mns64",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 9.897645235061646,
|
||||
"offered_request_rate": 5.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.011124187030024,
|
||||
"ttft_max_ms": 171.58529929215405,
|
||||
"ttft_p50_ms": 171.58529929215405,
|
||||
"ttft_p95_ms": 171.58529929215405
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp1_mns64",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 9.957297325134277,
|
||||
"offered_request_rate": 6.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 5.915042512446176,
|
||||
"ttft_max_ms": 432.43322440414465,
|
||||
"ttft_p50_ms": 282.618152646152,
|
||||
"ttft_p95_ms": 333.9001759332376
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp1_mns64",
|
||||
"tp": 1
|
||||
},
|
||||
"elapsed_seconds": 9.799486875534058,
|
||||
"offered_request_rate": 7.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 6.89299458621687,
|
||||
"ttft_max_ms": 432.97081304433016,
|
||||
"ttft_p50_ms": 297.34516049744997,
|
||||
"ttft_p95_ms": 427.64609771465524
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"contract": {
|
||||
"arrival": "open_loop_uniform",
|
||||
"input_tokens": 2048,
|
||||
"output_tokens": 1,
|
||||
"prefix_caching": false,
|
||||
"rates": [
|
||||
5.0,
|
||||
6.0,
|
||||
7.0
|
||||
],
|
||||
"requests_per_anchor": 64,
|
||||
"target_pass_rate": 0.95,
|
||||
"ttft_slo_ms": 1256.0
|
||||
},
|
||||
"frontier": {
|
||||
"git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n",
|
||||
"source": "/tmp/replayserve-frontier-rs1b"
|
||||
},
|
||||
"profiles": {
|
||||
"coverage": {
|
||||
"attention": {
|
||||
"1": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
},
|
||||
"2": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
},
|
||||
"4": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
}
|
||||
},
|
||||
"manifest": {
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection",
|
||||
"attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 132,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v2"
|
||||
}
|
||||
},
|
||||
"root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2",
|
||||
"sha256": {
|
||||
"attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb",
|
||||
"moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
}
|
||||
},
|
||||
"schema": "frontier-qwen30-prefill-surface-v1",
|
||||
"status": "partial_not_decision_bearing"
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
{
|
||||
"capacity": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp2_mns16",
|
||||
"tp": 2
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 14.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp2_mns32",
|
||||
"tp": 2
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 14.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp2_mns64",
|
||||
"tp": 2
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 14.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp2_mns8",
|
||||
"tp": 2
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 14.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 7.0,
|
||||
"upper_censored": true
|
||||
}
|
||||
],
|
||||
"config_results": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp2_mns8",
|
||||
"tp": 2
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp2_mns8",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.598209381103516,
|
||||
"offered_request_rate": 10.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 9.814433634929404,
|
||||
"ttft_max_ms": 300.1784228926625,
|
||||
"ttft_p50_ms": 211.39433537475938,
|
||||
"ttft_p95_ms": 297.1650738307572
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp2_mns8",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.660333633422852,
|
||||
"offered_request_rate": 12.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 11.55742285377119,
|
||||
"ttft_max_ms": 389.93612032123883,
|
||||
"ttft_p50_ms": 296.08352841701094,
|
||||
"ttft_p95_ms": 384.6764910356253
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp2_mns8",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.409451007843018,
|
||||
"offered_request_rate": 14.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 13.374732701347012,
|
||||
"ttft_max_ms": 388.3013907870674,
|
||||
"ttft_p50_ms": 309.4635231444016,
|
||||
"ttft_p95_ms": 386.4490667329015
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp2_mns16",
|
||||
"tp": 2
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp2_mns16",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.360321521759033,
|
||||
"offered_request_rate": 10.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 9.814433634929404,
|
||||
"ttft_max_ms": 300.1784228926625,
|
||||
"ttft_p50_ms": 211.39433537475938,
|
||||
"ttft_p95_ms": 297.1650738307572
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp2_mns16",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.402549028396606,
|
||||
"offered_request_rate": 12.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 11.55742285377119,
|
||||
"ttft_max_ms": 389.93612032123883,
|
||||
"ttft_p50_ms": 296.08352841701094,
|
||||
"ttft_p95_ms": 384.6764910356253
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp2_mns16",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.40966272354126,
|
||||
"offered_request_rate": 14.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 13.374732701347012,
|
||||
"ttft_max_ms": 388.3013907870674,
|
||||
"ttft_p50_ms": 309.4635231444016,
|
||||
"ttft_p95_ms": 386.4490667329015
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp2_mns32",
|
||||
"tp": 2
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp2_mns32",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.155991315841675,
|
||||
"offered_request_rate": 10.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 9.814433634929404,
|
||||
"ttft_max_ms": 300.1784228926625,
|
||||
"ttft_p50_ms": 211.39433537475938,
|
||||
"ttft_p95_ms": 297.1650738307572
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp2_mns32",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.103453874588013,
|
||||
"offered_request_rate": 12.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 11.55742285377119,
|
||||
"ttft_max_ms": 389.93612032123883,
|
||||
"ttft_p50_ms": 296.08352841701094,
|
||||
"ttft_p95_ms": 384.6764910356253
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp2_mns32",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.255317687988281,
|
||||
"offered_request_rate": 14.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 13.374732701347012,
|
||||
"ttft_max_ms": 388.3013907870674,
|
||||
"ttft_p50_ms": 309.4635231444016,
|
||||
"ttft_p95_ms": 386.4490667329015
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp2_mns64",
|
||||
"tp": 2
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp2_mns64",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.106109380722046,
|
||||
"offered_request_rate": 10.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 9.814433634929404,
|
||||
"ttft_max_ms": 300.1784228926625,
|
||||
"ttft_p50_ms": 211.39433537475938,
|
||||
"ttft_p95_ms": 297.1650738307572
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp2_mns64",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 10.112579107284546,
|
||||
"offered_request_rate": 12.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 11.55742285377119,
|
||||
"ttft_max_ms": 389.93612032123883,
|
||||
"ttft_p50_ms": 296.08352841701094,
|
||||
"ttft_p95_ms": 384.6764910356253
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp2_mns64",
|
||||
"tp": 2
|
||||
},
|
||||
"elapsed_seconds": 9.847446918487549,
|
||||
"offered_request_rate": 14.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 13.374732701347012,
|
||||
"ttft_max_ms": 388.3013907870674,
|
||||
"ttft_p50_ms": 309.4635231444016,
|
||||
"ttft_p95_ms": 386.4490667329015
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"contract": {
|
||||
"arrival": "open_loop_uniform",
|
||||
"input_tokens": 2048,
|
||||
"output_tokens": 1,
|
||||
"prefix_caching": false,
|
||||
"rates": [
|
||||
10.0,
|
||||
12.0,
|
||||
14.0
|
||||
],
|
||||
"requests_per_anchor": 64,
|
||||
"target_pass_rate": 0.95,
|
||||
"ttft_slo_ms": 1256.0
|
||||
},
|
||||
"frontier": {
|
||||
"git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n",
|
||||
"source": "/tmp/replayserve-frontier-rs1b"
|
||||
},
|
||||
"profiles": {
|
||||
"coverage": {
|
||||
"attention": {
|
||||
"1": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
},
|
||||
"2": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
},
|
||||
"4": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
}
|
||||
},
|
||||
"manifest": {
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection",
|
||||
"attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 132,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v2"
|
||||
}
|
||||
},
|
||||
"root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2",
|
||||
"sha256": {
|
||||
"attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb",
|
||||
"moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
}
|
||||
},
|
||||
"schema": "frontier-qwen30-prefill-surface-v1",
|
||||
"status": "partial_not_decision_bearing"
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
{
|
||||
"capacity": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp4_mns16",
|
||||
"tp": 4
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 24.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 6.0,
|
||||
"upper_censored": false
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp4_mns32",
|
||||
"tp": 4
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 24.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 6.0,
|
||||
"upper_censored": false
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp4_mns64",
|
||||
"tp": 4
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 24.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 6.0,
|
||||
"upper_censored": false
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp4_mns8",
|
||||
"tp": 4
|
||||
},
|
||||
"lower_censored": false,
|
||||
"maximum_tested_feasible_request_rate": 24.0,
|
||||
"maximum_tested_feasible_request_rate_per_gpu": 6.0,
|
||||
"upper_censored": false
|
||||
}
|
||||
],
|
||||
"config_results": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp4_mns8",
|
||||
"tp": 4
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp4_mns8",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 10.649231672286987,
|
||||
"offered_request_rate": 20.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.74355310200617,
|
||||
"ttft_max_ms": 779.2433711652614,
|
||||
"ttft_p50_ms": 476.39050138849325,
|
||||
"ttft_p95_ms": 718.1022232545545
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp4_mns8",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 10.245610237121582,
|
||||
"offered_request_rate": 24.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.93744325871021,
|
||||
"ttft_max_ms": 1219.4533647214066,
|
||||
"ttft_p50_ms": 708.2221064125775,
|
||||
"ttft_p95_ms": 1155.549457433053
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 8,
|
||||
"name": "tp4_mns8",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.953459978103638,
|
||||
"offered_request_rate": 28.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459",
|
||||
"score": {
|
||||
"feasible": false,
|
||||
"pass_rate": 0.765625,
|
||||
"passed": 49,
|
||||
"throughput_requests_per_second": 16.898601280793688,
|
||||
"ttft_max_ms": 1587.02950504691,
|
||||
"ttft_p50_ms": 885.3220562610811,
|
||||
"ttft_p95_ms": 1515.6009336189102
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp4_mns16",
|
||||
"tp": 4
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp4_mns16",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 10.059623956680298,
|
||||
"offered_request_rate": 20.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.74355310200617,
|
||||
"ttft_max_ms": 779.2433711652614,
|
||||
"ttft_p50_ms": 476.39050138849325,
|
||||
"ttft_p95_ms": 718.1022232545545
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp4_mns16",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 10.000702381134033,
|
||||
"offered_request_rate": 24.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.93744325871021,
|
||||
"ttft_max_ms": 1219.4533647214066,
|
||||
"ttft_p50_ms": 708.2221064125775,
|
||||
"ttft_p95_ms": 1155.549457433053
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 16,
|
||||
"name": "tp4_mns16",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.852763652801514,
|
||||
"offered_request_rate": 28.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459",
|
||||
"score": {
|
||||
"feasible": false,
|
||||
"pass_rate": 0.765625,
|
||||
"passed": 49,
|
||||
"throughput_requests_per_second": 16.898601280793688,
|
||||
"ttft_max_ms": 1587.02950504691,
|
||||
"ttft_p50_ms": 885.3220562610811,
|
||||
"ttft_p95_ms": 1515.6009336189102
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp4_mns32",
|
||||
"tp": 4
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp4_mns32",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.849465370178223,
|
||||
"offered_request_rate": 20.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.74355310200617,
|
||||
"ttft_max_ms": 779.2433711652614,
|
||||
"ttft_p50_ms": 476.39050138849325,
|
||||
"ttft_p95_ms": 718.1022232545545
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp4_mns32",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.80930495262146,
|
||||
"offered_request_rate": 24.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.93744325871021,
|
||||
"ttft_max_ms": 1219.4533647214066,
|
||||
"ttft_p50_ms": 708.2221064125775,
|
||||
"ttft_p95_ms": 1155.549457433053
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 32,
|
||||
"name": "tp4_mns32",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.798201560974121,
|
||||
"offered_request_rate": 28.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459",
|
||||
"score": {
|
||||
"feasible": false,
|
||||
"pass_rate": 0.765625,
|
||||
"passed": 49,
|
||||
"throughput_requests_per_second": 16.898601280793688,
|
||||
"ttft_max_ms": 1587.02950504691,
|
||||
"ttft_p50_ms": 885.3220562610811,
|
||||
"ttft_p95_ms": 1515.6009336189102
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp4_mns64",
|
||||
"tp": 4
|
||||
},
|
||||
"loads": [
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp4_mns64",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.800058603286743,
|
||||
"offered_request_rate": 20.0,
|
||||
"offered_request_rate_per_gpu": 5.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.74355310200617,
|
||||
"ttft_max_ms": 779.2433711652614,
|
||||
"ttft_p50_ms": 476.39050138849325,
|
||||
"ttft_p95_ms": 718.1022232545545
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp4_mns64",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.804483413696289,
|
||||
"offered_request_rate": 24.0,
|
||||
"offered_request_rate_per_gpu": 6.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7",
|
||||
"score": {
|
||||
"feasible": true,
|
||||
"pass_rate": 1.0,
|
||||
"passed": 64,
|
||||
"throughput_requests_per_second": 16.93744325871021,
|
||||
"ttft_max_ms": 1219.4533647214066,
|
||||
"ttft_p50_ms": 708.2221064125775,
|
||||
"ttft_p95_ms": 1155.549457433053
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa"
|
||||
},
|
||||
{
|
||||
"config": {
|
||||
"mns": 64,
|
||||
"name": "tp4_mns64",
|
||||
"tp": 4
|
||||
},
|
||||
"elapsed_seconds": 9.80262541770935,
|
||||
"offered_request_rate": 28.0,
|
||||
"offered_request_rate_per_gpu": 7.0,
|
||||
"request_count": 64,
|
||||
"request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459",
|
||||
"score": {
|
||||
"feasible": false,
|
||||
"pass_rate": 0.765625,
|
||||
"passed": 49,
|
||||
"throughput_requests_per_second": 16.898601280793688,
|
||||
"ttft_max_ms": 1587.02950504691,
|
||||
"ttft_p50_ms": 885.3220562610811,
|
||||
"ttft_p95_ms": 1515.6009336189102
|
||||
},
|
||||
"status": "completed",
|
||||
"trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"contract": {
|
||||
"arrival": "open_loop_uniform",
|
||||
"input_tokens": 2048,
|
||||
"output_tokens": 1,
|
||||
"prefix_caching": false,
|
||||
"rates": [
|
||||
20.0,
|
||||
24.0,
|
||||
28.0
|
||||
],
|
||||
"requests_per_anchor": 64,
|
||||
"target_pass_rate": 0.95,
|
||||
"ttft_slo_ms": 1256.0
|
||||
},
|
||||
"frontier": {
|
||||
"git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n",
|
||||
"source": "/tmp/replayserve-frontier-rs1b"
|
||||
},
|
||||
"profiles": {
|
||||
"coverage": {
|
||||
"attention": {
|
||||
"1": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
},
|
||||
"2": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
},
|
||||
"4": {
|
||||
"exact_prefill_2048_rows": 1,
|
||||
"profile_batch_size": 1
|
||||
}
|
||||
},
|
||||
"manifest": {
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection",
|
||||
"attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 132,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v2"
|
||||
}
|
||||
},
|
||||
"root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2",
|
||||
"sha256": {
|
||||
"attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb",
|
||||
"moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
}
|
||||
},
|
||||
"schema": "frontier-qwen30-prefill-surface-v1",
|
||||
"status": "partial_not_decision_bearing"
|
||||
}
|
||||
@@ -37,3 +37,9 @@ def test_grid_and_trace(tmp_path: Path) -> None:
|
||||
assert len(lines) == 4
|
||||
assert lines[1].split(",")[:3] == ["0.000000000000", "2048", "1"]
|
||||
assert lines[3].split(",")[:3] == ["0.500000000000", "2048", "1"]
|
||||
|
||||
|
||||
def test_kendall_tau_b() -> None:
|
||||
analysis = load("analyze_qwen30_prefill_fidelity.py")
|
||||
assert analysis.kendall_tau_b([1, 2, 3], [1, 2, 3])["kendall_tau_b"] == 1
|
||||
assert analysis.kendall_tau_b([1, 2, 3], [3, 2, 1])["kendall_tau_b"] == -1
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Frontier simulator fidelity：以 config ranking 为目标的阶段性评测
|
||||
|
||||
更新日期：2026-07-16。
|
||||
更新日期：2026-07-17。
|
||||
|
||||
统一实验平台：所有新增与重跑实验只使用 `dash0` 的 8×NVIDIA H20。Qwen30B 的真实 P1 artifacts 也来自 `dash0`；早期文档中的主机 provenance 标注错误，本版已按实验主机和远端 artifact 路径更正。
|
||||
|
||||
@@ -14,11 +14,12 @@
|
||||
- 将 operator profile 更新为与真实 serving 一致的 community vLLM `0.20.0`、BF16、H20、TP1/2/4 栈后，新的 profile-only Frontier **没有恢复排序**：92 个真实 anchors 在 simulator 中全部 SLO-infeasible，12 个 config 因而全部并列，最坏 tie-break regret 为 `60.91%`。这否证了“旧 profile 版本不一致是主要原因”这一简单解释，并把问题收敛到 execution context、operator composition 与 mixed-state schema。
|
||||
- 在 Qwen3-235B-A22B-FP8 prefill-only 上，补齐 FP8/MoE profile 与 serving semantics、但不做端到端 action calibration 后，Frontier 的最优集合与真机完全一致，Spearman ρ 为 `0.9487`，20/20 个可比较非 tie config pair 同序，选择 regret 为 `0`。
|
||||
- 在 Qwen3-235B-A22B-FP8 fixed-shape mixed 上，Frontier 同样给出了与真机完全相同的 TP4 top set，Kendall τ-b=`0.8944`、worst tie-break regret=`0`。但这次成功掩盖了一个明确的机制错误：Frontier 认为同一 TP family 内四个 MNS/MBT config 完全等价，真机 TP8 capacity 却形成对角为 0.30、非对角为 0.20 req/s/GPU 的 checkerboard interaction。34 个 config-load labels 中有 10 个 false-infeasible；20 个 real non-tie pairs 中只保持 16 个方向。
|
||||
- 现有结果表明：**在已对齐的受控 case 中，绝对 capacity 有 gap、甚至 action differential 错误，都不必然妨碍 Frontier 找到当前 surface 的最优 config。** 统一 dash0 campaign 已完整覆盖 Qwen235 FP8 prefill-only 与 fixed-shape mixed，但仍不能外推到 trace-faithful mixed 或 decode-only，也不能忽略达到这种 fidelity 所需的真机 profile、KV capacity、runtime-semantic patch 或 calibration 成本。
|
||||
- 新增的 Qwen3-30B-A3B BF16 prefill-only 对照在没有 decode、prefix reuse 和 true-mixed attention 的情况下仍然失败：真机 top set 是四个 TP4 config，Frontier top set 却是全部 TP1/TP2 config，top set 无交集，worst regret=`12.5%`、Kendall τ-b=`-1.0`，32 个 real non-tie pairs 全部反向。这直接否证了“prefill-only 是 simulator ranking 充分容易条件”。
|
||||
- 现有结果更支持一个 **margin-aware fidelity** 解释：action-differential residual 可以很大，但只有当它足以穿过 real decision margin 时才会导致选错。Qwen235 mixed 中真机 TP4 相对最好 TP8 的 margin 是 `2×`，足以掩盖同 TP family 内的机制错误；Qwen30 prefill-only 中 TP scaling 差分更细，Frontier 的 residual 直接反转了 topology ordering。模型大小或 execution phase 单独都不足以解释现有结果。
|
||||
|
||||
因此，现阶段最准确的 research statement 不是“simulator 排序一定错误”，而是：
|
||||
|
||||
> Simulator 的 usefulness 取决于 compatibility envelope。问题是需要多少、什么类型的真机信息，才能使其 config ranking 可被信任；以及当 workload、runtime 或 execution topology 变化时，如何低成本判断该 envelope 是否仍成立。
|
||||
> Simulator 的 usefulness 取决于 action-differential residual 是否小于真实 decision margin。问题不是简单区分 prefill 与 mixed，而是定位哪些 scheduler-state-conditioned execution residual 会随 TP、batch composition 和负载被放大，并用最少的真机信息判断 ranking 是否足够可信。
|
||||
|
||||
## 评测口径
|
||||
|
||||
@@ -89,6 +90,22 @@ Frontier 原生 profile schema 可以按 `num_tensor_parallel_workers` 选择 TP
|
||||
|
||||
该 case 是有意设计的 prefill mechanism-isolation workload：output 固定为 1，prefix cache 关闭，并使用固定 64-request cohort。它不保持原 trace 的 output-length distribution 和 prefix reuse，因此不能被称为 trace-faithful workload，也不能代表 mixed serving。
|
||||
|
||||
### Case D：Qwen3-30B-A3B BF16 prefill-only
|
||||
|
||||
| 项目 | 设置 |
|
||||
|---|---|
|
||||
| 真机 | `dash0`，8×NVIDIA H20；每个 job 按 TP 独占 GPU |
|
||||
| model/runtime | `Qwen/Qwen3-30B-A3B`；community vLLM `0.20.0+cu129`（source `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`），BF16 weights/activation/KV，FA3，runtime 默认 CUDA graph |
|
||||
| workload | 64 个不同 token-chain prompts；ISL=2,048、OSL=1、uniform open-loop QPS、prefix cache off |
|
||||
| config surface | `TP∈{1,2,4} × MNS∈{8,16,32,64}`，MBT=8,192，chunked prefill on，共 12 cells |
|
||||
| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + 当前 compatibility patches + vLLM 0.20 same-stack `profile-v2`；无 E2E calibration、decode graph=`none`、analytical collective |
|
||||
| load lattice | base system rates `{4,8,16,32,64}` req/s；事先声明的 common refinement 为 `{5,6,7}` req/s/GPU |
|
||||
| real anchor | 每个 `(config,rate,round)` fresh server；两轮独立运行且第二轮反转顺序；两轮都 pass 才算 feasible |
|
||||
| SLO | 至少 61/64 requests 满足 TTFT≤1,256 ms |
|
||||
| primary objective | 最大被测试可行 offered req/s / 实际 TP GPUs |
|
||||
|
||||
这个 case 是对“prefill-only 是 simulator 的容易区间”的直接可证伪测试。它刻意去掉 decode、true-mixed batch、prefix reuse 和 KV-residency feedback，但保留 TP 改变下的 local shard、collective、batching 与 scheduler queue。Frontier surface 在查看真机 ranking 前冻结，没有使用本 surface 的 serving latency 拟合 scale。
|
||||
|
||||
## Workload fidelity 修正
|
||||
|
||||
`thinking_w20260327_1000` 的 600 秒原始窗口包含 15,479 requests，字段包括 exact prompt、arrival、`input_length`、`output_length`、session parent/turn 和 block-size=64 的 `hash_ids`。只读审计得到：
|
||||
@@ -126,10 +143,11 @@ trace 的 source hash block size 为 64，而 community vLLM 0.10.2 的 CUDA KV
|
||||
| Qwen30 mixed / old profile-only | 12 | TP2,MNS32 | TP4,MNS32/64 | miss | τ-b=0.000；exact sign=37.88% | 25.63% | 排序错误 |
|
||||
| Qwen30 mixed / vLLM 0.20 profile-only | 12 | TP2,MNS32 | 全部 12 configs | 不可辨识 | τ-b=0.000；exact sign=7.58% | 60.91% | 同栈 raw profile 仍不足 |
|
||||
| Qwen30 mixed / frozen per-TP calibration | 12 | TP2,MNS32 | TP2,MNS32/64 | hit，非 exact | τ-b=0.9668；exact sign=93.94% | 0.76% | dash0 |
|
||||
| Qwen30 BF16 prefill-only / vLLM 0.20 profile-only | 12 | 全部 TP4 configs | 全部 TP1/TP2 configs | miss，无交集 | τ-b=-1.0000；real non-tie=0/32 | 12.50% | topology order 反转 |
|
||||
| Qwen235 FP8 prefill / best-effort | 8 | TP4,MBT16K,MNS64/128 | 与 real 完全相同 | exact match | ρ=0.9487；non-tied 20/20 | 0 | 足以选最优 config |
|
||||
| Qwen235 FP8 fixed-shape mixed / frozen full profile | 8 | 四个 TP4 configs | 与 real 完全相同 | exact match | τ-b=0.8944；exact sign=24/28；real non-tie=16/20 | 0 | 选对 topology；漏掉 TP8 MNS×MBT interaction |
|
||||
|
||||
最重要的区别是：Qwen30 的 high-fidelity 结果依赖 per-TP E2E calibration；Qwen235 的 high-fidelity 结果不依赖本 case 的 E2E calibration，但依赖同 runtime/hardware 的 operator profile、真实 KV capacity 和多处 compatibility fixes。二者都不能表述为“拿 stock Frontier 零成本预测即可”。
|
||||
最重要的区别是：Qwen30 的 high-fidelity mixed 结果依赖 per-TP E2E calibration，而新 prefill-only profile-only 实验证明“去掉 decode/mixed 状态”也不足以恢复 ranking。Qwen235 的 high-fidelity 结果不依赖本 case 的 E2E calibration，但依赖同 runtime/hardware 的 operator profile、真实 KV capacity 和多处 compatibility fixes。两个 model 的 stack 和精度不同，所以不能把差异直接归因为 model size；二者也都不能表述为“拿 stock Frontier 零成本预测即可”。
|
||||
|
||||
## Case A baseline 结果：Qwen30 mixed serving
|
||||
|
||||
@@ -311,6 +329,53 @@ MNS 128 fail pass
|
||||
|
||||
原预注册的 TPOT 40 ms 因最低负载已不可行而失去 capacity-ranking 可辨识性；本文已把 150 ms 明确记录为 post-pilot protocol amendment，并保留 40 ms 全部失败的 sensitivity 结果，没有把 SLO change 隐藏成预注册结论。
|
||||
|
||||
## Case D 结果：Qwen30 BF16 prefill-only
|
||||
|
||||
|
||||
|
||||
结果不是“绝对 capacity 有偏差但 rank 可用”，而是完整的 topology ordering reversal：
|
||||
|
||||
| TP | MNS | Real | Frontier profile-only | 差异 |
|
||||
|---:|---|---:|---:|---:|
|
||||
| 1 | 8/16/32/64 | 7.0 | **8.0** | +1.0 |
|
||||
| 2 | 8/16/32/64 | 7.0 | **8.0** | +1.0 |
|
||||
| 4 | 8/16/32/64 | **8.0** | 6.0 | -2.0 |
|
||||
|
||||
单位为最大被测试 SLO-feasible req/s/GPU。四个 TP4 configs 是真机 top set；Frontier 却把全部八个 TP1/TP2 configs 判为 top set，两者无交集。部署任一 simulator top config 都只能在真机上得到 7 req/s/GPU，因此 optimistic 与 worst tie-break regret 都为 `1-7/8=12.5%`。Kendall τ-b=`-1.0`；32 个 real non-tie pairs 中 0 个同序。96 个 config-load decisions 中 80 个一致，但同时有 8 个 false-feasible 和 8 个 false-infeasible，它们刚好跨过了不同 TP family 的 capacity boundary。
|
||||
|
||||
### 错误从哪里开始放大
|
||||
|
||||
代表性 `MNS=8` anchors 表明，低负载单次 execution time 并没有数倍偏差，错误主要在接近饱和时被 queue 放大：
|
||||
|
||||
| Config / system rate | Real TTFT p95（两轮） | Frontier TTFT p95 | Real / sim SLO |
|
||||
|---|---:|---:|---|
|
||||
| TP1 @ 4 req/s | 154.1 / 153.6 ms | 171.6 ms | pass / pass |
|
||||
| TP1 @ 8 req/s | 1289.2 / 1254.4 ms | 516.4 ms | fail / pass |
|
||||
| TP2 @ 8 req/s | 97.7 / 93.8 ms | 122.2 ms | pass / pass |
|
||||
| TP2 @ 16 req/s | 1389.9 / 1365.0 ms | 962.8 ms | fail / pass |
|
||||
| TP4 @ 4 req/s | 82.3 / 69.6 ms | 93.1 ms | pass / pass |
|
||||
| TP4 @ 32 req/s | 1136.6 / 1134.0 ms | 1787.9 ms | pass / fail |
|
||||
|
||||
这排除了“只要给所有 kernel 乘一个 global scale 就能修好”的解释：相同 stack 在低负载下接近，但在不同 TP 的饱和边界上向相反方向偏移。更符合数据的抽象是：小的 per-step composition residual 改变 service rate，再通过 scheduler queue 的非线性反馈放大成 TTFT 和 capacity 边界错误。
|
||||
|
||||
当前证据可以排除一些原因，但还不能唯一定位根因：
|
||||
|
||||
1. **Decode/true-mixed schema 不是必要条件。** 这个 workload 没有 decode、FULL decode CUDA graph、fused true-mixed attention 或 initial-KV state，排序仍然失败。这些机制可能在 mixed/decode 中进一步增大误差，但不是本次失败的前提。
|
||||
2. **Communication 可能解释部分 TP scaling，但不能单独解释全部误差。** `profile-v2/allreduce.json` 已有 24 个 TP2/TP4 实测 rows，但 base Frontier 为了保持历史对照使用 analytical 600-Gbps/1-µs collective，没有注入这些数据。下一步必须单独做 measured-collective ablation；但 TP1 无 collective 仍在 8 req/s 出现 773 ms 的 p95 低估，所以 collective 不会是唯一根因。
|
||||
3. **Profile 覆盖了 token count，却未必覆盖 scheduler 实际产生的 batch composition。** linear/MoE rows 覆盖到 8,192 tokens，但 exact-2,048 pure-prefill attention 在每个 TP 下只有 batch=1 的直接样本；实际 MBT=8,192 可以产生多个 2,048-token sequence 的组合。这是一个需要补测的 coverage hypothesis，不是已证明根因。
|
||||
4. **Routing、step composition 和 scheduler batch state 仍然是联合候选。** pure-prefill 只排除了 phase mixing，没有排除 MoE routing、TP-specific local shape、launch/fusion 和持续负载下 batch/queue trajectory 的耦合。
|
||||
|
||||
### 为什么 235B 能选对，30B 却选错
|
||||
|
||||
现有数据不支持“235B 大所以 simulator 更容易”这种模型大小因果。两个 case 的 precision、vLLM 版本、attention backend、graph mode、EP 和 profile closure 都不同。能被数据直接支持的区别是 **decision margin**：
|
||||
|
||||
- 235B mixed 真机中 TP4 最优 capacity 为 0.60 req/s/GPU，最好 TP8 为 0.30，有 `2×` margin；Frontier 虽然漏掉 TP8 的 MNS×MBT interaction，仍预测 TP4/TP8 为 0.40/0.15，所以 residual 没有穿过全局拓扑边界。
|
||||
- 30B prefill-only 真机 TP4 相对 TP1/TP2 只有 8 vs. 7 req/s/GPU 的 margin。Frontier 预测为 6 vs. 8，TP-dependent residual 大于真实 margin，因而把最优 topology 完整反转。
|
||||
|
||||
因此，“prefill-only 容易，decode/mixed 困难”的强假设已被否证。phase 仍可能改变 residual 的幅度，但它不是 fidelity 的充分条件。按预注册 decision rule，此时不应继续无区分度地扩展更多 phase cases，而应在 Qwen30 同一 model/workload 上按单变量顺序做：`measured collective injection` → `batch-composition-conditioned pure-prefill attention/step profile` → 在 TP1@8、TP2@16、TP4@32 对齐 real/sim scheduler batch、queue 与 per-step critical path → 最后再测 routing/graph。
|
||||
|
||||
本 case 接受的 ground truth 共 24 个 fleet jobs、192 个 fresh-server anchors、12,288 个 measured requests 和 4,512 个 warmup requests，消耗 12.07 H20-GPU-hours。运行中曾因 fleet controller 在 fresh-server 空窗误判 GPU 为空闲而产生重叠 launch；这些 attempts 未被合并到 accepted artifact root，已整体隔离并用独立 queue state 重跑。最终 comparison SHA256 为 `c9a9cac9f60c7be804d1cb9466c455f8fe9e3a8dc60b9cec3329bde6a8c19334`。
|
||||
|
||||
## 尚不能纳入 simulator ranking 的 case
|
||||
|
||||
### 原 internal-runtime Qwen235 prefill-only
|
||||
@@ -328,11 +393,12 @@ MNS 128 fail pass
|
||||
当前证据的结论是：
|
||||
|
||||
1. **绝对 gap 不是否决 simulator 的理由。** Qwen235 中 11%--50% 的 capacity 低估仍可保持 zero-regret config selection。
|
||||
2. **但 rank 成功不能证明模型机制正确。** Qwen235 prefill 的 TP8 MBT differential 错误，fixed-shape mixed 又漏掉 TP8 的 MNS×MBT 非加性交互；Qwen30 还有 28/92 anchor SLO labels 错误。只是当前 decision margin 足以容忍这些 residual。
|
||||
3. **alignment 是结果的一部分。** Qwen30 需要 per-TP E2E calibration 才能恢复排序；Qwen235 不需要本 case E2E calibration，但需要同栈 FP8 profiles、真机 KV capacity 和 simulator patches。论文必须报告这部分真机成本，不能把它隐藏在“offline profiles”中。
|
||||
4. **目前不能建立“simulator 全局选优普遍失败”的 premise。** 受控 prefill、fixed-shape mixed 和历史 mixed surface 上，经过 alignment 的 Frontier 已经能选到最优或 0.76% 内的近最优 config。更有证据的 research premise 是：aggregate selection success 会掩盖 action interaction 与 state-transition model 的错误；若要声称全局选优失败，仍必须在 dash0 的 trace-faithful mixed、decode-only、prefix cache 或跨 topology case 上得到稳定反例。
|
||||
2. **rank 成功不能证明模型机制正确，rank 失败则表明 residual 穿过了 decision margin。** Qwen235 prefill 的 TP8 MBT differential 错误，fixed-shape mixed 又漏掉 TP8 的 MNS×MBT 非加性交互，但大 topology margin 保住了 top set；Qwen30 prefill-only 的较小 margin 被 TP-dependent saturation residual 穿过，导致 τ-b=-1 和 12.5% regret。
|
||||
3. **prefill-only 不是 fidelity 的充分容易条件。** Qwen30 在没有 decode、prefix reuse、initial KV 和 true-mixed batch 时仍选错 topology。phase 可以改变 residual，但不能单独作为 compatibility-envelope 边界。
|
||||
4. **alignment 是结果的一部分。** Qwen30 mixed 需要 per-TP E2E calibration 才能恢复排序，同栈 operator profile 或去掉 mixed phase 都不足；Qwen235 不需要本 case E2E calibration，但需要同栈 FP8 profiles、真机 KV capacity 和 simulator patches。论文必须报告这部分真机成本，不能把它隐藏在“offline profiles”中。
|
||||
5. **现在有了可复现的全局选优反例，但尚不足以声称 simulator 普遍失败。** Qwen30 BF16 prefill-only 是一个无 top-set overlap 的稳定反例；Qwen235 两个 case 又证明 Frontier 在某些 envelope 内足以选优。更有研究价值的 premise 是：**什么 state-conditioned action residual 决定 ranking 是否能跨过 real margin，以及如何在不做全 surface 真机 sweep 的情况下检测这个风险。**
|
||||
|
||||
后续每个 case 建议使用同一 gate：worst selected-config regret ≤5%、tie-aware rank correlation ≥0.8、足够数量的 informative pairs、ground-truth bracket 不足以反转最优决策，并单独报告达到该结果所需的 real-GPU profiling/calibration cost。
|
||||
后续每个 case 建议使用同一 gate：worst selected-config regret ≤5%、tie-aware rank correlation ≥0.8、足够数量的 informative pairs、ground-truth bracket 不足以反转最优决策，并单独报告达到该结果所需的 real-GPU profiling/calibration cost。当前下一 gate 是在 Qwen30 prefill-only 上做同模型单变量 context ablation，而不是继续扩展跨模型 phase 矩阵。
|
||||
|
||||
## 数据与复现
|
||||
|
||||
@@ -340,6 +406,10 @@ MNS 128 fail pass
|
||||
- 画图脚本：[plot_simulator_fidelity.py](scripts/plot_simulator_fidelity.py)
|
||||
- Qwen30 audit：[report.md](runs/frontier-multicase-sufficiency-v0/results/qwen30-baseline/report.md)
|
||||
- Qwen30 aligned metrics：[metrics.json](runs/frontier-slo-alignment-v0/results/metrics.json)
|
||||
- Qwen30 prefill-only experiment card：[experiment-card.md](runs/frontier-phase-factorial-v0/experiment-card.md)
|
||||
- Qwen30 prefill-only comparison：[comparison.json](runs/frontier-phase-factorial-v0/results/final/comparison.json)
|
||||
- Qwen30 prefill-only capacity table：[capacity.csv](runs/frontier-phase-factorial-v0/results/final/capacity.csv)
|
||||
- Qwen30 prefill-only analyzer：[analyze_qwen30_prefill_fidelity.py](runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py)
|
||||
- Qwen235 fixed-cohort comparison：[v2_refined_comparison.json](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json)
|
||||
- Qwen235 full report：[report.md](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md)
|
||||
- Fixed-shape mixed comparison：[comparison.json](runs/frontier-multicase-sufficiency-v1/results/t0-final/comparison.json)
|
||||
@@ -357,4 +427,17 @@ MNS 128 fail pass
|
||||
|
||||
```bash
|
||||
python3 scripts/plot_simulator_fidelity.py
|
||||
|
||||
python3 runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py \
|
||||
--fleet-artifacts runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive \
|
||||
--simulator-manifest runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json \
|
||||
--simulator-manifest runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json \
|
||||
--simulator-manifest runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json \
|
||||
--simulator-manifest runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json \
|
||||
--simulator-manifest runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json \
|
||||
--simulator-manifest runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json \
|
||||
--output-root runs/frontier-phase-factorial-v0/results/final
|
||||
|
||||
cp runs/frontier-phase-factorial-v0/results/final/qwen30-prefill-ranking.png \
|
||||
docs/assets/simulator-fidelity/qwen30-prefill-ranking.png
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user