From cf610003ed1e6f1d0ba64429754f834dbbc3af53 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 23 Jul 2026 18:08:00 +0800 Subject: [PATCH] Close BC8 decode curve counterfactual --- runs/frontier-decode-batch-grid-v0/.gitignore | 1 + .../analyze_bc8_replay.py | 152 +++++++ .../experiment-card.md | 43 +- .../results/bc8-replay-verdict.json | 222 ++++++++++ .../results/bc8-replay-verdict.md | 13 + .../results/grid.json | 410 ++++++++++++++++++ .../results/tp4-b6-r3.json | 353 +++++++++++++++ .../run_bc8_replay.py | 222 ++++++++++ .../run_frontier_with_whole_decode_curve.py | 114 +++++ .../README.md | 97 +++++ .../results/three-experiment-verdict.md | 46 ++ 11 files changed, 1666 insertions(+), 7 deletions(-) create mode 100644 runs/frontier-decode-batch-grid-v0/analyze_bc8_replay.py create mode 100644 runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.json create mode 100644 runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.md create mode 100644 runs/frontier-decode-batch-grid-v0/results/grid.json create mode 100644 runs/frontier-decode-batch-grid-v0/results/tp4-b6-r3.json create mode 100644 runs/frontier-decode-batch-grid-v0/run_bc8_replay.py create mode 100644 runs/frontier-decode-batch-grid-v0/run_frontier_with_whole_decode_curve.py create mode 100644 runs/frontier-simulator-gap-campaign-v0/README.md create mode 100644 runs/frontier-simulator-gap-campaign-v0/results/three-experiment-verdict.md diff --git a/runs/frontier-decode-batch-grid-v0/.gitignore b/runs/frontier-decode-batch-grid-v0/.gitignore index 011358f..6ea11ec 100644 --- a/runs/frontier-decode-batch-grid-v0/.gitignore +++ b/runs/frontier-decode-batch-grid-v0/.gitignore @@ -1,3 +1,4 @@ fleet-artifacts/ fleet-state/ remote-outputs/ +replay/ diff --git a/runs/frontier-decode-batch-grid-v0/analyze_bc8_replay.py b/runs/frontier-decode-batch-grid-v0/analyze_bc8_replay.py new file mode 100644 index 0000000..086e327 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/analyze_bc8_replay.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Compare original and whole-layer-curve Frontier against the BC-8 real pilot.""" + +from __future__ import annotations + +import json +import statistics +from collections import Counter +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +CONFIGS = ("tp1_mns16", "tp2_mns16", "tp4_mns16", "tp4_mns32") +REPLAY = ROOT / "replay/bc8" +ORIGINAL = REPO / "runs/frontier-knee-sweep-v0/raw/fixed/rho0p02" +REAL = REPO / "runs/frontier-pilot-v0/results/pilot-surface.json" + + +def find_one(root: Path, name: str) -> Path: + matches = list(root.glob(f"**/{name}")) + if len(matches) != 1: + raise ValueError(f"expected one {name} under {root}, got {matches}") + return matches[0] + + +def summarize_sim(root: Path) -> dict: + metrics = json.loads(find_one(root, "system_metrics.json").read_text()) + ledger = find_one(root, "frontier_stage_batch_ledger.jsonl") + histogram: Counter[int] = Counter() + service_ms: dict[int, set[float]] = {} + for line in ledger.read_text().splitlines(): + row = json.loads(line) + tokens = row["request_num_tokens"] + if tokens and all(int(value) == 1 for value in tokens): + batch = len(tokens) + histogram[batch] += 1 + service_ms.setdefault(batch, set()).add( + float(row["execution_time"]["model_time_ms"]) + ) + return { + "tpot_mean_ms": metrics["tpot_statistics"]["mean"], + "ttft_mean_ms": metrics["ttft_statistics"]["mean"], + "e2e_mean_ms": metrics["request_e2e_time_statistics"]["mean"], + "decode_batch_histogram": dict(sorted(histogram.items())), + "decode_service_ms": { + str(batch): sorted(values) for batch, values in sorted(service_ms.items()) + }, + "decode_b_gt_1_fraction": sum( + count for batch, count in histogram.items() if batch > 1 + ) + / sum(histogram.values()), + } + + +def main() -> None: + real_rows = [ + row + for row in json.loads(REAL.read_text()) + if row["load"] == "p4" and row["canonical"] + ] + points = [] + for config in CONFIGS: + real = [row for row in real_rows if row["config"] == config] + original = summarize_sim(ORIGINAL / config) + corrected = summarize_sim(REPLAY / "raw" / config) + usage = json.loads((REPLAY / "raw" / config / "usage.json").read_text()) + points.append( + { + "config": config, + "tp": int(config[2]), + "mns": int(config.split("mns")[1]), + "real_tpot_mean_ms": statistics.fmean( + row["tpot_mean_ms"] for row in real + ), + "original": original, + "whole_curve": corrected, + "original_tpot_residual_ms": ( + original["tpot_mean_ms"] + - statistics.fmean(row["tpot_mean_ms"] for row in real) + ), + "whole_curve_tpot_residual_ms": ( + corrected["tpot_mean_ms"] + - statistics.fmean(row["tpot_mean_ms"] for row in real) + ), + "usage": usage, + } + ) + + def ranking(key): + return [ + point["config"] + for point in sorted(points, key=lambda point: (key(point), point["config"])) + if point["mns"] == 16 + ] + + real_ranking = ranking(lambda point: point["real_tpot_mean_ms"]) + original_ranking = ranking(lambda point: point["original"]["tpot_mean_ms"]) + corrected_ranking = ranking(lambda point: point["whole_curve"]["tpot_mean_ms"]) + payload = { + "schema": "frontier-decode-grid-bc8-verdict.v1", + "points": points, + "mns16_rankings_fast_to_slow": { + "real": real_ranking, + "original": original_ranking, + "whole_curve": corrected_ranking, + }, + "whole_curve_restores_real_mns16_ranking": corrected_ranking == real_ranking, + "all_corrected_batches_within_measured_support": all( + max(map(int, point["whole_curve"]["decode_batch_histogram"])) <= 8 + for point in points + ), + "decision": ( + "STATIC_WHOLE_CURVE_FIXES_BC8" + if corrected_ranking == real_ranking + else "ESCALATE_EVENT_LEVEL_STATE" + ), + } + output = ROOT / "results/bc8-replay-verdict.json" + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + lines = [ + "# BC-8 whole-layer decode-curve replay", + "", + "| Config | Real TPOT | Original sim | Whole-curve sim | Residual before -> after | b>1 |", + "|---|---:|---:|---:|---:|---:|", + ] + for point in points: + lines.append( + f"| {point['config']} | {point['real_tpot_mean_ms']:.3f} | " + f"{point['original']['tpot_mean_ms']:.3f} | " + f"{point['whole_curve']['tpot_mean_ms']:.3f} | " + f"{point['original_tpot_residual_ms']:+.3f} -> " + f"{point['whole_curve_tpot_residual_ms']:+.3f} | " + f"{point['whole_curve']['decode_b_gt_1_fraction']:.2%} |" + ) + lines.extend( + [ + "", + f"- Real MNS16: `{' < '.join(real_ranking)}`", + f"- Original: `{' < '.join(original_ranking)}`", + f"- Whole curve: `{' < '.join(corrected_ranking)}`", + f"- Decision: **{payload['decision']}**.", + "", + ] + ) + (ROOT / "results/bc8-replay-verdict.md").write_text("\n".join(lines)) + print(json.dumps({"output": str(output), "decision": payload["decision"]})) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-decode-batch-grid-v0/experiment-card.md b/runs/frontier-decode-batch-grid-v0/experiment-card.md index dc4071f..cd61d60 100644 --- a/runs/frontier-decode-batch-grid-v0/experiment-card.md +++ b/runs/frontier-decode-batch-grid-v0/experiment-card.md @@ -1,6 +1,6 @@ # 实验 EXP-DECODE-BATCH-GRID：BC-8 是否可由稳定 whole-layer b2--b8 curve 修复 -> **状态：** TP4/b2 stability gate PASS，扩展 full grid +> **状态：** COMPLETED；BC-8 排序工程修复成立，global curve merge 暂停 > > Parent campaign：[`../frontier-simulator-gap-campaign-v0/README.md`](../frontier-simulator-gap-campaign-v0/README.md) @@ -54,9 +54,10 @@ ## 复现信息 - **Frontier baseline：** `deadc4a321f0baaa534c6ebd17f974123733cdc2`； - joint serving-path curves + structured-attention analysis branch。 -- **Remote：** 复用 dash1 的 clean detached experiment worktree；canonical - checkout 的用户 dirty changes 不动。 + CPU replay 使用既有 joint serving-path curves，未合入 EXP-1 rejected patch。 +- **Remote：** dash1--dash4 共享 clean detached experiment worktree + `ecc559938132851d61bf22daa55cea9658807dfa`；canonical checkout 的用户 + dirty changes 未改动。 - **Known limits：** 固定 2048→128 state-matched workload；本实验只支持 decode curve 与 BC-8，不外推到 chat prefill shape 或其它模型。 @@ -68,6 +69,34 @@ `4.6263 ms` 和 `4.6201 ms`，跨 repeat CV=`0.067%`，稳定性 gate PASS，无需 repeat 3。median=`4.6232 ms`，比旧样本 `5.044 ms` 低 `0.421 ms`；方向支持 H1，但未达到事前 `>=0.5 ms` 的强证据阈值。 -- **观察事实：** 旧 TP4/b2 的高方差主要是测量不稳定；是否足以修复 - BC-8 仍须 full b2--b8 curve 与 trace replay。 -- **Decision：** 待 GPU。 +- **Full grid：** + + | Cell | fresh-process execute mean (ms) | median (ms) | initial CV | + |---|---:|---:|---:| + | TP2/b2 | 4.761 / 4.730 | 4.746 | 0.33% | + | TP2/b4 | 5.294 / 5.607 | 5.451 | 2.87% | + | TP2/b6 | 6.571 / 6.938 | 6.754 | 2.72% | + | TP2/b8 | 5.967 / 6.309 | 6.138 | 2.78% | + | TP4/b2 | 4.626 / 4.620 | 4.623 | 0.07% | + | TP4/b4 | 4.516 / 4.727 | 4.622 | 2.27% | + | TP4/b6 | 5.433 / 10.157 / 8.634 | 8.634 | 30.30% | + | TP4/b8 | 4.898 / 5.047 | 4.972 | 1.50% | + +- **TP4/b6 instability：** r2/r3 的 MoE 与 attention 基本不变，均值波动由 + 同步 collective 长尾造成；三个 run 的 step p50 仍约 `5.18--5.49 ms`。 + 依预注册规则追加 r3 并取三次中位数，但该 cell 不能视为稳定 deterministic + service constant。 +- **BC-8 exact replay：** replay 仅访问 b1/b2，因此没有使用 b6 或任何 + 插值/外推。TP4/b2 whole-layer override 将 TP4 sim TPOT + `5.864→5.367 ms`，残差 `+1.414→+0.918 ms`（MNS16）和 + `+1.498→+1.001 ms`（MNS32）；MNS16 完整排序从 + `TP21 stage fraction + `48.15%→37.47%`。 +- **7-cell support audit：** 最新 1h cells 的 decode batch 最大到 + `b6--b15`，而本实验 measured support 只到 b8；TP4/b6 又未通过 + deterministic-mean 稳定性。为避免重新引入 b8 constant extrapolation， + 不执行全局 7-cell injection，也不把整条 b2--b8 curve 合入默认 predictor。 +- **Decision：** H1 对 BC-8 成立，排序 gap 属于可工程修复的 TP4/b2 + whole-layer service residual；但绝对残差、MNS tie 和高 batch collective + tail 仍需 event/distribution-aware 模型。建议先合入受作用域保护的 TP4/b2 + correction，再补 b9--b16 与 collective-tail telemetry 后开启 global merge。 diff --git a/runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.json b/runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.json new file mode 100644 index 0000000..1e272f7 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.json @@ -0,0 +1,222 @@ +{ + "all_corrected_batches_within_measured_support": true, + "decision": "STATIC_WHOLE_CURVE_FIXES_BC8", + "mns16_rankings_fast_to_slow": { + "original": [ + "tp2_mns16", + "tp1_mns16", + "tp4_mns16" + ], + "real": [ + "tp2_mns16", + "tp4_mns16", + "tp1_mns16" + ], + "whole_curve": [ + "tp2_mns16", + "tp4_mns16", + "tp1_mns16" + ] + }, + "points": [ + { + "config": "tp1_mns16", + "mns": 16, + "original": { + "decode_b_gt_1_fraction": 0.0, + "decode_batch_histogram": { + "1": 16383 + }, + "decode_service_ms": { + "1": [ + 5.493362394 + ] + }, + "e2e_mean_ms": 864.3525078423435, + "tpot_mean_ms": 5.493362394020072, + "ttft_mean_ms": 166.6954838017941 + }, + "original_tpot_residual_ms": 0.616915450202109, + "real_tpot_mean_ms": 4.876446943817963, + "tp": 1, + "usage": { + "collective:moe:tp1-b1:structural-zero": 16383, + "moe:tp1-b1": 16383 + }, + "whole_curve": { + "decode_b_gt_1_fraction": 0.0, + "decode_batch_histogram": { + "1": 16383 + }, + "decode_service_ms": { + "1": [ + 5.493362394 + ] + }, + "e2e_mean_ms": 864.3525078423435, + "tpot_mean_ms": 5.493362394020072, + "ttft_mean_ms": 166.6954838017941 + }, + "whole_curve_tpot_residual_ms": 0.616915450202109 + }, + { + "config": "tp2_mns16", + "mns": 16, + "original": { + "decode_b_gt_1_fraction": 0.0, + "decode_batch_histogram": { + "1": 16383 + }, + "decode_service_ms": { + "1": [ + 4.928319445 + ] + }, + "e2e_mean_ms": 744.4521695355745, + "tpot_mean_ms": 4.9283194445308744, + "ttft_mean_ms": 118.55560008015348 + }, + "original_tpot_residual_ms": 0.6235698385840074, + "real_tpot_mean_ms": 4.304749605946867, + "tp": 2, + "usage": { + "collective:attention:tp2-b1": 16383, + "collective:moe:tp2-b1": 16383, + "fused_norm_deduction:attn:tp2-b1": 16383, + "fused_norm_deduction:mlp:tp2-b1": 16383, + "moe:tp2-b1": 16383 + }, + "whole_curve": { + "decode_b_gt_1_fraction": 0.0, + "decode_batch_histogram": { + "1": 16383 + }, + "decode_service_ms": { + "1": [ + 4.928319445 + ] + }, + "e2e_mean_ms": 744.4521695355745, + "tpot_mean_ms": 4.9283194445308744, + "ttft_mean_ms": 118.55560008015348 + }, + "whole_curve_tpot_residual_ms": 0.6235698385840074 + }, + { + "config": "tp4_mns16", + "mns": 16, + "original": { + "decode_b_gt_1_fraction": 0.4814983594604448, + "decode_batch_histogram": { + "1": 5689, + "2": 5283 + }, + "decode_service_ms": { + "1": [ + 4.738406304 + ], + "2": [ + 5.411344412 + ] + }, + "e2e_mean_ms": 840.3052136178922, + "tpot_mean_ms": 5.863803518571768, + "ttft_mean_ms": 95.60216675927755 + }, + "original_tpot_residual_ms": 1.4143723087101598, + "real_tpot_mean_ms": 4.449431209861608, + "tp": 4, + "usage": { + "collective:attention:tp4-b1": 7393, + "collective:attention:tp4-b2": 4431, + "collective:moe:tp4-b1": 7393, + "collective:moe:tp4-b2": 4431, + "fused_norm_deduction:attn:tp4-b1": 7393, + "fused_norm_deduction:attn:tp4-b2": 4431, + "fused_norm_deduction:mlp:tp4-b1": 7393, + "fused_norm_deduction:mlp:tp4-b2": 4431, + "moe:tp4-b1": 7393, + "moe:tp4-b2": 4431, + "whole_decode:tp4-b2:target_ms=4.623176437": 4431 + }, + "whole_curve": { + "decode_b_gt_1_fraction": 0.37474627875507444, + "decode_batch_histogram": { + "1": 7393, + "2": 4431 + }, + "decode_service_ms": { + "1": [ + 4.738406304 + ], + "2": [ + 4.623176437 + ] + }, + "e2e_mean_ms": 777.2050553866409, + "tpot_mean_ms": 5.367470055757525, + "ttft_mean_ms": 95.5363583054353 + }, + "whole_curve_tpot_residual_ms": 0.918038845895917 + }, + { + "config": "tp4_mns32", + "mns": 32, + "original": { + "decode_b_gt_1_fraction": 0.4814983594604448, + "decode_batch_histogram": { + "1": 5689, + "2": 5283 + }, + "decode_service_ms": { + "1": [ + 4.738406304 + ], + "2": [ + 5.411344412 + ] + }, + "e2e_mean_ms": 840.3052136178922, + "tpot_mean_ms": 5.863803518571768, + "ttft_mean_ms": 95.60216675927755 + }, + "original_tpot_residual_ms": 1.4977765516959103, + "real_tpot_mean_ms": 4.366026966875857, + "tp": 4, + "usage": { + "collective:attention:tp4-b1": 7393, + "collective:attention:tp4-b2": 4431, + "collective:moe:tp4-b1": 7393, + "collective:moe:tp4-b2": 4431, + "fused_norm_deduction:attn:tp4-b1": 7393, + "fused_norm_deduction:attn:tp4-b2": 4431, + "fused_norm_deduction:mlp:tp4-b1": 7393, + "fused_norm_deduction:mlp:tp4-b2": 4431, + "moe:tp4-b1": 7393, + "moe:tp4-b2": 4431, + "whole_decode:tp4-b2:target_ms=4.623176437": 4431 + }, + "whole_curve": { + "decode_b_gt_1_fraction": 0.37474627875507444, + "decode_batch_histogram": { + "1": 7393, + "2": 4431 + }, + "decode_service_ms": { + "1": [ + 4.738406304 + ], + "2": [ + 4.623176437 + ] + }, + "e2e_mean_ms": 777.2050553866409, + "tpot_mean_ms": 5.367470055757525, + "ttft_mean_ms": 95.5363583054353 + }, + "whole_curve_tpot_residual_ms": 1.0014430888816674 + } + ], + "schema": "frontier-decode-grid-bc8-verdict.v1", + "whole_curve_restores_real_mns16_ranking": true +} diff --git a/runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.md b/runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.md new file mode 100644 index 0000000..906bc16 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/results/bc8-replay-verdict.md @@ -0,0 +1,13 @@ +# BC-8 whole-layer decode-curve replay + +| Config | Real TPOT | Original sim | Whole-curve sim | Residual before -> after | b>1 | +|---|---:|---:|---:|---:|---:| +| tp1_mns16 | 4.876 | 5.493 | 5.493 | +0.617 -> +0.617 | 0.00% | +| tp2_mns16 | 4.305 | 4.928 | 4.928 | +0.624 -> +0.624 | 0.00% | +| tp4_mns16 | 4.449 | 5.864 | 5.367 | +1.414 -> +0.918 | 37.47% | +| tp4_mns32 | 4.366 | 5.864 | 5.367 | +1.498 -> +1.001 | 37.47% | + +- Real MNS16: `tp2_mns16 < tp4_mns16 < tp1_mns16` +- Original: `tp2_mns16 < tp1_mns16 < tp4_mns16` +- Whole curve: `tp2_mns16 < tp4_mns16 < tp1_mns16` +- Decision: **STATIC_WHOLE_CURVE_FIXES_BC8**. diff --git a/runs/frontier-decode-batch-grid-v0/results/grid.json b/runs/frontier-decode-batch-grid-v0/results/grid.json new file mode 100644 index 0000000..34e6669 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/results/grid.json @@ -0,0 +1,410 @@ +{ + "all_repeat_gates_resolved": true, + "cells": [ + { + "all_repeat_cv_pct": 0.33215115035261283, + "batch": 2, + "initial_repeat_cv_pct": 0.33215115035261283, + "median_component_ms": { + "attention": 0.7968654999999999, + "collective": 0.4589650625, + "linear_norm_rope": 0.9332466874999996, + "moe": 2.1423742343749987, + "other": 0.020707093750000002, + "output_head": 0.08698887500000001, + "router": 0.19049718749999994 + }, + "median_execute_ms": 4.74559925, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 0.7962288124999999, + "collective": 0.470928375, + "linear_norm_rope": 0.9330912187499996, + "moe": 2.136206281249999, + "other": 0.02065321875, + "output_head": 0.08696040625000001, + "router": 0.19041484374999995 + }, + "execute_mean_ms": 4.7613618125, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b2-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.7975021874999999, + "collective": 0.44700175, + "linear_norm_rope": 0.9334021562499997, + "moe": 2.1485421874999986, + "other": 0.02076096875, + "output_head": 0.08701734375, + "router": 0.19057953124999993 + }, + "execute_mean_ms": 4.7298366875, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b2-r2.json" + } + ], + "tp": 2 + }, + { + "all_repeat_cv_pct": 2.8664467148791393, + "batch": 4, + "initial_repeat_cv_pct": 2.8664467148791393, + "median_component_ms": { + "attention": 0.925814669047619, + "collective": 0.47613457857142855, + "linear_norm_rope": 0.926101651190476, + "moe": 2.5396719488095236, + "other": 0.021773490476190475, + "output_head": 0.08747840476190476, + "router": 0.18771020476190473 + }, + "median_execute_ms": 5.45066564047619, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 0.9284945714285713, + "collective": 0.4770398571428571, + "linear_norm_rope": 0.9251340357142854, + "moe": 2.5554174642857146, + "other": 0.021663214285714286, + "output_head": 0.08753164285714285, + "router": 0.1898966428571428 + }, + "execute_mean_ms": 5.294425214285715, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b4-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.9231347666666667, + "collective": 0.47522929999999997, + "linear_norm_rope": 0.9270692666666666, + "moe": 2.523926433333333, + "other": 0.021883766666666665, + "output_head": 0.08742516666666666, + "router": 0.18552376666666665 + }, + "execute_mean_ms": 5.606906066666666, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b4-r2.json" + } + ], + "tp": 2 + }, + { + "all_repeat_cv_pct": 2.718954157967049, + "batch": 6, + "initial_repeat_cv_pct": 2.718954157967049, + "median_component_ms": { + "attention": 1.0526737664835166, + "collective": 0.48269348763736264, + "linear_norm_rope": 0.9301222623626373, + "moe": 3.6929154436813176, + "other": 0.025437872252747254, + "output_head": 0.08800730082417582, + "router": 0.196665739010989 + }, + "median_execute_ms": 6.754197167582418, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 1.0590365714285717, + "collective": 0.48200882142857143, + "linear_norm_rope": 0.9312401785714285, + "moe": 3.678020964285713, + "other": 0.02528782142857143, + "output_head": 0.08801067857142857, + "router": 0.1985437857142856 + }, + "execute_mean_ms": 6.570553642857143, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b6-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 1.0463109615384618, + "collective": 0.48337815384615385, + "linear_norm_rope": 0.929004346153846, + "moe": 3.7078099230769217, + "other": 0.025587923076923078, + "output_head": 0.08800392307692309, + "router": 0.19478769230769236 + }, + "execute_mean_ms": 6.9378406923076925, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b6-r2.json" + } + ], + "tp": 2 + }, + { + "all_repeat_cv_pct": 2.7838528598972516, + "batch": 8, + "initial_repeat_cv_pct": 2.7838528598972516, + "median_component_ms": { + "attention": 1.408474403846154, + "collective": 0.5025935769230769, + "linear_norm_rope": 0.9324356730769228, + "moe": 2.6870017499999985, + "other": 0.02355823076923077, + "output_head": 0.08828742307692307, + "router": 0.19532132692307688 + }, + "median_execute_ms": 6.138216769230769, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 1.4162915000000003, + "collective": 0.5046254615384616, + "linear_norm_rope": 0.933206192307692, + "moe": 2.681186538461537, + "other": 0.023443884615384616, + "output_head": 0.08826711538461537, + "router": 0.19981553846153838 + }, + "execute_mean_ms": 5.967337846153845, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b8-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 1.400657307692308, + "collective": 0.5005616923076923, + "linear_norm_rope": 0.9316651538461536, + "moe": 2.69281696153846, + "other": 0.023672576923076925, + "output_head": 0.08830773076923076, + "router": 0.1908271153846154 + }, + "execute_mean_ms": 6.309095692307692, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp2-b8-r2.json" + } + ], + "tp": 2 + }, + { + "all_repeat_cv_pct": 0.06699668597711499, + "batch": 2, + "initial_repeat_cv_pct": 0.06699668597711499, + "median_component_ms": { + "attention": 0.721186765625, + "collective": 0.5570387890625, + "linear_norm_rope": 0.8363623671875, + "moe": 1.7957157499999985, + "other": 0.020733953125000003, + "output_head": 0.045581921875, + "router": 0.18678388281249997 + }, + "median_execute_ms": 4.6231764375, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 0.72195021875, + "collective": 0.553751203125, + "linear_norm_rope": 0.8375226249999999, + "moe": 1.7999987343749986, + "other": 0.020717078125, + "output_head": 0.0456011875, + "router": 0.18689503124999995 + }, + "execute_mean_ms": 4.6262738125, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b2-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.7204233125, + "collective": 0.560326375, + "linear_norm_rope": 0.8352021093750001, + "moe": 1.7914327656249986, + "other": 0.020750828125000002, + "output_head": 0.04556265625, + "router": 0.18667273437499995 + }, + "execute_mean_ms": 4.6200790625, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b2-r2.json" + } + ], + "tp": 4 + }, + { + "all_repeat_cv_pct": 2.2737660875555155, + "batch": 4, + "initial_repeat_cv_pct": 2.2737660875555155, + "median_component_ms": { + "attention": 0.8017847142857144, + "collective": 0.5969553392857143, + "linear_norm_rope": 0.8391979553571427, + "moe": 1.9608389732142864, + "other": 0.021788607142857144, + "output_head": 0.04581030357142857, + "router": 0.18947951785714276 + }, + "median_execute_ms": 4.6215097499999995, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 0.8023426607142858, + "collective": 0.5361581428571428, + "linear_norm_rope": 0.8392915178571427, + "moe": 1.9595749285714295, + "other": 0.021800214285714284, + "output_head": 0.04583946428571428, + "router": 0.1894944999999999 + }, + "execute_mean_ms": 4.516427428571428, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b4-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.801226767857143, + "collective": 0.6577525357142857, + "linear_norm_rope": 0.8391043928571427, + "moe": 1.9621030178571435, + "other": 0.021777, + "output_head": 0.045781142857142854, + "router": 0.18946453571428562 + }, + "execute_mean_ms": 4.726592071428572, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b4-r2.json" + } + ], + "tp": 4 + }, + { + "all_repeat_cv_pct": 24.383571784490655, + "batch": 6, + "initial_repeat_cv_pct": 30.304513890111306, + "median_component_ms": { + "attention": 0.926535269230769, + "collective": 3.192824375, + "linear_norm_rope": 0.8382221538461538, + "moe": 2.383938624999999, + "other": 0.025532517857142854, + "output_head": 0.04836294642857143, + "router": 0.19243757142857137 + }, + "median_execute_ms": 8.633801928571428, + "needs_repeat_3": true, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 0.926535269230769, + "collective": 0.5587644230769231, + "linear_norm_rope": 0.8382221538461538, + "moe": 2.378592788461537, + "other": 0.02582126923076923, + "output_head": 0.048266576923076926, + "router": 0.1886060961538461 + }, + "execute_mean_ms": 5.43262976923077, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b6-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.9215755535714284, + "collective": 4.099890464285714, + "linear_norm_rope": 0.8376546964285715, + "moe": 2.383938624999999, + "other": 0.02548019642857143, + "output_head": 0.04836294642857143, + "router": 0.19243757142857137 + }, + "execute_mean_ms": 10.1569875, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b6-r2.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.9331604285714286, + "collective": 3.192824375, + "linear_norm_rope": 0.8404031785714285, + "moe": 2.399789857142857, + "other": 0.025532517857142854, + "output_head": 0.048368125, + "router": 0.19489794642857136 + }, + "execute_mean_ms": 8.633801928571428, + "repeat": 3, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b6-r3.json" + } + ], + "tp": 4 + }, + { + "all_repeat_cv_pct": 1.4955512501139812, + "batch": 8, + "initial_repeat_cv_pct": 1.4955512501139812, + "median_component_ms": { + "attention": 0.9170377788461537, + "collective": 0.6205275096153846, + "linear_norm_rope": 0.8397761249999998, + "moe": 2.0145085576923076, + "other": 0.023612125, + "output_head": 0.04597551923076923, + "router": 0.1919322211538461 + }, + "median_execute_ms": 4.972498884615385, + "needs_repeat_3": false, + "repeat_gate_resolved": true, + "repeats": [ + { + "component_rank_mean_ms": { + "attention": 0.9206188846153844, + "collective": 0.6864052307692307, + "linear_norm_rope": 0.8407714807692306, + "moe": 2.0225017884615384, + "other": 0.023575519230769233, + "output_head": 0.04601186538461538, + "router": 0.19361455769230762 + }, + "execute_mean_ms": 4.898132615384616, + "repeat": 1, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b8-r1.json" + }, + { + "component_rank_mean_ms": { + "attention": 0.9134566730769228, + "collective": 0.5546497884615385, + "linear_norm_rope": 0.8387807692307689, + "moe": 2.0065153269230764, + "other": 0.02364873076923077, + "output_head": 0.04593917307692308, + "router": 0.19024988461538456 + }, + "execute_mean_ms": 5.046865153846154, + "repeat": 2, + "source": "runs/frontier-decode-batch-grid-v0/results/tp4-b8-r2.json" + } + ], + "tp": 4 + } + ], + "contract": { + "repeat_aggregation": "median of two fresh processes; three if unstable", + "stability_gate": "r1/r2 population CV <=10%; else require r3 and take median", + "timing": "slowest-rank execute mean over 16 pure-decode steps", + "workload": "Qwen3-30B-A3B BF16, 2048->128, graph-on, MNS=16" + }, + "schema": "frontier-decode-batch-grid.v1" +} diff --git a/runs/frontier-decode-batch-grid-v0/results/tp4-b6-r3.json b/runs/frontier-decode-batch-grid-v0/results/tp4-b6-r3.json new file mode 100644 index 0000000..ebad8f3 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/results/tp4-b6-r3.json @@ -0,0 +1,353 @@ +{ + "contract": { + "component_time": "sum of CUDA kernel durations inside execute range", + "timing": "CUDA graph-on GPU execute annotations", + "tp_aggregation": "per-rank; slowest rank mean approximates critical path" + }, + "label": "tp4-b6-r3", + "rank_summary": { + "component_rank_mean_ms": { + "attention": 0.9331604285714286, + "collective": 3.192824375, + "linear_norm_rope": 0.8404031785714285, + "moe": 2.399789857142857, + "other": 0.025532517857142854, + "output_head": 0.048368125, + "router": 0.19489794642857136 + }, + "ranks": 4, + "slowest_rank_execute_mean_ms": 8.633801928571428, + "slowest_rank_kernel_busy_mean_ms": 8.521951642857141 + }, + "ranks": [ + { + "components": { + "attention": { + "mean_ms": 0.9356066428571428, + "n": 14, + "p50_ms": 0.9354089999999998, + "p95_ms": 0.9385260000000003, + "population_std_ms": 0.001676932599682409 + }, + "collective": { + "mean_ms": 4.066270071428571, + "n": 14, + "p50_ms": 0.55129, + "p95_ms": 16.453366, + "population_std_ms": 5.619590739357225 + }, + "linear_norm_rope": { + "mean_ms": 0.8387703571428572, + "n": 14, + "p50_ms": 0.8386610000000003, + "p95_ms": 0.8405769999999998, + "population_std_ms": 0.001042224242880996 + }, + "moe": { + "mean_ms": 2.403109357142857, + "n": 14, + "p50_ms": 2.3945385000000012, + "p95_ms": 2.4938269999999996, + "population_std_ms": 0.04863547144920812 + }, + "other": { + "mean_ms": 0.025629785714285713, + "n": 14, + "p50_ms": 0.025696499999999997, + "p95_ms": 0.02624, + "population_std_ms": 0.00037226741129221103 + }, + "output_head": { + "mean_ms": 0.04842978571428571, + "n": 14, + "p50_ms": 0.048384, + "p95_ms": 0.049248, + "population_std_ms": 0.00035716626351871235 + }, + "router": { + "mean_ms": 0.19324292857142852, + "n": 14, + "p50_ms": 0.19292599999999996, + "p95_ms": 0.1981709999999999, + "population_std_ms": 0.0022171815591706862 + } + }, + "execute_annotation_histogram": { + "execute_context_0(0)_generation_6(6)": 14, + "execute_context_1(2048)_generation_2(2)": 1, + "execute_context_3(6144)_generation_3(3)": 1 + }, + "execute_wall": { + "mean_ms": 8.630969642857142, + "n": 14, + "p50_ms": 5.180047500000001, + "p95_ms": 20.987705000000002, + "population_std_ms": 5.5894495457541336 + }, + "gpu_kernel_busy": { + "mean_ms": 8.511058928571428, + "n": 14, + "p50_ms": 5.061531499999999, + "p95_ms": 20.864821999999997, + "population_std_ms": 5.589475958722368 + }, + "non_kernel_gap": { + "mean_ms": 0.11991071428571518, + "n": 14, + "p50_ms": 0.12010500000000235, + "p95_ms": 0.12316100000000052, + "population_std_ms": 0.002551737571274248 + }, + "selected_execute_annotation": "execute_context_0(0)_generation_6(6)", + "steps": 14, + "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784799692262262652.pt.trace.json.gz" + }, + { + "components": { + "attention": { + "mean_ms": 0.9304471428571427, + "n": 14, + "p50_ms": 0.930654, + "p95_ms": 0.9319719999999995, + "population_std_ms": 0.0009635016389000793 + }, + "collective": { + "mean_ms": 0.5773663571428572, + "n": 14, + "p50_ms": 0.5793035000000002, + "p95_ms": 0.59031, + "population_std_ms": 0.008760785031745086 + }, + "linear_norm_rope": { + "mean_ms": 0.8337531428571427, + "n": 14, + "p50_ms": 0.8337829999999997, + "p95_ms": 0.8351629999999995, + "population_std_ms": 0.0008970194182593586 + }, + "moe": { + "mean_ms": 2.3872937857142857, + "n": 14, + "p50_ms": 2.3779580000000005, + "p95_ms": 2.4758159999999987, + "population_std_ms": 0.04801953596369253 + }, + "other": { + "mean_ms": 0.02552642857142857, + "n": 14, + "p50_ms": 0.025569500000000002, + "p95_ms": 0.025981999999999998, + "population_std_ms": 0.00035502126984283196 + }, + "output_head": { + "mean_ms": 0.048183142857142855, + "n": 14, + "p50_ms": 0.048112, + "p95_ms": 0.049249, + "population_std_ms": 0.00048348686747460613 + }, + "router": { + "mean_ms": 0.19290549999999992, + "n": 14, + "p50_ms": 0.19270699999999996, + "p95_ms": 0.1958099999999999, + "population_std_ms": 0.0013893215584799379 + } + }, + "execute_annotation_histogram": { + "execute_context_0(0)_generation_6(6)": 14, + "execute_context_1(2048)_generation_2(2)": 1, + "execute_context_3(6144)_generation_3(3)": 1 + }, + "execute_wall": { + "mean_ms": 8.221614142857144, + "n": 14, + "p50_ms": 5.1788165, + "p95_ms": 21.941601, + "population_std_ms": 5.439025316289738 + }, + "gpu_kernel_busy": { + "mean_ms": 4.9954754999999995, + "n": 14, + "p50_ms": 4.988138500000001, + "p95_ms": 5.080260999999998, + "population_std_ms": 0.04987686944831562 + }, + "non_kernel_gap": { + "mean_ms": 3.2261386428571432, + "n": 14, + "p50_ms": 0.12194000000000038, + "p95_ms": 16.981787, + "population_std_ms": 5.465271976091736 + }, + "selected_execute_annotation": "execute_context_0(0)_generation_6(6)", + "steps": 14, + "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784799692310997248.pt.trace.json.gz" + }, + { + "components": { + "attention": { + "mean_ms": 0.9339937142857143, + "n": 14, + "p50_ms": 0.9335875, + "p95_ms": 0.9370900000000002, + "population_std_ms": 0.001917785904502957 + }, + "collective": { + "mean_ms": 4.071160642857143, + "n": 14, + "p50_ms": 0.5595504999999998, + "p95_ms": 16.441921000000004, + "population_std_ms": 5.617545177819675 + }, + "linear_norm_rope": { + "mean_ms": 0.8418829285714284, + "n": 14, + "p50_ms": 0.8420469999999998, + "p95_ms": 0.8443809999999999, + "population_std_ms": 0.0015434930914874625 + }, + "moe": { + "mean_ms": 2.3997612142857148, + "n": 14, + "p50_ms": 2.3921425000000003, + "p95_ms": 2.4903699999999995, + "population_std_ms": 0.04851648560788174 + }, + "other": { + "mean_ms": 0.025574571428571426, + "n": 14, + "p50_ms": 0.025535999999999996, + "p95_ms": 0.026114000000000002, + "population_std_ms": 0.0002516030644503687 + }, + "output_head": { + "mean_ms": 0.04837264285714286, + "n": 14, + "p50_ms": 0.048512, + "p95_ms": 0.048960000000000004, + "population_std_ms": 0.00043459432761120634 + }, + "router": { + "mean_ms": 0.1906739285714285, + "n": 14, + "p50_ms": 0.19033549999999994, + "p95_ms": 0.19376099999999996, + "population_std_ms": 0.0011997104093599232 + } + }, + "execute_annotation_histogram": { + "execute_context_0(0)_generation_6(6)": 14, + "execute_context_1(2048)_generation_2(2)": 1, + "execute_context_3(6144)_generation_3(3)": 1 + }, + "execute_wall": { + "mean_ms": 8.630474571428572, + "n": 14, + "p50_ms": 5.184212499999999, + "p95_ms": 20.985098999999998, + "population_std_ms": 5.5897660056633045 + }, + "gpu_kernel_busy": { + "mean_ms": 8.511419642857144, + "n": 14, + "p50_ms": 5.0612965, + "p95_ms": 20.866541000000005, + "population_std_ms": 5.589603699080663 + }, + "non_kernel_gap": { + "mean_ms": 0.11905492857142821, + "n": 14, + "p50_ms": 0.1185754999999995, + "p95_ms": 0.1234339999999996, + "population_std_ms": 0.0022576749881340562 + }, + "selected_execute_annotation": "execute_context_0(0)_generation_6(6)", + "steps": 14, + "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp2_dcp0_ep2_rank2.1784799692265838678.pt.trace.json.gz" + }, + { + "components": { + "attention": { + "mean_ms": 0.9325942142857142, + "n": 14, + "p50_ms": 0.9325534999999999, + "p95_ms": 0.9368339999999997, + "population_std_ms": 0.0022950169367122264 + }, + "collective": { + "mean_ms": 4.056500428571428, + "n": 14, + "p50_ms": 0.5449905, + "p95_ms": 16.437329999999996, + "population_std_ms": 5.618098886085954 + }, + "linear_norm_rope": { + "mean_ms": 0.8472062857142859, + "n": 14, + "p50_ms": 0.8469400000000002, + "p95_ms": 0.8496459999999999, + "population_std_ms": 0.001113636605808041 + }, + "moe": { + "mean_ms": 2.4089950714285697, + "n": 14, + "p50_ms": 2.400836999999998, + "p95_ms": 2.491555999999999, + "population_std_ms": 0.04692533473274991 + }, + "other": { + "mean_ms": 0.025399285714285715, + "n": 14, + "p50_ms": 0.025391999999999998, + "p95_ms": 0.025988, + "population_std_ms": 0.00023926925316514007 + }, + "output_head": { + "mean_ms": 0.04848692857142857, + "n": 14, + "p50_ms": 0.048528, + "p95_ms": 0.04912, + "population_std_ms": 0.0004430105552250606 + }, + "router": { + "mean_ms": 0.20276942857142852, + "n": 14, + "p50_ms": 0.2021774999999999, + "p95_ms": 0.20793499999999998, + "population_std_ms": 0.0026834677382150296 + } + }, + "execute_annotation_histogram": { + "execute_context_0(0)_generation_6(6)": 14, + "execute_context_1(2048)_generation_2(2)": 1, + "execute_context_3(6144)_generation_3(3)": 1 + }, + "execute_wall": { + "mean_ms": 8.633801928571428, + "n": 14, + "p50_ms": 5.179864499999999, + "p95_ms": 20.988474, + "population_std_ms": 5.589972337133016 + }, + "gpu_kernel_busy": { + "mean_ms": 8.521951642857141, + "n": 14, + "p50_ms": 5.071154499999998, + "p95_ms": 20.875197999999994, + "population_std_ms": 5.589281480492545 + }, + "non_kernel_gap": { + "mean_ms": 0.11185028571428772, + "n": 14, + "p50_ms": 0.11232300000000128, + "p95_ms": 0.11633899999999997, + "population_std_ms": 0.0023228722192460186 + }, + "selected_execute_annotation": "execute_context_0(0)_generation_6(6)", + "steps": 14, + "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp3_dcp0_ep3_rank3.1784799692262484339.pt.trace.json.gz" + } + ], + "schema": "frontier-decode-batch-trace.v1" +} diff --git a/runs/frontier-decode-batch-grid-v0/run_bc8_replay.py b/runs/frontier-decode-batch-grid-v0/run_bc8_replay.py new file mode 100644 index 0000000..d65ea94 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/run_bc8_replay.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Replay the exact BC-8 simulator cells with the whole-layer decode curve.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +EXPECTED_FRONTIER_COMMIT = "deadc4a321f0baaa534c6ebd17f974123733cdc2" +CONFIGS = ("tp1_mns16", "tp2_mns16", "tp4_mns16", "tp4_mns32") +SOURCE_MANIFEST = ( + REPO / "runs/frontier-collective-joint-v0/counterfactual/joint-r2/manifest.json" +) +JOINT_INPUTS = REPO / "runs/frontier-knee-sweep-v0/inputs" +GRID = ROOT / "results/grid.json" +WRAPPER = ROOT / "run_frontier_with_whole_decode_curve.py" +CACHE_ROOT = REPO / "runs/frontier-knee-sweep-v0/cache" +LOCAL_DEPENDENCY_ROOTS = ( + REPO / "runs/frontier-collective-joint-v0/python-deps", + Path("/home/gahow/.cache/uv/archive-v0/-_kzErLcPO5nASZFX8b9k"), + Path("/home/gahow/.cache/uv/archive-v0/FbaBs_QJ9QKEbQ9V_4aIR"), + Path("/home/gahow/.cache/uv/archive-v0/fuHsGXD0Lv_UjFC8yI4-7"), + Path("/home/gahow/.cache/uv/archive-v0/jFGdqQLpB1eopfm9VxT3j"), + Path("/home/gahow/.cache/uv/archive-v0/YWW6ExSJuPVvv4-qYQTin"), + Path("/home/gahow/.cache/uv/archive-v0/3_qxZ5Ll-EpVAGZfbksfe"), +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier-checkout", type=Path, required=True) + parser.add_argument("--output-root", type=Path, default=ROOT / "replay/bc8") + parser.add_argument("--jobs", type=int, default=2) + return parser.parse_args() + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_json(path: Path, payload) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def replace_flag(argv: list[str], flag: str, value: str) -> None: + index = argv.index(flag) + argv[index + 1] = value + + +def validate(frontier: Path) -> None: + commit = subprocess.check_output( + ["git", "-C", str(frontier), "rev-parse", "HEAD"], text=True + ).strip() + status = subprocess.check_output( + ["git", "-C", str(frontier), "status", "--porcelain"], text=True + ).strip() + if commit != EXPECTED_FRONTIER_COMMIT or status: + raise ValueError( + f"Frontier must be clean at {EXPECTED_FRONTIER_COMMIT}, " + f"got commit={commit}, dirty={bool(status)}" + ) + required = ( + SOURCE_MANIFEST, + GRID, + WRAPPER, + CACHE_ROOT, + *LOCAL_DEPENDENCY_ROOTS, + ) + missing = [str(path) for path in required if not path.exists()] + if missing: + raise ValueError(f"missing replay inputs: {missing}") + + +def trace_for(config: str) -> Path: + tp = int(config[2]) + return ( + JOINT_INPUTS + / f"traces-per-gpu-low/tp{tp}/w0-short-fixed-uniform-none/" + "rho0p02/public/frontier.csv" + ) + + +def run_one( + config: str, + *, + frontier: Path, + output_root: Path, + templates: dict, +) -> dict: + point = output_root / "raw" / config + metrics_root = point / "metrics" + expected = list(metrics_root.glob("**/system_metrics.json")) + if len(expected) == 1 and (point / "usage.json").is_file(): + return {"config": config, "status": "skipped_complete", "elapsed_s": 0.0} + + argv = list(templates[config]["argv"]) + argv[0] = sys.executable + argv[1] = str(WRAPPER.resolve()) + replace_flag(argv, "--trace_request_generator_config_trace_file", str(trace_for(config))) + replace_flag(argv, "--metrics_config_output_dir", str(metrics_root)) + replace_flag(argv, "--metrics_config_run_id", f"decode_grid_bc8_{config}") + replace_flag(argv, "--metrics_config_cache_dir", str(CACHE_ROOT / "model")) + replace_flag(argv, "--vidur_cc_backend_config_cache_dir", str(CACHE_ROOT / "cc")) + env = os.environ.copy() + env.update( + { + "CUDA_VISIBLE_DEVICES": "", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": os.pathsep.join( + [str(frontier), *(str(path) for path in LOCAL_DEPENDENCY_ROOTS)] + ), + "FRONTIER_COLLECTIVE_CURVE": str( + (JOINT_INPUTS / "collective-curve-b4-extrapolated.json").resolve() + ), + "FRONTIER_COLLECTIVE_CURVE_VARIANT": "drop_mean", + "FRONTIER_FUSED_MOE_CURVE": str( + (JOINT_INPUTS / "fused-moe-curve-b4-extrapolated.json").resolve() + ), + "FRONTIER_WHOLE_DECODE_GRID": str(GRID.resolve()), + "FRONTIER_CURVE_USAGE": str((point / "usage.json").resolve()), + } + ) + point.mkdir(parents=True, exist_ok=True) + write_json(point / "command.json", argv) + started = time.monotonic() + with (point / "run.log").open("w") as output: + completed = subprocess.run( + argv, + cwd=frontier, + env=env, + stdout=output, + stderr=subprocess.STDOUT, + check=False, + ) + elapsed = time.monotonic() - started + metrics = list(metrics_root.glob("**/system_metrics.json")) + status = ( + "completed" + if completed.returncode == 0 + and len(metrics) == 1 + and (point / "usage.json").is_file() + else "failed" + ) + record = { + "config": config, + "status": status, + "returncode": completed.returncode, + "elapsed_s": elapsed, + } + write_json(point / "run-status.json", record) + return record + + +def main() -> None: + args = parse_args() + if args.jobs < 1: + raise ValueError("--jobs must be positive") + frontier = args.frontier_checkout.resolve() + output_root = args.output_root.resolve() + validate(frontier) + source = json.loads(SOURCE_MANIFEST.read_text()) + manifest = { + "schema": "frontier-decode-grid-bc8-replay.v1", + "frontier_checkout": str(frontier), + "frontier_commit": EXPECTED_FRONTIER_COMMIT, + "configs": list(CONFIGS), + "rho_per_gpu": 0.02, + "wrapper": str(WRAPPER.resolve()), + "wrapper_sha256": sha256(WRAPPER), + "whole_decode_grid": str(GRID.resolve()), + "whole_decode_grid_sha256": sha256(GRID), + "collective_curve_sha256": sha256( + JOINT_INPUTS / "collective-curve-b4-extrapolated.json" + ), + "moe_curve_sha256": sha256( + JOINT_INPUTS / "fused-moe-curve-b4-extrapolated.json" + ), + "traces": { + config: { + "path": str(trace_for(config).resolve()), + "sha256": sha256(trace_for(config)), + } + for config in CONFIGS + }, + } + write_json(output_root / "manifest.json", manifest) + results = [] + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = { + pool.submit( + run_one, + config, + frontier=frontier, + output_root=output_root, + templates=source["cells"], + ): config + for config in CONFIGS + } + for future in as_completed(futures): + result = future.result() + results.append(result) + print(json.dumps(result, sort_keys=True), flush=True) + results.sort(key=lambda row: CONFIGS.index(row["config"])) + write_json(output_root / "run-summary.json", results) + failures = [row for row in results if row["status"] == "failed"] + if failures: + raise SystemExit(f"failed replay cells: {failures}") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-decode-batch-grid-v0/run_frontier_with_whole_decode_curve.py b/runs/frontier-decode-batch-grid-v0/run_frontier_with_whole_decode_curve.py new file mode 100644 index 0000000..f0e6199 --- /dev/null +++ b/runs/frontier-decode-batch-grid-v0/run_frontier_with_whole_decode_curve.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Run Frontier with the existing joint repair plus a whole-layer decode curve.""" + +from __future__ import annotations + +import json +import os +import runpy +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +GRID = json.loads(Path(os.environ["FRONTIER_WHOLE_DECODE_GRID"]).read_text()) +WHOLE_CURVE = { + str(tp): { + str(cell["batch"]): float(cell["median_execute_ms"]) + for cell in GRID["cells"] + if int(cell["tp"]) == tp + } + for tp in (2, 4) +} + +# Apply the existing serving-path collective/MoE correction first. +joint = runpy.run_path( + str(REPO / "runs/frontier-collective-joint-v0/run_frontier_with_curves.py") +) +USAGE = joint["USAGE"] +_pure_decode_point = joint["_pure_decode_point"] + +from frontier.entities import ExecutionTime # noqa: E402 +from frontier.execution_time_predictor.sklearn_moe_execution_time_predictor import ( # noqa: E402 + SklearnMoEExecutionTimePredictor, +) + + +_JOINT_STAGE_PREDICT = SklearnMoEExecutionTimePredictor.predict_stage_execution_time + + +def _whole_decode_stage_time( + self, + batch, + stage_id, + cluster_type, + num_layers=1, + layer_id=0, +): + base = _JOINT_STAGE_PREDICT( + self, + batch, + stage_id, + cluster_type, + num_layers=num_layers, + layer_id=layer_id, + ) + point = _pure_decode_point(self, batch) + if point is None or point[0] == "1" or point[1] == "1": + return base + tp, decode_batch = point + if tp not in WHOLE_CURVE or decode_batch not in WHOLE_CURVE[tp]: + raise ValueError( + "Whole-layer curve has no exact pure-decode point for " + f"TP={tp}, batch={decode_batch}; refusing to extrapolate" + ) + layers = int(base.num_layers) + if layers <= 0: + raise ValueError(f"invalid stage layer count: {layers}") + target_ms = WHOLE_CURVE[tp][decode_batch] + USAGE[f"whole_decode:tp{tp}-b{decode_batch}:target_ms={target_ms:.9f}"] += 1 + return ExecutionTime( + num_layers_per_pipeline_stage=layers, + attention_rope_execution_time=0.0, + attention_kv_cache_save_execution_time=0.0, + attention_decode_execution_time=0.0, + attention_prefill_execution_time=0.0, + attention_layer_pre_proj_execution_time=0.0, + attention_layer_post_proj_execution_time=0.0, + attn_norm_time=0.0, + mlp_norm_time=0.0, + add_time=0.0, + tensor_parallel_communication_time=0.0, + pipeline_parallel_communication_time=0.0, + expert_parallel_communication_time=0.0, + moe_gating_time=0.0, + moe_shuffling_time=0.0, + schedule_time=base._schedule_time, + sampler_e2e_time=base._sampler_e2e_time, + prepare_inputs_e2e_time=base._prepare_inputs_e2e_time, + process_model_outputs_time=base._process_model_outputs_time, + ray_comm_time=base._ray_comm_time, + is_moe=True, + moe_grouped_gemm_time=target_ms / layers, + pp_producer_send_path_runtime_time=( + base._pp_producer_send_path_runtime_time + ), + pp_receiver_head_runtime_time=base._pp_receiver_head_runtime_time, + pp_prefill_consumer_active_runtime_time=( + base._pp_prefill_consumer_active_runtime_time + ), + pp_stage_boundary_residual_runtime_time=( + base._pp_stage_boundary_residual_runtime_time + ), + ) + + +SklearnMoEExecutionTimePredictor.predict_stage_execution_time = ( + _whole_decode_stage_time +) + +from frontier.main import main # noqa: E402 + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-simulator-gap-campaign-v0/README.md b/runs/frontier-simulator-gap-campaign-v0/README.md new file mode 100644 index 0000000..db49528 --- /dev/null +++ b/runs/frontier-simulator-gap-campaign-v0/README.md @@ -0,0 +1,97 @@ +# Frontier simulator residual-gap campaign + +目标：在已经完成的 1h+ production chat-trace fidelity matrix 上，按信息增益和工程可修性依次关闭三个残余误差源。三个实验严格串行；前一项的产物是后一项的 baseline，不允许同时改多个 profile/component。 + +## 冻结基线 + +- Frontier：`deadc4a321f0baaa534c6ebd17f974123733cdc2` +- Profile：`runs/frontier-prefill-kvgrowth-fix-v0/profiles/profile-v5-kvgrowth` +- Workload：与 `docs/assets/frontier-fidelity/full-matrix.csv` 相同的 7 个 TP×load cells +- Real：复用现有每 cell 两次 60-min trial；不重新采集，不做逐 cell E2E calibration +- 指标：TTFT/TPOT/E2E 的 mean/p50/p90/p99；real trial 分别与同一 sim request ID 配对后汇总，不再把两次 real trial pool 成一个分布 +- 有效域：TP1 两个 cell 的 simulated waiting p99 已超过原 1 s subcritical gate，必须标成 `gate-fail diagnostic`，不能继续称为 subcritical evidence + +## 串行任务列表 + +### EXP-1：structured attention-prefill predictor + +- [x] 锁定 Frontier/profile/trace provenance +- [x] 审计训练与运行时 feature contract +- [x] 冻结 experiment card、事前判据和 mock figure +- [x] 实现最小 predictor patch： + - standard prefill 只训练 `batch_size=1`；多请求继续走既有 mixed predictor + - `base(q)` 使用单请求 `KV=0` profile 的单调分段插值 + - KV growth 使用非负 `KV + q×KV` 项 +- [x] 单元测试：exact anchors、q/KV 单调、非负、pickle round-trip +- [x] 离线 ablation：RF-all / RF-single / structured-single；held-out context MAPE +- [x] 7-cell CPU replay +- [x] trial-aware paired verdict 与旧 v5 baseline 对照 + +Decision：**profile gate PASS，trace/merge gate FAIL**。TP1/TP2 TTFT mean/p99 +绝对误差改善约 7--10 pp；TP4 三个 cell 的 TTFT mean 绝对误差稳定恶化 +5.3--5.8 pp。该 patch 保留为机制 ablation，不作为全局默认 predictor。 +EXP-2 先执行 entry audit；仅当 structured 分支上的 TP2 chunk residual 仍 ≥10% +才进入 GPU 三臂测量。 + +Go/no-go： + +- profile gate：held-out context MAPE ≤5%，且所有 TP 的 q/KV 单调检查通过 +- trace gate：TP1 TTFT mean/p99 的绝对偏差各改善 ≥5 pp；TP2/TP4 任一 TTFT/E2E quantile 不得恶化 >5 pp +- 若 profile gate 不过，不进入 trace replay；若 trace gate 不过，保留 profile diagnosis，回退 patch，不进入 EXP-2 + +### EXP-2：TP2 base-prefill serving-path profile + +- [x] 在 EXP-1 冻结输出上重新确认 chunk #1 residual：TP2 是否仍为孤立点 +- [x] probe `dash1`--`dash4`；只选择 8 张 H20 全部 idle/healthy 的主机 +- [x] 三臂中的 A/C 同形状 profile： + - A：当前 standalone microbenchmark + - C：真实 serving path extract +- [x] q8k smoke 与两个 TP rank 的 component contract +- [x] 注入 q8k ratio，重放 TP2 两个 load cell；其余 profile 冻结 +- [x] paired verdict + +Decision：**mechanism gate PASS，global constant injection FAIL**。serving q8k +execute=`408.19 ms`（real=`410 ms`），MoE=`214.52 ms` vs sim=`171.12 ms`， +解释 `80.75%` residual。全 prefill-domain constant `1.25366×` 会过校正： +subcritical TTFT mean `−4.54%→+5.23%` 且 E2E mean +`+12.70%→+15.55%`。保留的工程方向是 TP2 token/routing-conditioned MoE +curve，不合入 constant scale。B arm 合并为该 follow-up curve 的真实 routing +采样，不再为已被 C 直接确认的机制单独占一次 GPU run。 + +Go/no-go： + +- 若 B/C 相对 A 都没有稳定的 ≥10% shift，拒绝“tactic/warmup 或 serving composition”假设，不做 profile 注入 +- 若 C 能解释 real chunk residual，TP2 TTFT mean 目标收敛至 ±10%，且 E2E/TPOT 不恶化 >5 pp + +### EXP-3：decode whole-layer residual × batch curve + +- [x] 在 EXP-2 冻结输出上确认 TP4 low-load TPOT/E2E 正偏仍存在 +- [x] probe dash1--dash4；补 TP2/TP4 × batch 2/4/6/8 whole-layer grid， + b1 复用既有 serving anchor +- [x] 将 whole-layer time 与 component sum 对齐，定位 TP4/b6 collective tail +- [x] 稳定性 gate：TP4/b2 fresh-process CV=`0.067%`；TP4/b6 依规则加 r3 +- [x] 重放历史 BC-8 knee，检查 `TP25 pp + +## 统一输出格式 + +每个实验最终报告都分为： + +1. `Fact`：原始测量与 paired metric +2. `Interpretation`：支持/反驳哪条机制假设 +3. `Decision`：merge、回退、继续下一实验或停止 diff --git a/runs/frontier-simulator-gap-campaign-v0/results/three-experiment-verdict.md b/runs/frontier-simulator-gap-campaign-v0/results/three-experiment-verdict.md new file mode 100644 index 0000000..0c2dd88 --- /dev/null +++ b/runs/frontier-simulator-gap-campaign-v0/results/three-experiment-verdict.md @@ -0,0 +1,46 @@ +# Frontier residual-gap 三实验判决 + +## EXP-1 structured attention-prefill + +- **Fact：** held-out context MAPE 为 TP1/2/4 `0.84/1.60/3.01%`，profile + gate PASS；7-cell 中 TP1/TP2 TTFT 绝对误差改善约 7--10 pp，但 TP4 + TTFT mean 稳定恶化 5.3--5.8 pp。 +- **Interpretation：** structured feature contract 正确，但旧 TP4 的好结果含 + error cancellation；单独替换 attention 会暴露其它正偏。 +- **Decision：** 不合入默认 predictor，保留为机制 ablation。 + +## EXP-2 TP2 prefill serving-path + +- **Fact：** q8k TP2 serving execute=`408.19 ms`（real anchor=`410 ms`）； + MoE=`214.52 ms` vs sim=`171.12 ms`，解释 `80.75%` residual。全域 + `1.25366×` 常数注入使 subcritical TTFT mean `-4.54%→+5.23%`， + E2E mean `+12.70%→+15.55%`。 +- **Interpretation：** TP2 residual 的主机制是 serving-path MoE composition， + 但 correction 随 token/routing state 变化，不是常数。 +- **Decision：** 机制 PASS、constant merge FAIL；工程项为 TP2 + token/routing-conditioned MoE curve。 + +## EXP-3 decode whole-layer batch grid + +- **Fact：** TP4/b2 fresh-process=`4.626/4.620 ms`，CV=`0.067%`， + median=`4.623 ms`。BC-8 exact b2 replay 将 TP4 sim TPOT + `5.864→5.367 ms`，排序恢复 `TP2