Pass TP4 decode stability gate

2026-07-23 17:09:49 +08:00
parent c1c200b7cd
commit 1126d9be7d
8 changed files with 1002 additions and 2 deletions
--- a/runs/frontier-decode-batch-grid-v0/analyze_grid.py
+++ b/runs/frontier-decode-batch-grid-v0/analyze_grid.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Aggregate fresh-process decode profiles and enforce the repeat gate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+from pathlib import Path
+
+
+COMPONENT_NAMES = (
+    "attention",
+    "linear_norm_rope",
+    "router",
+    "moe",
+    "collective",
+    "output_head",
+    "other",
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--results-dir", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    cells = []
+    all_stable = True
+    for tp in (2, 4):
+        for batch in (2, 4, 6, 8):
+            repeats = []
+            for repeat in (1, 2):
+                path = args.results_dir / f"tp{tp}-b{batch}-r{repeat}.json"
+                payload = json.loads(path.read_text())
+                repeats.append(
+                    {
+                        "repeat": repeat,
+                        "source": str(path),
+                        "execute_mean_ms": payload["rank_summary"][
+                            "slowest_rank_execute_mean_ms"
+                        ],
+                        "component_rank_mean_ms": payload["rank_summary"][
+                            "component_rank_mean_ms"
+                        ],
+                    }
+                )
+            values = [row["execute_mean_ms"] for row in repeats]
+            mean = statistics.fmean(values)
+            cv_pct = statistics.pstdev(values) / mean * 100.0
+            stable = cv_pct <= 10.0
+            all_stable = all_stable and stable
+            cells.append(
+                {
+                    "tp": tp,
+                    "batch": batch,
+                    "repeats": repeats,
+                    "median_execute_ms": statistics.median(values),
+                    "repeat_cv_pct": cv_pct,
+                    "stable": stable,
+                    "median_component_ms": {
+                        name: statistics.median(
+                            row["component_rank_mean_ms"][name] for row in repeats
+                        )
+                        for name in COMPONENT_NAMES
+                    },
+                }
+            )
+    payload = {
+        "schema": "frontier-decode-batch-grid.v1",
+        "contract": {
+            "workload": "Qwen3-30B-A3B BF16, 2048->128, graph-on, MNS=16",
+            "timing": "slowest-rank execute mean over 16 pure-decode steps",
+            "repeat_aggregation": "median of two fresh engine processes",
+            "stability_gate": "population CV <= 10%; add repeat 3 otherwise",
+        },
+        "all_cells_stable": all_stable,
+        "cells": cells,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    if not all_stable:
+        raise SystemExit("one or more cells failed the repeat stability gate")
+
+
+if __name__ == "__main__":
+    main()
--- a/runs/frontier-decode-batch-grid-v0/experiment-card.md
+++ b/runs/frontier-decode-batch-grid-v0/experiment-card.md
@@ -1,6 +1,6 @@
 # 实验 EXP-DECODE-BATCH-GRID：BC-8 是否可由稳定 whole-layer b2--b8 curve 修复

-> **状态：** entry gate PASS，待 TP4/b2 GPU smoke
+> **状态：** TP4/b2 stability gate PASS，扩展 full grid
 >
 > Parent campaign：[`../frontier-simulator-gap-campaign-v0/README.md`](../frontier-simulator-gap-campaign-v0/README.md)

@@ -64,5 +64,10 @@

 - **Entry gate：** PASS。TP4 1h trace TPOT mean 正偏
  `+10.6%--+23.0%`，E2E mean `+9.8%--+19.2%`；BC-8 完整排序仍错。
- **观察事实：** 待 GPU。
+- **TP4/b2 smoke：** 两个 fresh-process critical-rank execute mean 分别为
+  `4.6263 ms` 和 `4.6201 ms`，跨 repeat CV=`0.067%`，稳定性 gate
+  PASS，无需 repeat 3。median=`4.6232 ms`，比旧样本 `5.044 ms` 低
+  `0.421 ms`；方向支持 H1，但未达到事前 `>=0.5 ms` 的强证据阈值。
+- **观察事实：** 旧 TP4/b2 的高方差主要是测量不稳定；是否足以修复
+  BC-8 仍须 full b2--b8 curve 与 trace replay。
 - **Decision：** 待 GPU。
--- a/runs/frontier-decode-batch-grid-v0/jobs-grid-r1a.toml
+++ b/runs/frontier-decode-batch-grid-v0/jobs-grid-r1a.toml
@@ -0,0 +1,61 @@
+version = 1
+
+[[jobs]]
+name = "decode-grid-tp2-b2-r1-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r1"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "2"
+SERVER_PORT = "9841"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp2-b4-r1-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash2"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b4-r1"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "4"
+SERVER_PORT = "9842"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b4-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp4-b4-r1-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash3"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b4-r1"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "4"
+SERVER_PORT = "9843"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b4-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp4-b6-r1-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash4"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r1"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "6"
+SERVER_PORT = "9844"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
--- a/runs/frontier-decode-batch-grid-v0/jobs-grid-r1b.toml
+++ b/runs/frontier-decode-batch-grid-v0/jobs-grid-r1b.toml
@@ -0,0 +1,46 @@
+version = 1
+
+[[jobs]]
+name = "decode-grid-tp2-b6-r1-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r1"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "6"
+SERVER_PORT = "9845"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp2-b8-r1-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash2"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b8-r1"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "8"
+SERVER_PORT = "9846"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b8-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp4-b8-r1-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash3"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b8-r1"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "8"
+SERVER_PORT = "9847"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b8-r1"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
--- a/runs/frontier-decode-batch-grid-v0/jobs-grid-r2a.toml
+++ b/runs/frontier-decode-batch-grid-v0/jobs-grid-r2a.toml
@@ -0,0 +1,61 @@
+version = 1
+
+[[jobs]]
+name = "decode-grid-tp2-b2-r2-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r2"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "2"
+SERVER_PORT = "9841"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp2-b4-r2-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash2"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b4-r2"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "4"
+SERVER_PORT = "9842"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b4-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp4-b4-r2-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash3"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b4-r2"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "4"
+SERVER_PORT = "9843"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b4-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp4-b6-r2-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash4"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r2"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "6"
+SERVER_PORT = "9844"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
--- a/runs/frontier-decode-batch-grid-v0/jobs-grid-r2b.toml
+++ b/runs/frontier-decode-batch-grid-v0/jobs-grid-r2b.toml
@@ -0,0 +1,46 @@
+version = 1
+
+[[jobs]]
+name = "decode-grid-tp2-b6-r2-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r2"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "6"
+SERVER_PORT = "9845"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp2-b8-r2-20260723"
+gpus = 2
+gpu_model = "H20"
+hosts = ["dash2"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b8-r2"]
+
+[jobs.env]
+TP = "2"
+PROFILE_BATCH = "8"
+SERVER_PORT = "9846"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b8-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
+
+[[jobs]]
+name = "decode-grid-tp4-b8-r2-20260723"
+gpus = 4
+gpu_model = "H20"
+hosts = ["dash3"]
+command = "timeout --signal=TERM --kill-after=60s 1800 bash runs/frontier-decode-batch-grid-v0/run_decode_profile.sh"
+artifacts = ["runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b8-r2"]
+
+[jobs.env]
+TP = "4"
+PROFILE_BATCH = "8"
+SERVER_PORT = "9847"
+OUTPUT_ROOT = "runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b8-r2"
+FLASHINFER_WORKSPACE_BASE = "/tmp/frontier-component-flashinfer-v4"
--- a/runs/frontier-decode-batch-grid-v0/results/tp4-b2-r1.json
+++ b/runs/frontier-decode-batch-grid-v0/results/tp4-b2-r1.json
@@ -0,0 +1,345 @@
+{
+  "contract": {
+    "component_time": "sum of CUDA kernel durations inside execute range",
+    "timing": "CUDA graph-on GPU execute annotations",
+    "tp_aggregation": "per-rank; slowest rank mean approximates critical path"
+  },
+  "label": "tp4-b2-r1",
+  "rank_summary": {
+    "component_rank_mean_ms": {
+      "attention": 0.72195021875,
+      "collective": 0.553751203125,
+      "linear_norm_rope": 0.8375226249999999,
+      "moe": 1.7999987343749986,
+      "other": 0.020717078125,
+      "output_head": 0.0456011875,
+      "router": 0.18689503124999995
+    },
+    "ranks": 4,
+    "slowest_rank_execute_mean_ms": 4.6262738125,
+    "slowest_rank_kernel_busy_mean_ms": 4.1718804999999985
+  },
+  "ranks": [
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.721057875,
+          "n": 16,
+          "p50_ms": 0.711759,
+          "p95_ms": 0.7497570000000005,
+          "population_std_ms": 0.014212609814329528
+        },
+        "collective": {
+          "mean_ms": 0.50711625,
+          "n": 16,
+          "p50_ms": 0.5068694999999999,
+          "p95_ms": 0.513504,
+          "population_std_ms": 0.004382496085280639
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8348696874999997,
+          "n": 16,
+          "p50_ms": 0.8344809999999998,
+          "p95_ms": 0.838978,
+          "population_std_ms": 0.001855606192014971
+        },
+        "moe": {
+          "mean_ms": 1.8552162499999982,
+          "n": 16,
+          "p50_ms": 1.8067699999999989,
+          "p95_ms": 1.9758739999999988,
+          "population_std_ms": 0.09932662894681066
+        },
+        "other": {
+          "mean_ms": 0.020664125,
+          "n": 16,
+          "p50_ms": 0.0206075,
+          "p95_ms": 0.02112,
+          "population_std_ms": 0.00022492384127744221
+        },
+        "output_head": {
+          "mean_ms": 0.0455501875,
+          "n": 16,
+          "p50_ms": 0.045616500000000004,
+          "p95_ms": 0.046176,
+          "population_std_ms": 0.0003880140491576959
+        },
+        "router": {
+          "mean_ms": 0.18695149999999994,
+          "n": 16,
+          "p50_ms": 0.18694599999999995,
+          "p95_ms": 0.18726899999999994,
+          "population_std_ms": 0.00018246506514945761
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.6235766875,
+        "n": 16,
+        "p50_ms": 4.5675045,
+        "p95_ms": 4.776305000000001,
+        "population_std_ms": 0.11774099993349754
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.171425874999998,
+        "n": 16,
+        "p50_ms": 4.111105499999998,
+        "p95_ms": 4.327753999999999,
+        "population_std_ms": 0.11827515014357118
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.4521508125000021,
+        "n": 16,
+        "p50_ms": 0.4516910000000025,
+        "p95_ms": 0.4587190000000012,
+        "population_std_ms": 0.003134758388192225
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r1-20260723-20260723T085748513290Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784797289133609908.pt.trace.json.gz"
+    },
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.7231355625,
+          "n": 16,
+          "p50_ms": 0.714801,
+          "p95_ms": 0.7511429999999999,
+          "population_std_ms": 0.01670948629135843
+        },
+        "collective": {
+          "mean_ms": 0.5030436875000001,
+          "n": 16,
+          "p50_ms": 0.5025615000000001,
+          "p95_ms": 0.5099899999999999,
+          "population_std_ms": 0.003726454090666345
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8384581250000002,
+          "n": 16,
+          "p50_ms": 0.8374080000000004,
+          "p95_ms": 0.8426070000000001,
+          "population_std_ms": 0.0022757014818238397
+        },
+        "moe": {
+          "mean_ms": 1.8538525624999982,
+          "n": 16,
+          "p50_ms": 1.8066204999999975,
+          "p95_ms": 1.980685999999996,
+          "population_std_ms": 0.0992721840478037
+        },
+        "other": {
+          "mean_ms": 0.02081575,
+          "n": 16,
+          "p50_ms": 0.020768000000000002,
+          "p95_ms": 0.021249000000000004,
+          "population_std_ms": 0.00024472216389203553
+        },
+        "output_head": {
+          "mean_ms": 0.0455743125,
+          "n": 16,
+          "p50_ms": 0.045568,
+          "p95_ms": 0.046335999999999995,
+          "population_std_ms": 0.00038872157753815117
+        },
+        "router": {
+          "mean_ms": 0.18700049999999993,
+          "n": 16,
+          "p50_ms": 0.18708599999999992,
+          "p95_ms": 0.18729799999999988,
+          "population_std_ms": 0.0002492310674855524
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.624022875,
+        "n": 16,
+        "p50_ms": 4.5670969999999995,
+        "p95_ms": 4.776058,
+        "population_std_ms": 0.11785700086167718
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.1718804999999985,
+        "n": 16,
+        "p50_ms": 4.113653999999998,
+        "p95_ms": 4.327779999999995,
+        "population_std_ms": 0.11872818612749433
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.4521423750000019,
+        "n": 16,
+        "p50_ms": 0.452637500000002,
+        "p95_ms": 0.4561850000000023,
+        "population_std_ms": 0.002606323557115182
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r1-20260723-20260723T085748513290Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784797289127939045.pt.trace.json.gz"
+    },
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.7228365624999997,
+          "n": 16,
+          "p50_ms": 0.7140979999999998,
+          "p95_ms": 0.7481270000000001,
+          "population_std_ms": 0.014826216615208781
+        },
+        "collective": {
+          "mean_ms": 0.6974953125000001,
+          "n": 16,
+          "p50_ms": 0.6933765000000001,
+          "p95_ms": 0.7296940000000001,
+          "population_std_ms": 0.019150322023136917
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8396519374999999,
+          "n": 16,
+          "p50_ms": 0.8394079999999997,
+          "p95_ms": 0.8423959999999997,
+          "population_std_ms": 0.0015175569786975635
+        },
+        "moe": {
+          "mean_ms": 1.6409965000000004,
+          "n": 16,
+          "p50_ms": 1.6039915000000007,
+          "p95_ms": 1.749915000000001,
+          "population_std_ms": 0.08420929796569343
+        },
+        "other": {
+          "mean_ms": 0.0206121875,
+          "n": 16,
+          "p50_ms": 0.020608,
+          "p95_ms": 0.021438,
+          "population_std_ms": 0.0003990141317594523
+        },
+        "output_head": {
+          "mean_ms": 0.0457240625,
+          "n": 16,
+          "p50_ms": 0.04584,
+          "p95_ms": 0.046816,
+          "population_std_ms": 0.0005658698468674145
+        },
+        "router": {
+          "mean_ms": 0.18694024999999992,
+          "n": 16,
+          "p50_ms": 0.18695999999999993,
+          "p95_ms": 0.18755099999999994,
+          "population_std_ms": 0.00021861224233788088
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.6262738125,
+        "n": 16,
+        "p50_ms": 4.569678,
+        "p95_ms": 4.780622,
+        "population_std_ms": 0.11869307020905795
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.1542568125,
+        "n": 16,
+        "p50_ms": 4.100302,
+        "p95_ms": 4.307204000000001,
+        "population_std_ms": 0.11747190161545976
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.47201699999999974,
+        "n": 16,
+        "p50_ms": 0.47202699999999975,
+        "p95_ms": 0.47818999999999967,
+        "population_std_ms": 0.0027272721719696818
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r1-20260723-20260723T085748513290Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1/traces/profile/dp0_pp0_tp2_dcp0_ep2_rank2.1784797289123687444.pt.trace.json.gz"
+    },
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.720770875,
+          "n": 16,
+          "p50_ms": 0.7128364999999999,
+          "p95_ms": 0.7470090000000001,
+          "population_std_ms": 0.01543277700170565
+        },
+        "collective": {
+          "mean_ms": 0.5073495625,
+          "n": 16,
+          "p50_ms": 0.5075825,
+          "p95_ms": 0.514207,
+          "population_std_ms": 0.004872701393589966
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8371107499999998,
+          "n": 16,
+          "p50_ms": 0.8368309999999997,
+          "p95_ms": 0.8408340000000005,
+          "population_std_ms": 0.0019278550742989576
+        },
+        "moe": {
+          "mean_ms": 1.849929624999998,
+          "n": 16,
+          "p50_ms": 1.8029354999999985,
+          "p95_ms": 1.9727399999999982,
+          "population_std_ms": 0.09810616143359367
+        },
+        "other": {
+          "mean_ms": 0.02077625,
+          "n": 16,
+          "p50_ms": 0.020671500000000002,
+          "p95_ms": 0.021408,
+          "population_std_ms": 0.00033703143399392235
+        },
+        "output_head": {
+          "mean_ms": 0.0455561875,
+          "n": 16,
+          "p50_ms": 0.045392,
+          "p95_ms": 0.046880000000000005,
+          "population_std_ms": 0.0004960692011642635
+        },
+        "router": {
+          "mean_ms": 0.18668787499999998,
+          "n": 16,
+          "p50_ms": 0.18664099999999995,
+          "p95_ms": 0.18707300000000002,
+          "population_std_ms": 0.00021251878593432893
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.623365625,
+        "n": 16,
+        "p50_ms": 4.5671315,
+        "p95_ms": 4.775658999999999,
+        "population_std_ms": 0.11841808903936239
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.168181124999998,
+        "n": 16,
+        "p50_ms": 4.109987499999998,
+        "p95_ms": 4.320808999999999,
+        "population_std_ms": 0.1170474600631484
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.45518450000000205,
+        "n": 16,
+        "p50_ms": 0.45484250000000115,
+        "p95_ms": 0.46047800000000283,
+        "population_std_ms": 0.0026128110101962454
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r1-20260723-20260723T085748513290Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r1/traces/profile/dp0_pp0_tp3_dcp0_ep3_rank3.1784797289132564461.pt.trace.json.gz"
+    }
+  ],
+  "schema": "frontier-decode-batch-trace.v1"
+}
--- a/runs/frontier-decode-batch-grid-v0/results/tp4-b2-r2.json
+++ b/runs/frontier-decode-batch-grid-v0/results/tp4-b2-r2.json
@@ -0,0 +1,345 @@
+{
+  "contract": {
+    "component_time": "sum of CUDA kernel durations inside execute range",
+    "timing": "CUDA graph-on GPU execute annotations",
+    "tp_aggregation": "per-rank; slowest rank mean approximates critical path"
+  },
+  "label": "tp4-b2-r2",
+  "rank_summary": {
+    "component_rank_mean_ms": {
+      "attention": 0.7204233125,
+      "collective": 0.560326375,
+      "linear_norm_rope": 0.8352021093750001,
+      "moe": 1.7914327656249986,
+      "other": 0.020750828125000002,
+      "output_head": 0.04556265625,
+      "router": 0.18667273437499995
+    },
+    "ranks": 4,
+    "slowest_rank_execute_mean_ms": 4.6200790625,
+    "slowest_rank_kernel_busy_mean_ms": 4.168571874999999
+  },
+  "ranks": [
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.7155257500000002,
+          "n": 16,
+          "p50_ms": 0.7066910000000002,
+          "p95_ms": 0.7447440000000003,
+          "population_std_ms": 0.01558176035265282
+        },
+        "collective": {
+          "mean_ms": 0.5219253125,
+          "n": 16,
+          "p50_ms": 0.5189115,
+          "p95_ms": 0.5383740000000001,
+          "population_std_ms": 0.008178805388921035
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8318194374999998,
+          "n": 16,
+          "p50_ms": 0.8315799999999995,
+          "p95_ms": 0.8361279999999999,
+          "population_std_ms": 0.001956002842813193
+        },
+        "moe": {
+          "mean_ms": 1.8462991249999985,
+          "n": 16,
+          "p50_ms": 1.8014489999999976,
+          "p95_ms": 1.9688019999999977,
+          "population_std_ms": 0.098837188149802
+        },
+        "other": {
+          "mean_ms": 0.020839937500000003,
+          "n": 16,
+          "p50_ms": 0.020830500000000002,
+          "p95_ms": 0.021311,
+          "population_std_ms": 0.00026370875714270555
+        },
+        "output_head": {
+          "mean_ms": 0.0455260625,
+          "n": 16,
+          "p50_ms": 0.045296,
+          "p95_ms": 0.047200000000000006,
+          "population_std_ms": 0.0005808763281402957
+        },
+        "router": {
+          "mean_ms": 0.1866362499999999,
+          "n": 16,
+          "p50_ms": 0.18664149999999985,
+          "p95_ms": 0.1869089999999999,
+          "population_std_ms": 0.0001586574533389501
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.6200790625,
+        "n": 16,
+        "p50_ms": 4.560737,
+        "p95_ms": 4.785777,
+        "population_std_ms": 0.1217774651913628
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.168571874999999,
+        "n": 16,
+        "p50_ms": 4.106530999999998,
+        "p95_ms": 4.335109999999998,
+        "population_std_ms": 0.12256936973397699
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.4515071875000016,
+        "n": 16,
+        "p50_ms": 0.45221700000000054,
+        "p95_ms": 0.4565129999999993,
+        "population_std_ms": 0.002988645119839536
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r2-20260723-20260723T090307191004Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784797602726202234.pt.trace.json.gz"
+    },
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.7221352500000001,
+          "n": 16,
+          "p50_ms": 0.7126265000000003,
+          "p95_ms": 0.7521960000000003,
+          "population_std_ms": 0.015924308283642965
+        },
+        "collective": {
+          "mean_ms": 0.5009256875,
+          "n": 16,
+          "p50_ms": 0.5015225,
+          "p95_ms": 0.5122899999999999,
+          "population_std_ms": 0.0051018101287526214
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8378076250000003,
+          "n": 16,
+          "p50_ms": 0.8377155000000002,
+          "p95_ms": 0.8417550000000004,
+          "population_std_ms": 0.0019228740167716244
+        },
+        "moe": {
+          "mean_ms": 1.851001874999998,
+          "n": 16,
+          "p50_ms": 1.8053344999999985,
+          "p95_ms": 1.9718849999999972,
+          "population_std_ms": 0.09850954145403519
+        },
+        "other": {
+          "mean_ms": 0.0207508125,
+          "n": 16,
+          "p50_ms": 0.0207035,
+          "p95_ms": 0.021217,
+          "population_std_ms": 0.00025922799297867157
+        },
+        "output_head": {
+          "mean_ms": 0.0454381875,
+          "n": 16,
+          "p50_ms": 0.045216,
+          "p95_ms": 0.046560000000000004,
+          "population_std_ms": 0.0004811869723337822
+        },
+        "router": {
+          "mean_ms": 0.1865970625,
+          "n": 16,
+          "p50_ms": 0.1865795,
+          "p95_ms": 0.18706799999999996,
+          "population_std_ms": 0.00018487748265743356
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.6191813125,
+        "n": 16,
+        "p50_ms": 4.5577369999999995,
+        "p95_ms": 4.785498,
+        "population_std_ms": 0.12056056733428579
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.164656499999998,
+        "n": 16,
+        "p50_ms": 4.105473,
+        "p95_ms": 4.326126999999998,
+        "population_std_ms": 0.11950220623590964
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.4545248125000018,
+        "n": 16,
+        "p50_ms": 0.45509500000000225,
+        "p95_ms": 0.45937100000000175,
+        "population_std_ms": 0.0031089089770119727
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r2-20260723-20260723T090307191004Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784797602726796969.pt.trace.json.gz"
+    },
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.7246866249999998,
+          "n": 16,
+          "p50_ms": 0.7141314999999997,
+          "p95_ms": 0.7533139999999997,
+          "population_std_ms": 0.016680379828840117
+        },
+        "collective": {
+          "mean_ms": 0.71273025,
+          "n": 16,
+          "p50_ms": 0.7052324999999999,
+          "p95_ms": 0.7373889999999999,
+          "population_std_ms": 0.01757743828214172
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8347086875000002,
+          "n": 16,
+          "p50_ms": 0.8344474999999999,
+          "p95_ms": 0.8372729999999998,
+          "population_std_ms": 0.0011941147724752546
+        },
+        "moe": {
+          "mean_ms": 1.6199748125000002,
+          "n": 16,
+          "p50_ms": 1.5786544999999998,
+          "p95_ms": 1.7316220000000002,
+          "population_std_ms": 0.08667323273307787
+        },
+        "other": {
+          "mean_ms": 0.0206221875,
+          "n": 16,
+          "p50_ms": 0.0206095,
+          "p95_ms": 0.021182,
+          "population_std_ms": 0.0003110010487823955
+        },
+        "output_head": {
+          "mean_ms": 0.0457541875,
+          "n": 16,
+          "p50_ms": 0.0456,
+          "p95_ms": 0.04736,
+          "population_std_ms": 0.0005682590978979142
+        },
+        "router": {
+          "mean_ms": 0.18685312499999993,
+          "n": 16,
+          "p50_ms": 0.1868349999999999,
+          "p95_ms": 0.18742899999999987,
+          "population_std_ms": 0.00024282758157795687
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.6185679375,
+        "n": 16,
+        "p50_ms": 4.559101999999999,
+        "p95_ms": 4.787310000000001,
+        "population_std_ms": 0.12219376749832045
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.145329875,
+        "n": 16,
+        "p50_ms": 4.0855665,
+        "p95_ms": 4.30906,
+        "population_std_ms": 0.12076811704868254
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.4732380625000001,
+        "n": 16,
+        "p50_ms": 0.47285649999999935,
+        "p95_ms": 0.47825000000000095,
+        "population_std_ms": 0.0030751450874057767
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r2-20260723-20260723T090307191004Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2/traces/profile/dp0_pp0_tp2_dcp0_ep2_rank2.1784797602710411107.pt.trace.json.gz"
+    },
+    {
+      "components": {
+        "attention": {
+          "mean_ms": 0.7193456250000001,
+          "n": 16,
+          "p50_ms": 0.7110120000000001,
+          "p95_ms": 0.7488160000000005,
+          "population_std_ms": 0.0156480688619834
+        },
+        "collective": {
+          "mean_ms": 0.5057242500000001,
+          "n": 16,
+          "p50_ms": 0.5041895000000001,
+          "p95_ms": 0.5194939999999999,
+          "population_std_ms": 0.00615956476242434
+        },
+        "linear_norm_rope": {
+          "mean_ms": 0.8364726875,
+          "n": 16,
+          "p50_ms": 0.8363539999999998,
+          "p95_ms": 0.84147,
+          "population_std_ms": 0.0026977789873977565
+        },
+        "moe": {
+          "mean_ms": 1.8484552499999982,
+          "n": 16,
+          "p50_ms": 1.8005024999999995,
+          "p95_ms": 1.9741669999999973,
+          "population_std_ms": 0.09999423839808207
+        },
+        "other": {
+          "mean_ms": 0.020790375,
+          "n": 16,
+          "p50_ms": 0.020800000000000003,
+          "p95_ms": 0.021249999999999998,
+          "population_std_ms": 0.000248934699017634
+        },
+        "output_head": {
+          "mean_ms": 0.0455321875,
+          "n": 16,
+          "p50_ms": 0.045424,
+          "p95_ms": 0.046304000000000005,
+          "population_std_ms": 0.00037697981026011215
+        },
+        "router": {
+          "mean_ms": 0.18660449999999998,
+          "n": 16,
+          "p50_ms": 0.1866215,
+          "p95_ms": 0.18684699999999987,
+          "population_std_ms": 0.00013768759929635568
+        }
+      },
+      "execute_annotation_histogram": {
+        "execute_context_0(0)_generation_2(2)": 16
+      },
+      "execute_wall": {
+        "mean_ms": 4.617897375,
+        "n": 16,
+        "p50_ms": 4.558091500000001,
+        "p95_ms": 4.783308,
+        "population_std_ms": 0.12193403131912922
+      },
+      "gpu_kernel_busy": {
+        "mean_ms": 4.162924874999998,
+        "n": 16,
+        "p50_ms": 4.102588,
+        "p95_ms": 4.328206999999998,
+        "population_std_ms": 0.12156658812810923
+      },
+      "non_kernel_gap": {
+        "mean_ms": 0.4549725000000015,
+        "n": 16,
+        "p50_ms": 0.45506600000000175,
+        "p95_ms": 0.4593380000000007,
+        "population_std_ms": 0.002668184377999502
+      },
+      "selected_execute_annotation": "execute_context_0(0)_generation_2(2)",
+      "steps": 16,
+      "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-batch-tp4-b2-r2-20260723-20260723T090307191004Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b2-r2/traces/profile/dp0_pp0_tp3_dcp0_ep3_rank3.1784797602728757539.pt.trace.json.gz"
+    }
+  ],
+  "schema": "frontier-decode-batch-trace.v1"
+}