Close BC8 decode curve counterfactual

2026-07-23 18:08:00 +08:00
parent ecc5599381
commit cf610003ed
11 changed files with 1666 additions and 7 deletions
--- a/runs/frontier-decode-batch-grid-v0/analyze_bc8_replay.py
+++ b/runs/frontier-decode-batch-grid-v0/analyze_bc8_replay.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Compare original and whole-layer-curve Frontier against the BC-8 real pilot."""
+
+from __future__ import annotations
+
+import json
+import statistics
+from collections import Counter
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent
+REPO = ROOT.parents[1]
+CONFIGS = ("tp1_mns16", "tp2_mns16", "tp4_mns16", "tp4_mns32")
+REPLAY = ROOT / "replay/bc8"
+ORIGINAL = REPO / "runs/frontier-knee-sweep-v0/raw/fixed/rho0p02"
+REAL = REPO / "runs/frontier-pilot-v0/results/pilot-surface.json"
+
+
+def find_one(root: Path, name: str) -> Path:
+    matches = list(root.glob(f"**/{name}"))
+    if len(matches) != 1:
+        raise ValueError(f"expected one {name} under {root}, got {matches}")
+    return matches[0]
+
+
+def summarize_sim(root: Path) -> dict:
+    metrics = json.loads(find_one(root, "system_metrics.json").read_text())
+    ledger = find_one(root, "frontier_stage_batch_ledger.jsonl")
+    histogram: Counter[int] = Counter()
+    service_ms: dict[int, set[float]] = {}
+    for line in ledger.read_text().splitlines():
+        row = json.loads(line)
+        tokens = row["request_num_tokens"]
+        if tokens and all(int(value) == 1 for value in tokens):
+            batch = len(tokens)
+            histogram[batch] += 1
+            service_ms.setdefault(batch, set()).add(
+                float(row["execution_time"]["model_time_ms"])
+            )
+    return {
+        "tpot_mean_ms": metrics["tpot_statistics"]["mean"],
+        "ttft_mean_ms": metrics["ttft_statistics"]["mean"],
+        "e2e_mean_ms": metrics["request_e2e_time_statistics"]["mean"],
+        "decode_batch_histogram": dict(sorted(histogram.items())),
+        "decode_service_ms": {
+            str(batch): sorted(values) for batch, values in sorted(service_ms.items())
+        },
+        "decode_b_gt_1_fraction": sum(
+            count for batch, count in histogram.items() if batch > 1
+        )
+        / sum(histogram.values()),
+    }
+
+
+def main() -> None:
+    real_rows = [
+        row
+        for row in json.loads(REAL.read_text())
+        if row["load"] == "p4" and row["canonical"]
+    ]
+    points = []
+    for config in CONFIGS:
+        real = [row for row in real_rows if row["config"] == config]
+        original = summarize_sim(ORIGINAL / config)
+        corrected = summarize_sim(REPLAY / "raw" / config)
+        usage = json.loads((REPLAY / "raw" / config / "usage.json").read_text())
+        points.append(
+            {
+                "config": config,
+                "tp": int(config[2]),
+                "mns": int(config.split("mns")[1]),
+                "real_tpot_mean_ms": statistics.fmean(
+                    row["tpot_mean_ms"] for row in real
+                ),
+                "original": original,
+                "whole_curve": corrected,
+                "original_tpot_residual_ms": (
+                    original["tpot_mean_ms"]
+                    - statistics.fmean(row["tpot_mean_ms"] for row in real)
+                ),
+                "whole_curve_tpot_residual_ms": (
+                    corrected["tpot_mean_ms"]
+                    - statistics.fmean(row["tpot_mean_ms"] for row in real)
+                ),
+                "usage": usage,
+            }
+        )
+
+    def ranking(key):
+        return [
+            point["config"]
+            for point in sorted(points, key=lambda point: (key(point), point["config"]))
+            if point["mns"] == 16
+        ]
+
+    real_ranking = ranking(lambda point: point["real_tpot_mean_ms"])
+    original_ranking = ranking(lambda point: point["original"]["tpot_mean_ms"])
+    corrected_ranking = ranking(lambda point: point["whole_curve"]["tpot_mean_ms"])
+    payload = {
+        "schema": "frontier-decode-grid-bc8-verdict.v1",
+        "points": points,
+        "mns16_rankings_fast_to_slow": {
+            "real": real_ranking,
+            "original": original_ranking,
+            "whole_curve": corrected_ranking,
+        },
+        "whole_curve_restores_real_mns16_ranking": corrected_ranking == real_ranking,
+        "all_corrected_batches_within_measured_support": all(
+            max(map(int, point["whole_curve"]["decode_batch_histogram"])) <= 8
+            for point in points
+        ),
+        "decision": (
+            "STATIC_WHOLE_CURVE_FIXES_BC8"
+            if corrected_ranking == real_ranking
+            else "ESCALATE_EVENT_LEVEL_STATE"
+        ),
+    }
+    output = ROOT / "results/bc8-replay-verdict.json"
+    output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+    lines = [
+        "# BC-8 whole-layer decode-curve replay",
+        "",
+        "| Config | Real TPOT | Original sim | Whole-curve sim | Residual before -> after | b>1 |",
+        "|---|---:|---:|---:|---:|---:|",
+    ]
+    for point in points:
+        lines.append(
+            f"| {point['config']} | {point['real_tpot_mean_ms']:.3f} | "
+            f"{point['original']['tpot_mean_ms']:.3f} | "
+            f"{point['whole_curve']['tpot_mean_ms']:.3f} | "
+            f"{point['original_tpot_residual_ms']:+.3f} -> "
+            f"{point['whole_curve_tpot_residual_ms']:+.3f} | "
+            f"{point['whole_curve']['decode_b_gt_1_fraction']:.2%} |"
+        )
+    lines.extend(
+        [
+            "",
+            f"- Real MNS16: `{' < '.join(real_ranking)}`",
+            f"- Original: `{' < '.join(original_ranking)}`",
+            f"- Whole curve: `{' < '.join(corrected_ranking)}`",
+            f"- Decision: **{payload['decision']}**.",
+            "",
+        ]
+    )
+    (ROOT / "results/bc8-replay-verdict.md").write_text("\n".join(lines))
+    print(json.dumps({"output": str(output), "decision": payload["decision"]}))
+
+
+if __name__ == "__main__":
+    main()