Add static-policy oracle gap experiment

2026-07-13 20:25:44 +08:00
parent a730b368d6
commit 34e1f4c144
6 changed files with 1431 additions and 0 deletions
--- a/scripts/oracle_gap/phase3_upper_bound.py
+++ b/scripts/oracle_gap/phase3_upper_bound.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Retrospective point-estimate oracle bound from accepted Phase-3 cells."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from analyze import CONFIGS, PHASES, atomic_json, numeric
+
+
+def build(metrics: dict[str, Any]) -> dict[str, Any]:
+    tables: dict[str, dict[str, dict[str, float]]] = {}
+    for load in ("saturation", "moderate"):
+        table: dict[str, dict[str, float]] = {}
+        for phase in PHASES:
+            row = {}
+            for config in CONFIGS:
+                run = metrics.get("runs", {}).get(f"{phase}-{config}-{load}")
+                if run is not None:
+                    row[config] = float(run["clean"]["completed_throughput_rps"])
+            table[phase] = row
+        tables[load] = table
+
+    analyses = {}
+    all_values = []
+    for load, table in tables.items():
+        complete = all(set(row) == set(CONFIGS) for row in table.values())
+        if not complete:
+            raise RuntimeError(f"missing sentinel throughput cell for {load}: {table}")
+        per_phase = {}
+        c00_regrets = []
+        for phase, row in table.items():
+            oracle_config = max(row, key=row.get)
+            oracle = row[oracle_config]
+            regret = oracle / row["C00"] - 1.0
+            c00_regrets.append(regret)
+            per_phase[phase] = {
+                "throughput_rps": row,
+                "oracle_config": oracle_config,
+                "oracle_rps": oracle,
+                "c00_regret": regret,
+            }
+            all_values.extend(row.values())
+        equal_oracle = sum(item["oracle_rps"] for item in per_phase.values()) / len(PHASES)
+        static = {
+            config: sum(table[p][config] for p in PHASES) / len(PHASES)
+            for config in CONFIGS
+        }
+        analyses[load] = {
+            "per_phase": per_phase,
+            "universal_c00_point_bound": max(c00_regrets),
+            "equal_time_oracle_rps": equal_oracle,
+            "equal_time_best_static_config": max(static, key=static.get),
+            "equal_time_best_static_rps": max(static.values()),
+            "equal_time_gap": equal_oracle / max(static.values()) - 1.0,
+            "interpretation": (
+                "valid additive point-estimate precheck"
+                if load == "saturation"
+                else "diagnostic only: each config used its own offered rate"
+            ),
+        }
+    return {
+        "schema": 1,
+        "scope": list(PHASES),
+        "analyses": analyses,
+        "sanity": {
+            "throughput_rps": numeric(all_values),
+            "invariants": {
+                "all_nonnegative": all(value >= 0 for value in all_values),
+                "all_cells_present": len(all_values) == 2 * len(PHASES) * len(CONFIGS),
+                "per_config_not_identical": len(set(all_values)) > len(CONFIGS),
+            },
+        },
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--metrics", required=True)
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args()
+    result = build(json.loads(Path(args.metrics).read_text()))
+    atomic_json(Path(args.out), result)
+    print(json.dumps(result, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()