Strengthen fidelity calibration baseline

runs/fidelity-headroom/test_strong_baseline.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from analyze_strong_baseline import analyze
+
+
+ROOT = Path(__file__).resolve().parents[2]
+REPLAYSERVE = ROOT.parent / "replayserve"
+
+
+def main() -> None:
+    result = analyze(
+        ROOT / "runs/opprof-phase6/phase6/metrics.json",
+        ROOT / "runs/opprof-phase6/phase6/solo-authoritative/cells",
+        REPLAYSERVE / "runs/simfid_s2rb/results/raw",
+        REPLAYSERVE / "runs/simfid_s2rb/results/metrics.json",
+    )
+    assert result["status"] == "PASS", json.dumps(result["sanity"], indent=2)
+    assert result["sanity"]["frozen_simulator_runs"] == 92
+    assert result["sanity"]["labels"]["n"] == 37
+    headline = result["headline"]
+    assert headline["sim_plus_outcome"]["policy_0p95"]["false_accept"] == 0
+    assert headline["sim_plus_outcome"]["policy_0p95"]["false_reject"] == 0
+    assert (
+        headline["sim_plus_outcome_plus_instrumentation"]["policy_0p95"][
+            "false_accept"
+        ]
+        == 0
+    )
+    print("fidelity strong baseline: PASS")
+
+
+if __name__ == "__main__":
+    main()