Add fidelity-aware verification pilot

runs/fidelity-headroom/test_analysis.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import importlib.util
+import math
+import sys
+from pathlib import Path
+
+
+HERE = Path(__file__).resolve().parent
+
+
+def load_analysis():
+    spec = importlib.util.spec_from_file_location("fidelity_headroom", HERE / "analyze_existing.py")
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def main() -> None:
+    analysis = load_analysis()
+    curve = analysis.topk_curve(
+        {"a": 3.0, "b": 2.0, "c": 1.0},
+        {"a": 1.0, "b": 2.0, "c": 2.0},
+        2e-6,
+    )
+    assert curve["points"][0]["expanded_k"] == 2
+    assert curve["points"][0]["candidates"] == ["b", "c"]
+    assert math.isclose(curve["points"][0]["real_regret"], 1.0 / 3.0)
+    assert curve["points"][2]["real_regret"] == 0.0
+    assert curve["minimum_k"]["five_percent"] == {"nominal_k": 3, "expanded_k": 3}
+    assert analysis._mcnemar_exact_p(0, 1) == 1.0
+    assert analysis._mcnemar_exact_p(0, 5) == 0.0625
+    print("fidelity headroom analysis: PASS")
+
+
+if __name__ == "__main__":
+    main()