41 lines
1.1 KiB
Python
41 lines
1.1 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import math
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
|
|
|
|
def load_analysis():
|
|
spec = importlib.util.spec_from_file_location("fidelity_headroom", HERE / "analyze_existing.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec.loader is not None
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def main() -> None:
|
|
analysis = load_analysis()
|
|
curve = analysis.topk_curve(
|
|
{"a": 3.0, "b": 2.0, "c": 1.0},
|
|
{"a": 1.0, "b": 2.0, "c": 2.0},
|
|
2e-6,
|
|
)
|
|
assert curve["points"][0]["expanded_k"] == 2
|
|
assert curve["points"][0]["candidates"] == ["b", "c"]
|
|
assert math.isclose(curve["points"][0]["real_regret"], 1.0 / 3.0)
|
|
assert curve["points"][2]["real_regret"] == 0.0
|
|
assert curve["minimum_k"]["five_percent"] == {"nominal_k": 3, "expanded_k": 3}
|
|
assert analysis._mcnemar_exact_p(0, 1) == 1.0
|
|
assert analysis._mcnemar_exact_p(0, 5) == 0.0625
|
|
print("fidelity headroom analysis: PASS")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|