38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from analyze_strong_baseline import analyze
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
REPLAYSERVE = ROOT.parent / "replayserve"
|
|
|
|
|
|
def main() -> None:
|
|
result = analyze(
|
|
ROOT / "runs/opprof-phase6/phase6/metrics.json",
|
|
ROOT / "runs/opprof-phase6/phase6/solo-authoritative/cells",
|
|
REPLAYSERVE / "runs/simfid_s2rb/results/raw",
|
|
REPLAYSERVE / "runs/simfid_s2rb/results/metrics.json",
|
|
)
|
|
assert result["status"] == "PASS", json.dumps(result["sanity"], indent=2)
|
|
assert result["sanity"]["frozen_simulator_runs"] == 92
|
|
assert result["sanity"]["labels"]["n"] == 37
|
|
headline = result["headline"]
|
|
assert headline["sim_plus_outcome"]["policy_0p95"]["false_accept"] == 0
|
|
assert headline["sim_plus_outcome"]["policy_0p95"]["false_reject"] == 0
|
|
assert (
|
|
headline["sim_plus_outcome_plus_instrumentation"]["policy_0p95"][
|
|
"false_accept"
|
|
]
|
|
== 0
|
|
)
|
|
print("fidelity strong baseline: PASS")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|