Track simulator fidelity experiment artifacts

runs/frontier-slo-alignment-v0/test_analysis.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import importlib.util
+import math
+import sys
+from pathlib import Path
+
+
+HERE = Path(__file__).resolve().parent
+
+
+def load_analysis():
+    spec = importlib.util.spec_from_file_location("frontier_slo_alignment", HERE / "analyze.py")
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def anchor(rate: float, real: bool, frontier: bool, probe: int) -> dict[str, object]:
+    return {
+        "cell_id": "tp1_mns8",
+        "tp": 1,
+        "mns": 8,
+        "probe_index": probe,
+        "offered_req_s_per_gpu": rate,
+        "real_feasible": real,
+        "frontier_feasible": frontier,
+    }
+
+
+def main() -> None:
+    analysis = load_analysis()
+    rows = [
+        anchor(1.0, True, True, 0),
+        anchor(2.0, False, True, 1),
+        anchor(3.0, True, False, 2),
+    ]
+    real_violations = analysis.monotonic_violations(rows, "real_feasible")
+    assert real_violations == [
+        {"lower_probe": 1, "lower_rate": 2.0, "upper_probe": 2, "upper_rate": 3.0}
+    ]
+    assert analysis.selected_anchor(rows, "real_feasible")["probe_index"] == 2
+    confusion = analysis.confusion_metrics(rows)
+    assert confusion["true_feasible"] == 1
+    assert confusion["false_feasible"] == 1
+    assert confusion["false_infeasible"] == 1
+    assert math.isclose(confusion["accuracy"], 1.0 / 3.0)
+
+    rank = analysis.ranking_metrics(
+        {"a": 3.0, "b": 2.0, "c": 1.0},
+        {"a": 2.0, "b": 3.0, "c": 3.0},
+    )
+    assert rank["top1_candidate_cells"] == ["b", "c"]
+    assert math.isclose(rank["top1_optimistic_regret"], 1.0 / 3.0)
+    assert math.isclose(rank["top1_worst_case_regret"], 2.0 / 3.0)
+    assert rank["pair_count"] == 3
+    print("frontier SLO alignment analysis: PASS")
+
+
+if __name__ == "__main__":
+    main()