Audit tuning cost and core challenges

runs/tuning-cost/test_analysis.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import importlib.util
+import math
+import sys
+from pathlib import Path
+
+
+HERE = Path(__file__).resolve().parent
+
+
+def load_analysis():
+    spec = importlib.util.spec_from_file_location("tuning_cost", HERE / "analyze.py")
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def main() -> None:
+    analysis = load_analysis()
+    duration, monotonic, count = analysis.timestamp_span(
+        "INFO 07-01 10:00:00 x\nINFO 07-01 10:05:30 y\n", 2026
+    )
+    assert duration == 330.0
+    assert monotonic
+    assert count == 2
+
+    trials = [
+        {"trial_id": "t1", "engine_h20_hours_lower_bound": 0.1, "score_req_s_per_gpu": 8.0},
+        {"trial_id": "t2", "engine_h20_hours_lower_bound": 0.2, "score_req_s_per_gpu": None},
+        {"trial_id": "t3", "engine_h20_hours_lower_bound": 0.3, "score_req_s_per_gpu": 9.6},
+    ]
+    curve = analysis.sequential_curve(trials, 10.0, [0.05, 0.04])
+    assert curve["cost_to_threshold"]["regret_le_0.05"]["trial_id"] == "t3"
+    assert curve["cost_to_threshold"]["regret_le_0.04"]["trial_id"] == "t3"
+    assert math.isclose(curve["total_engine_h20_hours_lower_bound"], 0.6)
+
+    candidates = analysis.tie_expanded_candidates(
+        {"a": 3.0, "b": 2.0, "c": 2.0, "d": 1.0}, 2
+    )
+    assert candidates == ["a", "b", "c"]
+    policy = analysis.real_final_policy(
+        candidates,
+        {"a": 1.0, "b": 4.0, "c": 2.0, "d": 3.0},
+        {"a": 0.1, "b": 0.2, "c": 0.3, "d": 0.4},
+    )
+    assert policy["selected_cell"] == "b"
+    assert policy["real_regret"] == 0.0
+    assert math.isclose(policy["engine_h20_hours_lower_bound"], 0.6)
+    print("tuning cost analysis: PASS")
+
+
+if __name__ == "__main__":
+    main()