Make telemetry audit replay-phase aware

runs/intervention-response-v2/test_analysis.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import importlib.util
+import math
+from pathlib import Path
+from types import SimpleNamespace
+
+
+HERE = Path(__file__).resolve().parent
+
+
+def load_module():
+    spec = importlib.util.spec_from_file_location(
+        "intervention_response_phase_aware_v2", HERE / "analyze_existing.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def load_prepare_module():
+    spec = importlib.util.spec_from_file_location(
+        "intervention_response_phase_aware_prepare", HERE / "prepare_pilot.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def load_controller_module():
+    spec = importlib.util.spec_from_file_location(
+        "intervention_response_phase_aware_controller", HERE / "pilot_controller.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def load_pilot_analysis_module():
+    spec = importlib.util.spec_from_file_location(
+        "intervention_response_phase_aware_pilot_analysis", HERE / "analyze_pilot.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def main() -> None:
+    module = load_module()
+    assert module.common_decile_fractions(
+        trace_duration_s=60.0, minimum_elapsed_s=19.448
+    ) == (0.1, 0.2, 0.3)
+    assert module.common_decile_fractions(
+        trace_duration_s=60.0, minimum_elapsed_s=60.0
+    )[-1] == 1.0
+    stats = module.numeric([0.0, 1.0, 2.0])
+    assert stats == {
+        "n": 3,
+        "min": 0.0,
+        "max": 2.0,
+        "distinct_n": 3,
+        "median": 1.0,
+    }
+    assert math.isclose(module._pearson([1.0, 2.0], [2.0, 4.0]), 1.0)
+    assert module._pearson([1.0, 1.0], [2.0, 3.0]) is None
+    prepare = load_prepare_module()
+    requests = [
+        SimpleNamespace(
+            sampling_u=index / 10.0,
+            row_id=f"r{index}",
+            arrival_s=float(index),
+            prompt_tokens_hint=100 + index,
+        )
+        for index in range(1, 6)
+    ]
+    _anchor, selected = prepare.attainable_anchor(requests, 3)
+    assert len(selected) == 3
+    record = prepare.selection_record(selected, duration_s=3.0)
+    assert record["selected_count"] == 3
+    assert record["offered_req_s_per_gpu"] == 0.25
+    assert len(prepare.SESSION_ORDER) == 6
+    assert {mns for _replicate, mns in prepare.SESSION_ORDER} == {16, 64}
+    controller = load_controller_module()
+    assert math.isclose(controller.remaining_projection(6, 0), 7.7)
+    assert math.isclose(controller.remaining_projection(6, 5), 1.45)
+    pilot_analysis = load_pilot_analysis_module()
+    stable = pilot_analysis.stable_adjacent_features(
+        [
+            {"end_fraction": 0.1, "qualifying_response_features": ["queue"]},
+            {
+                "end_fraction": 0.25,
+                "qualifying_response_features": ["kv", "queue"],
+            },
+            {"end_fraction": 0.5, "qualifying_response_features": ["queue"]},
+        ]
+    )
+    assert stable == {"0.10->0.25": ["queue"], "0.25->0.50": ["queue"]}
+    print("phase-aware intervention response v2 analysis: PASS")
+
+
+if __name__ == "__main__":
+    main()