Audit telemetry intervention response for tuning

runs/intervention-response-v0/test_analysis.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import importlib.util
+import math
+from pathlib import Path
+
+
+HERE = Path(__file__).resolve().parent
+
+
+def load_module():
+    spec = importlib.util.spec_from_file_location(
+        "intervention_response_v0", HERE / "analyze_phase6.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def pair(module, delta: dict[str, float]) -> dict[str, object]:
+    state = {feature: 0.0 for feature in module.ALL_FEATURES}
+    state.update(delta)
+    return {"delta_state": state}
+
+
+def main() -> None:
+    module = load_module()
+    assert module.numeric([0.0, 1.0, 1.0]) == {
+        "n": 3,
+        "min": 0.0,
+        "max": 1.0,
+        "distinct_n": 2,
+    }
+    assert math.isclose(module.quantile([0.0, 10.0], 0.95), 9.5)
+
+    actions = [
+        pair(module, {"queue_waiting_mean": -1.0 - 0.1 * index})
+        for index in range(8)
+    ]
+    repeats = [
+        pair(module, {"queue_waiting_mean": 0.01 * ((index % 3) - 1)})
+        for index in range(20)
+    ]
+    stats = module.response_statistics(actions, repeats)
+    waiting = stats["queue_waiting_mean"]
+    assert waiting["qualifies"]
+    assert waiting["action_signs"]["negative"] == 8
+    assert waiting["action_signs"]["consistency"] == 1.0
+    assert waiting["effect_to_repeat_median"] > 2.0
+    assert not stats["kv_usage_mean"]["qualifies"]
+    print("intervention response v0 analysis: PASS")
+
+
+if __name__ == "__main__":
+    main()