Replace undrainable telemetry load with fresh rerun

runs/intervention-response-v2/pilot_controller.py

@@ -12,7 +12,7 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import Any
+from typing import Any, Mapping
 
 
 HERE = Path(__file__).resolve().parent
@@ -22,8 +22,8 @@ sys.path.insert(0, str(PHASE6))
 import opprof_phase6_controller as base  # noqa: E402
 
 
-SCHEMA = "intervention-response-phase-aware-pilot-state-v2"
-SESSION_ESTIMATE_H20_HOURS = 1.25
+SCHEMA = "intervention-response-phase-aware-pilot-state-v3"
+SESSION_ESTIMATE_H20_HOURS = 1.0
 SAFETY_H20_HOURS = 0.20
 CLIENT_TIMEOUT_S = 450.0
 
@@ -55,7 +55,7 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None:
     base.MODEL = args.model
     base.CLIENT = args.client
     base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"])
-    base.MARKER = "intervention-response-phase-aware-v2"
+    base.MARKER = "intervention-response-phase-aware-v3"
     base.CELLS = {
         f"tp4_mns{mns}": {"tp": 4, "mns": int(mns)}
         for mns in manifest["engine"]["mns_endpoints"]
@@ -63,10 +63,15 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None:
 
 
 def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None:
-    if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2":
+    if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3":
         raise RuntimeError("unexpected phase-aware pilot manifest schema")
     if manifest.get("status") != "PASS":
         raise RuntimeError("phase-aware pilot manifest did not pass preflight")
+    if abs(
+        float(manifest["budget"]["session_estimate_h20_hours"])
+        - SESSION_ESTIMATE_H20_HOURS
+    ) > 1e-12:
+        raise RuntimeError("controller and manifest session-cost estimates disagree")
     failed_invariants = [
         name
         for name, passed in manifest.get("sanity", {}).get("invariants", {}).items()
@@ -95,6 +100,13 @@ def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None:
         raise RuntimeError(f"phase-aware pilot input paths missing: {missing}")
 
 
+def warmup_selection(repetition: Mapping[str, Any]) -> Mapping[str, Any]:
+    return max(
+        repetition["selections"].values(),
+        key=lambda selection: float(selection["offered_req_s_per_gpu"]),
+    )
+
+
 def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str, Any]:
     sessions = []
     for index, session in enumerate(manifest["sessions"]):
@@ -102,13 +114,13 @@ def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str
         entry = {"cell": cell, "gpus": (0, 1, 2, 3), "port": 8950 + index}
         repetition = manifest["repetitions"][str(session["replicate"])]
         session_root = args.run_root / "sessions" / str(session["session"])
-        high = repetition["selections"]["high"]
+        warmup = warmup_selection(repetition)
         commands = {
             "server": base.server_command(cell, entry["gpus"], entry["port"]),
             "warmup": client_command(
                 entry,
                 study=repetition["study"],
-                anchor=float(high["anchor"]),
+                anchor=float(warmup["anchor"]),
                 output=session_root / "warmup",
                 warmup=True,
             ),
@@ -145,7 +157,7 @@ def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str
             }
         )
     return {
-        "schema": "intervention-response-phase-aware-pilot-dry-run-v2",
+        "schema": "intervention-response-phase-aware-pilot-dry-run-v3",
         "status": "PASS",
         "manifest": str(args.manifest),
         "run_root": str(args.run_root),
@@ -423,14 +435,14 @@ def execute_session(
     failure: Exception | None = None
     try:
         base.wait_ready(entry)
-        high = repetition["selections"]["high"]
+        warmup = warmup_selection(repetition)
         session_state["status"] = "warmup"
         save_state(state_path, state)
         run_client(
             entry=entry,
             role="warmup",
             study=repetition["study"],
-            selection=high,
+            selection=warmup,
             output=entry["dir"] / "warmup",
             state=state,
             warmup=True,