Replace undrainable telemetry load with fresh rerun

runs/intervention-response-v2/prepare_pilot.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Prepare the uncensored 300-second TP4 matched pilot on dash0."""
+"""Prepare the uncensored 300-second TP4 two-load matched pilot on dash0."""
 
 from __future__ import annotations
 
@@ -20,18 +20,18 @@ from aituner.spec import load_study_spec  # noqa: E402
 from aituner.trace import load_trace_requests, select_requests_for_threshold  # noqa: E402
 
 
-SCHEMA = "intervention-response-phase-aware-pilot-manifest-v2"
+SCHEMA = "intervention-response-phase-aware-pilot-manifest-v3"
 TP = 4
 MNS_ENDPOINTS = (16, 64)
 REPLAY_TIME_SCALE = 0.5
 EXPECTED_DURATION_S = 300.0
 SAFETY_DEADLINE_S = 360.0
-LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125, "high": 3.125}
+LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125}
 REPLICATE_ROLE_PAIRS = (("low1", "high1"), ("low2", "high2"), ("low3", "high3"))
 LOAD_ORDERS = {
-    1: ("low", "mid", "high"),
-    2: ("high", "low", "mid"),
-    3: ("mid", "high", "low"),
+    1: ("low", "mid"),
+    2: ("mid", "low"),
+    3: ("low", "mid"),
 }
 SESSION_ORDER = (
     (1, 16),
@@ -118,7 +118,7 @@ def private_request_id(
     *, source_sha256: str, line_number: int, original_id: str
 ) -> str:
     payload = f"{source_sha256}:{line_number}:{original_id}".encode()
-    return f"phase-v2-{hashlib.sha256(payload).hexdigest()}"
+    return f"phase-v3-{hashlib.sha256(payload).hexdigest()}"
 
 
 def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]:
@@ -159,7 +159,7 @@ def materialize_study(
     source: Path, target: Path, *, replicate: int, trace_override: Path
 ) -> Path:
     payload = json.loads(source.read_text(encoding="utf-8"))
-    payload["study_id"] = f"phase-aware-telemetry-v2-rep{replicate}"
+    payload["study_id"] = f"phase-aware-telemetry-v3-rep{replicate}"
     payload["hardware"]["host_candidates"] = ["dash0"]
     payload["engine"]["engine_version"] = "0.24.1.dev3+opprof"
     trace = payload["trace"]
@@ -174,7 +174,7 @@ def materialize_study(
 
 
 def build_manifest(
-    *, base_manifest_path: Path, private_root: Path
+    *, base_manifest_path: Path, private_root: Path, prior_attempt_state: Path
 ) -> dict[str, Any]:
     base = json.loads(base_manifest_path.read_text(encoding="utf-8"))
     if base.get("schema") != "fidelity-prefix-pilot-manifest-v1":
@@ -223,7 +223,7 @@ def build_manifest(
             selections[level] = record
             selected_ids_by_level[level] = {request.row_id for request in selected}
             selection_hashes.append(record["request_id_order_sha256"])
-        selected_ids_by_replicate[replicate] = selected_ids_by_level["high"]
+        selected_ids_by_replicate[replicate] = selected_ids_by_level["mid"]
         repetitions[str(replicate)] = {
             "source_roles": list(roles),
             "merged_trace": merged_traces[str(replicate)],
@@ -250,8 +250,8 @@ def build_manifest(
     invariants = {
         "three_repetitions": len(repetitions) == 3,
         "six_sessions": len(sessions) == 6,
-        "load_levels_three": all(
-            len(item["selections"]) == 3 for item in repetitions.values()
+        "load_levels_two": all(
+            len(item["selections"]) == 2 for item in repetitions.values()
         ),
         "selection_hashes_unique": len(selection_hashes) == len(set(selection_hashes)),
         "selection_sets_disjoint_across_repetitions": all(
@@ -267,6 +267,13 @@ def build_manifest(
         ),
     }
     red_flags = [name for name, passed in invariants.items() if not passed]
+    prior_state = json.loads(prior_attempt_state.read_text(encoding="utf-8"))
+    if prior_state.get("status") != "failed":
+        raise ValueError("prior three-load attempt was not recorded as failed")
+    prior_h20_hours = float(prior_state["gpu_hours_total"])
+    incremental_cap_h20_hours = 8.0 - prior_h20_hours
+    if incremental_cap_h20_hours < 6.5:
+        raise ValueError("insufficient global H20-hour budget for the two-load rerun")
     return {
         "schema": SCHEMA,
         "status": "PASS" if not red_flags else "STOP",
@@ -276,6 +283,10 @@ def build_manifest(
             "window_id": base["source"]["window_id"],
             "source_trace": base["source"]["trace"],
             "source_trace_sha256": base["source"]["trace_sha256"],
+            "prior_attempt_state": str(prior_attempt_state),
+            "prior_attempt_state_sha256": sha256_file(prior_attempt_state),
+            "prior_attempt_h20_hours": prior_h20_hours,
+            "prior_attempt_failure": prior_state["failures"],
         },
         "engine": {
             "tp": TP,
@@ -294,9 +305,12 @@ def build_manifest(
             "seconds": [30.0, 75.0, 150.0, 225.0, 300.0],
         },
         "budget": {
-            "hard_cap_h20_hours": 8.0,
-            "expected_wall_minutes": [95, 120],
-            "expected_h20_hours": [6.3, 8.0],
+            "global_hard_cap_h20_hours": 8.0,
+            "prior_attempt_h20_hours": prior_h20_hours,
+            "hard_cap_h20_hours": incremental_cap_h20_hours,
+            "session_estimate_h20_hours": 1.0,
+            "expected_wall_minutes": [75, 95],
+            "expected_h20_hours": [4.8, incremental_cap_h20_hours],
         },
         "sanity": {
             "red_flags": red_flags,
@@ -312,9 +326,12 @@ def main() -> None:
     parser.add_argument("--base-manifest", type=Path, required=True)
     parser.add_argument("--private-root", type=Path, required=True)
     parser.add_argument("--public-manifest", type=Path, required=True)
+    parser.add_argument("--prior-attempt-state", type=Path, required=True)
     args = parser.parse_args()
     manifest = build_manifest(
-        base_manifest_path=args.base_manifest, private_root=args.private_root
+        base_manifest_path=args.base_manifest,
+        private_root=args.private_root,
+        prior_attempt_state=args.prior_attempt_state,
     )
     atomic_json(args.public_manifest, manifest)
     print(