Account for failed fidelity pilot attempts

runs/fidelity-headroom/analyze_pilot.py

@@ -37,6 +37,41 @@ def selection_for(
     return manifest["cells"][cell]["targets"][level]["selections"][role]
 
 
+def campaign_gpu_accounting(
+    primary_state_path: Path, prior_state_paths: tuple[Path, ...] = ()
+) -> dict[str, Any]:
+    attempts = []
+    for role, path in (
+        [("prior_failure", path) for path in prior_state_paths]
+        + [("primary", primary_state_path)]
+    ):
+        state = json.loads(path.read_text(encoding="utf-8"))
+        gpu_hours = float(state["gpu_hours_total"])
+        attempts.append(
+            {
+                "role": role,
+                "path": str(path.resolve()),
+                "sha256": sha256_file(path),
+                "status": state["status"],
+                "h20_hours": gpu_hours,
+            }
+        )
+    total = sum(attempt["h20_hours"] for attempt in attempts)
+    primary = json.loads(primary_state_path.read_text(encoding="utf-8"))
+    hard_cap = float(primary["hard_cap_h20_hours"])
+    return {
+        "attempts": attempts,
+        "aggregate_h20_hours": total,
+        "hard_cap_h20_hours": hard_cap,
+        "invariants": {
+            "costs_nonnegative": all(
+                attempt["h20_hours"] >= 0.0 for attempt in attempts
+            ),
+            "aggregate_below_cap": 0.0 <= total < hard_cap,
+        },
+    }
+
+
 def build_pilot_examples(
     manifest: dict[str, Any], run_root: Path, cutoff_s: float
 ) -> tuple[list[PrefixExample], list[dict[str, Any]], list[str]]:
@@ -125,11 +160,13 @@ def analyze(
     manifest_path: Path,
     model_path: Path,
     run_root: Path,
+    prior_state_paths: tuple[Path, ...] = (),
 ) -> dict[str, Any]:
     manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
     models = json.loads(model_path.read_text(encoding="utf-8"))
     state_path = run_root / "controller-state.json"
     state = json.loads(state_path.read_text(encoding="utf-8"))
+    gpu_accounting = campaign_gpu_accounting(state_path, prior_state_paths)
     cutoff_s = float(models["cutoff_s"])
     threshold = float(models["accept_probability"])
     examples, details, red_flags = build_pilot_examples(manifest, run_root, cutoff_s)
@@ -171,7 +208,7 @@ def analyze(
         detail["actual_timestamped_outcomes"] == 0 for detail in details
     ):
         red_flags.append("no_exact_request_timestamps")
-    if float(state["gpu_hours_total"]) >= float(state["hard_cap_h20_hours"]):
+    if not all(gpu_accounting["invariants"].values()):
         red_flags.append("hard_cap_exceeded")
 
     outcome_errors = outcome_policy["false_accept"] + outcome_policy["false_reject"]
@@ -232,8 +269,8 @@ def analyze(
             "opens_expanded_p2": pilot_pass,
         },
         "gpu": {
-            "actual_h20_hours": state["gpu_hours_total"],
-            "hard_cap_h20_hours": state["hard_cap_h20_hours"],
+            "primary_attempt_h20_hours": state["gpu_hours_total"],
+            **gpu_accounting,
         },
         "sanity": {
             "red_flags": red_flags,
@@ -277,9 +314,15 @@ def main() -> None:
     parser.add_argument("--manifest", type=Path, required=True)
     parser.add_argument("--frozen-models", type=Path, required=True)
     parser.add_argument("--run-root", type=Path, required=True)
+    parser.add_argument("--prior-state", type=Path, action="append", default=[])
     parser.add_argument("--output", type=Path, required=True)
     args = parser.parse_args()
-    result = analyze(args.manifest, args.frozen_models, args.run_root)
+    result = analyze(
+        args.manifest,
+        args.frozen_models,
+        args.run_root,
+        tuple(args.prior_state),
+    )
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
     print(json.dumps({