Account for failed fidelity pilot attempts

runs/fidelity-headroom/analyze_pilot.py

@@ -37,6 +37,41 @@ def selection_for(
     return manifest["cells"][cell]["targets"][level]["selections"][role]
 
 
+def campaign_gpu_accounting(
+    primary_state_path: Path, prior_state_paths: tuple[Path, ...] = ()
+) -> dict[str, Any]:
+    attempts = []
+    for role, path in (
+        [("prior_failure", path) for path in prior_state_paths]
+        + [("primary", primary_state_path)]
+    ):
+        state = json.loads(path.read_text(encoding="utf-8"))
+        gpu_hours = float(state["gpu_hours_total"])
+        attempts.append(
+            {
+                "role": role,
+                "path": str(path.resolve()),
+                "sha256": sha256_file(path),
+                "status": state["status"],
+                "h20_hours": gpu_hours,
+            }
+        )
+    total = sum(attempt["h20_hours"] for attempt in attempts)
+    primary = json.loads(primary_state_path.read_text(encoding="utf-8"))
+    hard_cap = float(primary["hard_cap_h20_hours"])
+    return {
+        "attempts": attempts,
+        "aggregate_h20_hours": total,
+        "hard_cap_h20_hours": hard_cap,
+        "invariants": {
+            "costs_nonnegative": all(
+                attempt["h20_hours"] >= 0.0 for attempt in attempts
+            ),
+            "aggregate_below_cap": 0.0 <= total < hard_cap,
+        },
+    }
+
+
 def build_pilot_examples(
     manifest: dict[str, Any], run_root: Path, cutoff_s: float
 ) -> tuple[list[PrefixExample], list[dict[str, Any]], list[str]]:
@@ -125,11 +160,13 @@ def analyze(
     manifest_path: Path,
     model_path: Path,
     run_root: Path,
+    prior_state_paths: tuple[Path, ...] = (),
 ) -> dict[str, Any]:
     manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
     models = json.loads(model_path.read_text(encoding="utf-8"))
     state_path = run_root / "controller-state.json"
     state = json.loads(state_path.read_text(encoding="utf-8"))
+    gpu_accounting = campaign_gpu_accounting(state_path, prior_state_paths)
     cutoff_s = float(models["cutoff_s"])
     threshold = float(models["accept_probability"])
     examples, details, red_flags = build_pilot_examples(manifest, run_root, cutoff_s)
@@ -171,7 +208,7 @@ def analyze(
         detail["actual_timestamped_outcomes"] == 0 for detail in details
     ):
         red_flags.append("no_exact_request_timestamps")
-    if float(state["gpu_hours_total"]) >= float(state["hard_cap_h20_hours"]):
+    if not all(gpu_accounting["invariants"].values()):
         red_flags.append("hard_cap_exceeded")
 
     outcome_errors = outcome_policy["false_accept"] + outcome_policy["false_reject"]
@@ -232,8 +269,8 @@ def analyze(
             "opens_expanded_p2": pilot_pass,
         },
         "gpu": {
-            "actual_h20_hours": state["gpu_hours_total"],
+            "primary_attempt_h20_hours": state["gpu_hours_total"],
-            "hard_cap_h20_hours": state["hard_cap_h20_hours"],
+            **gpu_accounting,
         },
         "sanity": {
             "red_flags": red_flags,
@@ -277,9 +314,15 @@ def main() -> None:
     parser.add_argument("--manifest", type=Path, required=True)
     parser.add_argument("--frozen-models", type=Path, required=True)
     parser.add_argument("--run-root", type=Path, required=True)
+    parser.add_argument("--prior-state", type=Path, action="append", default=[])
     parser.add_argument("--output", type=Path, required=True)
     args = parser.parse_args()
-    result = analyze(args.manifest, args.frozen_models, args.run_root)
+    result = analyze(
+        args.manifest,
+        args.frozen_models,
+        args.run_root,
+        tuple(args.prior_state),
+    )
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
     print(json.dumps({

runs/fidelity-headroom/analyze_strong_pilot.py

@@ -27,7 +27,7 @@ from analyze_existing import (
     _mcnemar_exact_p,
     _sigmoid,
 )
-from analyze_pilot import build_pilot_examples
+from analyze_pilot import build_pilot_examples, campaign_gpu_accounting
 from analyze_prefixes import (
     INSTRUMENTATION_FEATURES,
     OUTCOME_FEATURES,
@@ -241,11 +241,15 @@ def analyze(
     pilot_manifest_path: Path,
     pilot_run_root: Path,
     pilot_simulator_path: Path,
+    prior_state_paths: tuple[Path, ...] = (),
 ) -> dict[str, Any]:
     phase6 = json.loads(phase6_path.read_text(encoding="utf-8"))
     pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8"))
     pilot_state_path = pilot_run_root / "controller-state.json"
     pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8"))
+    gpu_accounting = campaign_gpu_accounting(
+        pilot_state_path, prior_state_paths
+    )
     training_examples = build_examples(phase6, phase6_raw_root, 5.0)
     training_simulator_map, training_simulator_sha256 = load_simulator_features(
         training_simulator_root
@@ -306,9 +310,7 @@ def analyze(
     )
     if not all_cell_validations:
         red_flags.append("pilot_cell_validation_failed")
-    if float(pilot_state.get("gpu_hours_total", math.inf)) >= float(
+    if not all(gpu_accounting["invariants"].values()):
-        pilot_state.get("hard_cap_h20_hours", -math.inf)
-    ):
         red_flags.append("pilot_hard_cap_exceeded")
     covariate_diagnostics = {
         "sim_plus_outcome": covariate_shift(
@@ -411,6 +413,10 @@ def analyze(
         ],
         "covariate_shift_diagnostic": covariate_diagnostics,
         "decision": decision,
+        "gpu": {
+            "primary_attempt_h20_hours": pilot_state["gpu_hours_total"],
+            **gpu_accounting,
+        },
         "analysis": {
             "script": str(Path(__file__).resolve()),
             "script_sha256": sha256_file(Path(__file__).resolve()),
@@ -455,8 +461,7 @@ def analyze(
                 ),
                 "all_cell_validations": all_cell_validations,
                 "gpu_cost_nonnegative_below_cap": (
-                    0.0 <= float(pilot_state["gpu_hours_total"])
+                    all(gpu_accounting["invariants"].values())
-                    < float(pilot_state["hard_cap_h20_hours"])
                 ),
             },
         },
@@ -471,6 +476,7 @@ def main() -> None:
     parser.add_argument("--pilot-manifest", type=Path, required=True)
     parser.add_argument("--pilot-run-root", type=Path, required=True)
     parser.add_argument("--pilot-simulator", type=Path, required=True)
+    parser.add_argument("--prior-state", type=Path, action="append", default=[])
     parser.add_argument("--output", type=Path, required=True)
     args = parser.parse_args()
     result = analyze(
@@ -480,6 +486,7 @@ def main() -> None:
         args.pilot_manifest,
         args.pilot_run_root,
         args.pilot_simulator,
+        tuple(args.prior_state),
     )
     args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
     print(

runs/fidelity-headroom/test_pilot_tools.py

@@ -14,6 +14,7 @@ sys.path.insert(0, str(HERE))
 
 import pilot_controller as controller  # noqa: E402
 import prepare_pilot as prepare  # noqa: E402
+from analyze_pilot import campaign_gpu_accounting  # noqa: E402
 
 
 @dataclass
@@ -69,6 +70,32 @@ def main() -> None:
             assert row["fidelity_pilot_band"] == role
             assert abs(float(row["sampling_u"]) - 0.5) < 1e-12
 
+        prior = root / "prior-state.json"
+        primary = root / "primary-state.json"
+        prior.write_text(
+            json.dumps(
+                {
+                    "status": "failed",
+                    "gpu_hours_total": 0.02,
+                    "hard_cap_h20_hours": 3.5,
+                }
+            ),
+            encoding="utf-8",
+        )
+        primary.write_text(
+            json.dumps(
+                {
+                    "status": "complete",
+                    "gpu_hours_total": 1.5,
+                    "hard_cap_h20_hours": 3.5,
+                }
+            ),
+            encoding="utf-8",
+        )
+        accounting = campaign_gpu_accounting(primary, (prior,))
+        assert math.isclose(accounting["aggregate_h20_hours"], 1.52)
+        assert all(accounting["invariants"].values())
+
     assert len(controller.ORDER) == 6
     assert set(controller.ORDER) == set(prepare.CELLS)
     assert math.isclose(