diff --git a/runs/fidelity-headroom/analyze_pilot.py b/runs/fidelity-headroom/analyze_pilot.py index 3797e18..b1293b3 100644 --- a/runs/fidelity-headroom/analyze_pilot.py +++ b/runs/fidelity-headroom/analyze_pilot.py @@ -37,6 +37,41 @@ def selection_for( return manifest["cells"][cell]["targets"][level]["selections"][role] +def campaign_gpu_accounting( + primary_state_path: Path, prior_state_paths: tuple[Path, ...] = () +) -> dict[str, Any]: + attempts = [] + for role, path in ( + [("prior_failure", path) for path in prior_state_paths] + + [("primary", primary_state_path)] + ): + state = json.loads(path.read_text(encoding="utf-8")) + gpu_hours = float(state["gpu_hours_total"]) + attempts.append( + { + "role": role, + "path": str(path.resolve()), + "sha256": sha256_file(path), + "status": state["status"], + "h20_hours": gpu_hours, + } + ) + total = sum(attempt["h20_hours"] for attempt in attempts) + primary = json.loads(primary_state_path.read_text(encoding="utf-8")) + hard_cap = float(primary["hard_cap_h20_hours"]) + return { + "attempts": attempts, + "aggregate_h20_hours": total, + "hard_cap_h20_hours": hard_cap, + "invariants": { + "costs_nonnegative": all( + attempt["h20_hours"] >= 0.0 for attempt in attempts + ), + "aggregate_below_cap": 0.0 <= total < hard_cap, + }, + } + + def build_pilot_examples( manifest: dict[str, Any], run_root: Path, cutoff_s: float ) -> tuple[list[PrefixExample], list[dict[str, Any]], list[str]]: @@ -125,11 +160,13 @@ def analyze( manifest_path: Path, model_path: Path, run_root: Path, + prior_state_paths: tuple[Path, ...] = (), ) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) models = json.loads(model_path.read_text(encoding="utf-8")) state_path = run_root / "controller-state.json" state = json.loads(state_path.read_text(encoding="utf-8")) + gpu_accounting = campaign_gpu_accounting(state_path, prior_state_paths) cutoff_s = float(models["cutoff_s"]) threshold = float(models["accept_probability"]) examples, details, red_flags = build_pilot_examples(manifest, run_root, cutoff_s) @@ -171,7 +208,7 @@ def analyze( detail["actual_timestamped_outcomes"] == 0 for detail in details ): red_flags.append("no_exact_request_timestamps") - if float(state["gpu_hours_total"]) >= float(state["hard_cap_h20_hours"]): + if not all(gpu_accounting["invariants"].values()): red_flags.append("hard_cap_exceeded") outcome_errors = outcome_policy["false_accept"] + outcome_policy["false_reject"] @@ -232,8 +269,8 @@ def analyze( "opens_expanded_p2": pilot_pass, }, "gpu": { - "actual_h20_hours": state["gpu_hours_total"], - "hard_cap_h20_hours": state["hard_cap_h20_hours"], + "primary_attempt_h20_hours": state["gpu_hours_total"], + **gpu_accounting, }, "sanity": { "red_flags": red_flags, @@ -277,9 +314,15 @@ def main() -> None: parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--frozen-models", type=Path, required=True) parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--prior-state", type=Path, action="append", default=[]) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() - result = analyze(args.manifest, args.frozen_models, args.run_root) + result = analyze( + args.manifest, + args.frozen_models, + args.run_root, + tuple(args.prior_state), + ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") print(json.dumps({ diff --git a/runs/fidelity-headroom/analyze_strong_pilot.py b/runs/fidelity-headroom/analyze_strong_pilot.py index 19752a7..05cebca 100644 --- a/runs/fidelity-headroom/analyze_strong_pilot.py +++ b/runs/fidelity-headroom/analyze_strong_pilot.py @@ -27,7 +27,7 @@ from analyze_existing import ( _mcnemar_exact_p, _sigmoid, ) -from analyze_pilot import build_pilot_examples +from analyze_pilot import build_pilot_examples, campaign_gpu_accounting from analyze_prefixes import ( INSTRUMENTATION_FEATURES, OUTCOME_FEATURES, @@ -241,11 +241,15 @@ def analyze( pilot_manifest_path: Path, pilot_run_root: Path, pilot_simulator_path: Path, + prior_state_paths: tuple[Path, ...] = (), ) -> dict[str, Any]: phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) pilot_state_path = pilot_run_root / "controller-state.json" pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8")) + gpu_accounting = campaign_gpu_accounting( + pilot_state_path, prior_state_paths + ) training_examples = build_examples(phase6, phase6_raw_root, 5.0) training_simulator_map, training_simulator_sha256 = load_simulator_features( training_simulator_root @@ -306,9 +310,7 @@ def analyze( ) if not all_cell_validations: red_flags.append("pilot_cell_validation_failed") - if float(pilot_state.get("gpu_hours_total", math.inf)) >= float( - pilot_state.get("hard_cap_h20_hours", -math.inf) - ): + if not all(gpu_accounting["invariants"].values()): red_flags.append("pilot_hard_cap_exceeded") covariate_diagnostics = { "sim_plus_outcome": covariate_shift( @@ -411,6 +413,10 @@ def analyze( ], "covariate_shift_diagnostic": covariate_diagnostics, "decision": decision, + "gpu": { + "primary_attempt_h20_hours": pilot_state["gpu_hours_total"], + **gpu_accounting, + }, "analysis": { "script": str(Path(__file__).resolve()), "script_sha256": sha256_file(Path(__file__).resolve()), @@ -455,8 +461,7 @@ def analyze( ), "all_cell_validations": all_cell_validations, "gpu_cost_nonnegative_below_cap": ( - 0.0 <= float(pilot_state["gpu_hours_total"]) - < float(pilot_state["hard_cap_h20_hours"]) + all(gpu_accounting["invariants"].values()) ), }, }, @@ -471,6 +476,7 @@ def main() -> None: parser.add_argument("--pilot-manifest", type=Path, required=True) parser.add_argument("--pilot-run-root", type=Path, required=True) parser.add_argument("--pilot-simulator", type=Path, required=True) + parser.add_argument("--prior-state", type=Path, action="append", default=[]) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() result = analyze( @@ -480,6 +486,7 @@ def main() -> None: args.pilot_manifest, args.pilot_run_root, args.pilot_simulator, + tuple(args.prior_state), ) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") print( diff --git a/runs/fidelity-headroom/test_pilot_tools.py b/runs/fidelity-headroom/test_pilot_tools.py index f5a674f..46d9cba 100644 --- a/runs/fidelity-headroom/test_pilot_tools.py +++ b/runs/fidelity-headroom/test_pilot_tools.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(HERE)) import pilot_controller as controller # noqa: E402 import prepare_pilot as prepare # noqa: E402 +from analyze_pilot import campaign_gpu_accounting # noqa: E402 @dataclass @@ -69,6 +70,32 @@ def main() -> None: assert row["fidelity_pilot_band"] == role assert abs(float(row["sampling_u"]) - 0.5) < 1e-12 + prior = root / "prior-state.json" + primary = root / "primary-state.json" + prior.write_text( + json.dumps( + { + "status": "failed", + "gpu_hours_total": 0.02, + "hard_cap_h20_hours": 3.5, + } + ), + encoding="utf-8", + ) + primary.write_text( + json.dumps( + { + "status": "complete", + "gpu_hours_total": 1.5, + "hard_cap_h20_hours": 3.5, + } + ), + encoding="utf-8", + ) + accounting = campaign_gpu_accounting(primary, (prior,)) + assert math.isclose(accounting["aggregate_h20_hours"], 1.52) + assert all(accounting["invariants"].values()) + assert len(controller.ORDER) == 6 assert set(controller.ORDER) == set(prepare.CELLS) assert math.isclose(