diff --git a/runs/fidelity-headroom/analyze_strong_pilot.py b/runs/fidelity-headroom/analyze_strong_pilot.py index de19dce..19752a7 100644 --- a/runs/fidelity-headroom/analyze_strong_pilot.py +++ b/runs/fidelity-headroom/analyze_strong_pilot.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse import json import math +import subprocess from pathlib import Path from typing import Any @@ -43,6 +44,19 @@ from analyze_strong_baseline import ( ) +AITUNER_ROOT = Path(__file__).resolve().parents[2] + + +def git_capture(*arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(AITUNER_ROOT), *arguments], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + def load_pilot_simulator( path: Path, ) -> tuple[dict[tuple[str, str], tuple[float, ...]], list[str]]: @@ -230,6 +244,8 @@ def analyze( ) -> dict[str, Any]: phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) + pilot_state_path = pilot_run_root / "controller-state.json" + pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8")) training_examples = build_examples(phase6, phase6_raw_root, 5.0) training_simulator_map, training_simulator_sha256 = load_simulator_features( training_simulator_root @@ -277,6 +293,23 @@ def analyze( red_flags.append("pilot_single_label") if len(set(simulator_pass_rates)) <= 1: red_flags.append("pilot_simulator_results_identical") + if pilot_state.get("status") != "complete" or int( + pilot_state.get("completed_cells", 0) + ) != 6: + red_flags.append("pilot_campaign_incomplete") + if any(detail["actual_timestamped_outcomes"] == 0 for detail in pilot_details): + red_flags.append("pilot_no_exact_request_timestamps") + all_cell_validations = all( + cell.get("validation") is not None + and all(cell["validation"]["invariants"].values()) + for cell in pilot_state.get("cells", {}).values() + ) + if not all_cell_validations: + red_flags.append("pilot_cell_validation_failed") + if float(pilot_state.get("gpu_hours_total", math.inf)) >= float( + pilot_state.get("hard_cap_h20_hours", -math.inf) + ): + red_flags.append("pilot_hard_cap_exceeded") covariate_diagnostics = { "sim_plus_outcome": covariate_shift( training_examples, @@ -378,6 +411,12 @@ def analyze( ], "covariate_shift_diagnostic": covariate_diagnostics, "decision": decision, + "analysis": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": git_capture("rev-parse", "HEAD").strip(), + "aituner_git_status_short": git_capture("status", "--short"), + }, "provenance": { "phase6_metrics": str(phase6_path.resolve()), "phase6_metrics_sha256": sha256_file(phase6_path), @@ -387,6 +426,8 @@ def analyze( "pilot_manifest": str(pilot_manifest_path.resolve()), "pilot_manifest_sha256": sha256_file(pilot_manifest_path), "pilot_run_root": str(pilot_run_root.resolve()), + "pilot_controller_state": str(pilot_state_path.resolve()), + "pilot_controller_state_sha256": sha256_file(pilot_state_path), "pilot_simulator": str(pilot_simulator_path.resolve()), "pilot_simulator_sha256": sha256_file(pilot_simulator_path), }, @@ -412,6 +453,11 @@ def analyze( example.completion_time_source in {"exact_monotonic", "none_completed"} for example in pilot_examples ), + "all_cell_validations": all_cell_validations, + "gpu_cost_nonnegative_below_cap": ( + 0.0 <= float(pilot_state["gpu_hours_total"]) + < float(pilot_state["hard_cap_h20_hours"]) + ), }, }, }