Harden strong fidelity pilot validation

runs/fidelity-headroom/analyze_strong_pilot.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import argparse
 import json
 import math
+import subprocess
 from pathlib import Path
 from typing import Any
 
@@ -43,6 +44,19 @@ from analyze_strong_baseline import (
 )
 
 
+AITUNER_ROOT = Path(__file__).resolve().parents[2]
+
+
+def git_capture(*arguments: str) -> str:
+    return subprocess.run(
+        ["git", "-C", str(AITUNER_ROOT), *arguments],
+        check=True,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    ).stdout
+
+
 def load_pilot_simulator(
     path: Path,
 ) -> tuple[dict[tuple[str, str], tuple[float, ...]], list[str]]:
@@ -230,6 +244,8 @@ def analyze(
 ) -> dict[str, Any]:
     phase6 = json.loads(phase6_path.read_text(encoding="utf-8"))
     pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8"))
+    pilot_state_path = pilot_run_root / "controller-state.json"
+    pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8"))
     training_examples = build_examples(phase6, phase6_raw_root, 5.0)
     training_simulator_map, training_simulator_sha256 = load_simulator_features(
         training_simulator_root
@@ -277,6 +293,23 @@ def analyze(
         red_flags.append("pilot_single_label")
     if len(set(simulator_pass_rates)) <= 1:
         red_flags.append("pilot_simulator_results_identical")
+    if pilot_state.get("status") != "complete" or int(
+        pilot_state.get("completed_cells", 0)
+    ) != 6:
+        red_flags.append("pilot_campaign_incomplete")
+    if any(detail["actual_timestamped_outcomes"] == 0 for detail in pilot_details):
+        red_flags.append("pilot_no_exact_request_timestamps")
+    all_cell_validations = all(
+        cell.get("validation") is not None
+        and all(cell["validation"]["invariants"].values())
+        for cell in pilot_state.get("cells", {}).values()
+    )
+    if not all_cell_validations:
+        red_flags.append("pilot_cell_validation_failed")
+    if float(pilot_state.get("gpu_hours_total", math.inf)) >= float(
+        pilot_state.get("hard_cap_h20_hours", -math.inf)
+    ):
+        red_flags.append("pilot_hard_cap_exceeded")
     covariate_diagnostics = {
         "sim_plus_outcome": covariate_shift(
             training_examples,
@@ -378,6 +411,12 @@ def analyze(
         ],
         "covariate_shift_diagnostic": covariate_diagnostics,
         "decision": decision,
+        "analysis": {
+            "script": str(Path(__file__).resolve()),
+            "script_sha256": sha256_file(Path(__file__).resolve()),
+            "aituner_git_head": git_capture("rev-parse", "HEAD").strip(),
+            "aituner_git_status_short": git_capture("status", "--short"),
+        },
         "provenance": {
             "phase6_metrics": str(phase6_path.resolve()),
             "phase6_metrics_sha256": sha256_file(phase6_path),
@@ -387,6 +426,8 @@ def analyze(
             "pilot_manifest": str(pilot_manifest_path.resolve()),
             "pilot_manifest_sha256": sha256_file(pilot_manifest_path),
             "pilot_run_root": str(pilot_run_root.resolve()),
+            "pilot_controller_state": str(pilot_state_path.resolve()),
+            "pilot_controller_state_sha256": sha256_file(pilot_state_path),
             "pilot_simulator": str(pilot_simulator_path.resolve()),
             "pilot_simulator_sha256": sha256_file(pilot_simulator_path),
         },
@@ -412,6 +453,11 @@ def analyze(
                     example.completion_time_source in {"exact_monotonic", "none_completed"}
                     for example in pilot_examples
                 ),
+                "all_cell_validations": all_cell_validations,
+                "gpu_cost_nonnegative_below_cap": (
+                    0.0 <= float(pilot_state["gpu_hours_total"])
+                    < float(pilot_state["hard_cap_h20_hours"])
+                ),
             },
         },
     }