gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Account for failed fidelity pilot attempts

Browse Source
This commit is contained in:
Gahow Wang
2026-07-14 13:41:46 +08:00
parent 1f32ae217e
commit 2261818994
3 changed files with 87 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
runs/fidelity-headroom/analyze_strong_pilot.py
View File

@@ -27,7 +27,7 @@ from analyze_existing import (
_mcnemar_exact_p,
_sigmoid,
)
from analyze_pilot import build_pilot_examples
from analyze_pilot import build_pilot_examples, campaign_gpu_accounting
from analyze_prefixes import (
INSTRUMENTATION_FEATURES,
OUTCOME_FEATURES,
@@ -241,11 +241,15 @@ def analyze(
pilot_manifest_path: Path,
pilot_run_root: Path,
pilot_simulator_path: Path,
prior_state_paths: tuple[Path, ...] = (),
) -> dict[str, Any]:
phase6 = json.loads(phase6_path.read_text(encoding="utf-8"))
pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8"))
pilot_state_path = pilot_run_root / "controller-state.json"
pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8"))
gpu_accounting = campaign_gpu_accounting(
pilot_state_path, prior_state_paths
)
training_examples = build_examples(phase6, phase6_raw_root, 5.0)
training_simulator_map, training_simulator_sha256 = load_simulator_features(
training_simulator_root
@@ -306,9 +310,7 @@ def analyze(
)
if not all_cell_validations:
red_flags.append("pilot_cell_validation_failed")
if float(pilot_state.get("gpu_hours_total", math.inf)) >= float(
pilot_state.get("hard_cap_h20_hours", -math.inf)
):
if not all(gpu_accounting["invariants"].values()):
red_flags.append("pilot_hard_cap_exceeded")
covariate_diagnostics = {
"sim_plus_outcome": covariate_shift(
@@ -411,6 +413,10 @@ def analyze(
],
"covariate_shift_diagnostic": covariate_diagnostics,
"decision": decision,
"gpu": {
"primary_attempt_h20_hours": pilot_state["gpu_hours_total"],
**gpu_accounting,
},
"analysis": {
"script": str(Path(__file__).resolve()),
"script_sha256": sha256_file(Path(__file__).resolve()),
@@ -455,8 +461,7 @@ def analyze(
),
"all_cell_validations": all_cell_validations,
"gpu_cost_nonnegative_below_cap": (
0.0 <= float(pilot_state["gpu_hours_total"])
< float(pilot_state["hard_cap_h20_hours"])
all(gpu_accounting["invariants"].values())
),
},
},
@@ -471,6 +476,7 @@ def main() -> None:
parser.add_argument("--pilot-manifest", type=Path, required=True)
parser.add_argument("--pilot-run-root", type=Path, required=True)
parser.add_argument("--pilot-simulator", type=Path, required=True)
parser.add_argument("--prior-state", type=Path, action="append", default=[])
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = analyze(
@@ -480,6 +486,7 @@ def main() -> None:
args.pilot_manifest,
args.pilot_run_root,
args.pilot_simulator,
tuple(args.prior_state),
)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
print(