gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Account for failed fidelity pilot attempts

Browse Source
This commit is contained in:
Gahow Wang
2026-07-14 13:41:46 +08:00
parent 1f32ae217e
commit 2261818994
3 changed files with 87 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
runs/fidelity-headroom/analyze_pilot.py
View File

@@ -37,6 +37,41 @@ def selection_for(
return manifest["cells"][cell]["targets"][level]["selections"][role]
def campaign_gpu_accounting(
primary_state_path: Path, prior_state_paths: tuple[Path, ...] = ()
) -> dict[str, Any]:
attempts = []
for role, path in (
[("prior_failure", path) for path in prior_state_paths]
+ [("primary", primary_state_path)]
):
state = json.loads(path.read_text(encoding="utf-8"))
gpu_hours = float(state["gpu_hours_total"])
attempts.append(
{
"role": role,
"path": str(path.resolve()),
"sha256": sha256_file(path),
"status": state["status"],
"h20_hours": gpu_hours,
}
)
total = sum(attempt["h20_hours"] for attempt in attempts)
primary = json.loads(primary_state_path.read_text(encoding="utf-8"))
hard_cap = float(primary["hard_cap_h20_hours"])
return {
"attempts": attempts,
"aggregate_h20_hours": total,
"hard_cap_h20_hours": hard_cap,
"invariants": {
"costs_nonnegative": all(
attempt["h20_hours"] >= 0.0 for attempt in attempts
),
"aggregate_below_cap": 0.0 <= total < hard_cap,
},
}
def build_pilot_examples(
manifest: dict[str, Any], run_root: Path, cutoff_s: float
) -> tuple[list[PrefixExample], list[dict[str, Any]], list[str]]:
@@ -125,11 +160,13 @@ def analyze(
manifest_path: Path,
model_path: Path,
run_root: Path,
prior_state_paths: tuple[Path, ...] = (),
) -> dict[str, Any]:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
models = json.loads(model_path.read_text(encoding="utf-8"))
state_path = run_root / "controller-state.json"
state = json.loads(state_path.read_text(encoding="utf-8"))
gpu_accounting = campaign_gpu_accounting(state_path, prior_state_paths)
cutoff_s = float(models["cutoff_s"])
threshold = float(models["accept_probability"])
examples, details, red_flags = build_pilot_examples(manifest, run_root, cutoff_s)
@@ -171,7 +208,7 @@ def analyze(
detail["actual_timestamped_outcomes"] == 0 for detail in details
):
red_flags.append("no_exact_request_timestamps")
if float(state["gpu_hours_total"]) >= float(state["hard_cap_h20_hours"]):
if not all(gpu_accounting["invariants"].values()):
red_flags.append("hard_cap_exceeded")
outcome_errors = outcome_policy["false_accept"] + outcome_policy["false_reject"]
@@ -232,8 +269,8 @@ def analyze(
"opens_expanded_p2": pilot_pass,
},
"gpu": {
"actual_h20_hours": state["gpu_hours_total"],
"hard_cap_h20_hours": state["hard_cap_h20_hours"],
"primary_attempt_h20_hours": state["gpu_hours_total"],
**gpu_accounting,
},
"sanity": {
"red_flags": red_flags,
@@ -277,9 +314,15 @@ def main() -> None:
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--frozen-models", type=Path, required=True)
parser.add_argument("--run-root", type=Path, required=True)
parser.add_argument("--prior-state", type=Path, action="append", default=[])
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = analyze(args.manifest, args.frozen_models, args.run_root)
result = analyze(
args.manifest,
args.frozen_models,
args.run_root,
tuple(args.prior_state),
)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
print(json.dumps({

19
runs/fidelity-headroom/analyze_strong_pilot.py
View File

@@ -27,7 +27,7 @@ from analyze_existing import (
_mcnemar_exact_p,
_sigmoid,
)
from analyze_pilot import build_pilot_examples
from analyze_pilot import build_pilot_examples, campaign_gpu_accounting
from analyze_prefixes import (
INSTRUMENTATION_FEATURES,
OUTCOME_FEATURES,
@@ -241,11 +241,15 @@ def analyze(
pilot_manifest_path: Path,
pilot_run_root: Path,
pilot_simulator_path: Path,
prior_state_paths: tuple[Path, ...] = (),
) -> dict[str, Any]:
phase6 = json.loads(phase6_path.read_text(encoding="utf-8"))
pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8"))
pilot_state_path = pilot_run_root / "controller-state.json"
pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8"))
gpu_accounting = campaign_gpu_accounting(
pilot_state_path, prior_state_paths
)
training_examples = build_examples(phase6, phase6_raw_root, 5.0)
training_simulator_map, training_simulator_sha256 = load_simulator_features(
training_simulator_root
@@ -306,9 +310,7 @@ def analyze(
)
if not all_cell_validations:
red_flags.append("pilot_cell_validation_failed")
if float(pilot_state.get("gpu_hours_total", math.inf)) >= float(
pilot_state.get("hard_cap_h20_hours", -math.inf)
):
if not all(gpu_accounting["invariants"].values()):
red_flags.append("pilot_hard_cap_exceeded")
covariate_diagnostics = {
"sim_plus_outcome": covariate_shift(
@@ -411,6 +413,10 @@ def analyze(
],
"covariate_shift_diagnostic": covariate_diagnostics,
"decision": decision,
"gpu": {
"primary_attempt_h20_hours": pilot_state["gpu_hours_total"],
**gpu_accounting,
},
"analysis": {
"script": str(Path(__file__).resolve()),
"script_sha256": sha256_file(Path(__file__).resolve()),
@@ -455,8 +461,7 @@ def analyze(
),
"all_cell_validations": all_cell_validations,
"gpu_cost_nonnegative_below_cap": (
0.0 <= float(pilot_state["gpu_hours_total"])
< float(pilot_state["hard_cap_h20_hours"])
all(gpu_accounting["invariants"].values())
),
},
},
@@ -471,6 +476,7 @@ def main() -> None:
parser.add_argument("--pilot-manifest", type=Path, required=True)
parser.add_argument("--pilot-run-root", type=Path, required=True)
parser.add_argument("--pilot-simulator", type=Path, required=True)
parser.add_argument("--prior-state", type=Path, action="append", default=[])
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = analyze(
@@ -480,6 +486,7 @@ def main() -> None:
args.pilot_manifest,
args.pilot_run_root,
args.pilot_simulator,
tuple(args.prior_state),
)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
print(

27
runs/fidelity-headroom/test_pilot_tools.py
View File

@@ -14,6 +14,7 @@ sys.path.insert(0, str(HERE))
import pilot_controller as controller # noqa: E402
import prepare_pilot as prepare # noqa: E402
from analyze_pilot import campaign_gpu_accounting # noqa: E402
@dataclass
@@ -69,6 +70,32 @@ def main() -> None:
assert row["fidelity_pilot_band"] == role
assert abs(float(row["sampling_u"]) - 0.5) < 1e-12
prior = root / "prior-state.json"
primary = root / "primary-state.json"
prior.write_text(
json.dumps(
{
"status": "failed",
"gpu_hours_total": 0.02,
"hard_cap_h20_hours": 3.5,
}
),
encoding="utf-8",
)
primary.write_text(
json.dumps(
{
"status": "complete",
"gpu_hours_total": 1.5,
"hard_cap_h20_hours": 3.5,
}
),
encoding="utf-8",
)
accounting = campaign_gpu_accounting(primary, (prior,))
assert math.isclose(accounting["aggregate_h20_hours"], 1.52)
assert all(accounting["invariants"].values())
assert len(controller.ORDER) == 6
assert set(controller.ORDER) == set(prepare.CELLS)
assert math.isclose(