#!/usr/bin/env python3 """Run the frozen 12-fixture P1 Frontier state-detail campaign, CPU only.""" from __future__ import annotations import argparse import json import sys import time from pathlib import Path from types import SimpleNamespace from typing import Any HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import run_frontier_state as state_runner # noqa: E402 def atomic_json(path: Path, payload: Any) -> None: state_runner.atomic_json(path, payload) def scorer_without_runtime(value: dict[str, Any]) -> dict[str, Any]: result = dict(value) result.pop("runtime_s", None) return result def execute(args: argparse.Namespace) -> dict[str, Any]: prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8")) committed = json.loads(args.committed_results.read_text(encoding="utf-8")) if prepared.get("status") != "PASS" or committed.get("status") != "PASS": raise RuntimeError("prepared or committed simulator evidence did not pass") committed_by_key = { (row["cell"], row["role"]): scorer_without_runtime(row["scorer"]) for row in committed["results"] } entries = prepared["entries"] if len(entries) != 12 or len(committed_by_key) != 12: raise ValueError("P1 state campaign requires exactly 12 fixtures") args.output.mkdir(parents=True, exist_ok=True) results = [] campaign_start = time.monotonic() for index, entry in enumerate(entries, start=1): key = (entry["cell"], entry["role"]) output = args.output / f"{entry['cell']}_{entry['role']}" result_path = output / "result.json" if args.resume and result_path.is_file(): result = json.loads(result_path.read_text(encoding="utf-8")) if result.get("status") != "PASS": raise RuntimeError(f"cannot resume failed state replay: {result_path}") resumed = True else: if output.exists() and any(output.iterdir()): raise FileExistsError(f"non-empty state replay output: {output}") print( f"RUN {index:02d}/12 {entry['cell']}/{entry['role']}", flush=True, ) result = state_runner.execute( SimpleNamespace( prepared_manifest=args.prepared_manifest, output=output, replayserve_root=args.replayserve_root, frontier_root=args.frontier_root, cell=entry["cell"], role=entry["role"], timeout_s=args.timeout_s, ) ) resumed = False observed_scorer = json.loads( (output / "scorer_output.json").read_text(encoding="utf-8") ) exact_scorer_match = scorer_without_runtime(observed_scorer) == committed_by_key[key] if not exact_scorer_match: raise ValueError(f"state-output replay changed the committed scorer: {key}") results.append( { "cell": entry["cell"], "role": entry["role"], "runtime_s": result["runtime_s"], "request_rows": result["sanity"]["request_rows"], "batch_rows": result["sanity"]["batch_rows"], "ledger_rows": result["sanity"]["ledger_rows"], "artifact_bytes": sum(result["bytes"].values()), "exact_committed_scorer_match": exact_scorer_match, "resumed": resumed, "result": str(result_path.resolve()), } ) print( f"DONE {index:02d}/12 {entry['cell']}/{entry['role']} " f"runtime={result['runtime_s']:.3f}s batches={result['sanity']['batch_rows']}", flush=True, ) runtimes = [float(row["runtime_s"]) for row in results] batches = [int(row["batch_rows"]) for row in results] bytes_values = [int(row["artifact_bytes"]) for row in results] red_flags = [] if len(results) != 12: red_flags.append("runs_not_12") if not all(row["exact_committed_scorer_match"] for row in results): red_flags.append("committed_scorer_mismatch") if any(value <= 0 for value in batches): red_flags.append("empty_batch_output") result = { "schema": "telemetry-residual-frontier-state-campaign-v1", "status": "PASS" if not red_flags else "STOP", "prepared_manifest": str(args.prepared_manifest.resolve()), "committed_results": str(args.committed_results.resolve()), "campaign_elapsed_s": time.monotonic() - campaign_start, "results": results, "red_flags": red_flags, "sanity": { "n": len(results), "runtime_s": state_runner.numeric(runtimes), "batch_rows": state_runner.numeric(batches), "artifact_bytes": state_runner.numeric(bytes_values), "invariants": { "runs_12": len(results) == 12, "zero_failures": not red_flags, "exact_committed_scorers": all( row["exact_committed_scorer_match"] for row in results ), "nonnegative_counts": all(value > 0 for value in batches), "per_config_not_identical": len(set(batches)) > 1, "gpu_visibility_disabled": True, }, }, } atomic_json(args.output / "campaign-metrics.json", result) if result["status"] != "PASS": raise RuntimeError(red_flags) return result def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser() result.add_argument("--prepared-manifest", type=Path, required=True) result.add_argument("--committed-results", type=Path, required=True) result.add_argument("--output", type=Path, required=True) result.add_argument("--replayserve-root", type=Path, required=True) result.add_argument("--frontier-root", type=Path, required=True) result.add_argument("--timeout-s", type=float, default=300.0) result.add_argument("--resume", action="store_true") return result def main() -> None: result = execute(parser().parse_args()) print( json.dumps( { "status": result["status"], "runs": len(result["results"]), "elapsed_s": result["campaign_elapsed_s"], "sanity": result["sanity"], "red_flags": result["red_flags"], }, sort_keys=True, ) ) if __name__ == "__main__": main()