aituner/runs/telemetry-residual/run_frontier_state_campaign.py

#!/usr/bin/env python3
"""Run the frozen 12-fixture P1 Frontier state-detail campaign, CPU only."""

from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path
from types import SimpleNamespace
from typing import Any


HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))

import run_frontier_state as state_runner  # noqa: E402


def atomic_json(path: Path, payload: Any) -> None:
    state_runner.atomic_json(path, payload)


def scorer_without_runtime(value: dict[str, Any]) -> dict[str, Any]:
    result = dict(value)
    result.pop("runtime_s", None)
    return result


def execute(args: argparse.Namespace) -> dict[str, Any]:
    prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8"))
    committed = json.loads(args.committed_results.read_text(encoding="utf-8"))
    if prepared.get("status") != "PASS" or committed.get("status") != "PASS":
        raise RuntimeError("prepared or committed simulator evidence did not pass")
    committed_by_key = {
        (row["cell"], row["role"]): scorer_without_runtime(row["scorer"])
        for row in committed["results"]
    }
    entries = prepared["entries"]
    if len(entries) != 12 or len(committed_by_key) != 12:
        raise ValueError("P1 state campaign requires exactly 12 fixtures")
    args.output.mkdir(parents=True, exist_ok=True)
    results = []
    campaign_start = time.monotonic()
    for index, entry in enumerate(entries, start=1):
        key = (entry["cell"], entry["role"])
        output = args.output / f"{entry['cell']}_{entry['role']}"
        result_path = output / "result.json"
        if args.resume and result_path.is_file():
            result = json.loads(result_path.read_text(encoding="utf-8"))
            if result.get("status") != "PASS":
                raise RuntimeError(f"cannot resume failed state replay: {result_path}")
            resumed = True
        else:
            if output.exists() and any(output.iterdir()):
                raise FileExistsError(f"non-empty state replay output: {output}")
            print(
                f"RUN {index:02d}/12 {entry['cell']}/{entry['role']}",
                flush=True,
            )
            result = state_runner.execute(
                SimpleNamespace(
                    prepared_manifest=args.prepared_manifest,
                    output=output,
                    replayserve_root=args.replayserve_root,
                    frontier_root=args.frontier_root,
                    cell=entry["cell"],
                    role=entry["role"],
                    timeout_s=args.timeout_s,
                )
            )
            resumed = False
        observed_scorer = json.loads(
            (output / "scorer_output.json").read_text(encoding="utf-8")
        )
        exact_scorer_match = scorer_without_runtime(observed_scorer) == committed_by_key[key]
        if not exact_scorer_match:
            raise ValueError(f"state-output replay changed the committed scorer: {key}")
        results.append(
            {
                "cell": entry["cell"],
                "role": entry["role"],
                "runtime_s": result["runtime_s"],
                "request_rows": result["sanity"]["request_rows"],
                "batch_rows": result["sanity"]["batch_rows"],
                "ledger_rows": result["sanity"]["ledger_rows"],
                "artifact_bytes": sum(result["bytes"].values()),
                "exact_committed_scorer_match": exact_scorer_match,
                "resumed": resumed,
                "result": str(result_path.resolve()),
            }
        )
        print(
            f"DONE {index:02d}/12 {entry['cell']}/{entry['role']} "
            f"runtime={result['runtime_s']:.3f}s batches={result['sanity']['batch_rows']}",
            flush=True,
        )
    runtimes = [float(row["runtime_s"]) for row in results]
    batches = [int(row["batch_rows"]) for row in results]
    bytes_values = [int(row["artifact_bytes"]) for row in results]
    red_flags = []
    if len(results) != 12:
        red_flags.append("runs_not_12")
    if not all(row["exact_committed_scorer_match"] for row in results):
        red_flags.append("committed_scorer_mismatch")
    if any(value <= 0 for value in batches):
        red_flags.append("empty_batch_output")
    result = {
        "schema": "telemetry-residual-frontier-state-campaign-v1",
        "status": "PASS" if not red_flags else "STOP",
        "prepared_manifest": str(args.prepared_manifest.resolve()),
        "committed_results": str(args.committed_results.resolve()),
        "campaign_elapsed_s": time.monotonic() - campaign_start,
        "results": results,
        "red_flags": red_flags,
        "sanity": {
            "n": len(results),
            "runtime_s": state_runner.numeric(runtimes),
            "batch_rows": state_runner.numeric(batches),
            "artifact_bytes": state_runner.numeric(bytes_values),
            "invariants": {
                "runs_12": len(results) == 12,
                "zero_failures": not red_flags,
                "exact_committed_scorers": all(
                    row["exact_committed_scorer_match"] for row in results
                ),
                "nonnegative_counts": all(value > 0 for value in batches),
                "per_config_not_identical": len(set(batches)) > 1,
                "gpu_visibility_disabled": True,
            },
        },
    }
    atomic_json(args.output / "campaign-metrics.json", result)
    if result["status"] != "PASS":
        raise RuntimeError(red_flags)
    return result


def parser() -> argparse.ArgumentParser:
    result = argparse.ArgumentParser()
    result.add_argument("--prepared-manifest", type=Path, required=True)
    result.add_argument("--committed-results", type=Path, required=True)
    result.add_argument("--output", type=Path, required=True)
    result.add_argument("--replayserve-root", type=Path, required=True)
    result.add_argument("--frontier-root", type=Path, required=True)
    result.add_argument("--timeout-s", type=float, default=300.0)
    result.add_argument("--resume", action="store_true")
    return result


def main() -> None:
    result = execute(parser().parse_args())
    print(
        json.dumps(
            {
                "status": result["status"],
                "runs": len(result["results"]),
                "elapsed_s": result["campaign_elapsed_s"],
                "sanity": result["sanity"],
                "red_flags": result["red_flags"],
            },
            sort_keys=True,
        )
    )


if __name__ == "__main__":
    main()