Evaluate vLLM 0.20 profiles against Frontier

runs/fidelity-headroom/run_pilot_simulator.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Run and score the 12 frozen Frontier P1 primary probes, CPU only."""
+"""Run and score a frozen Frontier probe manifest, CPU only."""
 
 from __future__ import annotations
 
@@ -54,9 +54,16 @@ def git_capture(root: Path, *arguments: str) -> str:
 
 
 def execute(args: argparse.Namespace) -> dict[str, Any]:
+    args.output = args.output.resolve()
     prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8"))
     if prepared["status"] != "PASS":
         raise RuntimeError("prepared simulator manifest did not pass")
+    expected_runs = int(prepared.get("expected_runs", len(prepared["entries"])))
+    if expected_runs != len(prepared["entries"]):
+        raise RuntimeError(
+            f"prepared manifest expected {expected_runs} runs but contains "
+            f"{len(prepared['entries'])} entries"
+        )
     driver = load_module(
         "simfid_execution_driver",
         args.replayserve_root
@@ -71,7 +78,7 @@ def execute(args: argparse.Namespace) -> dict[str, Any]:
     failures = []
     gpu_visibility_disabled = True
     for sequence, entry in enumerate(prepared["entries"]):
-        run_root = args.output / f"{sequence:02d}_{entry['fixture_id']}"
+        run_root = args.output / f"{sequence:03d}_{entry['fixture_id']}"
         scorer_path = run_root / "scorer_output.json"
         if scorer_path.is_file() and args.resume:
             scorer = json.loads(scorer_path.read_text(encoding="utf-8"))
@@ -205,8 +212,8 @@ def execute(args: argparse.Namespace) -> dict[str, Any]:
     red_flags = []
     if failures:
         red_flags.append("frontier_run_failure")
-    if len(results) != 12:
-        red_flags.append("runs_not_12")
+    if len(results) != expected_runs:
+        red_flags.append("runs_not_expected")
     if any(not 0.0 <= value <= 1.0 for value in pass_rates):
         red_flags.append("pass_rate_out_of_range")
     if any(value <= 0 for value in throughputs):
@@ -251,7 +258,8 @@ def execute(args: argparse.Namespace) -> dict[str, Any]:
                 "distinct_n": len(set(runtimes)),
             },
             "invariants": {
-                "runs_12": len(results) == 12,
+                "runs_expected": len(results) == expected_runs,
+                "expected_runs": expected_runs,
                 "zero_failures": not failures,
                 "ratios_bounded": all(0.0 <= value <= 1.0 for value in pass_rates),
                 "nonnegative_metrics": all(value > 0 for value in throughputs),