Add simulator-aware fidelity pilot audit
This commit is contained in:
292
runs/fidelity-headroom/run_pilot_simulator.py
Normal file
292
runs/fidelity-headroom/run_pilot_simulator.py
Normal file
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run and score the 12 frozen Frontier P1 primary probes, CPU only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_module(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def atomic_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def git_capture(root: Path, *arguments: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", "-C", str(root), *arguments],
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
).stdout
|
||||
|
||||
|
||||
def execute(args: argparse.Namespace) -> dict[str, Any]:
|
||||
prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8"))
|
||||
if prepared["status"] != "PASS":
|
||||
raise RuntimeError("prepared simulator manifest did not pass")
|
||||
driver = load_module(
|
||||
"simfid_execution_driver",
|
||||
args.replayserve_root
|
||||
/ "runs/simfid_s2rb/results/execution_driver.py",
|
||||
)
|
||||
head = git_capture(args.frontier_root, "rev-parse", "HEAD").strip()
|
||||
status_short = git_capture(args.frontier_root, "status", "--short")
|
||||
aituner_root = Path(__file__).resolve().parents[2]
|
||||
aituner_head = git_capture(aituner_root, "rev-parse", "HEAD").strip()
|
||||
aituner_status_short = git_capture(aituner_root, "status", "--short")
|
||||
results = []
|
||||
failures = []
|
||||
gpu_visibility_disabled = True
|
||||
for sequence, entry in enumerate(prepared["entries"]):
|
||||
run_root = args.output / f"{sequence:02d}_{entry['fixture_id']}"
|
||||
scorer_path = run_root / "scorer_output.json"
|
||||
if scorer_path.is_file() and args.resume:
|
||||
scorer = json.loads(scorer_path.read_text(encoding="utf-8"))
|
||||
results.append({**entry, "sequence": sequence, "scorer": scorer, "resumed": True})
|
||||
continue
|
||||
run_root.mkdir(parents=True, exist_ok=True)
|
||||
config_path = Path(entry["config"])
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
fixture_manifest_path = Path(entry["fixture_manifest"])
|
||||
fixture = json.loads(fixture_manifest_path.read_text(encoding="utf-8"))
|
||||
trace_path = Path(entry["frontier_csv"])
|
||||
sidecar_path = Path(entry["sidecar"])
|
||||
metrics_root = run_root / "frontier_metrics"
|
||||
run_id = f"fidelity_p1_frontier_{sequence:02d}_{entry['cell']}_{entry['role']}"
|
||||
knobs = config["frontier"]["knobs"]
|
||||
command = driver.build_command(
|
||||
trace_path=trace_path,
|
||||
metrics_root=metrics_root,
|
||||
run_id=run_id,
|
||||
knobs=knobs,
|
||||
)
|
||||
driver.audit_command(command, knobs)
|
||||
row = {
|
||||
"hook_path": config["calibration"]["hook_path"],
|
||||
"applied_a_tp": config["calibration"]["a_tp"],
|
||||
"sidecar_path": str(sidecar_path),
|
||||
"request_count": int(fixture["request_count"]),
|
||||
"tensor_parallel_size": int(fixture["tensor_parallel_size"]),
|
||||
}
|
||||
environment = driver.environment_for(row)
|
||||
gpu_visibility_disabled = gpu_visibility_disabled and (
|
||||
environment.get("CUDA_VISIBLE_DEVICES") == ""
|
||||
and environment.get("NVIDIA_VISIBLE_DEVICES") == "void"
|
||||
)
|
||||
run_manifest = {
|
||||
"schema": "fidelity-p1-frontier-run-v1",
|
||||
"sequence": sequence,
|
||||
"cell": entry["cell"],
|
||||
"role": entry["role"],
|
||||
"anchor": entry["anchor"],
|
||||
"request_count": entry["selected_count"],
|
||||
"frontier": {
|
||||
"root": str(args.frontier_root.resolve()),
|
||||
"git_head": head,
|
||||
"git_status_short": status_short,
|
||||
},
|
||||
"runner": {
|
||||
"script": str(Path(__file__).resolve()),
|
||||
"script_sha256": sha256_file(Path(__file__).resolve()),
|
||||
"aituner_git_head": aituner_head,
|
||||
"aituner_git_status_short": aituner_status_short,
|
||||
},
|
||||
"inputs": {
|
||||
"config": str(config_path),
|
||||
"config_sha256": sha256_file(config_path),
|
||||
"fixture_manifest": str(fixture_manifest_path),
|
||||
"fixture_manifest_sha256": sha256_file(fixture_manifest_path),
|
||||
"frontier_csv": str(trace_path),
|
||||
"frontier_csv_sha256": sha256_file(trace_path),
|
||||
"sidecar": str(sidecar_path),
|
||||
"sidecar_sha256": sha256_file(sidecar_path),
|
||||
},
|
||||
"environment": {
|
||||
key: environment[key]
|
||||
for key in (
|
||||
"PYTHONPATH",
|
||||
"FRONTIER_EXECUTION_TIME_SCALE",
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"NVIDIA_VISIBLE_DEVICES",
|
||||
"FRONTIER_LOG_LEVEL",
|
||||
)
|
||||
},
|
||||
"command": command,
|
||||
"contains_prompt_text": False,
|
||||
}
|
||||
atomic_json(run_root / "run_manifest.json", run_manifest)
|
||||
start = time.time()
|
||||
with (run_root / "stdout.log").open("w", encoding="utf-8") as stdout, (
|
||||
run_root / "stderr.log"
|
||||
).open("w", encoding="utf-8") as stderr:
|
||||
try:
|
||||
process = subprocess.run(
|
||||
command,
|
||||
cwd=args.frontier_root,
|
||||
env=environment,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
timeout=args.timeout_s,
|
||||
)
|
||||
return_code = int(process.returncode)
|
||||
except subprocess.TimeoutExpired:
|
||||
return_code = 124
|
||||
runtime = time.time() - start
|
||||
if return_code != 0:
|
||||
failure = {
|
||||
"sequence": sequence,
|
||||
"cell": entry["cell"],
|
||||
"role": entry["role"],
|
||||
"return_code": return_code,
|
||||
"runtime_s": runtime,
|
||||
}
|
||||
failures.append(failure)
|
||||
atomic_json(run_root / "failure.json", failure)
|
||||
break
|
||||
system_path, request_path = driver.find_metrics(run_root)
|
||||
scorer = driver.score_trial(row, system_path, request_path)
|
||||
scorer["runtime_s"] = runtime
|
||||
atomic_json(scorer_path, scorer)
|
||||
results.append({**entry, "sequence": sequence, "scorer": scorer, "resumed": False})
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"sequence": sequence,
|
||||
"cell": entry["cell"],
|
||||
"role": entry["role"],
|
||||
"runtime_s": runtime,
|
||||
"sim_pass_rate": scorer["slo"]["pass_rate"],
|
||||
"sim_feasible": scorer["slo"]["feasible"],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
pass_rates = [float(item["scorer"]["slo"]["pass_rate"]) for item in results]
|
||||
throughputs = [
|
||||
float(item["scorer"]["throughput_requests_per_second_per_gpu"])
|
||||
for item in results
|
||||
]
|
||||
runtimes = [float(item["scorer"]["runtime_s"]) for item in results]
|
||||
red_flags = []
|
||||
if failures:
|
||||
red_flags.append("frontier_run_failure")
|
||||
if len(results) != 12:
|
||||
red_flags.append("runs_not_12")
|
||||
if any(not 0.0 <= value <= 1.0 for value in pass_rates):
|
||||
red_flags.append("pass_rate_out_of_range")
|
||||
if any(value <= 0 for value in throughputs):
|
||||
red_flags.append("nonpositive_throughput")
|
||||
result = {
|
||||
"schema": "fidelity-p1-frontier-result-v1",
|
||||
"status": "PASS" if not red_flags else "STOP",
|
||||
"prepared_manifest": str(args.prepared_manifest.resolve()),
|
||||
"prepared_manifest_sha256": sha256_file(args.prepared_manifest),
|
||||
"frontier": {
|
||||
"root": str(args.frontier_root.resolve()),
|
||||
"git_head": head,
|
||||
"git_status_short": status_short,
|
||||
},
|
||||
"runner": {
|
||||
"script": str(Path(__file__).resolve()),
|
||||
"script_sha256": sha256_file(Path(__file__).resolve()),
|
||||
"aituner_git_head": aituner_head,
|
||||
"aituner_git_status_short": aituner_status_short,
|
||||
},
|
||||
"results": results,
|
||||
"failures": failures,
|
||||
"sanity": {
|
||||
"red_flags": red_flags,
|
||||
"n": len(results),
|
||||
"pass_rate": {
|
||||
"n": len(pass_rates),
|
||||
"min": min(pass_rates) if pass_rates else None,
|
||||
"max": max(pass_rates) if pass_rates else None,
|
||||
"distinct_n": len(set(pass_rates)),
|
||||
},
|
||||
"throughput_per_gpu": {
|
||||
"n": len(throughputs),
|
||||
"min": min(throughputs) if throughputs else None,
|
||||
"max": max(throughputs) if throughputs else None,
|
||||
"distinct_n": len(set(throughputs)),
|
||||
},
|
||||
"runtime_s": {
|
||||
"n": len(runtimes),
|
||||
"min": min(runtimes) if runtimes else None,
|
||||
"max": max(runtimes) if runtimes else None,
|
||||
"distinct_n": len(set(runtimes)),
|
||||
},
|
||||
"invariants": {
|
||||
"runs_12": len(results) == 12,
|
||||
"zero_failures": not failures,
|
||||
"ratios_bounded": all(0.0 <= value <= 1.0 for value in pass_rates),
|
||||
"nonnegative_metrics": all(value > 0 for value in throughputs),
|
||||
"per_config_not_identical": len(set(pass_rates)) > 1,
|
||||
"gpu_visibility_disabled": gpu_visibility_disabled,
|
||||
},
|
||||
},
|
||||
}
|
||||
atomic_json(args.output / "metrics.json", result)
|
||||
return result
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
result = argparse.ArgumentParser()
|
||||
result.add_argument("--prepared-manifest", type=Path, required=True)
|
||||
result.add_argument("--output", type=Path, required=True)
|
||||
result.add_argument("--replayserve-root", type=Path, required=True)
|
||||
result.add_argument("--frontier-root", type=Path, required=True)
|
||||
result.add_argument("--timeout-s", type=float, default=900.0)
|
||||
result.add_argument("--resume", action="store_true")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result = execute(parser().parse_args())
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": result["status"],
|
||||
"runs": len(result["results"]),
|
||||
"red_flags": result["sanity"]["red_flags"],
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
if result["status"] != "PASS":
|
||||
raise RuntimeError(result["sanity"]["red_flags"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user