#!/usr/bin/env python3 """Detached, resumable Phase-5 four-way primary/control controller.""" from __future__ import annotations import argparse import datetime as dt import hashlib import json import math import os import re import shlex import shutil import subprocess import time from pathlib import Path from typing import Any import opprof_phase3_matrix as m WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712") RUN_ROOT = WORKDIR / "runs/phase5" PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase5-private/manifests") P3_PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests") MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B") SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0") VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv") CLIENT = WORKDIR / "scripts/opprof_phase5_client.py" P3_CLIENT = WORKDIR / "scripts/opprof_phase3_client.py" STATE = RUN_ROOT / "controller-state.json" RATE = 0.4725 CAPTURE_SIZES = ( 1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, ) m.WORKDIR = WORKDIR m.RUN_ROOT = RUN_ROOT m.PRIVATE = PRIVATE m.MODEL = MODEL m.SOURCE = SOURCE m.VENV = VENV m.CLIENT = CLIENT m.STATE = STATE m.GPU_HOUR_LIMIT = 6.0 m.PRIOR_GPU_HOURS = 0.0 m.CONFIGS = { "C00": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []}, "A2": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []}, "A4": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []}, } def sha256_file(path: Path) -> str: return m.sha256_file(path) def arm_name(pattern: str) -> str: return pattern.split("-r", 1)[0] def manifest_for(pattern: str, burnin: bool) -> Path: if burnin or pattern.startswith("background"): return P3_PRIVATE / "P06.jsonl" if pattern.startswith("control-P03"): return P3_PRIVATE / "P03.jsonl" if pattern.startswith("control-P04"): return P3_PRIVATE / "P04.jsonl" arm = arm_name(pattern) return PRIVATE / ("P10-base.jsonl" if arm in {"base", "A2", "A4"} else f"P10-{arm}.jsonl") def saturation_result_for(pattern: str) -> Path: if pattern.startswith("control-P03"): cell = "P03-C00" elif pattern.startswith("control-P04"): cell = "P04-C00" else: cell = "P10-C00" return WORKDIR / f"runs/phase3/primary/{cell}/saturation/client/result.json" def drain_budget(_pattern: str) -> int: return 600 m.drain_budget = drain_budget def server_command(assignment: m.Assignment, port: int, _trace_dir: Path) -> list[str]: arm = arm_name(assignment.cell.pattern) command = [ "taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/vllm"), "serve", str(MODEL), "--host", "127.0.0.1", "--port", str(port), "--tensor-parallel-size", "1", "--enable-chunked-prefill", ] if arm != "A4" and assignment.cell.config != "A4": command.append("--enable-prefix-caching") command.extend(("--shutdown-timeout", "600")) if arm == "A2" or assignment.cell.config == "A2": command.extend(("--cudagraph-capture-sizes", *map(str, CAPTURE_SIZES))) return command def client_command( assignment: m.Assignment, port: int, run_dir: Path, _load_point: str, _profile: bool, burnin: bool, _saturation_result: Path | None, ) -> list[str]: pattern = assignment.cell.pattern common = [ "taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/python"), str(CLIENT), "run", "--manifest", str(manifest_for(pattern, burnin)), "--base-url", f"http://127.0.0.1:{port}", "--model", str(MODEL), "--max-concurrency", "256", "--ignore-eos", "--temperature", "0", "--workload-seed", "20260712", "--server-seed", "20260712", "--result-dir", str(run_dir / "client"), ] if burnin: return common + [ "--load-point", "saturation", "--request-rate", "inf", "--warmup-seconds", "0", "--clean-segment-seconds", "20", "--num-clean-segments", "3", "--post-clean-seconds", "0", "--drain-timeout-seconds", "600", ] if pattern.startswith("background"): return common + [ "--load-point", "saturation", "--request-rate", "inf", "--warmup-seconds", "60", "--clean-segment-seconds", "80", "--num-clean-segments", "3", "--post-clean-seconds", "0", "--drain-timeout-seconds", "600", ] if pattern.startswith("control-"): return common + [ "--load-point", "moderate", "--saturation-result", str(saturation_result_for(pattern)), "--rate-fraction", "0.60", "--warmup-seconds", "60", "--clean-segment-seconds", "80", "--num-clean-segments", "3", "--post-clean-seconds", "0", "--drain-timeout-seconds", "600", ] return common + [ "--load-point", "moderate", "--fixed-request-rate", str(RATE), "--warmup-seconds", "60", "--clean-segment-seconds", "80", "--num-clean-segments", "3", "--post-clean-seconds", "0", "--drain-timeout-seconds", "600", ] m.server_command = server_command m.client_command = client_command _ORIGINAL_VALIDATE_CLIENT = m.validate_client def _log_event_time(line: str, t0_wall_ns: int) -> float | None: match = re.search(r"INFO\s+(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})", line) if match is None: return None month, day, hour, minute, second = map(int, match.groups()) year = dt.datetime.fromtimestamp(t0_wall_ns / 1e9, tz=dt.timezone.utc).year stamp = dt.datetime(year, month, day, hour, minute, second, tzinfo=dt.timezone.utc) return stamp.timestamp() def cold_start_gate(run_dir: Path) -> dict[str, Any]: result = json.loads((run_dir / "client/result.json").read_text()) requests = [ json.loads(line) for line in (run_dir / "client/requests.jsonl").read_text().splitlines() ] warm = [ request for request in requests if request["success"] and 0 <= float(request["completed_s"]) < 60 ] warm_long = [request for request in warm if int(request["input_tokens"]) >= 8192] t0_mono_ns = int(result["t0_mono_ns"]) stream = next((run_dir / "opprof").glob("*.jsonl")) warm_descriptors: set[tuple[str, int]] = set() clean_descriptors: set[tuple[str, int]] = set() for line in stream.read_text().splitlines(): item = json.loads(line) if "step_index" not in item or not item.get("model_executed"): continue graph = item["cudagraph"] if not graph.get("hit"): continue relative_s = (int(item["submit_mono_ns"]) - t0_mono_ns) / 1e9 descriptor = (str(graph["runtime_mode"]), int(graph["bucket_tokens"])) if 0 <= relative_s < 60: warm_descriptors.add(descriptor) elif 60 <= relative_s < 300: clean_descriptors.add(descriptor) log_lines = (run_dir / "server.log").read_text(errors="replace").splitlines() ready_indices = [ index for index, line in enumerate(log_lines) if "Application startup complete" in line ] if len(ready_indices) != 1: raise RuntimeError(f"expected one server ready marker: {run_dir}: {ready_indices}") ready_index = ready_indices[0] event_pattern = re.compile( r"torch\.compile took|Directly load AOT compilation|\bCompiling\b|Capturing CUDA graphs", re.IGNORECASE, ) events = [] clean_boundary_wall_s = int(result["t0_wall_ns"]) / 1e9 + 60 clean_end_wall_s = int(result["t0_wall_ns"]) / 1e9 + 300 event_gate = True clean_events = 0 for index, line in enumerate(log_lines): if not event_pattern.search(line): continue timestamp = _log_event_time(line, int(result["t0_wall_ns"])) phase = "pre-ready" if index < ready_index else "post-ready" if index >= ready_index: if timestamp is None: event_gate = False phase = "post-ready-unparseable" elif timestamp >= clean_boundary_wall_s: event_gate = False phase = "clean" if timestamp < clean_end_wall_s else "post-clean" if timestamp < clean_end_wall_s: clean_events += 1 else: phase = "warmup" events.append( { "line": index + 1, "phase": phase, "timestamp_s": timestamp, "message_prefix": line[:200], } ) config_match = re.search( r"cudagraph_capture_sizes['\"]?:\s*\[([^\]]+)\]", "\n".join(log_lines[:ready_index]), ) startup_sizes = ( {int(value.strip()) for value in config_match.group(1).split(",")} if config_match else set() ) startup_modes = set() for line in log_lines[:ready_index]: if "Capturing CUDA graphs" not in line: continue if "PIECEWISE" in line: startup_modes.add("PIECEWISE") if "FULL" in line: startup_modes.add("FULL") uncovered = sorted( descriptor for descriptor in clean_descriptors if descriptor[0] not in startup_modes or descriptor[1] not in startup_sizes ) passed = ( event_gate and len(warm) >= 16 and len(warm_long) >= 1 and startup_modes == {"FULL", "PIECEWISE"} and bool(startup_sizes) and not uncovered and clean_events == 0 ) return { "amendment": "A-P5-1", "passed": passed, "warmup_completions": len(warm), "warmup_long_completions": len(warm_long), "warmup_long_min_tokens": min( (int(request["input_tokens"]) for request in warm_long), default=None ), "server_ready_line": ready_index + 1, "compile_capture_events": events, "event_gate_passed": event_gate, "clean_capture_events": clean_events, "startup_capture_sizes": sorted(startup_sizes), "startup_capture_modes": sorted(startup_modes), "warmup_descriptors": [list(item) for item in sorted(warm_descriptors)], "clean_descriptors": [list(item) for item in sorted(clean_descriptors)], "uncovered_clean_descriptors": [list(item) for item in uncovered], "invariants": { "events_preclean": event_gate, "warmup_completions_ge_16": len(warm) >= 16, "warmup_long_completion": len(warm_long) >= 1, "startup_modes_complete": startup_modes == {"FULL", "PIECEWISE"}, "startup_sizes_present": bool(startup_sizes), "clean_descriptors_covered": not uncovered, "zero_clean_capture_events": clean_events == 0, }, } def validate_rate_client(run_dir: Path) -> dict[str, Any]: result = json.loads((run_dir / "client/result.json").read_text()) sanity = json.loads((run_dir / "client/sanity.json").read_text()) requests = [ json.loads(line) for line in (run_dir / "client/requests.jsonl").read_text().splitlines() ] failed_sanity = [ key for key, value in sanity["invariants"].items() if not value and key != "drain_within_timeout" ] failure_summary = m.summarize_request_failures( requests, float(result["clean"]["start_s"]), float(result["clean"]["end_s"]) ) cold = cold_start_gate(run_dir) quarantined = float(result["drain_seconds"]) > 600 invariants = { "client_sanity": not failed_sanity, "clean_duration": math.isclose(float(result["clean"]["duration_s"]), 240.0), "clean_failures_zero": result["clean"]["failed"] == 0 and failure_summary["clean_failed"] == 0, "failed_records_accounted": result["failed_records"] == failure_summary["failed"], "manifest_no_wrap": not result["manifest_wrapped"] and not result["manifest_exhausted"], "warmup_cold_start_gate": cold["passed"], "profile_count": len(result["profiles"]) == 0, "drain_re_adjudicated": not quarantined, } non_drain = {key: value for key, value in invariants.items() if key != "drain_re_adjudicated"} if not all(non_drain.values()): raise RuntimeError( f"A-P5-1 rate-client invariant failure: {run_dir}: " f"invariants={invariants}; failed_sanity={failed_sanity}; cold={cold}" ) return { "result": result, "sanity": sanity, "request_count": len(requests), "warmup_completions": cold["warmup_completions"], "warmup_required": 16, "warmup_gate_branch": "A-P5-1-cold-start", "warmup_stability": None, "cold_start_gate": cold, "drain_budget_seconds": 600, "drain_quarantined": quarantined, "excluded_window_failures": failure_summary["excluded"], "excluded_window_failure_kinds": failure_summary["excluded_kinds"], "invariants": invariants, } def validate_run( entry: dict[str, Any], profile: bool, burnin: bool, allow_missing_traces: bool = False ) -> dict[str, Any]: del profile, allow_missing_traces pattern = entry["assignment"].cell.pattern if burnin or pattern.startswith("background"): validation_pattern = "P06" elif pattern.startswith("control-P03"): validation_pattern = "P03" elif pattern.startswith("control-P04"): validation_pattern = "P04" else: validation_pattern = "P10" if burnin or pattern.startswith("background"): client = _ORIGINAL_VALIDATE_CLIENT( entry["run_dir"], validation_pattern, False, burnin ) else: client = validate_rate_client(entry["run_dir"]) layer1 = m.validate_layer1(entry["run_dir"]) log = (entry["run_dir"] / "server.log").read_text(errors="replace") invariants = { "triton_moe": "Using TRITON Unquantized MoE backend" in log, "chunked_mbt": "Chunked prefill is enabled with max_num_batched_tokens=8192" in log, "tp1": "tensor_parallel_size=1" in log, "drain_shutdown": "mode=drain timeout=600s" in log, "a2_sizes": entry["assignment"].cell.config != "A2" or all(str(size) in log for size in (3, 5, 6, 7)), } if not all(invariants.values()): raise RuntimeError(f"server invariant failure: {entry['run_id']}: {invariants}") forbidden = re.compile(r'"(?:prompt|messages|content|text)"\s*:') for path in ( entry["run_dir"] / "client/requests.jsonl", entry["run_dir"] / "client/result.json", Path(layer1["stream"]), ): if forbidden.search(path.read_text(errors="replace")): raise RuntimeError(f"private text leaked: {path}") summary = { "schema": 1, "run_id": entry["run_id"], "pattern": pattern, "config": entry["assignment"].cell.config, "gpus": entry["assignment"].gpus, "client": client, "layer1": layer1, "traces": [], "missing_trace_files": 0, "layer2_missing_after_controller_cleanup": False, "drain_quarantined": client["drain_quarantined"], "server_invariants": invariants, } m.atomic_json(entry["run_dir"] / "run-complete.json", summary) return summary m.validate_run = validate_run def manifests() -> dict[str, Any]: result = {} for name in ("base", "A1", "A3"): path = PRIVATE / f"P10-{name}.jsonl" summary = json.loads(path.with_suffix(path.suffix + ".summary.json").read_text()) if summary["rows"] != 142 or summary["sha256"] != sha256_file(path): raise RuntimeError(f"manifest verification failed: {name}") result[name] = {"path": str(path), "sha256": summary["sha256"]} return result def fingerprint() -> dict[str, Any]: return { "source_commit": subprocess.check_output( ["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True ).strip(), "source_tree": subprocess.check_output( ["git", "-C", str(SOURCE), "rev-parse", "HEAD^{tree}"], text=True ).strip(), "client_sha256": sha256_file(CLIENT), "controller_sha256": sha256_file(Path(__file__)), "p3_client_sha256": sha256_file(P3_CLIENT), "p3_matrix_sha256": sha256_file(Path(m.__file__).resolve()), "manifests": manifests(), "capture_sizes": list(CAPTURE_SIZES), "rate": RATE, } def load_state(resume: bool) -> dict[str, Any]: if STATE.exists(): if not resume: raise RuntimeError("controller state exists; use --resume") return json.loads(STATE.read_text()) return { "schema": 1, "status": "created", "created_at": time.time(), "controller_pid": os.getpid(), "gpu_hours_total": 0.0, "gpu_hours_this_stage": 0.0, "completed_measured_runs": 0, "completed_burnins": 0, "drain_quarantined_runs": 0, "clean_window_failures": 0, "missing_trace_files": 0, "stages": {}, "fingerprint": {}, } def save_state(state: dict[str, Any]) -> None: state["controller_pid"] = os.getpid() state["updated_at"] = time.time() m.atomic_json(STATE, state) m.save_state = save_state def ensure_provenance() -> None: destination = RUN_ROOT / "provenance" destination.mkdir(parents=True, exist_ok=True) sources = [CLIENT, Path(__file__).resolve(), Path(m.__file__).resolve(), Path(m.common.__file__).resolve()] hashes = {} for source in sources: target = destination / source.name digest = sha256_file(source) if target.exists() and sha256_file(target) != digest: target = destination / f"{source.stem}.{digest[:12]}{source.suffix}" if target.exists() and sha256_file(target) != digest: raise RuntimeError(f"content-addressed provenance mismatch: {target}") if not target.exists(): shutil.copy2(source, target) hashes[target.name] = digest m.atomic_json(destination / "sha256.json", hashes) def primary_cells() -> list[m.Cell]: config = {"base": "C00", "A1": "C00", "A2": "A2", "A3": "C00", "A4": "A4"} items = [m.Cell(f"{arm}-r{replicate}", config[arm]) for arm in config for replicate in range(1, 4)] return sorted( items, key=lambda cell: hashlib.sha256( f"20260715:{cell.pattern.rsplit('-r',1)[1]}:{arm_name(cell.pattern)}".encode() ).hexdigest(), ) def pack_unique(items: list[m.Cell], prefix: str) -> list[list[m.Assignment]]: remaining = list(items) waves: list[list[m.Assignment]] = [] wave_index = 0 while remaining: selected: list[m.Cell] = [] used: set[str] = set() for cell in list(remaining): key = arm_name(cell.pattern) if key in used: continue selected.append(cell) used.add(key) remaining.remove(cell) if len(selected) == 4: break while len(selected) < 4: selected.append(m.Cell(f"background-{prefix}-{wave_index}-{len(selected)}", "C00")) assignments = [] for slot, cell in enumerate(selected): gpu = (slot + wave_index) % 4 assignments.append(m.Assignment(cell, (gpu,))) waves.append(assignments) wave_index += 1 return waves def pack_primary(items: list[m.Cell]) -> list[list[m.Assignment]]: """Pack SHA-ordered cells as 4/4/4/3 without duplicate arms per wave.""" capacities = (4, 4, 4, 3) waves: list[list[m.Cell]] = [[] for _ in capacities] def place(index: int) -> bool: if index == len(items): return all(len(wave) == capacity for wave, capacity in zip(waves, capacities, strict=True)) cell = items[index] arm = arm_name(cell.pattern) for wave_index, capacity in enumerate(capacities): if len(waves[wave_index]) >= capacity: continue if any(arm_name(existing.pattern) == arm for existing in waves[wave_index]): continue waves[wave_index].append(cell) if place(index + 1): return True waves[wave_index].pop() return False if not place(0): raise RuntimeError("cannot pack frozen primary assignments into 4/4/4/3") result: list[list[m.Assignment]] = [] for wave_index, cells in enumerate(waves): if len(cells) == 3: cells.append(m.Cell("background-primary-final", "C00")) result.append( [ m.Assignment(cell, ((slot + wave_index) % 4,)) for slot, cell in enumerate(cells) ] ) return result def execute_primary(resume: bool, amendment_a_p5_1: bool = False) -> None: RUN_ROOT.mkdir(parents=True, exist_ok=True) state = load_state(resume) if resume: m.cleanup_recorded(state) current = fingerprint() if state["fingerprint"] and state["fingerprint"] != current: failure = str(state.get("stages", {}).get("primary-01", {}).get("failure", "")) if not ( amendment_a_p5_1 and state.get("status") == "failed" and "warmup" in failure.lower() ): raise RuntimeError("resume fingerprint differs from frozen Phase-5 plan") state.setdefault("amendments", {})["A-P5-1"] = { "approved": True, "applied_at": time.time(), "reason": "replace rate-following drift gate with cold-start gates", "prior_fingerprint": state["fingerprint"], "replacement_fingerprint": current, "retained_gpu_hours": state["gpu_hours_total"], "burnins_reused": state["completed_burnins"], } state["fingerprint"] = current state["status"] = "amended_resume_A-P5-1" save_state(state) state["fingerprint"] = current state["status"] = "running_primary" save_state(state) ensure_provenance() burnins = [ m.Assignment(m.Cell("burnin-C00", "C00"), (0,)), m.Assignment(m.Cell("burnin-A2", "A2"), (1,)), m.Assignment(m.Cell("burnin-A4", "A4"), (2,)), ] m.run_stage(state, "burnins", burnins, "saturation", profile=False, burnin=True) waves = pack_primary(primary_cells()) for index, wave in enumerate(waves, 1): m.run_stage(state, f"primary-{index:02d}", wave, "moderate", profile=False) primary = [ path for path in (RUN_ROOT / "primary").glob("*-r*-*/moderate/run-complete.json") if "background" not in str(path) ] if len(primary) != 15: raise RuntimeError(f"primary completion mismatch: {len(primary)} != 15") state["primary_runs"] = 15 state["background_runs"] = state["completed_measured_runs"] - 15 state["status"] = "primary_complete" state["completed_at"] = time.time() save_state(state) def execute_controls(resume: bool) -> None: state = load_state(resume) m.cleanup_recorded(state) if state.get("fingerprint") != fingerprint(): raise RuntimeError("control resume fingerprint mismatch") cells = [ m.Cell(f"control-{pattern}-r{replicate}", "C00") for pattern in ("P03", "P04") for replicate in range(1, 4) ] for index, wave in enumerate(pack_unique(cells, "controls"), 1): m.run_stage(state, f"controls-{index:02d}", wave, "moderate", profile=False) state["status"] = "controls_complete" state["controls_completed_at"] = time.time() save_state(state) def plan() -> dict[str, Any]: return { "schema": 1, "primary_runs": 15, "burnins": 3, "waves": [[{"cell": a.cell.cell_id, "gpus": a.gpus} for a in wave] for wave in pack_primary(primary_cells())], "rate": RATE, "clean_seconds": 240, "drain_seconds": 600, "gpu_hour_limit": 6.0, } def main() -> None: parser = argparse.ArgumentParser() sub = parser.add_subparsers(dest="command", required=True) for name in ("primary", "controls"): item = sub.add_parser(name) item.add_argument("--resume", action="store_true") if name == "primary": item.add_argument("--amendment-a-p5-1", action="store_true") sub.add_parser("plan") sub.add_parser("status") args = parser.parse_args() if args.command == "primary": execute_primary(args.resume, args.amendment_a_p5_1) elif args.command == "controls": execute_controls(args.resume) elif args.command == "plan": print(json.dumps(plan(), sort_keys=True, indent=2)) else: print(STATE.read_text() if STATE.exists() else '{"status":"absent"}') if __name__ == "__main__": main()