Gahow Wang
d5b276180d
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
671 lines
24 KiB
Python
671 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Detached, resumable Phase-5 four-way primary/control controller."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import opprof_phase3_matrix as m
|
|
|
|
|
|
WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
|
|
RUN_ROOT = WORKDIR / "runs/phase5"
|
|
PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase5-private/manifests")
|
|
P3_PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests")
|
|
MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
|
|
SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
|
|
VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
|
|
CLIENT = WORKDIR / "scripts/opprof_phase5_client.py"
|
|
P3_CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
|
|
STATE = RUN_ROOT / "controller-state.json"
|
|
RATE = 0.4725
|
|
CAPTURE_SIZES = (
|
|
1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88,
|
|
96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192,
|
|
200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
|
|
352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
|
|
)
|
|
|
|
|
|
m.WORKDIR = WORKDIR
|
|
m.RUN_ROOT = RUN_ROOT
|
|
m.PRIVATE = PRIVATE
|
|
m.MODEL = MODEL
|
|
m.SOURCE = SOURCE
|
|
m.VENV = VENV
|
|
m.CLIENT = CLIENT
|
|
m.STATE = STATE
|
|
m.GPU_HOUR_LIMIT = 6.0
|
|
m.PRIOR_GPU_HOURS = 0.0
|
|
m.CONFIGS = {
|
|
"C00": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
|
|
"A2": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
|
|
"A4": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
|
|
}
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return m.sha256_file(path)
|
|
|
|
|
|
def arm_name(pattern: str) -> str:
|
|
return pattern.split("-r", 1)[0]
|
|
|
|
|
|
def manifest_for(pattern: str, burnin: bool) -> Path:
|
|
if burnin or pattern.startswith("background"):
|
|
return P3_PRIVATE / "P06.jsonl"
|
|
if pattern.startswith("control-P03"):
|
|
return P3_PRIVATE / "P03.jsonl"
|
|
if pattern.startswith("control-P04"):
|
|
return P3_PRIVATE / "P04.jsonl"
|
|
arm = arm_name(pattern)
|
|
return PRIVATE / ("P10-base.jsonl" if arm in {"base", "A2", "A4"} else f"P10-{arm}.jsonl")
|
|
|
|
|
|
def saturation_result_for(pattern: str) -> Path:
|
|
if pattern.startswith("control-P03"):
|
|
cell = "P03-C00"
|
|
elif pattern.startswith("control-P04"):
|
|
cell = "P04-C00"
|
|
else:
|
|
cell = "P10-C00"
|
|
return WORKDIR / f"runs/phase3/primary/{cell}/saturation/client/result.json"
|
|
|
|
|
|
def drain_budget(_pattern: str) -> int:
|
|
return 600
|
|
|
|
|
|
m.drain_budget = drain_budget
|
|
|
|
|
|
def server_command(assignment: m.Assignment, port: int, _trace_dir: Path) -> list[str]:
|
|
arm = arm_name(assignment.cell.pattern)
|
|
command = [
|
|
"taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/vllm"),
|
|
"serve", str(MODEL), "--host", "127.0.0.1", "--port", str(port),
|
|
"--tensor-parallel-size", "1", "--enable-chunked-prefill",
|
|
]
|
|
if arm != "A4" and assignment.cell.config != "A4":
|
|
command.append("--enable-prefix-caching")
|
|
command.extend(("--shutdown-timeout", "600"))
|
|
if arm == "A2" or assignment.cell.config == "A2":
|
|
command.extend(("--cudagraph-capture-sizes", *map(str, CAPTURE_SIZES)))
|
|
return command
|
|
|
|
|
|
def client_command(
|
|
assignment: m.Assignment,
|
|
port: int,
|
|
run_dir: Path,
|
|
_load_point: str,
|
|
_profile: bool,
|
|
burnin: bool,
|
|
_saturation_result: Path | None,
|
|
) -> list[str]:
|
|
pattern = assignment.cell.pattern
|
|
common = [
|
|
"taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/python"),
|
|
str(CLIENT), "run", "--manifest", str(manifest_for(pattern, burnin)),
|
|
"--base-url", f"http://127.0.0.1:{port}", "--model", str(MODEL),
|
|
"--max-concurrency", "256", "--ignore-eos", "--temperature", "0",
|
|
"--workload-seed", "20260712", "--server-seed", "20260712",
|
|
"--result-dir", str(run_dir / "client"),
|
|
]
|
|
if burnin:
|
|
return common + [
|
|
"--load-point", "saturation", "--request-rate", "inf",
|
|
"--warmup-seconds", "0", "--clean-segment-seconds", "20",
|
|
"--num-clean-segments", "3", "--post-clean-seconds", "0",
|
|
"--drain-timeout-seconds", "600",
|
|
]
|
|
if pattern.startswith("background"):
|
|
return common + [
|
|
"--load-point", "saturation", "--request-rate", "inf",
|
|
"--warmup-seconds", "60", "--clean-segment-seconds", "80",
|
|
"--num-clean-segments", "3", "--post-clean-seconds", "0",
|
|
"--drain-timeout-seconds", "600",
|
|
]
|
|
if pattern.startswith("control-"):
|
|
return common + [
|
|
"--load-point", "moderate", "--saturation-result",
|
|
str(saturation_result_for(pattern)), "--rate-fraction", "0.60",
|
|
"--warmup-seconds", "60", "--clean-segment-seconds", "80",
|
|
"--num-clean-segments", "3", "--post-clean-seconds", "0",
|
|
"--drain-timeout-seconds", "600",
|
|
]
|
|
return common + [
|
|
"--load-point", "moderate", "--fixed-request-rate", str(RATE),
|
|
"--warmup-seconds", "60", "--clean-segment-seconds", "80",
|
|
"--num-clean-segments", "3", "--post-clean-seconds", "0",
|
|
"--drain-timeout-seconds", "600",
|
|
]
|
|
|
|
|
|
m.server_command = server_command
|
|
m.client_command = client_command
|
|
|
|
|
|
_ORIGINAL_VALIDATE_CLIENT = m.validate_client
|
|
|
|
|
|
def _log_event_time(line: str, t0_wall_ns: int) -> float | None:
|
|
match = re.search(r"INFO\s+(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})", line)
|
|
if match is None:
|
|
return None
|
|
month, day, hour, minute, second = map(int, match.groups())
|
|
year = dt.datetime.fromtimestamp(t0_wall_ns / 1e9, tz=dt.timezone.utc).year
|
|
stamp = dt.datetime(year, month, day, hour, minute, second, tzinfo=dt.timezone.utc)
|
|
return stamp.timestamp()
|
|
|
|
|
|
def cold_start_gate(run_dir: Path) -> dict[str, Any]:
|
|
result = json.loads((run_dir / "client/result.json").read_text())
|
|
requests = [
|
|
json.loads(line)
|
|
for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
|
|
]
|
|
warm = [
|
|
request for request in requests
|
|
if request["success"] and 0 <= float(request["completed_s"]) < 60
|
|
]
|
|
warm_long = [request for request in warm if int(request["input_tokens"]) >= 8192]
|
|
t0_mono_ns = int(result["t0_mono_ns"])
|
|
stream = next((run_dir / "opprof").glob("*.jsonl"))
|
|
warm_descriptors: set[tuple[str, int]] = set()
|
|
clean_descriptors: set[tuple[str, int]] = set()
|
|
for line in stream.read_text().splitlines():
|
|
item = json.loads(line)
|
|
if "step_index" not in item or not item.get("model_executed"):
|
|
continue
|
|
graph = item["cudagraph"]
|
|
if not graph.get("hit"):
|
|
continue
|
|
relative_s = (int(item["submit_mono_ns"]) - t0_mono_ns) / 1e9
|
|
descriptor = (str(graph["runtime_mode"]), int(graph["bucket_tokens"]))
|
|
if 0 <= relative_s < 60:
|
|
warm_descriptors.add(descriptor)
|
|
elif 60 <= relative_s < 300:
|
|
clean_descriptors.add(descriptor)
|
|
|
|
log_lines = (run_dir / "server.log").read_text(errors="replace").splitlines()
|
|
ready_indices = [
|
|
index for index, line in enumerate(log_lines)
|
|
if "Application startup complete" in line
|
|
]
|
|
if len(ready_indices) != 1:
|
|
raise RuntimeError(f"expected one server ready marker: {run_dir}: {ready_indices}")
|
|
ready_index = ready_indices[0]
|
|
event_pattern = re.compile(
|
|
r"torch\.compile took|Directly load AOT compilation|\bCompiling\b|Capturing CUDA graphs",
|
|
re.IGNORECASE,
|
|
)
|
|
events = []
|
|
clean_boundary_wall_s = int(result["t0_wall_ns"]) / 1e9 + 60
|
|
clean_end_wall_s = int(result["t0_wall_ns"]) / 1e9 + 300
|
|
event_gate = True
|
|
clean_events = 0
|
|
for index, line in enumerate(log_lines):
|
|
if not event_pattern.search(line):
|
|
continue
|
|
timestamp = _log_event_time(line, int(result["t0_wall_ns"]))
|
|
phase = "pre-ready" if index < ready_index else "post-ready"
|
|
if index >= ready_index:
|
|
if timestamp is None:
|
|
event_gate = False
|
|
phase = "post-ready-unparseable"
|
|
elif timestamp >= clean_boundary_wall_s:
|
|
event_gate = False
|
|
phase = "clean" if timestamp < clean_end_wall_s else "post-clean"
|
|
if timestamp < clean_end_wall_s:
|
|
clean_events += 1
|
|
else:
|
|
phase = "warmup"
|
|
events.append(
|
|
{
|
|
"line": index + 1,
|
|
"phase": phase,
|
|
"timestamp_s": timestamp,
|
|
"message_prefix": line[:200],
|
|
}
|
|
)
|
|
|
|
config_match = re.search(
|
|
r"cudagraph_capture_sizes['\"]?:\s*\[([^\]]+)\]",
|
|
"\n".join(log_lines[:ready_index]),
|
|
)
|
|
startup_sizes = (
|
|
{int(value.strip()) for value in config_match.group(1).split(",")}
|
|
if config_match else set()
|
|
)
|
|
startup_modes = set()
|
|
for line in log_lines[:ready_index]:
|
|
if "Capturing CUDA graphs" not in line:
|
|
continue
|
|
if "PIECEWISE" in line:
|
|
startup_modes.add("PIECEWISE")
|
|
if "FULL" in line:
|
|
startup_modes.add("FULL")
|
|
uncovered = sorted(
|
|
descriptor for descriptor in clean_descriptors
|
|
if descriptor[0] not in startup_modes or descriptor[1] not in startup_sizes
|
|
)
|
|
passed = (
|
|
event_gate
|
|
and len(warm) >= 16
|
|
and len(warm_long) >= 1
|
|
and startup_modes == {"FULL", "PIECEWISE"}
|
|
and bool(startup_sizes)
|
|
and not uncovered
|
|
and clean_events == 0
|
|
)
|
|
return {
|
|
"amendment": "A-P5-1",
|
|
"passed": passed,
|
|
"warmup_completions": len(warm),
|
|
"warmup_long_completions": len(warm_long),
|
|
"warmup_long_min_tokens": min(
|
|
(int(request["input_tokens"]) for request in warm_long), default=None
|
|
),
|
|
"server_ready_line": ready_index + 1,
|
|
"compile_capture_events": events,
|
|
"event_gate_passed": event_gate,
|
|
"clean_capture_events": clean_events,
|
|
"startup_capture_sizes": sorted(startup_sizes),
|
|
"startup_capture_modes": sorted(startup_modes),
|
|
"warmup_descriptors": [list(item) for item in sorted(warm_descriptors)],
|
|
"clean_descriptors": [list(item) for item in sorted(clean_descriptors)],
|
|
"uncovered_clean_descriptors": [list(item) for item in uncovered],
|
|
"invariants": {
|
|
"events_preclean": event_gate,
|
|
"warmup_completions_ge_16": len(warm) >= 16,
|
|
"warmup_long_completion": len(warm_long) >= 1,
|
|
"startup_modes_complete": startup_modes == {"FULL", "PIECEWISE"},
|
|
"startup_sizes_present": bool(startup_sizes),
|
|
"clean_descriptors_covered": not uncovered,
|
|
"zero_clean_capture_events": clean_events == 0,
|
|
},
|
|
}
|
|
|
|
|
|
def validate_rate_client(run_dir: Path) -> dict[str, Any]:
|
|
result = json.loads((run_dir / "client/result.json").read_text())
|
|
sanity = json.loads((run_dir / "client/sanity.json").read_text())
|
|
requests = [
|
|
json.loads(line)
|
|
for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
|
|
]
|
|
failed_sanity = [
|
|
key for key, value in sanity["invariants"].items()
|
|
if not value and key != "drain_within_timeout"
|
|
]
|
|
failure_summary = m.summarize_request_failures(
|
|
requests, float(result["clean"]["start_s"]), float(result["clean"]["end_s"])
|
|
)
|
|
cold = cold_start_gate(run_dir)
|
|
quarantined = float(result["drain_seconds"]) > 600
|
|
invariants = {
|
|
"client_sanity": not failed_sanity,
|
|
"clean_duration": math.isclose(float(result["clean"]["duration_s"]), 240.0),
|
|
"clean_failures_zero": result["clean"]["failed"] == 0
|
|
and failure_summary["clean_failed"] == 0,
|
|
"failed_records_accounted": result["failed_records"] == failure_summary["failed"],
|
|
"manifest_no_wrap": not result["manifest_wrapped"]
|
|
and not result["manifest_exhausted"],
|
|
"warmup_cold_start_gate": cold["passed"],
|
|
"profile_count": len(result["profiles"]) == 0,
|
|
"drain_re_adjudicated": not quarantined,
|
|
}
|
|
non_drain = {key: value for key, value in invariants.items() if key != "drain_re_adjudicated"}
|
|
if not all(non_drain.values()):
|
|
raise RuntimeError(
|
|
f"A-P5-1 rate-client invariant failure: {run_dir}: "
|
|
f"invariants={invariants}; failed_sanity={failed_sanity}; cold={cold}"
|
|
)
|
|
return {
|
|
"result": result,
|
|
"sanity": sanity,
|
|
"request_count": len(requests),
|
|
"warmup_completions": cold["warmup_completions"],
|
|
"warmup_required": 16,
|
|
"warmup_gate_branch": "A-P5-1-cold-start",
|
|
"warmup_stability": None,
|
|
"cold_start_gate": cold,
|
|
"drain_budget_seconds": 600,
|
|
"drain_quarantined": quarantined,
|
|
"excluded_window_failures": failure_summary["excluded"],
|
|
"excluded_window_failure_kinds": failure_summary["excluded_kinds"],
|
|
"invariants": invariants,
|
|
}
|
|
|
|
|
|
def validate_run(
|
|
entry: dict[str, Any], profile: bool, burnin: bool, allow_missing_traces: bool = False
|
|
) -> dict[str, Any]:
|
|
del profile, allow_missing_traces
|
|
pattern = entry["assignment"].cell.pattern
|
|
if burnin or pattern.startswith("background"):
|
|
validation_pattern = "P06"
|
|
elif pattern.startswith("control-P03"):
|
|
validation_pattern = "P03"
|
|
elif pattern.startswith("control-P04"):
|
|
validation_pattern = "P04"
|
|
else:
|
|
validation_pattern = "P10"
|
|
if burnin or pattern.startswith("background"):
|
|
client = _ORIGINAL_VALIDATE_CLIENT(
|
|
entry["run_dir"], validation_pattern, False, burnin
|
|
)
|
|
else:
|
|
client = validate_rate_client(entry["run_dir"])
|
|
layer1 = m.validate_layer1(entry["run_dir"])
|
|
log = (entry["run_dir"] / "server.log").read_text(errors="replace")
|
|
invariants = {
|
|
"triton_moe": "Using TRITON Unquantized MoE backend" in log,
|
|
"chunked_mbt": "Chunked prefill is enabled with max_num_batched_tokens=8192" in log,
|
|
"tp1": "tensor_parallel_size=1" in log,
|
|
"drain_shutdown": "mode=drain timeout=600s" in log,
|
|
"a2_sizes": entry["assignment"].cell.config != "A2"
|
|
or all(str(size) in log for size in (3, 5, 6, 7)),
|
|
}
|
|
if not all(invariants.values()):
|
|
raise RuntimeError(f"server invariant failure: {entry['run_id']}: {invariants}")
|
|
forbidden = re.compile(r'"(?:prompt|messages|content|text)"\s*:')
|
|
for path in (
|
|
entry["run_dir"] / "client/requests.jsonl",
|
|
entry["run_dir"] / "client/result.json",
|
|
Path(layer1["stream"]),
|
|
):
|
|
if forbidden.search(path.read_text(errors="replace")):
|
|
raise RuntimeError(f"private text leaked: {path}")
|
|
summary = {
|
|
"schema": 1,
|
|
"run_id": entry["run_id"],
|
|
"pattern": pattern,
|
|
"config": entry["assignment"].cell.config,
|
|
"gpus": entry["assignment"].gpus,
|
|
"client": client,
|
|
"layer1": layer1,
|
|
"traces": [],
|
|
"missing_trace_files": 0,
|
|
"layer2_missing_after_controller_cleanup": False,
|
|
"drain_quarantined": client["drain_quarantined"],
|
|
"server_invariants": invariants,
|
|
}
|
|
m.atomic_json(entry["run_dir"] / "run-complete.json", summary)
|
|
return summary
|
|
|
|
|
|
m.validate_run = validate_run
|
|
|
|
|
|
def manifests() -> dict[str, Any]:
|
|
result = {}
|
|
for name in ("base", "A1", "A3"):
|
|
path = PRIVATE / f"P10-{name}.jsonl"
|
|
summary = json.loads(path.with_suffix(path.suffix + ".summary.json").read_text())
|
|
if summary["rows"] != 142 or summary["sha256"] != sha256_file(path):
|
|
raise RuntimeError(f"manifest verification failed: {name}")
|
|
result[name] = {"path": str(path), "sha256": summary["sha256"]}
|
|
return result
|
|
|
|
|
|
def fingerprint() -> dict[str, Any]:
|
|
return {
|
|
"source_commit": subprocess.check_output(
|
|
["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True
|
|
).strip(),
|
|
"source_tree": subprocess.check_output(
|
|
["git", "-C", str(SOURCE), "rev-parse", "HEAD^{tree}"], text=True
|
|
).strip(),
|
|
"client_sha256": sha256_file(CLIENT),
|
|
"controller_sha256": sha256_file(Path(__file__)),
|
|
"p3_client_sha256": sha256_file(P3_CLIENT),
|
|
"p3_matrix_sha256": sha256_file(Path(m.__file__).resolve()),
|
|
"manifests": manifests(),
|
|
"capture_sizes": list(CAPTURE_SIZES),
|
|
"rate": RATE,
|
|
}
|
|
|
|
|
|
def load_state(resume: bool) -> dict[str, Any]:
|
|
if STATE.exists():
|
|
if not resume:
|
|
raise RuntimeError("controller state exists; use --resume")
|
|
return json.loads(STATE.read_text())
|
|
return {
|
|
"schema": 1,
|
|
"status": "created",
|
|
"created_at": time.time(),
|
|
"controller_pid": os.getpid(),
|
|
"gpu_hours_total": 0.0,
|
|
"gpu_hours_this_stage": 0.0,
|
|
"completed_measured_runs": 0,
|
|
"completed_burnins": 0,
|
|
"drain_quarantined_runs": 0,
|
|
"clean_window_failures": 0,
|
|
"missing_trace_files": 0,
|
|
"stages": {},
|
|
"fingerprint": {},
|
|
}
|
|
|
|
|
|
def save_state(state: dict[str, Any]) -> None:
|
|
state["controller_pid"] = os.getpid()
|
|
state["updated_at"] = time.time()
|
|
m.atomic_json(STATE, state)
|
|
|
|
|
|
m.save_state = save_state
|
|
|
|
|
|
def ensure_provenance() -> None:
|
|
destination = RUN_ROOT / "provenance"
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
sources = [CLIENT, Path(__file__).resolve(), Path(m.__file__).resolve(), Path(m.common.__file__).resolve()]
|
|
hashes = {}
|
|
for source in sources:
|
|
target = destination / source.name
|
|
digest = sha256_file(source)
|
|
if target.exists() and sha256_file(target) != digest:
|
|
target = destination / f"{source.stem}.{digest[:12]}{source.suffix}"
|
|
if target.exists() and sha256_file(target) != digest:
|
|
raise RuntimeError(f"content-addressed provenance mismatch: {target}")
|
|
if not target.exists():
|
|
shutil.copy2(source, target)
|
|
hashes[target.name] = digest
|
|
m.atomic_json(destination / "sha256.json", hashes)
|
|
|
|
|
|
def primary_cells() -> list[m.Cell]:
|
|
config = {"base": "C00", "A1": "C00", "A2": "A2", "A3": "C00", "A4": "A4"}
|
|
items = [m.Cell(f"{arm}-r{replicate}", config[arm]) for arm in config for replicate in range(1, 4)]
|
|
return sorted(
|
|
items,
|
|
key=lambda cell: hashlib.sha256(
|
|
f"20260715:{cell.pattern.rsplit('-r',1)[1]}:{arm_name(cell.pattern)}".encode()
|
|
).hexdigest(),
|
|
)
|
|
|
|
|
|
def pack_unique(items: list[m.Cell], prefix: str) -> list[list[m.Assignment]]:
|
|
remaining = list(items)
|
|
waves: list[list[m.Assignment]] = []
|
|
wave_index = 0
|
|
while remaining:
|
|
selected: list[m.Cell] = []
|
|
used: set[str] = set()
|
|
for cell in list(remaining):
|
|
key = arm_name(cell.pattern)
|
|
if key in used:
|
|
continue
|
|
selected.append(cell)
|
|
used.add(key)
|
|
remaining.remove(cell)
|
|
if len(selected) == 4:
|
|
break
|
|
while len(selected) < 4:
|
|
selected.append(m.Cell(f"background-{prefix}-{wave_index}-{len(selected)}", "C00"))
|
|
assignments = []
|
|
for slot, cell in enumerate(selected):
|
|
gpu = (slot + wave_index) % 4
|
|
assignments.append(m.Assignment(cell, (gpu,)))
|
|
waves.append(assignments)
|
|
wave_index += 1
|
|
return waves
|
|
|
|
|
|
def pack_primary(items: list[m.Cell]) -> list[list[m.Assignment]]:
|
|
"""Pack SHA-ordered cells as 4/4/4/3 without duplicate arms per wave."""
|
|
capacities = (4, 4, 4, 3)
|
|
waves: list[list[m.Cell]] = [[] for _ in capacities]
|
|
|
|
def place(index: int) -> bool:
|
|
if index == len(items):
|
|
return all(len(wave) == capacity for wave, capacity in zip(waves, capacities, strict=True))
|
|
cell = items[index]
|
|
arm = arm_name(cell.pattern)
|
|
for wave_index, capacity in enumerate(capacities):
|
|
if len(waves[wave_index]) >= capacity:
|
|
continue
|
|
if any(arm_name(existing.pattern) == arm for existing in waves[wave_index]):
|
|
continue
|
|
waves[wave_index].append(cell)
|
|
if place(index + 1):
|
|
return True
|
|
waves[wave_index].pop()
|
|
return False
|
|
|
|
if not place(0):
|
|
raise RuntimeError("cannot pack frozen primary assignments into 4/4/4/3")
|
|
result: list[list[m.Assignment]] = []
|
|
for wave_index, cells in enumerate(waves):
|
|
if len(cells) == 3:
|
|
cells.append(m.Cell("background-primary-final", "C00"))
|
|
result.append(
|
|
[
|
|
m.Assignment(cell, ((slot + wave_index) % 4,))
|
|
for slot, cell in enumerate(cells)
|
|
]
|
|
)
|
|
return result
|
|
|
|
|
|
def execute_primary(resume: bool, amendment_a_p5_1: bool = False) -> None:
|
|
RUN_ROOT.mkdir(parents=True, exist_ok=True)
|
|
state = load_state(resume)
|
|
if resume:
|
|
m.cleanup_recorded(state)
|
|
current = fingerprint()
|
|
if state["fingerprint"] and state["fingerprint"] != current:
|
|
failure = str(state.get("stages", {}).get("primary-01", {}).get("failure", ""))
|
|
if not (
|
|
amendment_a_p5_1
|
|
and state.get("status") == "failed"
|
|
and "warmup" in failure.lower()
|
|
):
|
|
raise RuntimeError("resume fingerprint differs from frozen Phase-5 plan")
|
|
state.setdefault("amendments", {})["A-P5-1"] = {
|
|
"approved": True,
|
|
"applied_at": time.time(),
|
|
"reason": "replace rate-following drift gate with cold-start gates",
|
|
"prior_fingerprint": state["fingerprint"],
|
|
"replacement_fingerprint": current,
|
|
"retained_gpu_hours": state["gpu_hours_total"],
|
|
"burnins_reused": state["completed_burnins"],
|
|
}
|
|
state["fingerprint"] = current
|
|
state["status"] = "amended_resume_A-P5-1"
|
|
save_state(state)
|
|
state["fingerprint"] = current
|
|
state["status"] = "running_primary"
|
|
save_state(state)
|
|
ensure_provenance()
|
|
burnins = [
|
|
m.Assignment(m.Cell("burnin-C00", "C00"), (0,)),
|
|
m.Assignment(m.Cell("burnin-A2", "A2"), (1,)),
|
|
m.Assignment(m.Cell("burnin-A4", "A4"), (2,)),
|
|
]
|
|
m.run_stage(state, "burnins", burnins, "saturation", profile=False, burnin=True)
|
|
waves = pack_primary(primary_cells())
|
|
for index, wave in enumerate(waves, 1):
|
|
m.run_stage(state, f"primary-{index:02d}", wave, "moderate", profile=False)
|
|
primary = [
|
|
path for path in (RUN_ROOT / "primary").glob("*-r*-*/moderate/run-complete.json")
|
|
if "background" not in str(path)
|
|
]
|
|
if len(primary) != 15:
|
|
raise RuntimeError(f"primary completion mismatch: {len(primary)} != 15")
|
|
state["primary_runs"] = 15
|
|
state["background_runs"] = state["completed_measured_runs"] - 15
|
|
state["status"] = "primary_complete"
|
|
state["completed_at"] = time.time()
|
|
save_state(state)
|
|
|
|
|
|
def execute_controls(resume: bool) -> None:
|
|
state = load_state(resume)
|
|
m.cleanup_recorded(state)
|
|
if state.get("fingerprint") != fingerprint():
|
|
raise RuntimeError("control resume fingerprint mismatch")
|
|
cells = [
|
|
m.Cell(f"control-{pattern}-r{replicate}", "C00")
|
|
for pattern in ("P03", "P04") for replicate in range(1, 4)
|
|
]
|
|
for index, wave in enumerate(pack_unique(cells, "controls"), 1):
|
|
m.run_stage(state, f"controls-{index:02d}", wave, "moderate", profile=False)
|
|
state["status"] = "controls_complete"
|
|
state["controls_completed_at"] = time.time()
|
|
save_state(state)
|
|
|
|
|
|
def plan() -> dict[str, Any]:
|
|
return {
|
|
"schema": 1,
|
|
"primary_runs": 15,
|
|
"burnins": 3,
|
|
"waves": [[{"cell": a.cell.cell_id, "gpus": a.gpus} for a in wave] for wave in pack_primary(primary_cells())],
|
|
"rate": RATE,
|
|
"clean_seconds": 240,
|
|
"drain_seconds": 600,
|
|
"gpu_hour_limit": 6.0,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
for name in ("primary", "controls"):
|
|
item = sub.add_parser(name)
|
|
item.add_argument("--resume", action="store_true")
|
|
if name == "primary":
|
|
item.add_argument("--amendment-a-p5-1", action="store_true")
|
|
sub.add_parser("plan")
|
|
sub.add_parser("status")
|
|
args = parser.parse_args()
|
|
if args.command == "primary":
|
|
execute_primary(args.resume, args.amendment_a_p5_1)
|
|
elif args.command == "controls":
|
|
execute_controls(args.resume)
|
|
elif args.command == "plan":
|
|
print(json.dumps(plan(), sort_keys=True, indent=2))
|
|
else:
|
|
print(STATE.read_text() if STATE.exists() else '{"status":"absent"}')
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|