gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1611
runs/opprof-phase3/provenance/analyze_phase3.py Normal file
View File

File diff suppressed because it is too large Load Diff

794
runs/opprof-phase3/provenance/opprof_phase3_client.py Normal file
View File

@@ -0,0 +1,794 @@
#!/usr/bin/env python3
"""Token-exact fixed-duration client for the OpProf Phase-3 protocol."""
from __future__ import annotations
import argparse
import asyncio
import gzip
import hashlib
import json
import math
import os
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import aiohttp
SCHEMA = 1
TOKEN_BASE = 1000
TOKEN_SPAN = 100000
class ManifestExhausted(RuntimeError):
pass
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def atomic_json(path: Path, value: Any, mode: int = 0o640) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(value, f, sort_keys=True, indent=2)
f.write("\n")
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
def atomic_jsonl(path: Path, rows: list[dict[str, Any]], mode: int = 0o640) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
with os.fdopen(fd, "w", encoding="utf-8") as f:
for row in rows:
f.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
def parse_range(value: str) -> tuple[int, int]:
lo_text, hi_text = value.split(":", 1)
lo, hi = int(lo_text), int(hi_text)
if lo <= 0 or hi < lo:
raise argparse.ArgumentTypeError(f"invalid positive range: {value}")
return lo, hi
def _integer_counts(weights: list[float], total: int) -> list[int]:
raw = [w * total for w in weights]
counts = [math.floor(x) for x in raw]
order = sorted(
range(len(raw)), key=lambda i: raw[i] - counts[i], reverse=True
)
for idx in order[: total - sum(counts)]:
counts[idx] += 1
return counts
def numeric_sanity(values: list[float | int]) -> dict[str, Any]:
finite = [float(x) for x in values if math.isfinite(float(x))]
return {
"n": len(values),
"finite_n": len(finite),
"missing_n": len(values) - len(finite),
"min": min(finite) if finite else None,
"max": max(finite) if finite else None,
"distinct_n": len(set(finite)),
}
def manifest_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
return {
"schema": SCHEMA,
"rows": len(rows),
"input_tokens": numeric_sanity([int(r["input_tokens"]) for r in rows]),
"output_tokens": numeric_sanity([int(r["output_tokens"]) for r in rows]),
"arrival_values": sorted({str(r["arrival"]) for r in rows}),
"pattern_values": sorted({str(r["pattern_id"]) for r in rows}),
}
def materialize(args: argparse.Namespace) -> dict[str, Any]:
import numpy as np
rng = np.random.default_rng(args.workload_seed)
n = args.num_requests
if args.kind == "prefix-pool":
if args.num_prefixes <= 0 or args.prefix_len <= 0 or args.suffix_fixed <= 0:
raise ValueError("prefix-pool requires positive pool/prefix/suffix")
lengths = np.full(n, args.prefix_len + args.suffix_fixed, dtype=np.int64)
prefix_ids = np.arange(n, dtype=np.int64) % args.num_prefixes
rng.shuffle(prefix_ids)
else:
prefix_ids = np.full(n, -1, dtype=np.int64)
if args.input_uniform:
lo, hi = parse_range(args.input_uniform)
lengths = rng.integers(lo, hi + 1, n, dtype=np.int64)
elif args.input_fixed:
lengths = np.full(n, args.input_fixed, dtype=np.int64)
elif args.input_mixture:
spec = json.loads(args.input_mixture)
if not isinstance(spec, dict) or not spec:
raise ValueError("input mixture must be a non-empty JSON object")
keys = list(spec)
weights = [float(spec[key]) for key in keys]
if any(w < 0 for w in weights) or not math.isclose(sum(weights), 1.0):
raise ValueError("mixture weights must be non-negative and sum to 1")
pieces = []
for key, count in zip(
keys, _integer_counts(weights, n), strict=True
):
kind, lo_text, hi_text = key.split(":")
if kind != "uniform":
raise ValueError(f"unsupported mixture component: {key}")
pieces.append(
rng.integers(
int(lo_text), int(hi_text) + 1, count, dtype=np.int64
)
)
lengths = np.concatenate(pieces)
rng.shuffle(lengths)
else:
raise ValueError("exactly one input distribution is required")
if args.output_fixed <= 0 or args.arrival not in {"steady", "burst:8"}:
raise ValueError("invalid output length or arrival class")
rows = []
for i in range(n):
row = {
"schema": SCHEMA,
"request_id": f"{args.id}-{i:05d}",
"pattern_id": args.id,
"kind": args.kind,
"input_tokens": int(lengths[i]),
"output_tokens": args.output_fixed,
"arrival": args.arrival,
"token_seed": int(args.workload_seed * 1000003 + i),
}
if args.kind == "prefix-pool":
row.update(
{
"prefix_id": int(prefix_ids[i]),
"num_prefixes": args.num_prefixes,
"prefix_tokens": args.prefix_len,
}
)
rows.append(row)
out = Path(args.out)
atomic_jsonl(out, rows, mode=0o600)
summary = manifest_summary(rows)
summary.update({"sha256": sha256_file(out), "path": str(out)})
atomic_json(out.with_suffix(out.suffix + ".summary.json"), summary, mode=0o600)
print(json.dumps(summary, sort_keys=True))
return summary
def materialize_private(args: argparse.Namespace) -> dict[str, Any]:
from transformers import AutoTokenizer
source = Path(args.source)
selected: list[dict[str, Any]] = []
with source.open(encoding="utf-8") as f:
for source_index, line in enumerate(f):
row = json.loads(line)
if (
float(row["sampling_u"]) <= args.sampling_u_max
and int(row["input_length"]) <= args.max_input_tokens
):
selected.append(
{
"schema": SCHEMA,
"request_id": f"{args.id}-{len(selected):05d}",
"pattern_id": args.id,
"kind": "private-trace",
"input_tokens": int(row["input_length"]),
"output_tokens": min(
int(row["output_length"]), args.output_cap
),
"arrival": args.arrival,
"source_index": source_index,
"prompt": row["prompt"],
}
)
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
diffs = [
len(tokenizer.encode(row["prompt"], add_special_tokens=False))
- row["input_tokens"]
for row in selected
]
exact = sum(diff == 0 for diff in diffs)
exact_fraction = exact / len(diffs) if diffs else 0.0
max_abs = max((abs(diff) for diff in diffs), default=-1)
if exact_fraction < 0.99 or max_abs > 1:
raise RuntimeError(
"tokenizer parity gate failed: "
f"exact_fraction={exact_fraction:.6f} max_abs_error={max_abs}"
)
out = Path(args.out)
atomic_jsonl(out, selected, mode=0o600)
summary = manifest_summary(selected)
summary.update(
{
"sha256": sha256_file(out),
"source_sha256": sha256_file(source),
"tokenizer_exact_n": exact,
"tokenizer_exact_fraction": exact_fraction,
"tokenizer_max_abs_error": max_abs,
"path": str(out),
}
)
atomic_json(out.with_suffix(out.suffix + ".summary.json"), summary, mode=0o600)
print(json.dumps(summary, sort_keys=True))
return summary
def load_manifest(path: Path) -> list[dict[str, Any]]:
rows = [json.loads(line) for line in path.read_text().splitlines() if line]
required = {
"request_id",
"pattern_id",
"input_tokens",
"output_tokens",
"arrival",
}
if not rows:
raise ValueError("empty manifest")
for row in rows:
if not required.issubset(row):
raise ValueError(f"manifest row lacks {sorted(required - set(row))}")
if len({row["request_id"] for row in rows}) != len(rows):
raise ValueError("duplicate request_id")
return rows
def _token_stream(seed: int, count: int) -> list[int]:
state = seed & 0xFFFFFFFF
out = []
for _ in range(count):
state = (1664525 * state + 1013904223) & 0xFFFFFFFF
out.append(TOKEN_BASE + state % TOKEN_SPAN)
return out
def synthetic_prompt(row: dict[str, Any]) -> list[int]:
length = int(row["input_tokens"])
seed = int(row.get("token_seed", 0))
if row.get("kind") == "prefix-pool":
prefix_n = int(row["prefix_tokens"])
tokens = _token_stream(0xA5A50000 + int(row["prefix_id"]), prefix_n)
tokens += _token_stream(seed, length - prefix_n)
offset = prefix_n
else:
tokens = _token_stream(seed, length)
offset = 0
if length - offset >= 3:
index = int(row["request_id"].rsplit("-", 1)[1])
tokens[offset : offset + 3] = [
TOKEN_BASE + index % 100,
TOKEN_BASE + (index // 100) % 100,
TOKEN_BASE + (index // 10000) % 100,
]
return tokens
@dataclass
class RunContext:
args: argparse.Namespace
rows: list[dict[str, Any]]
t0: float
clean_end: float
stop_event: asyncio.Event
lock: asyncio.Lock
next_index: int = 0
in_flight: int = 0
max_in_flight: int = 0
exhausted: bool = False
admission_stop_s: float | None = None
async def next_row(self) -> dict[str, Any]:
async with self.lock:
if self.next_index >= len(self.rows):
self.exhausted = True
raise ManifestExhausted(
f"manifest exhausted after {self.next_index} admissions"
)
row = self.rows[self.next_index]
self.next_index += 1
return row
async def request_one(
ctx: RunContext,
session: aiohttp.ClientSession,
row: dict[str, Any],
scheduled: float,
) -> dict[str, Any]:
loop = asyncio.get_running_loop()
admitted = loop.time()
ctx.in_flight += 1
ctx.max_in_flight = max(ctx.max_in_flight, ctx.in_flight)
status = 0
actual_output: int | None = None
first_token: float | None = None
error_kind: str | None = None
try:
prompt: str | list[int] = (
row["prompt"]
if row.get("kind") == "private-trace"
else synthetic_prompt(row)
)
if not isinstance(prompt, str) and len(prompt) != int(row["input_tokens"]):
raise AssertionError("synthetic prompt length drift")
payload = {
"model": ctx.args.model,
"prompt": prompt,
"max_tokens": int(row["output_tokens"]),
"temperature": ctx.args.temperature,
"ignore_eos": ctx.args.ignore_eos,
"stream": True,
"stream_options": {"include_usage": True},
"add_special_tokens": False,
"seed": ctx.args.server_seed,
}
headers = {
"Content-Type": "application/json",
"x-request-id": str(row["request_id"]),
}
async with session.post(
ctx.args.base_url.rstrip("/") + "/v1/completions",
json=payload,
headers=headers,
) as response:
status = response.status
if status != 200:
error_kind = f"http_{status}"
else:
buf = b""
async for chunk in response.content.iter_any():
buf += chunk
while b"\n" in buf:
line, buf = buf.split(b"\n", 1)
line = line.strip()
if not line.startswith(b"data:"):
continue
data = line[5:].strip()
if data == b"[DONE]":
continue
event = json.loads(data)
if event.get("choices") and first_token is None:
first_token = loop.time()
if event.get("usage") is not None:
actual_output = int(event["usage"]["completion_tokens"])
if actual_output is None:
error_kind = "missing_usage"
except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
error_kind = type(exc).__name__
except Exception as exc:
error_kind = type(exc).__name__
finally:
completed = loop.time()
ctx.in_flight -= 1
success = (
status == 200
and error_kind is None
and actual_output == int(row["output_tokens"])
)
if status == 200 and actual_output is not None and not success:
error_kind = "output_token_mismatch"
return {
"schema": SCHEMA,
"request_id": row["request_id"],
"scheduled_s": scheduled - ctx.t0,
"admitted_s": admitted - ctx.t0,
"first_token_s": None if first_token is None else first_token - ctx.t0,
"completed_s": completed - ctx.t0,
"input_tokens": int(row["input_tokens"]),
"requested_output_tokens": int(row["output_tokens"]),
"actual_output_tokens": actual_output,
"http_status": status,
"success": success,
"error_kind": error_kind,
}
async def saturation_load(
ctx: RunContext, session: aiohttp.ClientSession
) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
async def worker() -> None:
while not ctx.stop_event.is_set():
try:
row = await ctx.next_row()
except ManifestExhausted:
ctx.stop_event.set()
return
results.append(
await request_one(ctx, session, row, asyncio.get_running_loop().time())
)
tasks = [
asyncio.create_task(worker()) for _ in range(ctx.args.max_concurrency)
]
await asyncio.gather(*tasks)
return results
async def finite_load(
ctx: RunContext, session: aiohttp.ClientSession, rate: float
) -> list[dict[str, Any]]:
sem = asyncio.Semaphore(ctx.args.max_concurrency)
tasks: list[asyncio.Task[dict[str, Any]]] = []
batch = 8 if str(ctx.rows[0]["arrival"]) == "burst:8" else 1
period = batch / rate
event_index = 0
async def limited(row: dict[str, Any], scheduled: float) -> dict[str, Any]:
async with sem:
return await request_one(ctx, session, row, scheduled)
while not ctx.stop_event.is_set():
scheduled = ctx.t0 + event_index * period
delay = scheduled - asyncio.get_running_loop().time()
if delay > 0:
try:
await asyncio.wait_for(ctx.stop_event.wait(), timeout=delay)
break
except asyncio.TimeoutError:
pass
if ctx.stop_event.is_set():
break
try:
for _ in range(batch):
tasks.append(
asyncio.create_task(limited(await ctx.next_row(), scheduled))
)
except ManifestExhausted:
ctx.stop_event.set()
break
event_index += 1
return await asyncio.gather(*tasks) if tasks else []
async def post_profile(
session: aiohttp.ClientSession, base_url: str, endpoint: str
) -> tuple[float, float, int]:
loop = asyncio.get_running_loop()
before = loop.time()
async with session.post(base_url.rstrip("/") + endpoint) as response:
status = response.status
await response.read()
return before, loop.time(), status
def _trace_loadable(path: Path) -> bool:
try:
opener = gzip.open if path.suffix == ".gz" else open
with opener(path, "rt", encoding="utf-8") as f:
parsed = json.load(f)
return isinstance(parsed, dict) and isinstance(parsed.get("traceEvents"), list)
except (OSError, EOFError, json.JSONDecodeError):
return False
async def wait_new_trace(
trace_dir: Path, before: set[Path], timeout: float
) -> Path:
deadline = asyncio.get_running_loop().time() + timeout
while asyncio.get_running_loop().time() < deadline:
for path in sorted(set(trace_dir.glob("*.pt.trace.json*")) - before):
if _trace_loadable(path):
return path
await asyncio.sleep(0.25)
raise TimeoutError(f"no new loadable trace within {timeout}s")
async def timeline(
ctx: RunContext, session: aiohttp.ClientSession
) -> list[dict[str, Any]]:
args = ctx.args
profiles: list[dict[str, Any]] = []
await asyncio.sleep(max(0, ctx.clean_end - asyncio.get_running_loop().time()))
if args.profile_after_clean:
trace_dir = Path(args.profile_trace_dir)
for window in range(args.num_profile_windows):
prior = set(trace_dir.glob("*.pt.trace.json*"))
start_before, start_after, start_status = await post_profile(
session, args.base_url, "/start_profile"
)
trace = await wait_new_trace(
trace_dir, prior, args.profile_timeout_seconds
)
trace_ready = asyncio.get_running_loop().time()
stop_before, stop_after, stop_status = await post_profile(
session, args.base_url, "/stop_profile"
)
profiles.append(
{
"window": window + 1,
"start_call_s": start_before - ctx.t0,
"start_return_s": start_after - ctx.t0,
"trace_ready_s": trace_ready - ctx.t0,
"stop_call_s": stop_before - ctx.t0,
"stop_return_s": stop_after - ctx.t0,
"start_status": start_status,
"stop_status": stop_status,
"trace_file": trace.name,
"trace_sha256": sha256_file(trace),
}
)
if start_status != 200 or stop_status != 200:
raise RuntimeError("profile endpoint returned non-200")
await asyncio.sleep(args.recovery_seconds)
else:
await asyncio.sleep(args.post_clean_seconds)
ctx.admission_stop_s = asyncio.get_running_loop().time() - ctx.t0
ctx.stop_event.set()
return profiles
def segment_summary(
records: list[dict[str, Any]], start: float, end: float
) -> dict[str, Any]:
admitted = [r for r in records if start <= r["admitted_s"] < end]
completed = [r for r in records if start <= r["completed_s"] < end]
successes = [r for r in completed if r["success"]]
duration = end - start
return {
"start_s": start,
"end_s": end,
"duration_s": duration,
"admitted": len(admitted),
"completed": len(successes),
"failed": len(completed) - len(successes),
"offered_rps": len(admitted) / duration,
"completed_throughput_rps": len(successes) / duration,
"input_tokens": sum(r["input_tokens"] for r in successes),
"output_tokens": sum(r["actual_output_tokens"] or 0 for r in successes),
}
async def run_load(args: argparse.Namespace) -> dict[str, Any]:
manifest = Path(args.manifest)
rows = load_manifest(manifest)
arrivals = {row["arrival"] for row in rows}
if len(arrivals) != 1:
raise ValueError("a manifest must have one arrival class")
if args.load_point == "saturation":
if args.request_rate != "inf":
raise ValueError("saturation requires --request-rate inf")
rate = math.inf
else:
if not args.saturation_result:
raise ValueError("moderate requires --saturation-result")
sat = json.loads(Path(args.saturation_result).read_text())
rate = args.rate_fraction * float(sat["clean"]["completed_throughput_rps"])
if not math.isfinite(rate) or rate <= 0:
raise ValueError("derived moderate rate must be positive and finite")
loop = asyncio.get_running_loop()
t0 = loop.time()
t0_mono_ns = int(t0 * 1e9)
t0_wall_ns = time.time_ns()
clean_seconds = args.clean_segment_seconds * args.num_clean_segments
ctx = RunContext(
args=args,
rows=rows,
t0=t0,
clean_end=t0 + args.warmup_seconds + clean_seconds,
stop_event=asyncio.Event(),
lock=asyncio.Lock(),
)
timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=600)
connector = aiohttp.TCPConnector(limit=args.max_concurrency)
control_connector = aiohttp.TCPConnector(limit=2)
async with (
aiohttp.ClientSession(timeout=timeout, connector=connector) as session,
aiohttp.ClientSession(
timeout=timeout, connector=control_connector
) as control_session,
):
profile_task = asyncio.create_task(timeline(ctx, control_session))
load_task = asyncio.create_task(
saturation_load(ctx, session)
if math.isinf(rate)
else finite_load(ctx, session, rate)
)
try:
profiles = await profile_task
except Exception:
ctx.stop_event.set()
await load_task
raise
records = await load_task
clean_start = args.warmup_seconds
clean_end = clean_start + clean_seconds
clean = segment_summary(records, clean_start, clean_end)
segments = []
for i in range(args.num_clean_segments):
start = clean_start + i * args.clean_segment_seconds
segments.append(
{
"name": chr(ord("A") + i),
**segment_summary(
records, start, start + args.clean_segment_seconds
),
}
)
successful = [r for r in records if r["success"]]
elapsed_seconds = loop.time() - t0
if ctx.admission_stop_s is None:
raise RuntimeError("admission stop timestamp was not recorded")
drain_seconds = elapsed_seconds - ctx.admission_stop_s
result = {
"schema": SCHEMA,
"manifest_sha256": sha256_file(manifest),
"manifest_rows": len(rows),
"manifest_admitted": ctx.next_index,
"manifest_wrapped": False,
"manifest_exhausted": ctx.exhausted,
"load_point": args.load_point,
"t0_mono_ns": t0_mono_ns,
"t0_wall_ns": t0_wall_ns,
"request_rate": "inf" if math.isinf(rate) else rate,
"rate_fraction": None if math.isinf(rate) else args.rate_fraction,
"arrival": next(iter(arrivals)),
"warmup_seconds": args.warmup_seconds,
"clean_segment_seconds": args.clean_segment_seconds,
"num_clean_segments": args.num_clean_segments,
"elapsed_seconds": elapsed_seconds,
"admission_stop_s": ctx.admission_stop_s,
"drain_seconds": drain_seconds,
"max_in_flight": ctx.max_in_flight,
"records": len(records),
"successful_records": len(successful),
"failed_records": len(records) - len(successful),
"clean": clean,
"segments": segments,
"profiles": profiles,
}
sanity = {
"schema": SCHEMA,
"numeric": {
"input_tokens": numeric_sanity([r["input_tokens"] for r in records]),
"requested_output_tokens": numeric_sanity(
[r["requested_output_tokens"] for r in records]
),
"actual_output_tokens": numeric_sanity(
[
r["actual_output_tokens"]
for r in records
if r["actual_output_tokens"] is not None
]
),
"scheduled_s": numeric_sanity([r["scheduled_s"] for r in records]),
"admitted_s": numeric_sanity([r["admitted_s"] for r in records]),
"completed_s": numeric_sanity([r["completed_s"] for r in records]),
},
"invariants": {
"clean_duration_exact": math.isclose(clean["duration_s"], clean_seconds),
"segment_count_exact": len(segments) == args.num_clean_segments,
"manifest_no_wrap": ctx.next_index <= len(rows),
"manifest_not_exhausted": not ctx.exhausted,
"concurrency_bounded": ctx.max_in_flight <= args.max_concurrency,
"drain_within_timeout": drain_seconds <= args.drain_timeout_seconds,
"output_tokens_exact": all(
r["actual_output_tokens"] == r["requested_output_tokens"]
for r in successful
),
"clean_failures_zero": clean["failed"] == 0,
"profile_count_exact": len(profiles)
== (args.num_profile_windows if args.profile_after_clean else 0),
"profile_status_ok": all(
p["start_status"] == 200 and p["stop_status"] == 200
for p in profiles
),
},
}
if not math.isinf(rate):
sanity["invariants"]["moderate_offered_within_5pct"] = (
abs(clean["offered_rps"] / rate - 1) <= 0.05
)
out = Path(args.result_dir)
out.mkdir(parents=True, exist_ok=True)
atomic_jsonl(out / "requests.jsonl", sorted(records, key=lambda r: r["admitted_s"]))
atomic_jsonl(out / "segments.jsonl", segments)
atomic_json(out / "result.json", result)
atomic_json(out / "sanity.json", sanity)
if ctx.exhausted:
raise ManifestExhausted("manifest exhausted; result retained for diagnosis")
failed = [name for name, ok in sanity["invariants"].items() if not ok]
if failed:
raise RuntimeError(f"client sanity failure: {failed}")
return result
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
mat = sub.add_parser("materialize")
mat.add_argument("--id", required=True)
mat.add_argument("--kind", choices=("synthetic", "prefix-pool"), required=True)
group = mat.add_mutually_exclusive_group()
group.add_argument("--input-uniform")
group.add_argument("--input-fixed", type=int)
group.add_argument("--input-mixture")
mat.add_argument("--output-fixed", type=int, required=True)
mat.add_argument("--prefix", default="none")
mat.add_argument("--arrival", required=True)
mat.add_argument("--num-requests", type=int, required=True)
mat.add_argument("--workload-seed", type=int, required=True)
mat.add_argument("--num-prefixes", type=int, default=0)
mat.add_argument("--prefix-len", type=int, default=0)
mat.add_argument("--suffix-fixed", type=int, default=0)
mat.add_argument("--out", required=True)
private = sub.add_parser("materialize-private")
private.add_argument("--id", required=True)
private.add_argument("--source", required=True)
private.add_argument("--sampling-u-max", type=float, required=True)
private.add_argument("--max-input-tokens", type=int, required=True)
private.add_argument("--output-cap", type=int, required=True)
private.add_argument("--preserve-prompts", action="store_true", required=True)
private.add_argument("--disable-shuffle", action="store_true", required=True)
private.add_argument("--arrival", required=True)
private.add_argument("--model", required=True)
private.add_argument("--out", required=True)
run = sub.add_parser("run")
run.add_argument("--manifest", required=True)
run.add_argument("--base-url", required=True)
run.add_argument("--model", required=True)
run.add_argument("--load-point", choices=("saturation", "moderate"), required=True)
run.add_argument("--request-rate")
run.add_argument("--saturation-result")
run.add_argument("--rate-fraction", type=float, default=0.60)
run.add_argument("--max-concurrency", type=int, default=256)
run.add_argument("--ignore-eos", action="store_true")
run.add_argument("--temperature", type=float, default=0.0)
run.add_argument("--warmup-seconds", type=float, default=60)
run.add_argument("--clean-segment-seconds", type=float, default=80)
run.add_argument("--num-clean-segments", type=int, default=3)
run.add_argument("--profile-after-clean", action="store_true")
run.add_argument("--num-profile-windows", type=int, default=0)
run.add_argument("--profile-warmup-iterations", type=int, default=2)
run.add_argument("--profile-active-iterations", type=int, default=8)
run.add_argument("--profile-trace-dir")
run.add_argument("--profile-timeout-seconds", type=float, default=120)
run.add_argument("--recovery-seconds", type=float, default=30)
run.add_argument("--post-clean-seconds", type=float, default=0)
run.add_argument("--drain-timeout-seconds", type=float, default=120)
run.add_argument("--workload-seed", type=int, default=20260712)
run.add_argument("--server-seed", type=int, default=20260712)
run.add_argument("--result-dir", required=True)
return parser
def main() -> None:
args = build_parser().parse_args()
if args.command == "materialize":
materialize(args)
elif args.command == "materialize-private":
materialize_private(args)
else:
if args.profile_after_clean and not args.profile_trace_dir:
raise ValueError("--profile-after-clean requires --profile-trace-dir")
print(json.dumps(asyncio.run(run_load(args)), sort_keys=True))
if __name__ == "__main__":
main()

1056
runs/opprof-phase3/provenance/opprof_phase3_controller.py Normal file
View File

File diff suppressed because it is too large Load Diff

1045
runs/opprof-phase3/provenance/opprof_phase3_controller_ea.py Normal file
View File

File diff suppressed because it is too large Load Diff

1252
runs/opprof-phase3/provenance/opprof_phase3_matrix.py Normal file
View File

File diff suppressed because it is too large Load Diff

109
runs/opprof-phase3/provenance/test_phase3_analysis.py Normal file
View File

@@ -0,0 +1,109 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
import analyze_phase3 as analysis
class Phase3AnalysisTests(unittest.TestCase):
def test_ap36_stability_formula(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "client").mkdir()
(root / "opprof").mkdir()
(root / "client/result.json").write_text(
json.dumps({"t0_mono_ns": 0, "warmup_seconds": 60})
)
(root / "client/requests.jsonl").write_text(
"".join(
json.dumps({"success": True, "completed_s": index + 1}) + "\n"
for index in range(16)
)
)
records = []
for bin_index in range(3):
for step in range(16):
records.append(
{
"step_index": len(records),
"model_executed": True,
"submit_mono_ns": int(
(45 + 5 * bin_index + (step + 0.5) / 16 * 5)
* 1e9
),
"prefill_tokens": 100,
"decode_tokens": 0,
}
)
(root / "opprof/test.jsonl").write_text(
"".join(json.dumps(item) + "\n" for item in records)
)
result = analysis.ap36_warmup_stability(root)
self.assertTrue(result["passes"])
self.assertEqual(result["normalized_drift"], 0)
def test_ap37_partial_verdict_can_confirm_but_not_refute(self):
self.assertEqual(analysis.partial_verdict(True), "PASS")
self.assertEqual(analysis.partial_verdict(False), "INCONCLUSIVE")
def test_accepted_markers_come_only_from_complete_stages(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
complete = root / "stages/primary-01-saturation"
complete.mkdir(parents=True)
(complete / "stage-complete.json").write_text(
json.dumps({"runs": ["P01-C00-saturation"]})
)
accepted = root / "primary/P01-C00/saturation"
accepted.mkdir(parents=True)
(accepted / "run-complete.json").write_text(
json.dumps({"run_id": "P01-C00-saturation"})
)
unaccepted = root / "primary/P05-C00/saturation"
unaccepted.mkdir(parents=True)
(unaccepted / "run-complete.json").write_text(
json.dumps({"run_id": "P05-C00-saturation"})
)
primary, confirmations, stages, excluded = analysis.accepted_marker_paths(
root
)
self.assertEqual(primary, [accepted / "run-complete.json"])
self.assertEqual(confirmations, [])
self.assertEqual(stages, [complete])
self.assertEqual(excluded, ["P05-C00-saturation"])
def test_r64_is_ratio_of_cohort_sums(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "manifest.jsonl"
rows = [{"input_tokens": value} for value in (1, 3, 2, 2)]
path.write_text("".join(json.dumps(row) + "\n" for row in rows))
value, pieces = analysis.manifest_raggedness(path, 2)
self.assertEqual(pieces, [(2.0, 6.0), (0.0, 4.0)])
self.assertAlmostEqual(value, 0.2)
def test_one_percentage_point_ranking_ties(self):
shares = dict.fromkeys(analysis.FAMILIES, 0.0)
shares.update(attention=0.40, moe_gemm=0.395, moe_router=0.20)
ranked = {item["family"]: item["rank"] for item in analysis.ranked_families(shares)}
self.assertEqual(ranked["attention"], ranked["moe_gemm"])
self.assertGreater(ranked["moe_router"], ranked["attention"])
def test_holm_uses_declared_total_test_family(self):
values = [{"p": 0.001}, {"p": 0.01}]
analysis.holm(values, total_tests=10)
self.assertAlmostEqual(values[0]["p_holm"], 0.01)
self.assertAlmostEqual(values[1]["p_holm"], 0.09)
def test_robust_spline_prediction_is_nonnegative(self):
rows = [(float(x), float(n), float(2 * x + n)) for x in range(1, 20) for n in (1, 4)]
predict, hull = analysis.fit_nonnegative_robust(rows)
self.assertGreaterEqual(predict(3, 2), 0)
self.assertTrue(analysis.inside_convex(hull, (3, 2)))
self.assertFalse(analysis.inside_convex(hull, (100, 2)))
if __name__ == "__main__":
unittest.main()

486
runs/opprof-phase3/provenance/test_phase3_tools.py Normal file
View File

@@ -0,0 +1,486 @@
#!/usr/bin/env python3
from __future__ import annotations
import asyncio
import importlib.util
import json
import sys
import tempfile
import unittest
from unittest import mock
from pathlib import Path
from types import SimpleNamespace
from aiohttp import web
HERE = Path(__file__).parent
def load_module(name: str, filename: str):
spec = importlib.util.spec_from_file_location(name, HERE / filename)
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
assert spec.loader is not None
spec.loader.exec_module(module)
return module
client = load_module("phase3_client", "opprof_phase3_client.py")
controller = load_module("phase3_controller", "opprof_phase3_controller.py")
sys.modules["opprof_phase3_controller"] = controller
matrix = load_module("phase3_matrix", "opprof_phase3_matrix.py")
def rows(n: int, arrival: str = "steady") -> list[dict]:
return [
{
"schema": 1,
"request_id": f"T-{i:05d}",
"pattern_id": "T",
"kind": "synthetic",
"input_tokens": 8,
"output_tokens": 2,
"arrival": arrival,
"token_seed": i + 1,
}
for i in range(n)
]
class MockServer:
def __init__(self, delay: float = 0.01, trace_dir: Path | None = None) -> None:
self.active = 0
self.max_active = 0
self.payloads = []
self.runner = None
self.port = None
self.delay = delay
self.trace_dir = trace_dir
self.profile_count = 0
async def completion(self, request):
payload = await request.json()
self.payloads.append(payload)
self.active += 1
self.max_active = max(self.max_active, self.active)
await asyncio.sleep(self.delay)
response = web.StreamResponse(
status=200, headers={"Content-Type": "text/event-stream"}
)
await response.prepare(request)
await response.write(
b'data: {"choices":[{"text":"x"}],"usage":null}\n\n'
)
usage = json.dumps(
{
"choices": [],
"usage": {
"prompt_tokens": len(payload["prompt"]),
"completion_tokens": payload["max_tokens"],
},
}
).encode()
await response.write(b"data: " + usage + b"\n\n")
await response.write(b"data: [DONE]\n\n")
await response.write_eof()
self.active -= 1
return response
async def start_profile(self, request):
self.profile_count += 1
path = self.trace_dir / f"window-{self.profile_count}.pt.trace.json"
path.write_text('{"traceEvents": []}')
return web.Response(status=200)
async def stop_profile(self, request):
return web.Response(status=200)
async def start(self):
app = web.Application()
app.router.add_post("/v1/completions", self.completion)
if self.trace_dir is not None:
app.router.add_post("/start_profile", self.start_profile)
app.router.add_post("/stop_profile", self.stop_profile)
self.runner = web.AppRunner(app)
await self.runner.setup()
site = web.TCPSite(self.runner, "127.0.0.1", 0)
await site.start()
self.port = site._server.sockets[0].getsockname()[1]
async def stop(self):
await self.runner.cleanup()
def run_args(
manifest: Path,
result_dir: Path,
port: int,
load_point: str = "saturation",
saturation_result: Path | None = None,
):
return SimpleNamespace(
manifest=str(manifest),
base_url=f"http://127.0.0.1:{port}",
model="mock",
load_point=load_point,
request_rate="inf" if load_point == "saturation" else None,
saturation_result=None if saturation_result is None else str(saturation_result),
rate_fraction=0.60,
max_concurrency=3,
ignore_eos=True,
temperature=0,
warmup_seconds=0.04,
clean_segment_seconds=0.04,
num_clean_segments=3,
profile_after_clean=False,
num_profile_windows=0,
profile_warmup_iterations=2,
profile_active_iterations=8,
profile_trace_dir=None,
profile_timeout_seconds=1,
recovery_seconds=0,
post_clean_seconds=0,
drain_timeout_seconds=1,
workload_seed=20260712,
server_seed=20260712,
result_dir=str(result_dir),
)
class Phase3ToolTests(unittest.TestCase):
def write_warmup_stream(self, root: Path, tokens: list[int]) -> None:
stream_dir = root / "opprof"
stream_dir.mkdir()
records = []
step = 0
for bin_index, token_count in enumerate(tokens):
per_step, remainder = divmod(token_count, 16)
for in_bin in range(16):
records.append(
{
"step_index": step,
"model_executed": True,
"submit_mono_ns": int(
1e9
+ (45 + bin_index * 5 + (in_bin + 0.5) / 16 * 5)
* 1e9
),
"prefill_tokens": per_step + (in_bin < remainder),
"decode_tokens": 0,
}
)
step += 1
(stream_dir / "test.jsonl").write_text(
"".join(json.dumps(record) + "\n" for record in records)
)
def test_p10_warmup_stability_accepts_flat_trailing_quartile(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.write_warmup_stream(root, [1600, 1600, 1600])
result = matrix.p10_warmup_stability(root, int(1e9))
self.assertTrue(result["passed"])
self.assertEqual(result["step_counts"], [16, 16, 16])
self.assertEqual(result["scheduled_token_throughput"], [320, 320, 320])
self.assertEqual(result["normalized_drift"], 0)
def test_p10_warmup_stability_rejects_large_drift(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.write_warmup_stream(root, [1600, 3200, 4800])
result = matrix.p10_warmup_stability(root, int(1e9))
self.assertFalse(result["passed"])
self.assertGreater(result["normalized_drift"], 0.10)
def test_only_clean_window_request_failures_are_hard_failures(self):
requests = [
{"success": False, "completed_s": 59.9, "error_kind": "warmup"},
{"success": False, "completed_s": 60.0, "error_kind": "clean"},
{"success": False, "completed_s": 299.9, "error_kind": "clean"},
{"success": False, "completed_s": 300.0, "error_kind": "recovery"},
{"success": True, "completed_s": 100.0, "error_kind": None},
]
summary = matrix.summarize_request_failures(requests, 60.0, 300.0)
self.assertEqual(summary["failed"], 4)
self.assertEqual(summary["clean_failed"], 2)
self.assertEqual(summary["excluded"], 2)
self.assertEqual(summary["excluded_kinds"], {"warmup": 1, "recovery": 1})
def test_synthetic_prompt_exact_and_unique_early_prefix(self):
generated = [client.synthetic_prompt(row) for row in rows(200)]
self.assertTrue(all(len(value) == 8 for value in generated))
self.assertEqual(len({tuple(value[:3]) for value in generated}), 200)
def test_p05_manifest_has_exact_half_modes(self):
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp) / "P05.jsonl"
args = SimpleNamespace(
id="P05",
kind="synthetic",
input_uniform=None,
input_fixed=None,
input_mixture='{"uniform:128:512":0.5,"uniform:4096:8192":0.5}',
output_fixed=64,
prefix="none",
arrival="steady",
num_requests=100,
workload_seed=20260712,
num_prefixes=0,
prefix_len=0,
suffix_fixed=0,
out=str(out),
)
client.materialize(args)
manifest = client.load_manifest(out)
self.assertEqual(sum(r["input_tokens"] <= 512 for r in manifest), 50)
self.assertEqual(sum(r["input_tokens"] >= 4096 for r in manifest), 50)
def test_prefix_pool_balanced(self):
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp) / "P08.jsonl"
args = SimpleNamespace(
id="P08",
kind="prefix-pool",
input_uniform=None,
input_fixed=None,
input_mixture=None,
output_fixed=512,
prefix="none",
arrival="burst:8",
num_requests=80,
workload_seed=20260712,
num_prefixes=8,
prefix_len=1024,
suffix_fixed=256,
out=str(out),
)
client.materialize(args)
manifest = client.load_manifest(out)
counts = {i: 0 for i in range(8)}
for row in manifest:
counts[row["prefix_id"]] += 1
self.assertEqual(row["input_tokens"], 1280)
self.assertEqual(set(counts.values()), {10})
def test_fixed_duration_saturation_and_redaction(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(1000))
mock = MockServer()
await mock.start()
try:
result = await client.run_load(
run_args(manifest, root / "result", mock.port)
)
finally:
await mock.stop()
self.assertEqual(result["max_in_flight"], 3)
self.assertEqual(mock.max_active, 3)
self.assertAlmostEqual(result["clean"]["duration_s"], 0.12, places=9)
self.assertEqual(len(result["segments"]), 3)
self.assertLessEqual(result["drain_seconds"], 1)
self.assertTrue(
all(p["max_tokens"] == 2 and p["ignore_eos"] for p in mock.payloads)
)
text = (root / "result/requests.jsonl").read_text()
self.assertNotIn('"prompt":', text)
self.assertNotIn('"text":', text)
asyncio.run(case())
def test_profile_control_plane_is_not_starved_by_data_connector(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
trace_dir = root / "traces"
trace_dir.mkdir()
client.atomic_jsonl(manifest, rows(1000))
server = MockServer(delay=0.5, trace_dir=trace_dir)
await server.start()
args = run_args(manifest, root / "result", server.port)
args.max_concurrency = 1
args.warmup_seconds = 0.01
args.clean_segment_seconds = 0.01
args.num_clean_segments = 1
args.profile_after_clean = True
args.num_profile_windows = 1
args.profile_trace_dir = str(trace_dir)
args.recovery_seconds = 0
try:
result = await client.run_load(args)
finally:
await server.stop()
self.assertEqual(server.max_active, 1)
self.assertEqual(server.profile_count, 1)
self.assertLess(result["profiles"][0]["start_return_s"], 0.25)
asyncio.run(case())
def test_drain_timeout_is_a_hard_sanity_failure(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(1000))
mock = MockServer()
await mock.start()
try:
args = run_args(manifest, root / "result", mock.port)
args.drain_timeout_seconds = 0
with self.assertRaisesRegex(RuntimeError, "drain_within_timeout"):
await client.run_load(args)
finally:
await mock.stop()
sanity = json.loads((root / "result/sanity.json").read_text())
self.assertFalse(sanity["invariants"]["drain_within_timeout"])
asyncio.run(case())
def test_burst_schedule_is_eight_at_once(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(1000, "burst:8"))
sat = root / "sat.json"
client.atomic_json(
sat, {"clean": {"completed_throughput_rps": 100.0}}
)
mock = MockServer()
await mock.start()
try:
args = run_args(
manifest, root / "result", mock.port, "moderate", sat
)
args.warmup_seconds = 0
args.clean_segment_seconds = 0.4 / 3
result = await client.run_load(args)
finally:
await mock.stop()
records = [
json.loads(line)
for line in (root / "result/requests.jsonl").read_text().splitlines()
]
groups = {}
for record in records:
groups.setdefault(round(record["scheduled_s"], 6), 0)
groups[round(record["scheduled_s"], 6)] += 1
self.assertTrue(all(value == 8 for value in groups.values()))
starts = sorted(groups)
if len(starts) > 1:
self.assertAlmostEqual(starts[1] - starts[0], 8 / 60, places=5)
self.assertAlmostEqual(result["request_rate"], 60.0)
asyncio.run(case())
def test_manifest_exhaustion_stops_instead_of_wrapping(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(2))
mock = MockServer()
await mock.start()
try:
with self.assertRaises(client.ManifestExhausted):
await client.run_load(
run_args(manifest, root / "result", mock.port)
)
finally:
await mock.stop()
asyncio.run(case())
def test_cpu_affinity_sets_are_disjoint_and_cover_host(self):
values = []
for spec in controller.CPU_MAP.values():
lo, hi = [int(x) for x in spec.split("-")]
values.extend(range(lo, hi + 1))
self.assertEqual(sorted(values), list(range(160)))
self.assertEqual(len(values), len(set(values)))
def test_preflight_waits_for_three_stable_zero_samples(self):
def sample(memory):
return [
{
"index": 0,
"uuid": "GPU-test",
"memory_used_mib": memory,
"utilization_pct": 0,
}
]
with tempfile.TemporaryDirectory() as tmp:
with (
mock.patch.object(
controller,
"gpu_query",
side_effect=[sample(4), sample(0), sample(0), sample(0)],
),
mock.patch.object(controller, "compute_apps", return_value=[]),
mock.patch.object(controller, "run_text", return_value=""),
mock.patch.object(controller.time, "sleep"),
):
root = Path(tmp)
controller.preflight([0], root)
samples = json.loads((root / "gpu-before-samples.json").read_text())
self.assertEqual(len(samples), 4)
self.assertEqual(samples[0]["gpus"][0]["memory_used_mib"], 4)
self.assertEqual(samples[-1]["gpus"][0]["memory_used_mib"], 0)
def test_kernel_mapping_priority(self):
cases = {
"void vllm::moe::topkGating": "moe_router",
"ncclDevKernel_AllReduce": "collective",
"flash_fwd_kernel": "attention",
"nvjet_sm90_tst": "moe_gemm",
"argmax_kernel": "sampler",
"cutlass_gemm": "dense_gemm",
"triton_red_fused_add_rms_norm": "norm_elementwise",
"cache_swap_kernel": "kv_memory",
"unknown": "other",
}
self.assertEqual(
{name: controller.classify_kernel(name) for name in cases}, cases
)
def test_atomic_state_replacement(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "state.json"
controller.atomic_json(path, {"schema": 1, "value": 1})
controller.atomic_json(path, {"schema": 1, "value": 2})
self.assertEqual(json.loads(path.read_text())["value"], 2)
self.assertFalse(list(path.parent.glob("*.tmp.*")))
def test_fingerprint_survives_json_roundtrip(self):
original_run_text = controller.run_text
original_hash = controller.sha256_file
try:
controller.run_text = lambda *args, **kwargs: "deadbeef\n"
controller.sha256_file = lambda path: "a" * 64
fingerprint = controller.make_fingerprint()
finally:
controller.run_text = original_run_text
controller.sha256_file = original_hash
self.assertEqual(json.loads(json.dumps(fingerprint)), fingerprint)
self.assertEqual(set(fingerprint["cpu_map"]), {str(i) for i in range(8)})
def test_server_shutdown_signals_parent_before_group(self):
process = SimpleNamespace(pid=12345, poll=lambda: None)
with (
mock.patch.object(controller.os, "kill") as kill,
mock.patch.object(controller.os, "killpg") as killpg,
mock.patch.object(controller, "_process_group_alive", return_value=False),
):
controller.stop_servers([process])
kill.assert_called_once_with(12345, controller.signal.SIGINT)
killpg.assert_not_called()
if __name__ == "__main__":
unittest.main(verbosity=2)

353
runs/opprof-phase3/provenance/test_phase3_tools_ea.py Normal file
View File

@@ -0,0 +1,353 @@
#!/usr/bin/env python3
from __future__ import annotations
import asyncio
import importlib.util
import json
import sys
import tempfile
import unittest
from unittest import mock
from pathlib import Path
from types import SimpleNamespace
from aiohttp import web
HERE = Path(__file__).parent
def load_module(name: str, filename: str):
spec = importlib.util.spec_from_file_location(name, HERE / filename)
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
assert spec.loader is not None
spec.loader.exec_module(module)
return module
client = load_module("phase3_client", "opprof_phase3_client.py")
controller = load_module("phase3_controller", "opprof_phase3_controller.py")
def rows(n: int, arrival: str = "steady") -> list[dict]:
return [
{
"schema": 1,
"request_id": f"T-{i:05d}",
"pattern_id": "T",
"kind": "synthetic",
"input_tokens": 8,
"output_tokens": 2,
"arrival": arrival,
"token_seed": i + 1,
}
for i in range(n)
]
class MockServer:
def __init__(self) -> None:
self.active = 0
self.max_active = 0
self.payloads = []
self.runner = None
self.port = None
async def completion(self, request):
payload = await request.json()
self.payloads.append(payload)
self.active += 1
self.max_active = max(self.max_active, self.active)
await asyncio.sleep(0.01)
response = web.StreamResponse(
status=200, headers={"Content-Type": "text/event-stream"}
)
await response.prepare(request)
await response.write(
b'data: {"choices":[{"text":"x"}],"usage":null}\n\n'
)
usage = json.dumps(
{
"choices": [],
"usage": {
"prompt_tokens": len(payload["prompt"]),
"completion_tokens": payload["max_tokens"],
},
}
).encode()
await response.write(b"data: " + usage + b"\n\n")
await response.write(b"data: [DONE]\n\n")
await response.write_eof()
self.active -= 1
return response
async def start(self):
app = web.Application()
app.router.add_post("/v1/completions", self.completion)
self.runner = web.AppRunner(app)
await self.runner.setup()
site = web.TCPSite(self.runner, "127.0.0.1", 0)
await site.start()
self.port = site._server.sockets[0].getsockname()[1]
async def stop(self):
await self.runner.cleanup()
def run_args(
manifest: Path,
result_dir: Path,
port: int,
load_point: str = "saturation",
saturation_result: Path | None = None,
):
return SimpleNamespace(
manifest=str(manifest),
base_url=f"http://127.0.0.1:{port}",
model="mock",
load_point=load_point,
request_rate="inf" if load_point == "saturation" else None,
saturation_result=None if saturation_result is None else str(saturation_result),
rate_fraction=0.60,
max_concurrency=3,
ignore_eos=True,
temperature=0,
warmup_seconds=0.04,
clean_segment_seconds=0.04,
num_clean_segments=3,
profile_after_clean=False,
num_profile_windows=0,
profile_warmup_iterations=2,
profile_active_iterations=8,
profile_trace_dir=None,
profile_timeout_seconds=1,
recovery_seconds=0,
post_clean_seconds=0,
drain_timeout_seconds=1,
workload_seed=20260712,
server_seed=20260712,
result_dir=str(result_dir),
)
class Phase3ToolTests(unittest.TestCase):
def test_synthetic_prompt_exact_and_unique_early_prefix(self):
generated = [client.synthetic_prompt(row) for row in rows(200)]
self.assertTrue(all(len(value) == 8 for value in generated))
self.assertEqual(len({tuple(value[:3]) for value in generated}), 200)
def test_p05_manifest_has_exact_half_modes(self):
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp) / "P05.jsonl"
args = SimpleNamespace(
id="P05",
kind="synthetic",
input_uniform=None,
input_fixed=None,
input_mixture='{"uniform:128:512":0.5,"uniform:4096:8192":0.5}',
output_fixed=64,
prefix="none",
arrival="steady",
num_requests=100,
workload_seed=20260712,
num_prefixes=0,
prefix_len=0,
suffix_fixed=0,
out=str(out),
)
client.materialize(args)
manifest = client.load_manifest(out)
self.assertEqual(sum(r["input_tokens"] <= 512 for r in manifest), 50)
self.assertEqual(sum(r["input_tokens"] >= 4096 for r in manifest), 50)
def test_prefix_pool_balanced(self):
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp) / "P08.jsonl"
args = SimpleNamespace(
id="P08",
kind="prefix-pool",
input_uniform=None,
input_fixed=None,
input_mixture=None,
output_fixed=512,
prefix="none",
arrival="burst:8",
num_requests=80,
workload_seed=20260712,
num_prefixes=8,
prefix_len=1024,
suffix_fixed=256,
out=str(out),
)
client.materialize(args)
manifest = client.load_manifest(out)
counts = {i: 0 for i in range(8)}
for row in manifest:
counts[row["prefix_id"]] += 1
self.assertEqual(row["input_tokens"], 1280)
self.assertEqual(set(counts.values()), {10})
def test_fixed_duration_saturation_and_redaction(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(1000))
mock = MockServer()
await mock.start()
try:
result = await client.run_load(
run_args(manifest, root / "result", mock.port)
)
finally:
await mock.stop()
self.assertEqual(result["max_in_flight"], 3)
self.assertEqual(mock.max_active, 3)
self.assertAlmostEqual(result["clean"]["duration_s"], 0.12, places=9)
self.assertEqual(len(result["segments"]), 3)
self.assertLessEqual(result["drain_seconds"], 1)
self.assertTrue(
all(p["max_tokens"] == 2 and p["ignore_eos"] for p in mock.payloads)
)
text = (root / "result/requests.jsonl").read_text()
self.assertNotIn('"prompt":', text)
self.assertNotIn('"text":', text)
asyncio.run(case())
def test_drain_timeout_is_a_hard_sanity_failure(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(1000))
mock = MockServer()
await mock.start()
try:
args = run_args(manifest, root / "result", mock.port)
args.drain_timeout_seconds = 0
with self.assertRaisesRegex(RuntimeError, "drain_within_timeout"):
await client.run_load(args)
finally:
await mock.stop()
sanity = json.loads((root / "result/sanity.json").read_text())
self.assertFalse(sanity["invariants"]["drain_within_timeout"])
asyncio.run(case())
def test_burst_schedule_is_eight_at_once(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(1000, "burst:8"))
sat = root / "sat.json"
client.atomic_json(
sat, {"clean": {"completed_throughput_rps": 100.0}}
)
mock = MockServer()
await mock.start()
try:
args = run_args(
manifest, root / "result", mock.port, "moderate", sat
)
args.warmup_seconds = 0
args.clean_segment_seconds = 0.4 / 3
result = await client.run_load(args)
finally:
await mock.stop()
records = [
json.loads(line)
for line in (root / "result/requests.jsonl").read_text().splitlines()
]
groups = {}
for record in records:
groups.setdefault(round(record["scheduled_s"], 6), 0)
groups[round(record["scheduled_s"], 6)] += 1
self.assertTrue(all(value == 8 for value in groups.values()))
starts = sorted(groups)
if len(starts) > 1:
self.assertAlmostEqual(starts[1] - starts[0], 8 / 60, places=5)
self.assertAlmostEqual(result["request_rate"], 60.0)
asyncio.run(case())
def test_manifest_exhaustion_stops_instead_of_wrapping(self):
async def case():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manifest = root / "m.jsonl"
client.atomic_jsonl(manifest, rows(2))
mock = MockServer()
await mock.start()
try:
with self.assertRaises(client.ManifestExhausted):
await client.run_load(
run_args(manifest, root / "result", mock.port)
)
finally:
await mock.stop()
asyncio.run(case())
def test_cpu_affinity_sets_are_disjoint_and_cover_host(self):
values = []
for spec in controller.CPU_MAP.values():
lo, hi = [int(x) for x in spec.split("-")]
values.extend(range(lo, hi + 1))
self.assertEqual(sorted(values), list(range(160)))
self.assertEqual(len(values), len(set(values)))
def test_kernel_mapping_priority(self):
cases = {
"void vllm::moe::topkGating": "moe_router",
"ncclDevKernel_AllReduce": "collective",
"flash_fwd_kernel": "attention",
"nvjet_sm90_tst": "moe_gemm",
"argmax_kernel": "sampler",
"cutlass_gemm": "dense_gemm",
"triton_red_fused_add_rms_norm": "norm_elementwise",
"cache_swap_kernel": "kv_memory",
"unknown": "other",
}
self.assertEqual(
{name: controller.classify_kernel(name) for name in cases}, cases
)
def test_atomic_state_replacement(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "state.json"
controller.atomic_json(path, {"schema": 1, "value": 1})
controller.atomic_json(path, {"schema": 1, "value": 2})
self.assertEqual(json.loads(path.read_text())["value"], 2)
self.assertFalse(list(path.parent.glob("*.tmp.*")))
def test_fingerprint_survives_json_roundtrip(self):
original_run_text = controller.run_text
original_hash = controller.sha256_file
try:
controller.run_text = lambda *args, **kwargs: "deadbeef\n"
controller.sha256_file = lambda path: "a" * 64
fingerprint = controller.make_fingerprint()
finally:
controller.run_text = original_run_text
controller.sha256_file = original_hash
self.assertEqual(json.loads(json.dumps(fingerprint)), fingerprint)
self.assertEqual(set(fingerprint["cpu_map"]), {str(i) for i in range(8)})
def test_server_shutdown_signals_parent_before_group(self):
process = SimpleNamespace(pid=12345, poll=lambda: None)
with (
mock.patch.object(controller.os, "kill") as kill,
mock.patch.object(controller.os, "killpg") as killpg,
mock.patch.object(controller, "_process_group_alive", return_value=False),
):
controller.stop_servers([process])
kill.assert_called_once_with(12345, controller.signal.SIGINT)
killpg.assert_not_called()
if __name__ == "__main__":
unittest.main(verbosity=2)

353
runs/opprof-phase3/provenance/verify_shutdown_footer.py Normal file
View File

@@ -0,0 +1,353 @@
#!/usr/bin/env python3
"""One-GPU verification of API-parent-first OpProf shutdown."""
from __future__ import annotations
import json
import os
import shlex
import shutil
import signal
import subprocess
import time
import urllib.request
from pathlib import Path
from typing import Any
WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
RUN_DIR = WORKDIR / "runs/e-b-shutdown-verification"
SOURCE = Path(
"/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0"
)
VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
MANIFEST = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl")
PORT = 8010
def atomic_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
with tmp.open("w", encoding="utf-8") as f:
json.dump(value, f, sort_keys=True, indent=2)
f.write("\n")
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
def run_text(command: list[str], check: bool = True) -> str:
result = subprocess.run(
command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
)
if check and result.returncode:
raise RuntimeError(
f"command failed ({result.returncode}): {shlex.join(command)}\n"
f"{result.stdout}"
)
return result.stdout
def gpu_rows() -> list[dict[str, int]]:
output = run_text(
[
"nvidia-smi",
"--query-gpu=index,memory.used,utilization.gpu",
"--format=csv,noheader,nounits",
]
)
rows = []
for line in output.strip().splitlines():
index, memory, utilization = [int(part.strip()) for part in line.split(",")]
rows.append(
{"index": index, "memory_mib": memory, "utilization_pct": utilization}
)
return rows
def compute_apps() -> str:
return run_text(
[
"nvidia-smi",
"--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
"--format=csv,noheader,nounits",
],
check=False,
).strip()
def group_alive(pgid: int, process: subprocess.Popen[Any] | None = None) -> bool:
if process is not None:
process.poll() # reap an exited API parent before probing its group
try:
os.killpg(pgid, 0)
return True
except ProcessLookupError:
return False
def wait_ready(process: subprocess.Popen[Any], timeout: float = 300) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if process.poll() is not None:
raise RuntimeError("server exited before readiness")
try:
with urllib.request.urlopen(
f"http://127.0.0.1:{PORT}/health", timeout=1
) as response:
if response.status == 200:
return
except Exception:
pass
time.sleep(1)
raise TimeoutError("server readiness timeout")
def shutdown_parent_first(process: subprocess.Popen[Any]) -> dict[str, Any]:
called_at = time.time()
os.kill(process.pid, signal.SIGINT)
deadline = time.monotonic() + 120
while time.monotonic() < deadline and group_alive(process.pid, process):
time.sleep(1)
graceful = not group_alive(process.pid, process)
if not graceful:
for sig in (signal.SIGTERM, signal.SIGKILL):
try:
os.killpg(process.pid, sig)
except ProcessLookupError:
break
time.sleep(5)
return {
"api_parent_pid": process.pid,
"sigint_wall_time": called_at,
"group_gone_wall_time": time.time(),
"graceful": graceful,
"returncode": process.poll(),
}
def wait_stable_zero() -> list[list[dict[str, int]]]:
samples = []
consecutive = 0
deadline = time.monotonic() + 120
while time.monotonic() < deadline and consecutive < 3:
rows = gpu_rows()
samples.append(rows)
zero = rows[0]["memory_mib"] == 0 and not compute_apps()
consecutive = consecutive + 1 if zero else 0
if consecutive < 3:
time.sleep(5)
if consecutive < 3:
raise RuntimeError("GPU0 did not reach three stable zero samples")
return samples
def validate_footer() -> dict[str, Any]:
files = sorted((RUN_DIR / "opprof").glob("*.jsonl"))
if len(files) != 1:
raise RuntimeError(f"expected one OpProf JSONL, found {len(files)}")
lines = files[0].read_text().splitlines()
decoded = [json.loads(line) for line in lines]
if not decoded or decoded[-1].get("record_type") != "footer":
raise RuntimeError("Layer-1 footer missing")
records, footer = decoded[:-1], decoded[-1]
indices = [record["step_index"] for record in records]
invariants = {
"all_schema_1": all(item.get("schema") == 1 for item in decoded),
"one_footer_last": sum(
item.get("record_type") == "footer" for item in decoded
)
== 1,
"steps_contiguous": indices == list(range(len(indices))),
"written_matches_records": footer["written_records"] == len(records),
"encoded_balanced": footer["encoded_records"]
== footer["written_records"] + footer["dropped_records"],
"zero_drops": footer["dropped_records"] == 0
and all(record["dropped_records_before"] == 0 for record in records),
}
if not all(invariants.values()):
raise RuntimeError(f"footer accounting invalid: {invariants}, {footer}")
return {
"path": str(files[0]),
"bytes": files[0].stat().st_size,
"records": len(records),
"footer": footer,
"invariants": invariants,
}
def main() -> None:
if RUN_DIR.exists():
raise RuntimeError(f"refusing to overwrite {RUN_DIR}")
RUN_DIR.mkdir(parents=True)
state: dict[str, Any] = {
"schema": 1,
"status": "preflight",
"started_at": time.time(),
"gpu": 0,
}
atomic_json(RUN_DIR / "state.json", state)
before = gpu_rows()
if before[0]["memory_mib"] != 0 or before[0]["utilization_pct"] != 0:
raise RuntimeError(f"GPU0 is not idle: {before[0]}")
if compute_apps():
raise RuntimeError("a compute process exists before verification")
(RUN_DIR / "gpu-before.json").write_text(json.dumps(before, indent=2) + "\n")
(RUN_DIR / "clocks-before.txt").write_text(
run_text(["nvidia-smi", "-q", "-d", "CLOCK"])
)
profile_config = {
"profiler": "torch",
"torch_profiler_dir": "/tmp/wjh-opprof-p3-footer-verify",
"ignore_frontend": True,
"wait_iterations": 0,
"warmup_iterations": 2,
"active_iterations": 8,
}
trace_dir = Path(profile_config["torch_profiler_dir"])
if trace_dir.exists():
shutil.rmtree(trace_dir)
trace_dir.mkdir()
server_command = [
"taskset",
"-c",
"0-19",
str(VENV / "bin/vllm"),
"serve",
str(MODEL),
"--host",
"127.0.0.1",
"--port",
str(PORT),
"--tensor-parallel-size",
"1",
"--enable-chunked-prefill",
"--enable-prefix-caching",
"--profiler-config",
json.dumps(profile_config, separators=(",", ":")),
]
client_command = [
"taskset",
"-c",
"0-19",
str(VENV / "bin/python"),
str(CLIENT),
"run",
"--manifest",
str(MANIFEST),
"--base-url",
f"http://127.0.0.1:{PORT}",
"--model",
str(MODEL),
"--load-point",
"saturation",
"--request-rate",
"inf",
"--max-concurrency",
"256",
"--ignore-eos",
"--temperature",
"0",
"--warmup-seconds",
"30",
"--clean-segment-seconds",
"40",
"--num-clean-segments",
"3",
"--drain-timeout-seconds",
"120",
"--workload-seed",
"20260712",
"--result-dir",
str(RUN_DIR / "client"),
]
with (RUN_DIR / "commands.log").open("w") as f:
f.write(
"GPU_COMMAND shutdown footer server: "
+ shlex.join(server_command)
+ " ; expected=60-180s startup + 150-180s load\n"
)
f.write(
"GPU_COMMAND shutdown footer client: "
+ shlex.join(client_command)
+ " ; expected=30s warmup + 120s clean + drain\n"
)
server_log = (RUN_DIR / "server.log").open("ab", buffering=0)
env = os.environ.copy()
env.update(
{
"CUDA_VISIBLE_DEVICES": "0",
"VLLM_OPPROF_DIR": str(RUN_DIR / "opprof"),
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"PYTHONUNBUFFERED": "1",
}
)
server: subprocess.Popen[Any] | None = None
try:
print(
"GPU_COMMAND shutdown-fix verification: P01/C00 GPU0, "
"30s warmup + 120s clean, expected 4-6 wall-min",
flush=True,
)
server = subprocess.Popen(
server_command,
cwd=SOURCE,
env=env,
stdout=server_log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
state.update({"status": "server_starting", "server_pid": server.pid})
atomic_json(RUN_DIR / "state.json", state)
wait_ready(server)
state.update({"status": "client_running", "server_ready_at": time.time()})
atomic_json(RUN_DIR / "state.json", state)
with (RUN_DIR / "client.log").open("ab", buffering=0) as client_log:
result = subprocess.run(
client_command,
cwd=WORKDIR,
stdout=client_log,
stderr=subprocess.STDOUT,
)
if result.returncode:
raise RuntimeError(f"client exited {result.returncode}")
state["status"] = "parent_first_shutdown"
atomic_json(RUN_DIR / "state.json", state)
shutdown = shutdown_parent_first(server)
if not shutdown["graceful"]:
raise RuntimeError(f"server required escalation: {shutdown}")
footer = validate_footer()
zero_samples = wait_stable_zero()
client_result = json.loads((RUN_DIR / "client/result.json").read_text())
result_json = {
"schema": 1,
"status": "pass",
"gpu": 0,
"shutdown": shutdown,
"footer": footer,
"clean": client_result["clean"],
"failed_records": client_result["failed_records"],
"gpu_zero_samples": zero_samples,
"gpu_seconds": zero_samples and time.time() - state["started_at"],
}
atomic_json(RUN_DIR / "result.json", result_json)
state.update({"status": "complete", "completed_at": time.time()})
atomic_json(RUN_DIR / "state.json", state)
print(json.dumps(result_json, sort_keys=True), flush=True)
except Exception as error:
state.update({"status": "failed", "failure": repr(error)})
atomic_json(RUN_DIR / "state.json", state)
if server is not None and group_alive(server.pid, server):
try:
os.killpg(server.pid, signal.SIGKILL)
except ProcessLookupError:
pass
raise
finally:
server_log.close()
if __name__ == "__main__":
main()

392
runs/opprof-phase3/provenance/verify_sidecar_shutdown.py Normal file
View File

@@ -0,0 +1,392 @@
#!/usr/bin/env python3
"""GPU verification of graceful and hard-kill OpProf accounting."""
from __future__ import annotations
import json
import os
import shlex
import signal
import subprocess
import time
import urllib.request
from pathlib import Path
from typing import Any
WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
RUN_ROOT = WORKDIR / "runs/e-b-sidecar-verification"
SOURCE = Path(
"/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0"
)
VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
MANIFEST = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl")
FLUSH_INTERVAL_SECONDS = 1.0
CHECKPOINT_TOLERANCE_SECONDS = 0.1
def atomic_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f"{path.name}.tmp-{os.getpid()}")
with temporary.open("w", encoding="utf-8") as output:
json.dump(value, output, indent=2, sort_keys=True)
output.write("\n")
output.flush()
os.fsync(output.fileno())
os.replace(temporary, path)
def run_text(command: list[str], check: bool = True) -> str:
result = subprocess.run(
command,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
if check and result.returncode:
raise RuntimeError(
f"command failed ({result.returncode}): {shlex.join(command)}\n"
f"{result.stdout}"
)
return result.stdout
def gpu_rows() -> list[dict[str, int]]:
output = run_text(
[
"nvidia-smi",
"--query-gpu=index,memory.used,utilization.gpu",
"--format=csv,noheader,nounits",
]
)
rows = []
for line in output.strip().splitlines():
index, memory, utilization = (int(value.strip()) for value in line.split(","))
rows.append(
{
"index": index,
"memory_mib": memory,
"utilization_pct": utilization,
}
)
return rows
def compute_apps() -> str:
return run_text(
[
"nvidia-smi",
"--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
"--format=csv,noheader,nounits",
],
check=False,
).strip()
def assert_idle() -> list[dict[str, int]]:
rows = gpu_rows()
if any(row["memory_mib"] or row["utilization_pct"] for row in rows):
raise RuntimeError(f"dash0 is not GPU-idle: {rows}")
applications = compute_apps()
if applications:
raise RuntimeError(f"compute applications present: {applications}")
return rows
def wait_ready(process: subprocess.Popen[Any], port: int) -> None:
deadline = time.monotonic() + 300
while time.monotonic() < deadline:
if process.poll() is not None:
raise RuntimeError("server exited before readiness")
try:
with urllib.request.urlopen(
f"http://127.0.0.1:{port}/health", timeout=1
) as response:
if response.status == 200:
return
except Exception:
pass
time.sleep(1)
raise TimeoutError("server readiness timeout")
def wait_zero() -> tuple[list[list[dict[str, int]]], float]:
samples: list[list[dict[str, int]]] = []
consecutive = 0
deadline = time.monotonic() + 180
while time.monotonic() < deadline and consecutive < 3:
rows = gpu_rows()
samples.append(rows)
zero = all(row["memory_mib"] == 0 for row in rows) and not compute_apps()
consecutive = consecutive + 1 if zero else 0
if consecutive < 3:
time.sleep(2)
if consecutive < 3:
raise RuntimeError("GPUs did not reach three stable zero samples")
return samples, time.time()
def wait_fresh_sidecar(run_dir: Path) -> dict[str, Any]:
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
files = sorted((run_dir / "opprof").glob("*.jsonl.footer.json"))
if len(files) == 1:
sidecar = json.loads(files[0].read_text())
age = time.time_ns() - sidecar["checkpoint_wall_ns"]
if 0 <= age <= 250_000_000:
return sidecar
time.sleep(0.02)
raise TimeoutError("could not observe a fresh OpProf sidecar")
def validate_accounting(
run_dir: Path, mode: str, termination_wall_ns: int
) -> dict[str, Any]:
streams = sorted((run_dir / "opprof").glob("*.jsonl"))
sidecars = sorted((run_dir / "opprof").glob("*.jsonl.footer.json"))
if len(streams) != 1 or len(sidecars) != 1:
raise RuntimeError(
f"expected one stream/sidecar, got {len(streams)}/{len(sidecars)}"
)
raw = streams[0].read_bytes()
if not raw.endswith(b"\n"):
raise RuntimeError("partial final JSONL line")
decoded = [json.loads(line) for line in raw.splitlines()]
footers = [row for row in decoded if row.get("record_type") == "footer"]
records = [row for row in decoded if row.get("record_type") != "footer"]
sidecar = json.loads(sidecars[0].read_text())
indices = [row["step_index"] for row in records]
checkpoint_age = (
termination_wall_ns - sidecar["checkpoint_wall_ns"]
) / 1e9
common = {
"all_schema_1": all(row.get("schema") == 1 for row in decoded)
and sidecar.get("schema") == 1,
"steps_contiguous": indices == list(range(len(indices))),
"written_matches_records": sidecar["written_records"] == len(records),
"encoded_balanced": sidecar["encoded_records"]
== sidecar["written_records"] + sidecar["dropped_records"],
"last_step_matches": bool(records)
and sidecar["last_step_index"] == records[-1]["step_index"],
"zero_drops": sidecar["dropped_records"] == 0
and all(row["dropped_records_before"] == 0 for row in records),
}
if mode == "graceful":
footer_ok = len(footers) == 1 and decoded[-1] is footers[0]
agreement = footer_ok and all(
footers[0][counter] == sidecar[counter]
for counter in ("encoded_records", "written_records", "dropped_records")
)
specific = {
"one_footer_last": footer_ok,
"final_sidecar": sidecar["final"] is True,
"footer_sidecar_agree": agreement,
}
else:
specific = {
"no_in_stream_footer": not footers,
"checkpoint_sidecar": sidecar["final"] is False,
"checkpoint_within_bound": checkpoint_age
<= FLUSH_INTERVAL_SECONDS + CHECKPOINT_TOLERANCE_SECONDS,
}
invariants = {**common, **specific}
if not all(invariants.values()):
raise RuntimeError(
f"{mode} accounting invalid: {invariants}; sidecar={sidecar}"
)
return {
"stream": str(streams[0]),
"sidecar": str(sidecars[0]),
"bytes": len(raw),
"records": len(records),
"footer_count": len(footers),
"checkpoint_age_seconds": checkpoint_age,
"counters": {
key: sidecar[key]
for key in ("encoded_records", "written_records", "dropped_records")
},
"last_step_index": sidecar["last_step_index"],
"invariants": invariants,
}
def run_trial(mode: str, port: int) -> dict[str, Any]:
run_dir = RUN_ROOT / mode
if run_dir.exists():
raise RuntimeError(f"refusing to overwrite {run_dir}")
run_dir.mkdir(parents=True)
before = assert_idle()
atomic_json(run_dir / "gpu-before.json", before)
(run_dir / "clocks-before.txt").write_text(
run_text(["nvidia-smi", "-q", "-d", "CLOCK"])
)
server_command = [
"taskset",
"-c",
"0-19",
str(VENV / "bin/vllm"),
"serve",
str(MODEL),
"--host",
"127.0.0.1",
"--port",
str(port),
"--tensor-parallel-size",
"1",
"--enable-chunked-prefill",
"--enable-prefix-caching",
"--shutdown-timeout",
"120",
]
client_command = [
"taskset",
"-c",
"0-19",
str(VENV / "bin/python"),
str(CLIENT),
"run",
"--manifest",
str(MANIFEST),
"--base-url",
f"http://127.0.0.1:{port}",
"--model",
str(MODEL),
"--load-point",
"saturation",
"--request-rate",
"inf",
"--max-concurrency",
"256",
"--ignore-eos",
"--temperature",
"0",
"--warmup-seconds",
"20",
"--clean-segment-seconds",
"40",
"--num-clean-segments",
"3",
"--drain-timeout-seconds",
"120",
"--workload-seed",
"20260712",
"--result-dir",
str(run_dir / "client"),
]
with (run_dir / "commands.log").open("w", encoding="utf-8") as output:
output.write(
f"GPU_COMMAND {mode} server: {shlex.join(server_command)}; "
"expected=60-180s startup + 140-180s load\n"
)
output.write(
f"GPU_COMMAND {mode} client: {shlex.join(client_command)}; "
"expected=20s warmup + 120s clean + drain\n"
)
environment = os.environ.copy()
environment.update(
{
"CUDA_VISIBLE_DEVICES": "0",
"VLLM_OPPROF_DIR": str(run_dir / "opprof"),
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"PYTHONUNBUFFERED": "1",
}
)
started = time.time()
server_log = (run_dir / "server.log").open("ab", buffering=0)
server: subprocess.Popen[Any] | None = None
try:
print(
f"GPU_COMMAND sidecar-{mode}: P01/C00 GPU0, 20s warmup + "
"120s clean, expected 4-6 wall-min",
flush=True,
)
server = subprocess.Popen(
server_command,
cwd=SOURCE,
env=environment,
stdout=server_log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
atomic_json(
run_dir / "state.json",
{"status": "server_starting", "server_pid": server.pid},
)
wait_ready(server, port)
with (run_dir / "client.log").open("ab", buffering=0) as client_log:
client = subprocess.run(
client_command,
cwd=WORKDIR,
stdout=client_log,
stderr=subprocess.STDOUT,
)
if client.returncode:
raise RuntimeError(f"client exited {client.returncode}")
if mode == "graceful":
termination_wall_ns = time.time_ns()
os.kill(server.pid, signal.SIGINT)
server.wait(timeout=150)
log_text = (run_dir / "server.log").read_text(errors="replace")
if "mode=drain timeout=120s" not in log_text:
raise RuntimeError("official drain-mode log not observed")
else:
wait_fresh_sidecar(run_dir)
termination_wall_ns = time.time_ns()
os.killpg(server.pid, signal.SIGKILL)
server.wait(timeout=30)
zero_samples, zero_at = wait_zero()
accounting = validate_accounting(run_dir, mode, termination_wall_ns)
client_result = json.loads((run_dir / "client/result.json").read_text())
result = {
"schema": 1,
"status": "pass",
"mode": mode,
"server_returncode": server.returncode,
"termination_wall_ns": termination_wall_ns,
"accounting": accounting,
"clean": client_result["clean"],
"failed_records": client_result["failed_records"],
"gpu_zero_samples": zero_samples,
"gpu_seconds": zero_at - started,
}
atomic_json(run_dir / "result.json", result)
atomic_json(run_dir / "state.json", {"status": "complete"})
return result
except Exception as error:
atomic_json(
run_dir / "state.json",
{"status": "failed", "failure": repr(error)},
)
if server is not None and server.poll() is None:
try:
os.killpg(server.pid, signal.SIGKILL)
except ProcessLookupError:
pass
wait_zero()
raise
finally:
server_log.close()
def main() -> None:
if RUN_ROOT.exists():
raise RuntimeError(f"refusing to overwrite {RUN_ROOT}")
RUN_ROOT.mkdir(parents=True)
state: dict[str, Any] = {"schema": 1, "status": "running", "results": {}}
atomic_json(RUN_ROOT / "state.json", state)
for offset, mode in enumerate(("graceful", "hard-kill")):
result = run_trial(mode, 8010 + offset)
state["results"][mode] = result
atomic_json(RUN_ROOT / "state.json", state)
state["status"] = "complete"
state["gpu_seconds"] = sum(
result["gpu_seconds"] for result in state["results"].values()
)
atomic_json(RUN_ROOT / "state.json", state)
print(json.dumps(state, sort_keys=True), flush=True)
if __name__ == "__main__":
main()