Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

runs/opprof-phase3-ea/provenance/opprof_phase3_client.py

@@ -0,0 +1,784 @@
+#!/usr/bin/env python3
+"""Token-exact fixed-duration client for the OpProf Phase-3 protocol."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import gzip
+import hashlib
+import json
+import math
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+SCHEMA = 1
+TOKEN_BASE = 1000
+TOKEN_SPAN = 100000
+
+
+class ManifestExhausted(RuntimeError):
+    pass
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, value: Any, mode: int = 0o640) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        json.dump(value, f, sort_keys=True, indent=2)
+        f.write("\n")
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)
+
+
+def atomic_jsonl(path: Path, rows: list[dict[str, Any]], mode: int = 0o640) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)
+
+
+def parse_range(value: str) -> tuple[int, int]:
+    lo_text, hi_text = value.split(":", 1)
+    lo, hi = int(lo_text), int(hi_text)
+    if lo <= 0 or hi < lo:
+        raise argparse.ArgumentTypeError(f"invalid positive range: {value}")
+    return lo, hi
+
+
+def _integer_counts(weights: list[float], total: int) -> list[int]:
+    raw = [w * total for w in weights]
+    counts = [math.floor(x) for x in raw]
+    order = sorted(
+        range(len(raw)), key=lambda i: raw[i] - counts[i], reverse=True
+    )
+    for idx in order[: total - sum(counts)]:
+        counts[idx] += 1
+    return counts
+
+
+def numeric_sanity(values: list[float | int]) -> dict[str, Any]:
+    finite = [float(x) for x in values if math.isfinite(float(x))]
+    return {
+        "n": len(values),
+        "finite_n": len(finite),
+        "missing_n": len(values) - len(finite),
+        "min": min(finite) if finite else None,
+        "max": max(finite) if finite else None,
+        "distinct_n": len(set(finite)),
+    }
+
+
+def manifest_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    return {
+        "schema": SCHEMA,
+        "rows": len(rows),
+        "input_tokens": numeric_sanity([int(r["input_tokens"]) for r in rows]),
+        "output_tokens": numeric_sanity([int(r["output_tokens"]) for r in rows]),
+        "arrival_values": sorted({str(r["arrival"]) for r in rows}),
+        "pattern_values": sorted({str(r["pattern_id"]) for r in rows}),
+    }
+
+
+def materialize(args: argparse.Namespace) -> dict[str, Any]:
+    import numpy as np
+
+    rng = np.random.default_rng(args.workload_seed)
+    n = args.num_requests
+    if args.kind == "prefix-pool":
+        if args.num_prefixes <= 0 or args.prefix_len <= 0 or args.suffix_fixed <= 0:
+            raise ValueError("prefix-pool requires positive pool/prefix/suffix")
+        lengths = np.full(n, args.prefix_len + args.suffix_fixed, dtype=np.int64)
+        prefix_ids = np.arange(n, dtype=np.int64) % args.num_prefixes
+        rng.shuffle(prefix_ids)
+    else:
+        prefix_ids = np.full(n, -1, dtype=np.int64)
+        if args.input_uniform:
+            lo, hi = parse_range(args.input_uniform)
+            lengths = rng.integers(lo, hi + 1, n, dtype=np.int64)
+        elif args.input_fixed:
+            lengths = np.full(n, args.input_fixed, dtype=np.int64)
+        elif args.input_mixture:
+            spec = json.loads(args.input_mixture)
+            if not isinstance(spec, dict) or not spec:
+                raise ValueError("input mixture must be a non-empty JSON object")
+            keys = list(spec)
+            weights = [float(spec[key]) for key in keys]
+            if any(w < 0 for w in weights) or not math.isclose(sum(weights), 1.0):
+                raise ValueError("mixture weights must be non-negative and sum to 1")
+            pieces = []
+            for key, count in zip(
+                keys, _integer_counts(weights, n), strict=True
+            ):
+                kind, lo_text, hi_text = key.split(":")
+                if kind != "uniform":
+                    raise ValueError(f"unsupported mixture component: {key}")
+                pieces.append(
+                    rng.integers(
+                        int(lo_text), int(hi_text) + 1, count, dtype=np.int64
+                    )
+                )
+            lengths = np.concatenate(pieces)
+            rng.shuffle(lengths)
+        else:
+            raise ValueError("exactly one input distribution is required")
+    if args.output_fixed <= 0 or args.arrival not in {"steady", "burst:8"}:
+        raise ValueError("invalid output length or arrival class")
+
+    rows = []
+    for i in range(n):
+        row = {
+            "schema": SCHEMA,
+            "request_id": f"{args.id}-{i:05d}",
+            "pattern_id": args.id,
+            "kind": args.kind,
+            "input_tokens": int(lengths[i]),
+            "output_tokens": args.output_fixed,
+            "arrival": args.arrival,
+            "token_seed": int(args.workload_seed * 1000003 + i),
+        }
+        if args.kind == "prefix-pool":
+            row.update(
+                {
+                    "prefix_id": int(prefix_ids[i]),
+                    "num_prefixes": args.num_prefixes,
+                    "prefix_tokens": args.prefix_len,
+                }
+            )
+        rows.append(row)
+
+    out = Path(args.out)
+    atomic_jsonl(out, rows, mode=0o600)
+    summary = manifest_summary(rows)
+    summary.update({"sha256": sha256_file(out), "path": str(out)})
+    atomic_json(out.with_suffix(out.suffix + ".summary.json"), summary, mode=0o600)
+    print(json.dumps(summary, sort_keys=True))
+    return summary
+
+
+def materialize_private(args: argparse.Namespace) -> dict[str, Any]:
+    from transformers import AutoTokenizer
+
+    source = Path(args.source)
+    selected: list[dict[str, Any]] = []
+    with source.open(encoding="utf-8") as f:
+        for source_index, line in enumerate(f):
+            row = json.loads(line)
+            if (
+                float(row["sampling_u"]) <= args.sampling_u_max
+                and int(row["input_length"]) <= args.max_input_tokens
+            ):
+                selected.append(
+                    {
+                        "schema": SCHEMA,
+                        "request_id": f"{args.id}-{len(selected):05d}",
+                        "pattern_id": args.id,
+                        "kind": "private-trace",
+                        "input_tokens": int(row["input_length"]),
+                        "output_tokens": min(
+                            int(row["output_length"]), args.output_cap
+                        ),
+                        "arrival": args.arrival,
+                        "source_index": source_index,
+                        "prompt": row["prompt"],
+                    }
+                )
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    diffs = [
+        len(tokenizer.encode(row["prompt"], add_special_tokens=False))
+        - row["input_tokens"]
+        for row in selected
+    ]
+    exact = sum(diff == 0 for diff in diffs)
+    exact_fraction = exact / len(diffs) if diffs else 0.0
+    max_abs = max((abs(diff) for diff in diffs), default=-1)
+    if exact_fraction < 0.99 or max_abs > 1:
+        raise RuntimeError(
+            "tokenizer parity gate failed: "
+            f"exact_fraction={exact_fraction:.6f} max_abs_error={max_abs}"
+        )
+
+    out = Path(args.out)
+    atomic_jsonl(out, selected, mode=0o600)
+    summary = manifest_summary(selected)
+    summary.update(
+        {
+            "sha256": sha256_file(out),
+            "source_sha256": sha256_file(source),
+            "tokenizer_exact_n": exact,
+            "tokenizer_exact_fraction": exact_fraction,
+            "tokenizer_max_abs_error": max_abs,
+            "path": str(out),
+        }
+    )
+    atomic_json(out.with_suffix(out.suffix + ".summary.json"), summary, mode=0o600)
+    print(json.dumps(summary, sort_keys=True))
+    return summary
+
+
+def load_manifest(path: Path) -> list[dict[str, Any]]:
+    rows = [json.loads(line) for line in path.read_text().splitlines() if line]
+    required = {
+        "request_id",
+        "pattern_id",
+        "input_tokens",
+        "output_tokens",
+        "arrival",
+    }
+    if not rows:
+        raise ValueError("empty manifest")
+    for row in rows:
+        if not required.issubset(row):
+            raise ValueError(f"manifest row lacks {sorted(required - set(row))}")
+    if len({row["request_id"] for row in rows}) != len(rows):
+        raise ValueError("duplicate request_id")
+    return rows
+
+
+def _token_stream(seed: int, count: int) -> list[int]:
+    state = seed & 0xFFFFFFFF
+    out = []
+    for _ in range(count):
+        state = (1664525 * state + 1013904223) & 0xFFFFFFFF
+        out.append(TOKEN_BASE + state % TOKEN_SPAN)
+    return out
+
+
+def synthetic_prompt(row: dict[str, Any]) -> list[int]:
+    length = int(row["input_tokens"])
+    seed = int(row.get("token_seed", 0))
+    if row.get("kind") == "prefix-pool":
+        prefix_n = int(row["prefix_tokens"])
+        tokens = _token_stream(0xA5A50000 + int(row["prefix_id"]), prefix_n)
+        tokens += _token_stream(seed, length - prefix_n)
+        offset = prefix_n
+    else:
+        tokens = _token_stream(seed, length)
+        offset = 0
+    if length - offset >= 3:
+        index = int(row["request_id"].rsplit("-", 1)[1])
+        tokens[offset : offset + 3] = [
+            TOKEN_BASE + index % 100,
+            TOKEN_BASE + (index // 100) % 100,
+            TOKEN_BASE + (index // 10000) % 100,
+        ]
+    return tokens
+
+
+@dataclass
+class RunContext:
+    args: argparse.Namespace
+    rows: list[dict[str, Any]]
+    t0: float
+    clean_end: float
+    stop_event: asyncio.Event
+    lock: asyncio.Lock
+    next_index: int = 0
+    in_flight: int = 0
+    max_in_flight: int = 0
+    exhausted: bool = False
+    admission_stop_s: float | None = None
+
+    async def next_row(self) -> dict[str, Any]:
+        async with self.lock:
+            if self.next_index >= len(self.rows):
+                self.exhausted = True
+                raise ManifestExhausted(
+                    f"manifest exhausted after {self.next_index} admissions"
+                )
+            row = self.rows[self.next_index]
+            self.next_index += 1
+            return row
+
+
+async def request_one(
+    ctx: RunContext,
+    session: aiohttp.ClientSession,
+    row: dict[str, Any],
+    scheduled: float,
+) -> dict[str, Any]:
+    loop = asyncio.get_running_loop()
+    admitted = loop.time()
+    ctx.in_flight += 1
+    ctx.max_in_flight = max(ctx.max_in_flight, ctx.in_flight)
+    status = 0
+    actual_output: int | None = None
+    first_token: float | None = None
+    error_kind: str | None = None
+    try:
+        prompt: str | list[int] = (
+            row["prompt"]
+            if row.get("kind") == "private-trace"
+            else synthetic_prompt(row)
+        )
+        if not isinstance(prompt, str) and len(prompt) != int(row["input_tokens"]):
+            raise AssertionError("synthetic prompt length drift")
+        payload = {
+            "model": ctx.args.model,
+            "prompt": prompt,
+            "max_tokens": int(row["output_tokens"]),
+            "temperature": ctx.args.temperature,
+            "ignore_eos": ctx.args.ignore_eos,
+            "stream": True,
+            "stream_options": {"include_usage": True},
+            "add_special_tokens": False,
+            "seed": ctx.args.server_seed,
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "x-request-id": str(row["request_id"]),
+        }
+        async with session.post(
+            ctx.args.base_url.rstrip("/") + "/v1/completions",
+            json=payload,
+            headers=headers,
+        ) as response:
+            status = response.status
+            if status != 200:
+                error_kind = f"http_{status}"
+            else:
+                buf = b""
+                async for chunk in response.content.iter_any():
+                    buf += chunk
+                    while b"\n" in buf:
+                        line, buf = buf.split(b"\n", 1)
+                        line = line.strip()
+                        if not line.startswith(b"data:"):
+                            continue
+                        data = line[5:].strip()
+                        if data == b"[DONE]":
+                            continue
+                        event = json.loads(data)
+                        if event.get("choices") and first_token is None:
+                            first_token = loop.time()
+                        if event.get("usage") is not None:
+                            actual_output = int(event["usage"]["completion_tokens"])
+                if actual_output is None:
+                    error_kind = "missing_usage"
+    except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
+        error_kind = type(exc).__name__
+    except Exception as exc:
+        error_kind = type(exc).__name__
+    finally:
+        completed = loop.time()
+        ctx.in_flight -= 1
+    success = (
+        status == 200
+        and error_kind is None
+        and actual_output == int(row["output_tokens"])
+    )
+    if status == 200 and actual_output is not None and not success:
+        error_kind = "output_token_mismatch"
+    return {
+        "schema": SCHEMA,
+        "request_id": row["request_id"],
+        "scheduled_s": scheduled - ctx.t0,
+        "admitted_s": admitted - ctx.t0,
+        "first_token_s": None if first_token is None else first_token - ctx.t0,
+        "completed_s": completed - ctx.t0,
+        "input_tokens": int(row["input_tokens"]),
+        "requested_output_tokens": int(row["output_tokens"]),
+        "actual_output_tokens": actual_output,
+        "http_status": status,
+        "success": success,
+        "error_kind": error_kind,
+    }
+
+
+async def saturation_load(
+    ctx: RunContext, session: aiohttp.ClientSession
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    async def worker() -> None:
+        while not ctx.stop_event.is_set():
+            try:
+                row = await ctx.next_row()
+            except ManifestExhausted:
+                ctx.stop_event.set()
+                return
+            results.append(
+                await request_one(ctx, session, row, asyncio.get_running_loop().time())
+            )
+
+    tasks = [
+        asyncio.create_task(worker()) for _ in range(ctx.args.max_concurrency)
+    ]
+    await asyncio.gather(*tasks)
+    return results
+
+
+async def finite_load(
+    ctx: RunContext, session: aiohttp.ClientSession, rate: float
+) -> list[dict[str, Any]]:
+    sem = asyncio.Semaphore(ctx.args.max_concurrency)
+    tasks: list[asyncio.Task[dict[str, Any]]] = []
+    batch = 8 if str(ctx.rows[0]["arrival"]) == "burst:8" else 1
+    period = batch / rate
+    event_index = 0
+
+    async def limited(row: dict[str, Any], scheduled: float) -> dict[str, Any]:
+        async with sem:
+            return await request_one(ctx, session, row, scheduled)
+
+    while not ctx.stop_event.is_set():
+        scheduled = ctx.t0 + event_index * period
+        delay = scheduled - asyncio.get_running_loop().time()
+        if delay > 0:
+            try:
+                await asyncio.wait_for(ctx.stop_event.wait(), timeout=delay)
+                break
+            except asyncio.TimeoutError:
+                pass
+        if ctx.stop_event.is_set():
+            break
+        try:
+            for _ in range(batch):
+                tasks.append(
+                    asyncio.create_task(limited(await ctx.next_row(), scheduled))
+                )
+        except ManifestExhausted:
+            ctx.stop_event.set()
+            break
+        event_index += 1
+    return await asyncio.gather(*tasks) if tasks else []
+
+
+async def post_profile(
+    session: aiohttp.ClientSession, base_url: str, endpoint: str
+) -> tuple[float, float, int]:
+    loop = asyncio.get_running_loop()
+    before = loop.time()
+    async with session.post(base_url.rstrip("/") + endpoint) as response:
+        status = response.status
+        await response.read()
+    return before, loop.time(), status
+
+
+def _trace_loadable(path: Path) -> bool:
+    try:
+        opener = gzip.open if path.suffix == ".gz" else open
+        with opener(path, "rt", encoding="utf-8") as f:
+            parsed = json.load(f)
+        return isinstance(parsed, dict) and isinstance(parsed.get("traceEvents"), list)
+    except (OSError, EOFError, json.JSONDecodeError):
+        return False
+
+
+async def wait_new_trace(
+    trace_dir: Path, before: set[Path], timeout: float
+) -> Path:
+    deadline = asyncio.get_running_loop().time() + timeout
+    while asyncio.get_running_loop().time() < deadline:
+        for path in sorted(set(trace_dir.glob("*.pt.trace.json*")) - before):
+            if _trace_loadable(path):
+                return path
+        await asyncio.sleep(0.25)
+    raise TimeoutError(f"no new loadable trace within {timeout}s")
+
+
+async def timeline(
+    ctx: RunContext, session: aiohttp.ClientSession
+) -> list[dict[str, Any]]:
+    args = ctx.args
+    profiles: list[dict[str, Any]] = []
+    await asyncio.sleep(max(0, ctx.clean_end - asyncio.get_running_loop().time()))
+    if args.profile_after_clean:
+        trace_dir = Path(args.profile_trace_dir)
+        for window in range(args.num_profile_windows):
+            prior = set(trace_dir.glob("*.pt.trace.json*"))
+            start_before, start_after, start_status = await post_profile(
+                session, args.base_url, "/start_profile"
+            )
+            trace = await wait_new_trace(
+                trace_dir, prior, args.profile_timeout_seconds
+            )
+            trace_ready = asyncio.get_running_loop().time()
+            stop_before, stop_after, stop_status = await post_profile(
+                session, args.base_url, "/stop_profile"
+            )
+            profiles.append(
+                {
+                    "window": window + 1,
+                    "start_call_s": start_before - ctx.t0,
+                    "start_return_s": start_after - ctx.t0,
+                    "trace_ready_s": trace_ready - ctx.t0,
+                    "stop_call_s": stop_before - ctx.t0,
+                    "stop_return_s": stop_after - ctx.t0,
+                    "start_status": start_status,
+                    "stop_status": stop_status,
+                    "trace_file": trace.name,
+                    "trace_sha256": sha256_file(trace),
+                }
+            )
+            if start_status != 200 or stop_status != 200:
+                raise RuntimeError("profile endpoint returned non-200")
+            await asyncio.sleep(args.recovery_seconds)
+    else:
+        await asyncio.sleep(args.post_clean_seconds)
+    ctx.admission_stop_s = asyncio.get_running_loop().time() - ctx.t0
+    ctx.stop_event.set()
+    return profiles
+
+
+def segment_summary(
+    records: list[dict[str, Any]], start: float, end: float
+) -> dict[str, Any]:
+    admitted = [r for r in records if start <= r["admitted_s"] < end]
+    completed = [r for r in records if start <= r["completed_s"] < end]
+    successes = [r for r in completed if r["success"]]
+    duration = end - start
+    return {
+        "start_s": start,
+        "end_s": end,
+        "duration_s": duration,
+        "admitted": len(admitted),
+        "completed": len(successes),
+        "failed": len(completed) - len(successes),
+        "offered_rps": len(admitted) / duration,
+        "completed_throughput_rps": len(successes) / duration,
+        "input_tokens": sum(r["input_tokens"] for r in successes),
+        "output_tokens": sum(r["actual_output_tokens"] or 0 for r in successes),
+    }
+
+
+async def run_load(args: argparse.Namespace) -> dict[str, Any]:
+    manifest = Path(args.manifest)
+    rows = load_manifest(manifest)
+    arrivals = {row["arrival"] for row in rows}
+    if len(arrivals) != 1:
+        raise ValueError("a manifest must have one arrival class")
+    if args.load_point == "saturation":
+        if args.request_rate != "inf":
+            raise ValueError("saturation requires --request-rate inf")
+        rate = math.inf
+    else:
+        if not args.saturation_result:
+            raise ValueError("moderate requires --saturation-result")
+        sat = json.loads(Path(args.saturation_result).read_text())
+        rate = args.rate_fraction * float(sat["clean"]["completed_throughput_rps"])
+        if not math.isfinite(rate) or rate <= 0:
+            raise ValueError("derived moderate rate must be positive and finite")
+
+    loop = asyncio.get_running_loop()
+    t0 = loop.time()
+    clean_seconds = args.clean_segment_seconds * args.num_clean_segments
+    ctx = RunContext(
+        args=args,
+        rows=rows,
+        t0=t0,
+        clean_end=t0 + args.warmup_seconds + clean_seconds,
+        stop_event=asyncio.Event(),
+        lock=asyncio.Lock(),
+    )
+    timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=600)
+    connector = aiohttp.TCPConnector(limit=args.max_concurrency)
+    async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
+        profile_task = asyncio.create_task(timeline(ctx, session))
+        load_task = asyncio.create_task(
+            saturation_load(ctx, session)
+            if math.isinf(rate)
+            else finite_load(ctx, session, rate)
+        )
+        try:
+            profiles = await profile_task
+        except Exception:
+            ctx.stop_event.set()
+            await load_task
+            raise
+        records = await load_task
+
+    clean_start = args.warmup_seconds
+    clean_end = clean_start + clean_seconds
+    clean = segment_summary(records, clean_start, clean_end)
+    segments = []
+    for i in range(args.num_clean_segments):
+        start = clean_start + i * args.clean_segment_seconds
+        segments.append(
+            {
+                "name": chr(ord("A") + i),
+                **segment_summary(
+                    records, start, start + args.clean_segment_seconds
+                ),
+            }
+        )
+    successful = [r for r in records if r["success"]]
+    elapsed_seconds = loop.time() - t0
+    if ctx.admission_stop_s is None:
+        raise RuntimeError("admission stop timestamp was not recorded")
+    drain_seconds = elapsed_seconds - ctx.admission_stop_s
+    result = {
+        "schema": SCHEMA,
+        "manifest_sha256": sha256_file(manifest),
+        "manifest_rows": len(rows),
+        "manifest_admitted": ctx.next_index,
+        "manifest_wrapped": False,
+        "manifest_exhausted": ctx.exhausted,
+        "load_point": args.load_point,
+        "request_rate": "inf" if math.isinf(rate) else rate,
+        "rate_fraction": None if math.isinf(rate) else args.rate_fraction,
+        "arrival": next(iter(arrivals)),
+        "warmup_seconds": args.warmup_seconds,
+        "clean_segment_seconds": args.clean_segment_seconds,
+        "num_clean_segments": args.num_clean_segments,
+        "elapsed_seconds": elapsed_seconds,
+        "admission_stop_s": ctx.admission_stop_s,
+        "drain_seconds": drain_seconds,
+        "max_in_flight": ctx.max_in_flight,
+        "records": len(records),
+        "successful_records": len(successful),
+        "failed_records": len(records) - len(successful),
+        "clean": clean,
+        "segments": segments,
+        "profiles": profiles,
+    }
+    sanity = {
+        "schema": SCHEMA,
+        "numeric": {
+            "input_tokens": numeric_sanity([r["input_tokens"] for r in records]),
+            "requested_output_tokens": numeric_sanity(
+                [r["requested_output_tokens"] for r in records]
+            ),
+            "actual_output_tokens": numeric_sanity(
+                [
+                    r["actual_output_tokens"]
+                    for r in records
+                    if r["actual_output_tokens"] is not None
+                ]
+            ),
+            "scheduled_s": numeric_sanity([r["scheduled_s"] for r in records]),
+            "admitted_s": numeric_sanity([r["admitted_s"] for r in records]),
+            "completed_s": numeric_sanity([r["completed_s"] for r in records]),
+        },
+        "invariants": {
+            "clean_duration_exact": math.isclose(clean["duration_s"], clean_seconds),
+            "segment_count_exact": len(segments) == args.num_clean_segments,
+            "manifest_no_wrap": ctx.next_index <= len(rows),
+            "manifest_not_exhausted": not ctx.exhausted,
+            "concurrency_bounded": ctx.max_in_flight <= args.max_concurrency,
+            "drain_within_timeout": drain_seconds <= args.drain_timeout_seconds,
+            "output_tokens_exact": all(
+                r["actual_output_tokens"] == r["requested_output_tokens"]
+                for r in successful
+            ),
+            "clean_failures_zero": clean["failed"] == 0,
+            "profile_count_exact": len(profiles)
+            == (args.num_profile_windows if args.profile_after_clean else 0),
+            "profile_status_ok": all(
+                p["start_status"] == 200 and p["stop_status"] == 200
+                for p in profiles
+            ),
+        },
+    }
+    if not math.isinf(rate):
+        sanity["invariants"]["moderate_offered_within_5pct"] = (
+            abs(clean["offered_rps"] / rate - 1) <= 0.05
+        )
+    out = Path(args.result_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    atomic_jsonl(out / "requests.jsonl", sorted(records, key=lambda r: r["admitted_s"]))
+    atomic_jsonl(out / "segments.jsonl", segments)
+    atomic_json(out / "result.json", result)
+    atomic_json(out / "sanity.json", sanity)
+    if ctx.exhausted:
+        raise ManifestExhausted("manifest exhausted; result retained for diagnosis")
+    failed = [name for name, ok in sanity["invariants"].items() if not ok]
+    if failed:
+        raise RuntimeError(f"client sanity failure: {failed}")
+    return result
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="command", required=True)
+    mat = sub.add_parser("materialize")
+    mat.add_argument("--id", required=True)
+    mat.add_argument("--kind", choices=("synthetic", "prefix-pool"), required=True)
+    group = mat.add_mutually_exclusive_group()
+    group.add_argument("--input-uniform")
+    group.add_argument("--input-fixed", type=int)
+    group.add_argument("--input-mixture")
+    mat.add_argument("--output-fixed", type=int, required=True)
+    mat.add_argument("--prefix", default="none")
+    mat.add_argument("--arrival", required=True)
+    mat.add_argument("--num-requests", type=int, required=True)
+    mat.add_argument("--workload-seed", type=int, required=True)
+    mat.add_argument("--num-prefixes", type=int, default=0)
+    mat.add_argument("--prefix-len", type=int, default=0)
+    mat.add_argument("--suffix-fixed", type=int, default=0)
+    mat.add_argument("--out", required=True)
+    private = sub.add_parser("materialize-private")
+    private.add_argument("--id", required=True)
+    private.add_argument("--source", required=True)
+    private.add_argument("--sampling-u-max", type=float, required=True)
+    private.add_argument("--max-input-tokens", type=int, required=True)
+    private.add_argument("--output-cap", type=int, required=True)
+    private.add_argument("--preserve-prompts", action="store_true", required=True)
+    private.add_argument("--disable-shuffle", action="store_true", required=True)
+    private.add_argument("--arrival", required=True)
+    private.add_argument("--model", required=True)
+    private.add_argument("--out", required=True)
+    run = sub.add_parser("run")
+    run.add_argument("--manifest", required=True)
+    run.add_argument("--base-url", required=True)
+    run.add_argument("--model", required=True)
+    run.add_argument("--load-point", choices=("saturation", "moderate"), required=True)
+    run.add_argument("--request-rate")
+    run.add_argument("--saturation-result")
+    run.add_argument("--rate-fraction", type=float, default=0.60)
+    run.add_argument("--max-concurrency", type=int, default=256)
+    run.add_argument("--ignore-eos", action="store_true")
+    run.add_argument("--temperature", type=float, default=0.0)
+    run.add_argument("--warmup-seconds", type=float, default=60)
+    run.add_argument("--clean-segment-seconds", type=float, default=80)
+    run.add_argument("--num-clean-segments", type=int, default=3)
+    run.add_argument("--profile-after-clean", action="store_true")
+    run.add_argument("--num-profile-windows", type=int, default=0)
+    run.add_argument("--profile-warmup-iterations", type=int, default=2)
+    run.add_argument("--profile-active-iterations", type=int, default=8)
+    run.add_argument("--profile-trace-dir")
+    run.add_argument("--profile-timeout-seconds", type=float, default=120)
+    run.add_argument("--recovery-seconds", type=float, default=30)
+    run.add_argument("--post-clean-seconds", type=float, default=0)
+    run.add_argument("--drain-timeout-seconds", type=float, default=120)
+    run.add_argument("--workload-seed", type=int, default=20260712)
+    run.add_argument("--server-seed", type=int, default=20260712)
+    run.add_argument("--result-dir", required=True)
+    return parser
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.command == "materialize":
+        materialize(args)
+    elif args.command == "materialize-private":
+        materialize_private(args)
+    else:
+        if args.profile_after_clean and not args.profile_trace_dir:
+            raise ValueError("--profile-after-clean requires --profile-trace-dir")
+        print(json.dumps(asyncio.run(run_load(args)), sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase3-ea/provenance/test_phase3_tools.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from pathlib import Path
+from types import SimpleNamespace
+
+from aiohttp import web
+
+
+HERE = Path(__file__).parent
+
+
+def load_module(name: str, filename: str):
+    spec = importlib.util.spec_from_file_location(name, HERE / filename)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+client = load_module("phase3_client", "opprof_phase3_client.py")
+controller = load_module("phase3_controller", "opprof_phase3_controller.py")
+
+
+def rows(n: int, arrival: str = "steady") -> list[dict]:
+    return [
+        {
+            "schema": 1,
+            "request_id": f"T-{i:05d}",
+            "pattern_id": "T",
+            "kind": "synthetic",
+            "input_tokens": 8,
+            "output_tokens": 2,
+            "arrival": arrival,
+            "token_seed": i + 1,
+        }
+        for i in range(n)
+    ]
+
+
+class MockServer:
+    def __init__(self) -> None:
+        self.active = 0
+        self.max_active = 0
+        self.payloads = []
+        self.runner = None
+        self.port = None
+
+    async def completion(self, request):
+        payload = await request.json()
+        self.payloads.append(payload)
+        self.active += 1
+        self.max_active = max(self.max_active, self.active)
+        await asyncio.sleep(0.01)
+        response = web.StreamResponse(
+            status=200, headers={"Content-Type": "text/event-stream"}
+        )
+        await response.prepare(request)
+        await response.write(
+            b'data: {"choices":[{"text":"x"}],"usage":null}\n\n'
+        )
+        usage = json.dumps(
+            {
+                "choices": [],
+                "usage": {
+                    "prompt_tokens": len(payload["prompt"]),
+                    "completion_tokens": payload["max_tokens"],
+                },
+            }
+        ).encode()
+        await response.write(b"data: " + usage + b"\n\n")
+        await response.write(b"data: [DONE]\n\n")
+        await response.write_eof()
+        self.active -= 1
+        return response
+
+    async def start(self):
+        app = web.Application()
+        app.router.add_post("/v1/completions", self.completion)
+        self.runner = web.AppRunner(app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, "127.0.0.1", 0)
+        await site.start()
+        self.port = site._server.sockets[0].getsockname()[1]
+
+    async def stop(self):
+        await self.runner.cleanup()
+
+
+def run_args(
+    manifest: Path,
+    result_dir: Path,
+    port: int,
+    load_point: str = "saturation",
+    saturation_result: Path | None = None,
+):
+    return SimpleNamespace(
+        manifest=str(manifest),
+        base_url=f"http://127.0.0.1:{port}",
+        model="mock",
+        load_point=load_point,
+        request_rate="inf" if load_point == "saturation" else None,
+        saturation_result=None if saturation_result is None else str(saturation_result),
+        rate_fraction=0.60,
+        max_concurrency=3,
+        ignore_eos=True,
+        temperature=0,
+        warmup_seconds=0.04,
+        clean_segment_seconds=0.04,
+        num_clean_segments=3,
+        profile_after_clean=False,
+        num_profile_windows=0,
+        profile_warmup_iterations=2,
+        profile_active_iterations=8,
+        profile_trace_dir=None,
+        profile_timeout_seconds=1,
+        recovery_seconds=0,
+        post_clean_seconds=0,
+        drain_timeout_seconds=1,
+        workload_seed=20260712,
+        server_seed=20260712,
+        result_dir=str(result_dir),
+    )
+
+
+class Phase3ToolTests(unittest.TestCase):
+    def test_synthetic_prompt_exact_and_unique_early_prefix(self):
+        generated = [client.synthetic_prompt(row) for row in rows(200)]
+        self.assertTrue(all(len(value) == 8 for value in generated))
+        self.assertEqual(len({tuple(value[:3]) for value in generated}), 200)
+
+    def test_p05_manifest_has_exact_half_modes(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "P05.jsonl"
+            args = SimpleNamespace(
+                id="P05",
+                kind="synthetic",
+                input_uniform=None,
+                input_fixed=None,
+                input_mixture='{"uniform:128:512":0.5,"uniform:4096:8192":0.5}',
+                output_fixed=64,
+                prefix="none",
+                arrival="steady",
+                num_requests=100,
+                workload_seed=20260712,
+                num_prefixes=0,
+                prefix_len=0,
+                suffix_fixed=0,
+                out=str(out),
+            )
+            client.materialize(args)
+            manifest = client.load_manifest(out)
+            self.assertEqual(sum(r["input_tokens"] <= 512 for r in manifest), 50)
+            self.assertEqual(sum(r["input_tokens"] >= 4096 for r in manifest), 50)
+
+    def test_prefix_pool_balanced(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "P08.jsonl"
+            args = SimpleNamespace(
+                id="P08",
+                kind="prefix-pool",
+                input_uniform=None,
+                input_fixed=None,
+                input_mixture=None,
+                output_fixed=512,
+                prefix="none",
+                arrival="burst:8",
+                num_requests=80,
+                workload_seed=20260712,
+                num_prefixes=8,
+                prefix_len=1024,
+                suffix_fixed=256,
+                out=str(out),
+            )
+            client.materialize(args)
+            manifest = client.load_manifest(out)
+            counts = {i: 0 for i in range(8)}
+            for row in manifest:
+                counts[row["prefix_id"]] += 1
+                self.assertEqual(row["input_tokens"], 1280)
+            self.assertEqual(set(counts.values()), {10})
+
+    def test_fixed_duration_saturation_and_redaction(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    result = await client.run_load(
+                        run_args(manifest, root / "result", mock.port)
+                    )
+                finally:
+                    await mock.stop()
+                self.assertEqual(result["max_in_flight"], 3)
+                self.assertEqual(mock.max_active, 3)
+                self.assertAlmostEqual(result["clean"]["duration_s"], 0.12, places=9)
+                self.assertEqual(len(result["segments"]), 3)
+                self.assertLessEqual(result["drain_seconds"], 1)
+                self.assertTrue(
+                    all(p["max_tokens"] == 2 and p["ignore_eos"] for p in mock.payloads)
+                )
+                text = (root / "result/requests.jsonl").read_text()
+                self.assertNotIn('"prompt":', text)
+                self.assertNotIn('"text":', text)
+
+        asyncio.run(case())
+
+    def test_drain_timeout_is_a_hard_sanity_failure(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    args = run_args(manifest, root / "result", mock.port)
+                    args.drain_timeout_seconds = 0
+                    with self.assertRaisesRegex(RuntimeError, "drain_within_timeout"):
+                        await client.run_load(args)
+                finally:
+                    await mock.stop()
+                sanity = json.loads((root / "result/sanity.json").read_text())
+                self.assertFalse(sanity["invariants"]["drain_within_timeout"])
+
+        asyncio.run(case())
+
+    def test_burst_schedule_is_eight_at_once(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000, "burst:8"))
+                sat = root / "sat.json"
+                client.atomic_json(
+                    sat, {"clean": {"completed_throughput_rps": 100.0}}
+                )
+                mock = MockServer()
+                await mock.start()
+                try:
+                    args = run_args(
+                        manifest, root / "result", mock.port, "moderate", sat
+                    )
+                    args.warmup_seconds = 0
+                    args.clean_segment_seconds = 0.4 / 3
+                    result = await client.run_load(args)
+                finally:
+                    await mock.stop()
+                records = [
+                    json.loads(line)
+                    for line in (root / "result/requests.jsonl").read_text().splitlines()
+                ]
+                groups = {}
+                for record in records:
+                    groups.setdefault(round(record["scheduled_s"], 6), 0)
+                    groups[round(record["scheduled_s"], 6)] += 1
+                self.assertTrue(all(value == 8 for value in groups.values()))
+                starts = sorted(groups)
+                if len(starts) > 1:
+                    self.assertAlmostEqual(starts[1] - starts[0], 8 / 60, places=5)
+                self.assertAlmostEqual(result["request_rate"], 60.0)
+
+        asyncio.run(case())
+
+    def test_manifest_exhaustion_stops_instead_of_wrapping(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(2))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    with self.assertRaises(client.ManifestExhausted):
+                        await client.run_load(
+                            run_args(manifest, root / "result", mock.port)
+                        )
+                finally:
+                    await mock.stop()
+
+        asyncio.run(case())
+
+    def test_cpu_affinity_sets_are_disjoint_and_cover_host(self):
+        values = []
+        for spec in controller.CPU_MAP.values():
+            lo, hi = [int(x) for x in spec.split("-")]
+            values.extend(range(lo, hi + 1))
+        self.assertEqual(sorted(values), list(range(160)))
+        self.assertEqual(len(values), len(set(values)))
+
+    def test_kernel_mapping_priority(self):
+        cases = {
+            "void vllm::moe::topkGating": "moe_router",
+            "ncclDevKernel_AllReduce": "collective",
+            "flash_fwd_kernel": "attention",
+            "nvjet_sm90_tst": "moe_gemm",
+            "argmax_kernel": "sampler",
+            "cutlass_gemm": "dense_gemm",
+            "triton_red_fused_add_rms_norm": "norm_elementwise",
+            "cache_swap_kernel": "kv_memory",
+            "unknown": "other",
+        }
+        self.assertEqual(
+            {name: controller.classify_kernel(name) for name in cases}, cases
+        )
+
+    def test_atomic_state_replacement(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "state.json"
+            controller.atomic_json(path, {"schema": 1, "value": 1})
+            controller.atomic_json(path, {"schema": 1, "value": 2})
+            self.assertEqual(json.loads(path.read_text())["value"], 2)
+            self.assertFalse(list(path.parent.glob("*.tmp.*")))
+
+    def test_fingerprint_survives_json_roundtrip(self):
+        original_run_text = controller.run_text
+        original_hash = controller.sha256_file
+        try:
+            controller.run_text = lambda *args, **kwargs: "deadbeef\n"
+            controller.sha256_file = lambda path: "a" * 64
+            fingerprint = controller.make_fingerprint()
+        finally:
+            controller.run_text = original_run_text
+            controller.sha256_file = original_hash
+        self.assertEqual(json.loads(json.dumps(fingerprint)), fingerprint)
+        self.assertEqual(set(fingerprint["cpu_map"]), {str(i) for i in range(8)})
+
+    def test_server_shutdown_signals_parent_before_group(self):
+        process = SimpleNamespace(pid=12345, poll=lambda: None)
+        with (
+            mock.patch.object(controller.os, "kill") as kill,
+            mock.patch.object(controller.os, "killpg") as killpg,
+            mock.patch.object(controller, "_process_group_alive", return_value=False),
+        ):
+            controller.stop_servers([process])
+        kill.assert_called_once_with(12345, controller.signal.SIGINT)
+        killpg.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

runs/opprof-phase3/phase3/access-blocker-20260712.json

@@ -0,0 +1,47 @@
+{
+  "schema": 1,
+  "status": "external_access_blocked",
+  "host": "dash0",
+  "last_reachability_probe_local": "2026-07-12T16:16:39+08:00",
+  "failure": "SSH connection timed out during banner exchange before any remote command executed",
+  "last_durable_remote_state": {
+    "observed_before_outage": true,
+    "controller_pid": 2237019,
+    "controller_status": "running",
+    "active_stage": "primary-02-saturation",
+    "active_stage_phase": "starting_servers",
+    "completed_measured_runs": 8,
+    "drain_quarantined_runs": 0,
+    "clean_window_failures": 0,
+    "missing_trace_files": 8,
+    "gpu_hours_total": 6.8156048206885655
+  },
+  "resume": {
+    "state": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/controller-state.json",
+    "log": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/phase3-matrix-controller.log",
+    "command": "cd /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712 && /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python scripts/opprof_phase3_matrix.py run --resume",
+    "rule": "inspect state/PID/GPU ownership first; do not relaunch a live controller"
+  },
+  "profile_control_repair": {
+    "reason": "P03/C01 saturated all data-plane connector slots, so /start_profile timed out before reaching the server",
+    "change": "dedicated aiohttp TCPConnector(limit=2) and ClientSession for profile control calls",
+    "old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
+    "new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
+    "remote_record": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/repairs/repair-004-profile-control-connector.json",
+    "tests": "13/13 unittest PASS; ruff PASS; py_compile PASS"
+  },
+  "analysis_ready": {
+    "script_sha256": "205d9012d9462d84403a0f1435c8a449389aa4bef448d8d6dbced707a11559b0",
+    "test_sha256": "eac6f55042865c9a24811eef9c1a13c23a26dd1504b54348513d8e6a7ba2b939",
+    "tests": "4/4 unittest PASS; ruff PASS; py_compile PASS",
+    "executed_on_matrix": false
+  },
+  "not_claimed": [
+    "matrix completion",
+    "final quarantine count",
+    "H1a verdict",
+    "H1b verdict",
+    "final GPU hours",
+    "final GPU cleanup"
+  ]
+}

runs/opprof-phase3/phase3/analysis/launch-ap37.log

@@ -0,0 +1 @@
+CPU_ANALYSIS A-P3-7: accepted_runs=40, run_root=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3, private_manifests=/home/admin/cpfs/wjh/opprof-phase3-private/manifests, bootstrap=100000 seed=20260714, output=runs/phase3/analysis/metrics-ap37.json, expected=10-30m CPU-only, GPU_cost=0

runs/opprof-phase3/phase3/controller-state-stop-ap36.json

@@ -0,0 +1,737 @@
+{
+  "clean_window_failures": 0,
+  "completed_burnins": 5,
+  "completed_measured_runs": 40,
+  "controller_pid": 2438791,
+  "created_at": 1783833885.960389,
+  "drain_quarantined_runs": 0,
+  "fingerprint": {
+    "cells": [
+      "P08-C00",
+      "P03-C10",
+      "P07-C00",
+      "P10-C01",
+      "P01-C10",
+      "P01-C01",
+      "P10-C10",
+      "P03-C01",
+      "P09-C00",
+      "P06-C01",
+      "P10-C11",
+      "P01-C00",
+      "P01-C11",
+      "P10-C00",
+      "P04-C00",
+      "P03-C00",
+      "P06-C10",
+      "P02-C00",
+      "P06-C00",
+      "P06-C11",
+      "P11-C00",
+      "P10-C00-TP2",
+      "P03-C11",
+      "P05-C00"
+    ],
+    "client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
+    "common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
+    "controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
+    "cpu_map": {
+      "0": "0-19",
+      "1": "20-39",
+      "2": "40-59",
+      "3": "60-79",
+      "4": "80-99",
+      "5": "100-119",
+      "6": "120-139",
+      "7": "140-159"
+    },
+    "manifests": {
+      "P01": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl",
+        "rows": 32768,
+        "sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4"
+      },
+      "P02": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl",
+        "rows": 32768,
+        "sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9"
+      },
+      "P03": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl",
+        "rows": 32768,
+        "sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1"
+      },
+      "P04": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl",
+        "rows": 32768,
+        "sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247"
+      },
+      "P05": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl",
+        "rows": 32768,
+        "sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7"
+      },
+      "P06": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl",
+        "rows": 32768,
+        "sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96"
+      },
+      "P07": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl",
+        "rows": 32768,
+        "sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6"
+      },
+      "P08": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl",
+        "rows": 32768,
+        "sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9"
+      },
+      "P09": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
+        "rows": 32768,
+        "sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e"
+      },
+      "P10": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl",
+        "rows": 4011,
+        "sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33"
+      },
+      "P11": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl",
+        "rows": 32768,
+        "sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18"
+      }
+    },
+    "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
+    "source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
+    "source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
+  },
+  "gpu_hours_this_stage": 0.7173924536837472,
+  "gpu_hours_total": 14.025875418755744,
+  "missing_trace_files": 8,
+  "repairs": [
+    {
+      "change": "dedicated aiohttp TCPConnector(limit=2) for profile endpoint session; request stream unchanged",
+      "evidence": "/start_profile connection acquisition timed out before server receipt while 256 data-plane connections were occupied",
+      "failed_run": "P03-C01-saturation",
+      "failed_stage": "primary-02-saturation",
+      "new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
+      "old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
+      "repair": "profile-control-connector-isolation",
+      "retry": "one exact whole-wave retry; failed directories retained as .interrupted-*",
+      "schema": 1,
+      "tests": {
+        "py_compile": "PASS",
+        "ruff": "PASS",
+        "unittest": "13/13 PASS"
+      },
+      "timestamp": 1783837763.431071
+    },
+    {
+      "budget": {
+        "hard_cap": 16.0,
+        "projected_remaining_h20_hours": 6.3857560787267165,
+        "projected_total_h20_hours": 14.513826778670154,
+        "projected_total_with_15pct_remaining_contingency": 15.471690190479162,
+        "used_h20_hours": 8.128070699943438
+      },
+      "cause": "controller required failed_records==0 across excluded profile/recovery time although the registered hard gate is zero clean-window failures",
+      "change": "replace all_failures_zero with clean-window boundary validation plus failed-record accounting; excluded failures remain reported",
+      "evidence": {
+        "clean_completed": 5828,
+        "clean_failed": 0,
+        "error_kind": "ServerDisconnectedError",
+        "excluded_window_failures": 2,
+        "failure_completion_s": [
+          373.1505718010012,
+          373.9740898209857
+        ],
+        "run": "P01-C01-moderate"
+      },
+      "failed_stage": "primary-02-moderate",
+      "gpu_rerun": false,
+      "new_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
+      "old_controller_sha256": "167e48f98f307e16ee44b321068a82a13813b3bf9a4d76882f774f15fe85e595",
+      "repair": "clean-window-failure-scope",
+      "schema": 1,
+      "tests": {
+        "py_compile": "PASS",
+        "ruff": "PASS",
+        "unittest": "14/14 PASS"
+      },
+      "timestamp": 1783847092.6920242
+    },
+    {
+      "cause": "single preflight NVML sample observed 4 MiB on GPU0-3 with 0% utilization and no compute apps immediately after a verified cleanup",
+      "change": "bounded 60s preflight requiring three consecutive samples with zero memory, zero utilization, and no compute apps; no nonzero-memory tolerance",
+      "failed_stage": "primary-06-saturation",
+      "gpu_hours_total_unchanged": 11.920448313262728,
+      "gpu_launch_before_failure": false,
+      "new_common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
+      "old_common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5",
+      "repair": "stable-zero-preflight",
+      "schema": 1,
+      "tests": {
+        "py_compile": "PASS",
+        "ruff": "PASS",
+        "unittest": "15/15 PASS"
+      },
+      "timestamp": 1783850898.1496143
+    },
+    {
+      "code_change": false,
+      "gpu_hours_used": 12.606695837948058,
+      "hard_cap": 16.0,
+      "projected_remaining_h20_hours": 2.1455133807990285,
+      "projected_total_with_15pct_contingency": 15.07403622586694,
+      "protocol_action": "single exact whole-wave infrastructure retry under identical 4-way placement; gate unchanged",
+      "reason": "first TP2 P10 inference-shape cold start/autotune after readiness yielded 15 successful warmup completions against required 32; clean window and artifacts otherwise valid",
+      "repair": "tp2-warmup-exact-retry",
+      "run": "P10-C00-TP2-saturation",
+      "schema": 1,
+      "stage": "primary-06-saturation",
+      "timestamp": 1783851710.3100822,
+      "warmup_required": 32,
+      "warmup_success": 15
+    },
+    {
+      "accepted_runs_preserved": 40,
+      "amendment": "A-P3-6",
+      "changed_fingerprint_keys": [
+        "controller_sha256"
+      ],
+      "created_at": 1783853258.3098204,
+      "failed_stage_preserved": "primary-06-saturation",
+      "new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
+      "old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
+      "projected_remaining_h20_hours": 2.1,
+      "projected_total_h20_hours": 15.408482965071997,
+      "repair_id": "repair-008-ap36",
+      "retained_attempt_re_adjudicated": false,
+      "retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44",
+      "schema": 1
+    }
+  ],
+  "schema": 1,
+  "stages": {
+    "burnin-01": {
+      "assignments": [
+        {
+          "cell": "P06-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P06-C10",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P06-C01",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P06-C11",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": true,
+      "clients": {},
+      "completed_at": 1783834288.1238313,
+      "confirmation": false,
+      "gpu_hours": 0.30808498481909435,
+      "load_point": "saturation",
+      "profile": false,
+      "servers": {},
+      "started_at": 1783833886.4448946,
+      "status": "complete",
+      "validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")"
+    },
+    "burnin-02": {
+      "assignments": [
+        {
+          "cell": "P06-C00-TP2",
+          "gpus": [
+            0,
+            1
+          ]
+        }
+      ],
+      "burnin": true,
+      "clients": {},
+      "completed_at": 1783834671.921921,
+      "confirmation": false,
+      "gpu_hours": 0.19903395679261948,
+      "load_point": "saturation",
+      "profile": false,
+      "servers": {},
+      "started_at": 1783834301.7813473,
+      "status": "complete"
+    },
+    "primary-01-moderate": {
+      "assignments": [
+        {
+          "cell": "P08-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P03-C10",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P07-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P10-C01",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783836723.3639715,
+      "confirmation": false,
+      "gpu_hours": 0.5195140059126748,
+      "load_point": "moderate",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783836150.2169476,
+      "status": "complete",
+      "validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
+      "warmup_feasibility": "P10 target=min(32,floor(0.445*60))=26; observed=27"
+    },
+    "primary-01-saturation": {
+      "assignments": [
+        {
+          "cell": "P08-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P03-C10",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P07-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P10-C01",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783836130.8498049,
+      "confirmation": false,
+      "drain_readjudication": "P10/C01 288.619107924s <= 600s",
+      "gpu_hours": 0.8836704309119119,
+      "load_point": "saturation",
+      "missing_trace_files": 8,
+      "profile": true,
+      "servers": {},
+      "started_at": 1783834671.9563339,
+      "status": "complete",
+      "validation_attempt1_failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")"
+    },
+    "primary-02-moderate": {
+      "assignments": [
+        {
+          "cell": "P01-C10",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P01-C01",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P10-C10",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P03-C01",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783847092.7229948,
+      "confirmation": false,
+      "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
+      "gpu_hours": 0.5151156750652525,
+      "load_point": "moderate",
+      "profile": true,
+      "readjudicated_without_gpu": true,
+      "servers": {},
+      "started_at": 1783838513.4472992,
+      "status": "complete",
+      "validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")"
+    },
+    "primary-02-saturation": {
+      "assignments": [
+        {
+          "cell": "P01-C10",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P01-C01",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P10-C10",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P03-C01",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783838513.4027848,
+      "confirmation": false,
+      "gpu_hours": 0.7973502041896184,
+      "load_point": "saturation",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783837773.6088765,
+      "status": "complete"
+    },
+    "primary-03-moderate": {
+      "assignments": [
+        {
+          "cell": "P09-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P06-C01",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P10-C11",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P01-C00",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783848369.3507695,
+      "confirmation": false,
+      "gpu_hours": 0.5205460974905226,
+      "load_point": "moderate",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783847881.06146,
+      "status": "complete"
+    },
+    "primary-03-saturation": {
+      "assignments": [
+        {
+          "cell": "P09-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P06-C01",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P10-C11",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P01-C00",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783847881.0197804,
+      "confirmation": false,
+      "gpu_hours": 0.8273645816908942,
+      "load_point": "saturation",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783847109.5683346,
+      "status": "complete"
+    },
+    "primary-04-moderate": {
+      "assignments": [
+        {
+          "cell": "P01-C11",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P10-C00",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P04-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P03-C00",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783849583.5228572,
+      "confirmation": false,
+      "gpu_hours": 0.5205216948853598,
+      "load_point": "moderate",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783849096.4363403,
+      "status": "complete"
+    },
+    "primary-04-saturation": {
+      "assignments": [
+        {
+          "cell": "P01-C11",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P10-C00",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P04-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P03-C00",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783849096.3948114,
+      "confirmation": false,
+      "gpu_hours": 0.7872003297011058,
+      "load_point": "saturation",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783848369.390857,
+      "status": "complete"
+    },
+    "primary-05-moderate": {
+      "assignments": [
+        {
+          "cell": "P06-C10",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P02-C00",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P06-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P06-C11",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783850641.6781306,
+      "confirmation": false,
+      "gpu_hours": 0.5138198028008143,
+      "load_point": "moderate",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783850163.1211865,
+      "status": "complete"
+    },
+    "primary-05-saturation": {
+      "assignments": [
+        {
+          "cell": "P06-C10",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P02-C00",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P06-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P06-C11",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {},
+      "completed_at": 1783850163.0472832,
+      "confirmation": false,
+      "gpu_hours": 0.6229251067505942,
+      "load_point": "saturation",
+      "profile": true,
+      "servers": {},
+      "started_at": 1783849583.5652869,
+      "status": "complete"
+    },
+    "primary-06-saturation": {
+      "assignments": [
+        {
+          "cell": "P11-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P10-C00-TP2",
+          "gpus": [
+            1,
+            2
+          ]
+        },
+        {
+          "cell": "P03-C11",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {
+        "P03-C11-saturation": {
+          "pgid": 2440965,
+          "pid": 2440965
+        },
+        "P10-C00-TP2-saturation": {
+          "pgid": 2440964,
+          "pid": 2440964
+        },
+        "P11-C00-saturation": {
+          "pgid": 2440962,
+          "pid": 2440962
+        }
+      },
+      "confirmation": false,
+      "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C00-TP2/saturation: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=17; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [11, 10, 16], 'scheduled_tokens': [90112, 81920, 70313], 'scheduled_token_throughput': [18022.4, 16384.0, 14062.6], 'mean_scheduled_token_throughput': 16156.333333333334, 'slope_tokens_per_second_squared': -395.98000000000013, 'normalized_drift': 0.367639109533929, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")",
+      "gpu_hours": 0.7173924536837472,
+      "load_point": "saturation",
+      "profile": true,
+      "servers": {
+        "P03-C11-saturation": {
+          "gpus": [
+            3
+          ],
+          "pgid": 2438868,
+          "pid": 2438868
+        },
+        "P10-C00-TP2-saturation": {
+          "gpus": [
+            1,
+            2
+          ],
+          "pgid": 2438867,
+          "pid": 2438867
+        },
+        "P11-C00-saturation": {
+          "gpus": [
+            0
+          ],
+          "pgid": 2438866,
+          "pid": 2438866
+        }
+      },
+      "started_at": 1783853279.9695158,
+      "status": "failed"
+    }
+  },
+  "status": "failed",
+  "updated_at": 1783853946.066117
+}

runs/opprof-phase3/phase3/remote-evidence/P03-C10/result.json

@@ -0,0 +1,103 @@
+{
+  "admission_stop_s": 417.3262625900097,
+  "arrival": "steady",
+  "clean": {
+    "admitted": 559,
+    "completed": 559,
+    "completed_throughput_rps": 2.3291666666666666,
+    "duration_s": 240.0,
+    "end_s": 300.0,
+    "failed": 0,
+    "input_tokens": 3429894,
+    "offered_rps": 2.3291666666666666,
+    "output_tokens": 35776,
+    "start_s": 60.0
+  },
+  "clean_segment_seconds": 80.0,
+  "drain_seconds": 91.29178713698639,
+  "elapsed_seconds": 508.6180497269961,
+  "failed_records": 0,
+  "load_point": "saturation",
+  "manifest_admitted": 1094,
+  "manifest_exhausted": false,
+  "manifest_rows": 32768,
+  "manifest_sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1",
+  "manifest_wrapped": false,
+  "max_in_flight": 256,
+  "num_clean_segments": 3,
+  "profiles": [
+    {
+      "start_call_s": 300.0043962339987,
+      "start_return_s": 301.3819164079905,
+      "start_status": 200,
+      "stop_call_s": 321.8697677869932,
+      "stop_return_s": 326.31530767300865,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835059474707213.pt.trace.json.gz",
+      "trace_ready_s": 321.86976291501196,
+      "trace_sha256": "527ddb81b1d54a12a463508bc163d85a2c443d94c8a897959a39735e030b90a2",
+      "window": 1
+    },
+    {
+      "start_call_s": 356.32926020701416,
+      "start_return_s": 361.34978177401354,
+      "start_status": 200,
+      "stop_call_s": 381.577829301008,
+      "stop_return_s": 387.31277802900877,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835119198018909.pt.trace.json.gz",
+      "trace_ready_s": 381.57782580400817,
+      "trace_sha256": "0882d9705f1208580d5e693f94ba64d89ab12ef0aae2731a82d460a46a3e4cb1",
+      "window": 2
+    }
+  ],
+  "rate_fraction": null,
+  "records": 1094,
+  "request_rate": "inf",
+  "schema": 1,
+  "segments": [
+    {
+      "admitted": 191,
+      "completed": 191,
+      "completed_throughput_rps": 2.3875,
+      "duration_s": 80.0,
+      "end_s": 140.0,
+      "failed": 0,
+      "input_tokens": 1154628,
+      "name": "A",
+      "offered_rps": 2.3875,
+      "output_tokens": 12224,
+      "start_s": 60.0
+    },
+    {
+      "admitted": 184,
+      "completed": 184,
+      "completed_throughput_rps": 2.3,
+      "duration_s": 80.0,
+      "end_s": 220.0,
+      "failed": 0,
+      "input_tokens": 1135217,
+      "name": "B",
+      "offered_rps": 2.3,
+      "output_tokens": 11776,
+      "start_s": 140.0
+    },
+    {
+      "admitted": 184,
+      "completed": 184,
+      "completed_throughput_rps": 2.3,
+      "duration_s": 80.0,
+      "end_s": 300.0,
+      "failed": 0,
+      "input_tokens": 1140049,
+      "name": "C",
+      "offered_rps": 2.3,
+      "output_tokens": 11776,
+      "start_s": 220.0
+    }
+  ],
+  "successful_records": 1094,
+  "t0_mono_ns": 166323993366724,
+  "t0_wall_ns": 1783834751119796663,
+  "warmup_seconds": 60.0
+}

runs/opprof-phase3/phase3/remote-evidence/P03-C10/sanity.json

@@ -0,0 +1,65 @@
+{
+  "invariants": {
+    "clean_duration_exact": true,
+    "clean_failures_zero": true,
+    "concurrency_bounded": true,
+    "drain_within_timeout": true,
+    "manifest_no_wrap": true,
+    "manifest_not_exhausted": true,
+    "output_tokens_exact": true,
+    "profile_count_exact": true,
+    "profile_status_ok": true,
+    "segment_count_exact": true
+  },
+  "numeric": {
+    "actual_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 1094,
+      "max": 64.0,
+      "min": 64.0,
+      "missing_n": 0,
+      "n": 1094
+    },
+    "admitted_s": {
+      "distinct_n": 1094,
+      "finite_n": 1094,
+      "max": 416.7779763180006,
+      "min": 0.0008771540015004575,
+      "missing_n": 0,
+      "n": 1094
+    },
+    "completed_s": {
+      "distinct_n": 1094,
+      "finite_n": 1094,
+      "max": 508.61568894199445,
+      "min": 22.18080371801625,
+      "missing_n": 0,
+      "n": 1094
+    },
+    "input_tokens": {
+      "distinct_n": 964,
+      "finite_n": 1094,
+      "max": 8190.0,
+      "min": 4097.0,
+      "missing_n": 0,
+      "n": 1094
+    },
+    "requested_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 1094,
+      "max": 64.0,
+      "min": 64.0,
+      "missing_n": 0,
+      "n": 1094
+    },
+    "scheduled_s": {
+      "distinct_n": 1094,
+      "finite_n": 1094,
+      "max": 416.7779758319957,
+      "min": 0.0008756940078455955,
+      "missing_n": 0,
+      "n": 1094
+    }
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase3/remote-evidence/P07-C00/result.json

@@ -0,0 +1,103 @@
+{
+  "admission_stop_s": 417.66287675101194,
+  "arrival": "burst:8",
+  "clean": {
+    "admitted": 1226,
+    "completed": 1226,
+    "completed_throughput_rps": 5.108333333333333,
+    "duration_s": 240.0,
+    "end_s": 300.0,
+    "failed": 0,
+    "input_tokens": 1569280,
+    "offered_rps": 5.108333333333333,
+    "output_tokens": 627712,
+    "start_s": 60.0
+  },
+  "clean_segment_seconds": 80.0,
+  "drain_seconds": 26.098238860984566,
+  "elapsed_seconds": 443.7611156119965,
+  "failed_records": 0,
+  "load_point": "saturation",
+  "manifest_admitted": 2078,
+  "manifest_exhausted": false,
+  "manifest_rows": 32768,
+  "manifest_sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6",
+  "manifest_wrapped": false,
+  "max_in_flight": 256,
+  "num_clean_segments": 3,
+  "profiles": [
+    {
+      "start_call_s": 300.0039349270228,
+      "start_return_s": 300.9924320260179,
+      "start_status": 200,
+      "stop_call_s": 323.90002606701455,
+      "stop_return_s": 328.078528626007,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835058771060551.pt.trace.json.gz",
+      "trace_ready_s": 323.9000201699964,
+      "trace_sha256": "2d12038b03c6f9c28d1a488abd880ff2763e1f3c0ab502cfdd2507b649de2af7",
+      "window": 1
+    },
+    {
+      "start_call_s": 358.0940292410087,
+      "start_return_s": 358.8905565890018,
+      "start_status": 200,
+      "stop_call_s": 382.04741394601297,
+      "stop_return_s": 387.64796055300394,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835116964699401.pt.trace.json.gz",
+      "trace_ready_s": 382.04741024502437,
+      "trace_sha256": "2184d4f83ea22c0e5455dd3914922548ce8d3d5eba34f89562df69326e3832b5",
+      "window": 2
+    }
+  ],
+  "rate_fraction": null,
+  "records": 2078,
+  "request_rate": "inf",
+  "schema": 1,
+  "segments": [
+    {
+      "admitted": 394,
+      "completed": 394,
+      "completed_throughput_rps": 4.925,
+      "duration_s": 80.0,
+      "end_s": 140.0,
+      "failed": 0,
+      "input_tokens": 504320,
+      "name": "A",
+      "offered_rps": 4.925,
+      "output_tokens": 201728,
+      "start_s": 60.0
+    },
+    {
+      "admitted": 397,
+      "completed": 397,
+      "completed_throughput_rps": 4.9625,
+      "duration_s": 80.0,
+      "end_s": 220.0,
+      "failed": 0,
+      "input_tokens": 508160,
+      "name": "B",
+      "offered_rps": 4.9625,
+      "output_tokens": 203264,
+      "start_s": 140.0
+    },
+    {
+      "admitted": 435,
+      "completed": 435,
+      "completed_throughput_rps": 5.4375,
+      "duration_s": 80.0,
+      "end_s": 300.0,
+      "failed": 0,
+      "input_tokens": 556800,
+      "name": "C",
+      "offered_rps": 5.4375,
+      "output_tokens": 222720,
+      "start_s": 220.0
+    }
+  ],
+  "successful_records": 2078,
+  "t0_mono_ns": 166323999932716,
+  "t0_wall_ns": 1783834751126361482,
+  "warmup_seconds": 60.0
+}

runs/opprof-phase3/phase3/remote-evidence/P07-C00/sanity.json

@@ -0,0 +1,65 @@
+{
+  "invariants": {
+    "clean_duration_exact": true,
+    "clean_failures_zero": true,
+    "concurrency_bounded": true,
+    "drain_within_timeout": true,
+    "manifest_no_wrap": true,
+    "manifest_not_exhausted": true,
+    "output_tokens_exact": true,
+    "profile_count_exact": true,
+    "profile_status_ok": true,
+    "segment_count_exact": true
+  },
+  "numeric": {
+    "actual_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 2078,
+      "max": 512.0,
+      "min": 512.0,
+      "missing_n": 0,
+      "n": 2078
+    },
+    "admitted_s": {
+      "distinct_n": 2078,
+      "finite_n": 2078,
+      "max": 417.40899016702315,
+      "min": 0.0008179180149454623,
+      "missing_n": 0,
+      "n": 2078
+    },
+    "completed_s": {
+      "distinct_n": 2078,
+      "finite_n": 2078,
+      "max": 443.75592052700813,
+      "min": 35.15151289102505,
+      "missing_n": 0,
+      "n": 2078
+    },
+    "input_tokens": {
+      "distinct_n": 1,
+      "finite_n": 2078,
+      "max": 1280.0,
+      "min": 1280.0,
+      "missing_n": 0,
+      "n": 2078
+    },
+    "requested_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 2078,
+      "max": 512.0,
+      "min": 512.0,
+      "missing_n": 0,
+      "n": 2078
+    },
+    "scheduled_s": {
+      "distinct_n": 2078,
+      "finite_n": 2078,
+      "max": 417.4089898300008,
+      "min": 0.0008165129984263331,
+      "missing_n": 0,
+      "n": 2078
+    }
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase3/remote-evidence/P08-C00/result.json

@@ -0,0 +1,103 @@
+{
+  "admission_stop_s": 493.2576180040196,
+  "arrival": "burst:8",
+  "clean": {
+    "admitted": 2233,
+    "completed": 2233,
+    "completed_throughput_rps": 9.304166666666667,
+    "duration_s": 240.0,
+    "end_s": 300.0,
+    "failed": 0,
+    "input_tokens": 2858240,
+    "offered_rps": 9.304166666666667,
+    "output_tokens": 1143296,
+    "start_s": 60.0
+  },
+  "clean_segment_seconds": 80.0,
+  "drain_seconds": 14.923095890000695,
+  "elapsed_seconds": 508.1807138940203,
+  "failed_records": 0,
+  "load_point": "saturation",
+  "manifest_admitted": 3941,
+  "manifest_exhausted": false,
+  "manifest_rows": 32768,
+  "manifest_sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9",
+  "manifest_wrapped": false,
+  "max_in_flight": 256,
+  "num_clean_segments": 3,
+  "profiles": [
+    {
+      "start_call_s": 300.0054247789958,
+      "start_return_s": 323.45228312400286,
+      "start_status": 200,
+      "stop_call_s": 354.6066670610162,
+      "stop_return_s": 381.28106322701205,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835082061801278.pt.trace.json.gz",
+      "trace_ready_s": 354.6066622030048,
+      "trace_sha256": "53d36238e82e5d031d8ed7841094c56d715b7cd5009368b64e380bf89a0f0dec",
+      "window": 1
+    },
+    {
+      "start_call_s": 411.30330635100836,
+      "start_return_s": 411.4401313569979,
+      "start_status": 200,
+      "stop_call_s": 436.0742275560042,
+      "stop_return_s": 463.23921855099616,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835169024306574.pt.trace.json.gz",
+      "trace_ready_s": 436.0742245099973,
+      "trace_sha256": "e088bd800a4e43446effbbcd4fce7d9ba1defaceb378a6e0db04d64780e1d09a",
+      "window": 2
+    }
+  ],
+  "rate_fraction": null,
+  "records": 3941,
+  "request_rate": "inf",
+  "schema": 1,
+  "segments": [
+    {
+      "admitted": 697,
+      "completed": 697,
+      "completed_throughput_rps": 8.7125,
+      "duration_s": 80.0,
+      "end_s": 140.0,
+      "failed": 0,
+      "input_tokens": 892160,
+      "name": "A",
+      "offered_rps": 8.7125,
+      "output_tokens": 356864,
+      "start_s": 60.0
+    },
+    {
+      "admitted": 768,
+      "completed": 768,
+      "completed_throughput_rps": 9.6,
+      "duration_s": 80.0,
+      "end_s": 220.0,
+      "failed": 0,
+      "input_tokens": 983040,
+      "name": "B",
+      "offered_rps": 9.6,
+      "output_tokens": 393216,
+      "start_s": 140.0
+    },
+    {
+      "admitted": 768,
+      "completed": 768,
+      "completed_throughput_rps": 9.6,
+      "duration_s": 80.0,
+      "end_s": 300.0,
+      "failed": 0,
+      "input_tokens": 983040,
+      "name": "C",
+      "offered_rps": 9.6,
+      "output_tokens": 393216,
+      "start_s": 220.0
+    }
+  ],
+  "successful_records": 3941,
+  "t0_mono_ns": 166324009670273,
+  "t0_wall_ns": 1783834751136099481,
+  "warmup_seconds": 60.0
+}

runs/opprof-phase3/phase3/remote-evidence/P08-C00/sanity.json

@@ -0,0 +1,65 @@
+{
+  "invariants": {
+    "clean_duration_exact": true,
+    "clean_failures_zero": true,
+    "concurrency_bounded": true,
+    "drain_within_timeout": true,
+    "manifest_no_wrap": true,
+    "manifest_not_exhausted": true,
+    "output_tokens_exact": true,
+    "profile_count_exact": true,
+    "profile_status_ok": true,
+    "segment_count_exact": true
+  },
+  "numeric": {
+    "actual_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 3941,
+      "max": 512.0,
+      "min": 512.0,
+      "missing_n": 0,
+      "n": 3941
+    },
+    "admitted_s": {
+      "distinct_n": 3941,
+      "finite_n": 3941,
+      "max": 493.25117621201207,
+      "min": 0.0009094980196096003,
+      "missing_n": 0,
+      "n": 3941
+    },
+    "completed_s": {
+      "distinct_n": 3941,
+      "finite_n": 3941,
+      "max": 508.1755032700021,
+      "min": 30.030000179016497,
+      "missing_n": 0,
+      "n": 3941
+    },
+    "input_tokens": {
+      "distinct_n": 1,
+      "finite_n": 3941,
+      "max": 1280.0,
+      "min": 1280.0,
+      "missing_n": 0,
+      "n": 3941
+    },
+    "requested_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 3941,
+      "max": 512.0,
+      "min": 512.0,
+      "missing_n": 0,
+      "n": 3941
+    },
+    "scheduled_s": {
+      "distinct_n": 3941,
+      "finite_n": 3941,
+      "max": 493.25117589501315,
+      "min": 0.000907983019715175,
+      "missing_n": 0,
+      "n": 3941
+    }
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase3/remote-evidence/P10-C01/result.json

@@ -0,0 +1,103 @@
+{
+  "admission_stop_s": 420.6437278209778,
+  "arrival": "steady",
+  "clean": {
+    "admitted": 178,
+    "completed": 178,
+    "completed_throughput_rps": 0.7416666666666667,
+    "duration_s": 240.0,
+    "end_s": 300.0,
+    "failed": 0,
+    "input_tokens": 1755158,
+    "offered_rps": 0.7416666666666667,
+    "output_tokens": 40658,
+    "start_s": 60.0
+  },
+  "clean_segment_seconds": 80.0,
+  "drain_seconds": 288.6191079240234,
+  "elapsed_seconds": 709.2628357450012,
+  "failed_records": 0,
+  "load_point": "saturation",
+  "manifest_admitted": 558,
+  "manifest_exhausted": false,
+  "manifest_rows": 4011,
+  "manifest_sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33",
+  "manifest_wrapped": false,
+  "max_in_flight": 256,
+  "num_clean_segments": 3,
+  "profiles": [
+    {
+      "start_call_s": 300.0035294489935,
+      "start_return_s": 310.3520467719936,
+      "start_status": 200,
+      "stop_call_s": 325.0655262139917,
+      "stop_return_s": 338.54296391498065,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835066783986388.pt.trace.json.gz",
+      "trace_ready_s": 325.0655210709956,
+      "trace_sha256": "d457af416fa91f44e617c7259167fca467bea8466a3e4c4b916f811fb5899c73",
+      "window": 1
+    },
+    {
+      "start_call_s": 368.55468072599615,
+      "start_return_s": 372.31818850699347,
+      "start_status": 200,
+      "stop_call_s": 386.0271748309897,
+      "stop_return_s": 390.6334540609969,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835128341345321.pt.trace.json.gz",
+      "trace_ready_s": 386.02717122397735,
+      "trace_sha256": "41e8b34a0464cfd92d16b21c6242113b23594081697f5d1e88cdc3647542a96a",
+      "window": 2
+    }
+  ],
+  "rate_fraction": null,
+  "records": 558,
+  "request_rate": "inf",
+  "schema": 1,
+  "segments": [
+    {
+      "admitted": 48,
+      "completed": 48,
+      "completed_throughput_rps": 0.6,
+      "duration_s": 80.0,
+      "end_s": 140.0,
+      "failed": 0,
+      "input_tokens": 548329,
+      "name": "A",
+      "offered_rps": 0.6,
+      "output_tokens": 11027,
+      "start_s": 60.0
+    },
+    {
+      "admitted": 70,
+      "completed": 70,
+      "completed_throughput_rps": 0.875,
+      "duration_s": 80.0,
+      "end_s": 220.0,
+      "failed": 0,
+      "input_tokens": 574251,
+      "name": "B",
+      "offered_rps": 0.875,
+      "output_tokens": 16298,
+      "start_s": 140.0
+    },
+    {
+      "admitted": 60,
+      "completed": 60,
+      "completed_throughput_rps": 0.75,
+      "duration_s": 80.0,
+      "end_s": 300.0,
+      "failed": 0,
+      "input_tokens": 632578,
+      "name": "C",
+      "offered_rps": 0.75,
+      "output_tokens": 13333,
+      "start_s": 220.0
+    }
+  ],
+  "successful_records": 558,
+  "t0_mono_ns": 166325149689000,
+  "t0_wall_ns": 1783834752276118250,
+  "warmup_seconds": 60.0
+}

runs/opprof-phase3/phase3/remote-evidence/P10-C01/sanity.json

@@ -0,0 +1,65 @@
+{
+  "invariants": {
+    "clean_duration_exact": true,
+    "clean_failures_zero": true,
+    "concurrency_bounded": true,
+    "drain_within_timeout": false,
+    "manifest_no_wrap": true,
+    "manifest_not_exhausted": true,
+    "output_tokens_exact": true,
+    "profile_count_exact": true,
+    "profile_status_ok": true,
+    "segment_count_exact": true
+  },
+  "numeric": {
+    "actual_output_tokens": {
+      "distinct_n": 92,
+      "finite_n": 558,
+      "max": 256.0,
+      "min": 6.0,
+      "missing_n": 0,
+      "n": 558
+    },
+    "admitted_s": {
+      "distinct_n": 558,
+      "finite_n": 558,
+      "max": 420.08466253298684,
+      "min": 0.000850101001560688,
+      "missing_n": 0,
+      "n": 558
+    },
+    "completed_s": {
+      "distinct_n": 558,
+      "finite_n": 558,
+      "max": 709.2613093179825,
+      "min": 14.001143716974184,
+      "missing_n": 0,
+      "n": 558
+    },
+    "input_tokens": {
+      "distinct_n": 535,
+      "finite_n": 558,
+      "max": 32527.0,
+      "min": 70.0,
+      "missing_n": 0,
+      "n": 558
+    },
+    "requested_output_tokens": {
+      "distinct_n": 92,
+      "finite_n": 558,
+      "max": 256.0,
+      "min": 6.0,
+      "missing_n": 0,
+      "n": 558
+    },
+    "scheduled_s": {
+      "distinct_n": 558,
+      "finite_n": 558,
+      "max": 420.08466206499725,
+      "min": 0.0008485929865855724,
+      "missing_n": 0,
+      "n": 558
+    }
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase3/remote-evidence/controller-state.json

@@ -0,0 +1,258 @@
+{
+  "completed_burnins": 5,
+  "completed_measured_runs": 0,
+  "controller_pid": 2187729,
+  "created_at": 1783833885.960389,
+  "fingerprint": {
+    "cells": [
+      "P08-C00",
+      "P03-C10",
+      "P07-C00",
+      "P10-C01",
+      "P01-C10",
+      "P01-C01",
+      "P10-C10",
+      "P03-C01",
+      "P09-C00",
+      "P06-C01",
+      "P10-C11",
+      "P01-C00",
+      "P01-C11",
+      "P10-C00",
+      "P04-C00",
+      "P03-C00",
+      "P06-C10",
+      "P02-C00",
+      "P06-C00",
+      "P06-C11",
+      "P11-C00",
+      "P10-C00-TP2",
+      "P03-C11",
+      "P05-C00"
+    ],
+    "client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
+    "common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5",
+    "controller_sha256": "c2d3232fb99c66cea55e30e6cb1aa6a84d3a816434ed8155a309f41df2837f73",
+    "cpu_map": {
+      "0": "0-19",
+      "1": "20-39",
+      "2": "40-59",
+      "3": "60-79",
+      "4": "80-99",
+      "5": "100-119",
+      "6": "120-139",
+      "7": "140-159"
+    },
+    "manifests": {
+      "P01": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl",
+        "rows": 32768,
+        "sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4"
+      },
+      "P02": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl",
+        "rows": 32768,
+        "sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9"
+      },
+      "P03": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl",
+        "rows": 32768,
+        "sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1"
+      },
+      "P04": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl",
+        "rows": 32768,
+        "sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247"
+      },
+      "P05": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl",
+        "rows": 32768,
+        "sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7"
+      },
+      "P06": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl",
+        "rows": 32768,
+        "sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96"
+      },
+      "P07": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl",
+        "rows": 32768,
+        "sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6"
+      },
+      "P08": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl",
+        "rows": 32768,
+        "sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9"
+      },
+      "P09": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
+        "rows": 32768,
+        "sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e"
+      },
+      "P10": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl",
+        "rows": 4011,
+        "sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33"
+      },
+      "P11": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl",
+        "rows": 32768,
+        "sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18"
+      }
+    },
+    "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
+    "source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
+    "source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
+  },
+  "gpu_hours_this_stage": 0.8836704309119119,
+  "gpu_hours_total": 5.473521912336349,
+  "schema": 1,
+  "stages": {
+    "burnin-01": {
+      "assignments": [
+        {
+          "cell": "P06-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P06-C10",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P06-C01",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P06-C11",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": true,
+      "clients": {},
+      "completed_at": 1783834288.1238313,
+      "confirmation": false,
+      "gpu_hours": 0.30808498481909435,
+      "load_point": "saturation",
+      "profile": false,
+      "servers": {},
+      "started_at": 1783833886.4448946,
+      "status": "complete",
+      "validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")"
+    },
+    "burnin-02": {
+      "assignments": [
+        {
+          "cell": "P06-C00-TP2",
+          "gpus": [
+            0,
+            1
+          ]
+        }
+      ],
+      "burnin": true,
+      "clients": {},
+      "completed_at": 1783834671.921921,
+      "confirmation": false,
+      "gpu_hours": 0.19903395679261948,
+      "load_point": "saturation",
+      "profile": false,
+      "servers": {},
+      "started_at": 1783834301.7813473,
+      "status": "complete"
+    },
+    "primary-01-saturation": {
+      "assignments": [
+        {
+          "cell": "P08-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "P03-C10",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "P07-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "P10-C01",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {
+        "P03-C10-saturation": {
+          "pgid": 2195599,
+          "pid": 2195599
+        },
+        "P07-C00-saturation": {
+          "pgid": 2195600,
+          "pid": 2195600
+        },
+        "P08-C00-saturation": {
+          "pgid": 2195597,
+          "pid": 2195597
+        },
+        "P10-C01-saturation": {
+          "pgid": 2195601,
+          "pid": 2195601
+        }
+      },
+      "confirmation": false,
+      "failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")",
+      "gpu_hours": 0.8836704309119119,
+      "load_point": "saturation",
+      "profile": true,
+      "servers": {
+        "P03-C10-saturation": {
+          "gpus": [
+            1
+          ],
+          "pgid": 2193680,
+          "pid": 2193680
+        },
+        "P07-C00-saturation": {
+          "gpus": [
+            2
+          ],
+          "pgid": 2193681,
+          "pid": 2193681
+        },
+        "P08-C00-saturation": {
+          "gpus": [
+            0
+          ],
+          "pgid": 2193679,
+          "pid": 2193679
+        },
+        "P10-C01-saturation": {
+          "gpus": [
+            3
+          ],
+          "pgid": 2193682,
+          "pid": 2193682
+        }
+      },
+      "started_at": 1783834671.9563339,
+      "status": "failed"
+    }
+  },
+  "status": "failed",
+  "updated_at": 1783835479.1651883
+}

runs/opprof-phase3/phase3/remote-evidence/repair-001-explicit-mbt-log.json

@@ -0,0 +1,16 @@
+{
+  "at": 1783834288.13871,
+  "finding": "Explicit MBT=2048 is logged in non-default args; scheduler.py emits the separate Chunked-prefill line only for default MBT.",
+  "gpu_rerun": false,
+  "kind": "validator-only",
+  "new_controller_sha256": "c2d3232fb99c66cea55e30e6cb1aa6a84d3a816434ed8155a309f41df2837f73",
+  "old_controller_sha256": "74859614a8da3341bf827bb75b667db7d219f059d1d29c1182b9a77c2ee59e92",
+  "protocol_or_workload_changed": false,
+  "revalidated_runs": [
+    "P06-C00-burnin",
+    "P06-C10-burnin",
+    "P06-C01-burnin",
+    "P06-C11-burnin"
+  ],
+  "schema": 1
+}

runs/opprof-phase3/phase3/remote-evidence/sidecar-verification-state.json

@@ -0,0 +1,355 @@
+{
+  "gpu_seconds": 427.43714332580566,
+  "results": {
+    "graceful": {
+      "accounting": {
+        "bytes": 1462585,
+        "checkpoint_age_seconds": -0.382308841,
+        "counters": {
+          "dropped_records": 0,
+          "encoded_records": 1598,
+          "written_records": 1598
+        },
+        "footer_count": 1,
+        "invariants": {
+          "all_schema_1": true,
+          "encoded_balanced": true,
+          "final_sidecar": true,
+          "footer_sidecar_agree": true,
+          "last_step_matches": true,
+          "one_footer_last": true,
+          "steps_contiguous": true,
+          "written_matches_records": true,
+          "zero_drops": true
+        },
+        "last_step_index": 1597,
+        "records": 1598,
+        "sidecar": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/graceful/opprof/opprof-v1-dp0-pid2163417-1783832631507578426.jsonl.footer.json",
+        "stream": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/graceful/opprof/opprof-v1-dp0-pid2163417-1783832631507578426.jsonl"
+      },
+      "clean": {
+        "admitted": 5202,
+        "completed": 5202,
+        "completed_throughput_rps": 43.35,
+        "duration_s": 120.0,
+        "end_s": 140.0,
+        "failed": 0,
+        "input_tokens": 1662118,
+        "offered_rps": 43.35,
+        "output_tokens": 332928,
+        "start_s": 20.0
+      },
+      "failed_records": 0,
+      "gpu_seconds": 219.92383646965027,
+      "gpu_zero_samples": [
+        [
+          {
+            "index": 0,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 1,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 2,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 3,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 4,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 5,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 6,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 7,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          }
+        ],
+        [
+          {
+            "index": 0,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 1,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 2,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 3,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 4,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 5,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 6,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 7,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          }
+        ],
+        [
+          {
+            "index": 0,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 1,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 2,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 3,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 4,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 5,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 6,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 7,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          }
+        ]
+      ],
+      "mode": "graceful",
+      "schema": 1,
+      "server_returncode": 0,
+      "status": "pass",
+      "termination_wall_ns": 1783832775735239240
+    },
+    "hard-kill": {
+      "accounting": {
+        "bytes": 1460860,
+        "checkpoint_age_seconds": 0.02436319,
+        "counters": {
+          "dropped_records": 0,
+          "encoded_records": 1596,
+          "written_records": 1596
+        },
+        "footer_count": 0,
+        "invariants": {
+          "all_schema_1": true,
+          "checkpoint_sidecar": true,
+          "checkpoint_within_bound": true,
+          "encoded_balanced": true,
+          "last_step_matches": true,
+          "no_in_stream_footer": true,
+          "steps_contiguous": true,
+          "written_matches_records": true,
+          "zero_drops": true
+        },
+        "last_step_index": 1595,
+        "records": 1596,
+        "sidecar": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/hard-kill/opprof/opprof-v1-dp0-pid2166360-1783832841962667333.jsonl.footer.json",
+        "stream": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/hard-kill/opprof/opprof-v1-dp0-pid2166360-1783832841962667333.jsonl"
+      },
+      "clean": {
+        "admitted": 5179,
+        "completed": 5179,
+        "completed_throughput_rps": 43.15833333333333,
+        "duration_s": 120.0,
+        "end_s": 140.0,
+        "failed": 0,
+        "input_tokens": 1655384,
+        "offered_rps": 43.15833333333333,
+        "output_tokens": 331456,
+        "start_s": 20.0
+      },
+      "failed_records": 0,
+      "gpu_seconds": 207.5133068561554,
+      "gpu_zero_samples": [
+        [
+          {
+            "index": 0,
+            "memory_mib": 0,
+            "utilization_pct": 1
+          },
+          {
+            "index": 1,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 2,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 3,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 4,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 5,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 6,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 7,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          }
+        ],
+        [
+          {
+            "index": 0,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 1,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 2,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 3,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 4,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 5,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 6,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 7,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          }
+        ],
+        [
+          {
+            "index": 0,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 1,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 2,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 3,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 4,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 5,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 6,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          },
+          {
+            "index": 7,
+            "memory_mib": 0,
+            "utilization_pct": 0
+          }
+        ]
+      ],
+      "mode": "hard-kill",
+      "schema": 1,
+      "server_returncode": -9,
+      "status": "pass",
+      "termination_wall_ns": 1783832986410202706
+    }
+  },
+  "schema": 1,
+  "status": "complete"
+}

runs/opprof-phase3/phase3/repair-008-ap36.json

@@ -0,0 +1,17 @@
+{
+  "accepted_runs_preserved": 40,
+  "amendment": "A-P3-6",
+  "changed_fingerprint_keys": [
+    "controller_sha256"
+  ],
+  "created_at": 1783853258.3098204,
+  "failed_stage_preserved": "primary-06-saturation",
+  "new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
+  "old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
+  "projected_remaining_h20_hours": 2.1,
+  "projected_total_h20_hours": 15.408482965071997,
+  "repair_id": "repair-008-ap36",
+  "retained_attempt_re_adjudicated": false,
+  "retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44",
+  "schema": 1
+}

runs/opprof-phase3/phase4/capture-p09/controller-state.json

@@ -0,0 +1,469 @@
+{
+  "arms": {
+    "OFF": {
+      "client_pid": 2487398,
+      "server_pid": 2486295,
+      "started_at": 1783856664.5207698,
+      "status": "complete",
+      "summary": {
+        "arm": "OFF",
+        "bucket_tokens": 237911,
+        "capture_sizes": "default",
+        "clean_completed": 1184,
+        "clean_completed_throughput_rps": 4.933333333333334,
+        "clean_failed": 0,
+        "clean_offered_rps": 4.920833333333333,
+        "drain_seconds": 0.7853007119847462,
+        "e2e_latency_mean_s": 1.681152764688729,
+        "e2e_latency_p95_s": 3.8763178953449815,
+        "gpu_hours": 0.13751606033907995,
+        "graph_hit_steps": 12252,
+        "graph_miss_rate": 0.045199501246882795,
+        "layer1_invariants": {
+          "cudagraph_identity": true,
+          "footer_balanced": true,
+          "footer_written_matches": true,
+          "schema_1": true,
+          "sidecar_agrees": true,
+          "sidecar_final": true,
+          "steps_unique_contiguous": true,
+          "token_composition": true,
+          "zero_drops": true
+        },
+        "layer1_records": 16491,
+        "model_step_duration_ms": 479367.749483,
+        "model_steps": 12832,
+        "padding_fraction": 0.08545632610514016,
+        "padding_tokens": 20331,
+        "schema": 1,
+        "token_efficiency_per_ms": 4.5540506267984115,
+        "useful_tokens": 2183065
+      }
+    },
+    "ON": {
+      "client_pid": 2479800,
+      "server_pid": 2477652,
+      "started_at": 1783856086.549906,
+      "status": "complete",
+      "summary": {
+        "arm": "ON",
+        "bucket_tokens": 227714,
+        "capture_sizes": [
+          1,
+          2,
+          3,
+          4,
+          5,
+          6,
+          7,
+          8,
+          9,
+          16,
+          24,
+          32,
+          40,
+          48,
+          56,
+          64,
+          72,
+          80,
+          88,
+          96,
+          104,
+          112,
+          120,
+          128,
+          136,
+          144,
+          152,
+          160,
+          168,
+          176,
+          184,
+          192,
+          200,
+          208,
+          216,
+          224,
+          232,
+          240,
+          248,
+          256,
+          272,
+          288,
+          304,
+          320,
+          336,
+          352,
+          368,
+          384,
+          400,
+          416,
+          432,
+          448,
+          464,
+          480,
+          496,
+          512
+        ],
+        "clean_completed": 1190,
+        "clean_completed_throughput_rps": 4.958333333333333,
+        "clean_failed": 0,
+        "clean_offered_rps": 4.920833333333333,
+        "drain_seconds": 0.7876331160077825,
+        "e2e_latency_mean_s": 1.612280680370388,
+        "e2e_latency_p95_s": 3.9930475192406423,
+        "gpu_hours": 0.15887338949574364,
+        "graph_hit_steps": 14253,
+        "graph_miss_rate": 0.0389724226282786,
+        "layer1_invariants": {
+          "cudagraph_identity": true,
+          "footer_balanced": true,
+          "footer_written_matches": true,
+          "schema_1": true,
+          "sidecar_agrees": true,
+          "sidecar_final": true,
+          "steps_unique_contiguous": true,
+          "token_composition": true,
+          "zero_drops": true
+        },
+        "layer1_records": 17579,
+        "model_step_duration_ms": 478655.543996,
+        "model_steps": 14831,
+        "padding_fraction": 0.035658764941988635,
+        "padding_tokens": 8120,
+        "schema": 1,
+        "token_efficiency_per_ms": 4.562222306607711,
+        "useful_tokens": 2183733
+      }
+    }
+  },
+  "controller_pid": 2477456,
+  "created_at": 1783856081.3481774,
+  "gpu_hours_increment": 0.2963894498348236,
+  "plan": {
+    "added_capture_sizes": [
+      3,
+      5,
+      6,
+      7,
+      9
+    ],
+    "clean_seconds": 240,
+    "gpu": 0,
+    "gpu_hour_limit": 16.0,
+    "load": "moderate",
+    "manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
+    "measured_padding_recovery_bound": 0.049776380388728225,
+    "on_capture_sizes": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      16,
+      24,
+      32,
+      40,
+      48,
+      56,
+      64,
+      72,
+      80,
+      88,
+      96,
+      104,
+      112,
+      120,
+      128,
+      136,
+      144,
+      152,
+      160,
+      168,
+      176,
+      184,
+      192,
+      200,
+      208,
+      216,
+      224,
+      232,
+      240,
+      248,
+      256,
+      272,
+      288,
+      304,
+      320,
+      336,
+      352,
+      368,
+      384,
+      400,
+      416,
+      432,
+      448,
+      464,
+      480,
+      496,
+      512
+    ],
+    "order": [
+      "ON",
+      "OFF"
+    ],
+    "pattern": "P09",
+    "primary_metric": "clean graph-hit padding_fraction",
+    "prior_gpu_hours": 14.025875418755744,
+    "profile": false,
+    "projected_increment_gpu_hours": 0.5,
+    "projected_total_gpu_hours": 14.525875418755744,
+    "rate_fraction": 0.6,
+    "saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json",
+    "schema": 1,
+    "secondary_metrics": [
+      "Layer-1 useful scheduled tokens/model-step millisecond",
+      "clean completed request throughput",
+      "clean request latency"
+    ],
+    "warmup_seconds": 60
+  },
+  "result": {
+    "delta": {
+      "completed_throughput_relative": 0.0050675675675675436,
+      "e2e_mean_latency_relative": -0.04096717785851722,
+      "e2e_p95_latency_relative": 0.03011353223527924,
+      "padding_fraction_points": -0.049797561163151524,
+      "padding_reduction_fraction": 0.5827252753866776,
+      "token_efficiency_relative": 0.0017943761453185214
+    },
+    "gpu_hours_increment": 0.2963894498348236,
+    "gpu_hours_total": 14.322264868590567,
+    "off": {
+      "arm": "OFF",
+      "bucket_tokens": 237911,
+      "capture_sizes": "default",
+      "clean_completed": 1184,
+      "clean_completed_throughput_rps": 4.933333333333334,
+      "clean_failed": 0,
+      "clean_offered_rps": 4.920833333333333,
+      "drain_seconds": 0.7853007119847462,
+      "e2e_latency_mean_s": 1.681152764688729,
+      "e2e_latency_p95_s": 3.8763178953449815,
+      "gpu_hours": 0.13751606033907995,
+      "graph_hit_steps": 12252,
+      "graph_miss_rate": 0.045199501246882795,
+      "layer1_invariants": {
+        "cudagraph_identity": true,
+        "footer_balanced": true,
+        "footer_written_matches": true,
+        "schema_1": true,
+        "sidecar_agrees": true,
+        "sidecar_final": true,
+        "steps_unique_contiguous": true,
+        "token_composition": true,
+        "zero_drops": true
+      },
+      "layer1_records": 16491,
+      "model_step_duration_ms": 479367.749483,
+      "model_steps": 12832,
+      "padding_fraction": 0.08545632610514016,
+      "padding_tokens": 20331,
+      "schema": 1,
+      "token_efficiency_per_ms": 4.5540506267984115,
+      "useful_tokens": 2183065
+    },
+    "on": {
+      "arm": "ON",
+      "bucket_tokens": 227714,
+      "capture_sizes": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        16,
+        24,
+        32,
+        40,
+        48,
+        56,
+        64,
+        72,
+        80,
+        88,
+        96,
+        104,
+        112,
+        120,
+        128,
+        136,
+        144,
+        152,
+        160,
+        168,
+        176,
+        184,
+        192,
+        200,
+        208,
+        216,
+        224,
+        232,
+        240,
+        248,
+        256,
+        272,
+        288,
+        304,
+        320,
+        336,
+        352,
+        368,
+        384,
+        400,
+        416,
+        432,
+        448,
+        464,
+        480,
+        496,
+        512
+      ],
+      "clean_completed": 1190,
+      "clean_completed_throughput_rps": 4.958333333333333,
+      "clean_failed": 0,
+      "clean_offered_rps": 4.920833333333333,
+      "drain_seconds": 0.7876331160077825,
+      "e2e_latency_mean_s": 1.612280680370388,
+      "e2e_latency_p95_s": 3.9930475192406423,
+      "gpu_hours": 0.15887338949574364,
+      "graph_hit_steps": 14253,
+      "graph_miss_rate": 0.0389724226282786,
+      "layer1_invariants": {
+        "cudagraph_identity": true,
+        "footer_balanced": true,
+        "footer_written_matches": true,
+        "schema_1": true,
+        "sidecar_agrees": true,
+        "sidecar_final": true,
+        "steps_unique_contiguous": true,
+        "token_composition": true,
+        "zero_drops": true
+      },
+      "layer1_records": 17579,
+      "model_step_duration_ms": 478655.543996,
+      "model_steps": 14831,
+      "padding_fraction": 0.035658764941988635,
+      "padding_tokens": 8120,
+      "schema": 1,
+      "token_efficiency_per_ms": 4.562222306607711,
+      "useful_tokens": 2183733
+    },
+    "plan": {
+      "added_capture_sizes": [
+        3,
+        5,
+        6,
+        7,
+        9
+      ],
+      "clean_seconds": 240,
+      "gpu": 0,
+      "gpu_hour_limit": 16.0,
+      "load": "moderate",
+      "manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
+      "measured_padding_recovery_bound": 0.049776380388728225,
+      "on_capture_sizes": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        16,
+        24,
+        32,
+        40,
+        48,
+        56,
+        64,
+        72,
+        80,
+        88,
+        96,
+        104,
+        112,
+        120,
+        128,
+        136,
+        144,
+        152,
+        160,
+        168,
+        176,
+        184,
+        192,
+        200,
+        208,
+        216,
+        224,
+        232,
+        240,
+        248,
+        256,
+        272,
+        288,
+        304,
+        320,
+        336,
+        352,
+        368,
+        384,
+        400,
+        416,
+        432,
+        448,
+        464,
+        480,
+        496,
+        512
+      ],
+      "order": [
+        "ON",
+        "OFF"
+      ],
+      "pattern": "P09",
+      "primary_metric": "clean graph-hit padding_fraction",
+      "prior_gpu_hours": 14.025875418755744,
+      "profile": false,
+      "projected_increment_gpu_hours": 0.5,
+      "projected_total_gpu_hours": 14.525875418755744,
+      "rate_fraction": 0.6,
+      "saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json",
+      "schema": 1,
+      "secondary_metrics": [
+        "Layer-1 useful scheduled tokens/model-step millisecond",
+        "clean completed request throughput",
+        "clean request latency"
+      ],
+      "warmup_seconds": 60
+    },
+    "schema": 1
+  },
+  "schema": 1,
+  "status": "complete",
+  "updated_at": 1783857160.3890784
+}

runs/opprof-phase3/phase4/capture-p09/controller.log

@@ -0,0 +1,5 @@
+GPU_COMMAND P09-capture-ON-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120 --cudagraph-capture-sizes 1 2 3 4 5 6 7 8 9 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512; expected=6-9m
+GPU_COMMAND P09-capture-ON-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/on/client; expected=5-7m
+GPU_COMMAND P09-capture-OFF-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120; expected=6-9m
+GPU_COMMAND P09-capture-OFF-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/off/client; expected=5-7m
+{"delta": {"completed_throughput_relative": 0.0050675675675675436, "e2e_mean_latency_relative": -0.04096717785851722, "e2e_p95_latency_relative": 0.03011353223527924, "padding_fraction_points": -0.049797561163151524, "padding_reduction_fraction": 0.5827252753866776, "token_efficiency_relative": 0.0017943761453185214}, "gpu_hours_increment": 0.2963894498348236, "gpu_hours_total": 14.322264868590567, "off": {"arm": "OFF", "bucket_tokens": 237911, "capture_sizes": "default", "clean_completed": 1184, "clean_completed_throughput_rps": 4.933333333333334, "clean_failed": 0, "clean_offered_rps": 4.920833333333333, "drain_seconds": 0.7853007119847462, "e2e_latency_mean_s": 1.681152764688729, "e2e_latency_p95_s": 3.8763178953449815, "gpu_hours": 0.13751606033907995, "graph_hit_steps": 12252, "graph_miss_rate": 0.045199501246882795, "layer1_invariants": {"cudagraph_identity": true, "footer_balanced": true, "footer_written_matches": true, "schema_1": true, "sidecar_agrees": true, "sidecar_final": true, "steps_unique_contiguous": true, "token_composition": true, "zero_drops": true}, "layer1_records": 16491, "model_step_duration_ms": 479367.749483, "model_steps": 12832, "padding_fraction": 0.08545632610514016, "padding_tokens": 20331, "schema": 1, "token_efficiency_per_ms": 4.5540506267984115, "useful_tokens": 2183065}, "on": {"arm": "ON", "bucket_tokens": 227714, "capture_sizes": [1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "clean_completed": 1190, "clean_completed_throughput_rps": 4.958333333333333, "clean_failed": 0, "clean_offered_rps": 4.920833333333333, "drain_seconds": 0.7876331160077825, "e2e_latency_mean_s": 1.612280680370388, "e2e_latency_p95_s": 3.9930475192406423, "gpu_hours": 0.15887338949574364, "graph_hit_steps": 14253, "graph_miss_rate": 0.0389724226282786, "layer1_invariants": {"cudagraph_identity": true, "footer_balanced": true, "footer_written_matches": true, "schema_1": true, "sidecar_agrees": true, "sidecar_final": true, "steps_unique_contiguous": true, "token_composition": true, "zero_drops": true}, "layer1_records": 17579, "model_step_duration_ms": 478655.543996, "model_steps": 14831, "padding_fraction": 0.035658764941988635, "padding_tokens": 8120, "schema": 1, "token_efficiency_per_ms": 4.562222306607711, "useful_tokens": 2183733}, "plan": {"added_capture_sizes": [3, 5, 6, 7, 9], "clean_seconds": 240, "gpu": 0, "gpu_hour_limit": 16.0, "load": "moderate", "manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl", "measured_padding_recovery_bound": 0.049776380388728225, "on_capture_sizes": [1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "order": ["ON", "OFF"], "pattern": "P09", "primary_metric": "clean graph-hit padding_fraction", "prior_gpu_hours": 14.025875418755744, "profile": false, "projected_increment_gpu_hours": 0.5, "projected_total_gpu_hours": 14.525875418755744, "rate_fraction": 0.6, "saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json", "schema": 1, "secondary_metrics": ["Layer-1 useful scheduled tokens/model-step millisecond", "clean completed request throughput", "clean request latency"], "warmup_seconds": 60}, "schema": 1}

runs/opprof-phase3/phase4/capture-p09/launch.log

@@ -0,0 +1 @@
+GPU_VALIDATION Phase4 P09 capture sizes: order=ON,OFF, GPU0 TP1, manifest=/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl, saturation_source=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json, added_sizes=3,5,6,7,9, warmup=60s, clean=240s/arm, projected=0.50 H20-hours, cumulative_projection=14.525875/16, expected_wall=15-22m

runs/opprof-phase3/phase4/capture-p09/off/client-sanity.json

@@ -0,0 +1,66 @@
+{
+  "invariants": {
+    "clean_duration_exact": true,
+    "clean_failures_zero": true,
+    "concurrency_bounded": true,
+    "drain_within_timeout": true,
+    "manifest_no_wrap": true,
+    "manifest_not_exhausted": true,
+    "moderate_offered_within_5pct": true,
+    "output_tokens_exact": true,
+    "profile_count_exact": true,
+    "profile_status_ok": true,
+    "segment_count_exact": true
+  },
+  "numeric": {
+    "actual_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 1477,
+      "max": 64.0,
+      "min": 64.0,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "admitted_s": {
+      "distinct_n": 1477,
+      "finite_n": 1477,
+      "max": 299.8483776419889,
+      "min": 0.00022695399820804596,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "completed_s": {
+      "distinct_n": 1477,
+      "finite_n": 1477,
+      "max": 300.78334441498737,
+      "min": 0.6682249770092312,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "input_tokens": {
+      "distinct_n": 944,
+      "finite_n": 1477,
+      "max": 8161.0,
+      "min": 128.0,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "requested_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 1477,
+      "max": 64.0,
+      "min": 64.0,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "scheduled_s": {
+      "distinct_n": 1477,
+      "finite_n": 1477,
+      "max": 299.84763839512016,
+      "min": 0.0,
+      "missing_n": 0,
+      "n": 1477
+    }
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase4/capture-p09/off/commands.log

@@ -0,0 +1,2 @@
+GPU_COMMAND P09-capture-OFF-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120 ; expected=6-9m
+GPU_COMMAND P09-capture-OFF-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/off/client ; expected=5-7m

runs/opprof-phase3/phase4/capture-p09/off/summary.json

@@ -0,0 +1,34 @@
+{
+  "arm": "OFF",
+  "bucket_tokens": 237911,
+  "capture_sizes": "default",
+  "clean_completed": 1184,
+  "clean_completed_throughput_rps": 4.933333333333334,
+  "clean_failed": 0,
+  "clean_offered_rps": 4.920833333333333,
+  "drain_seconds": 0.7853007119847462,
+  "e2e_latency_mean_s": 1.681152764688729,
+  "e2e_latency_p95_s": 3.8763178953449815,
+  "gpu_hours": 0.13751606033907995,
+  "graph_hit_steps": 12252,
+  "graph_miss_rate": 0.045199501246882795,
+  "layer1_invariants": {
+    "cudagraph_identity": true,
+    "footer_balanced": true,
+    "footer_written_matches": true,
+    "schema_1": true,
+    "sidecar_agrees": true,
+    "sidecar_final": true,
+    "steps_unique_contiguous": true,
+    "token_composition": true,
+    "zero_drops": true
+  },
+  "layer1_records": 16491,
+  "model_step_duration_ms": 479367.749483,
+  "model_steps": 12832,
+  "padding_fraction": 0.08545632610514016,
+  "padding_tokens": 20331,
+  "schema": 1,
+  "token_efficiency_per_ms": 4.5540506267984115,
+  "useful_tokens": 2183065
+}

runs/opprof-phase3/phase4/capture-p09/on/client-sanity.json

@@ -0,0 +1,66 @@
+{
+  "invariants": {
+    "clean_duration_exact": true,
+    "clean_failures_zero": true,
+    "concurrency_bounded": true,
+    "drain_within_timeout": true,
+    "manifest_no_wrap": true,
+    "manifest_not_exhausted": true,
+    "moderate_offered_within_5pct": true,
+    "output_tokens_exact": true,
+    "profile_count_exact": true,
+    "profile_status_ok": true,
+    "segment_count_exact": true
+  },
+  "numeric": {
+    "actual_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 1477,
+      "max": 64.0,
+      "min": 64.0,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "admitted_s": {
+      "distinct_n": 1477,
+      "finite_n": 1477,
+      "max": 299.8487557930057,
+      "min": 0.00024887899053283036,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "completed_s": {
+      "distinct_n": 1477,
+      "finite_n": 1477,
+      "max": 300.78616064199014,
+      "min": 15.648636110010557,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "input_tokens": {
+      "distinct_n": 944,
+      "finite_n": 1477,
+      "max": 8161.0,
+      "min": 128.0,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "requested_output_tokens": {
+      "distinct_n": 1,
+      "finite_n": 1477,
+      "max": 64.0,
+      "min": 64.0,
+      "missing_n": 0,
+      "n": 1477
+    },
+    "scheduled_s": {
+      "distinct_n": 1477,
+      "finite_n": 1477,
+      "max": 299.84763839512016,
+      "min": 0.0,
+      "missing_n": 0,
+      "n": 1477
+    }
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase4/capture-p09/on/commands.log

@@ -0,0 +1,2 @@
+GPU_COMMAND P09-capture-ON-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120 --cudagraph-capture-sizes 1 2 3 4 5 6 7 8 9 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512 ; expected=6-9m
+GPU_COMMAND P09-capture-ON-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/on/client ; expected=5-7m

runs/opprof-phase3/phase4/capture-p09/on/summary.json

@@ -0,0 +1,91 @@
+{
+  "arm": "ON",
+  "bucket_tokens": 227714,
+  "capture_sizes": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    16,
+    24,
+    32,
+    40,
+    48,
+    56,
+    64,
+    72,
+    80,
+    88,
+    96,
+    104,
+    112,
+    120,
+    128,
+    136,
+    144,
+    152,
+    160,
+    168,
+    176,
+    184,
+    192,
+    200,
+    208,
+    216,
+    224,
+    232,
+    240,
+    248,
+    256,
+    272,
+    288,
+    304,
+    320,
+    336,
+    352,
+    368,
+    384,
+    400,
+    416,
+    432,
+    448,
+    464,
+    480,
+    496,
+    512
+  ],
+  "clean_completed": 1190,
+  "clean_completed_throughput_rps": 4.958333333333333,
+  "clean_failed": 0,
+  "clean_offered_rps": 4.920833333333333,
+  "drain_seconds": 0.7876331160077825,
+  "e2e_latency_mean_s": 1.612280680370388,
+  "e2e_latency_p95_s": 3.9930475192406423,
+  "gpu_hours": 0.15887338949574364,
+  "graph_hit_steps": 14253,
+  "graph_miss_rate": 0.0389724226282786,
+  "layer1_invariants": {
+    "cudagraph_identity": true,
+    "footer_balanced": true,
+    "footer_written_matches": true,
+    "schema_1": true,
+    "sidecar_agrees": true,
+    "sidecar_final": true,
+    "steps_unique_contiguous": true,
+    "token_composition": true,
+    "zero_drops": true
+  },
+  "layer1_records": 17579,
+  "model_step_duration_ms": 478655.543996,
+  "model_steps": 14831,
+  "padding_fraction": 0.035658764941988635,
+  "padding_tokens": 8120,
+  "schema": 1,
+  "token_efficiency_per_ms": 4.562222306607711,
+  "useful_tokens": 2183733
+}

runs/opprof-phase3/phase4/capture-p09/result.json

@@ -0,0 +1,230 @@
+{
+  "delta": {
+    "completed_throughput_relative": 0.0050675675675675436,
+    "e2e_mean_latency_relative": -0.04096717785851722,
+    "e2e_p95_latency_relative": 0.03011353223527924,
+    "padding_fraction_points": -0.049797561163151524,
+    "padding_reduction_fraction": 0.5827252753866776,
+    "token_efficiency_relative": 0.0017943761453185214
+  },
+  "gpu_hours_increment": 0.2963894498348236,
+  "gpu_hours_total": 14.322264868590567,
+  "off": {
+    "arm": "OFF",
+    "bucket_tokens": 237911,
+    "capture_sizes": "default",
+    "clean_completed": 1184,
+    "clean_completed_throughput_rps": 4.933333333333334,
+    "clean_failed": 0,
+    "clean_offered_rps": 4.920833333333333,
+    "drain_seconds": 0.7853007119847462,
+    "e2e_latency_mean_s": 1.681152764688729,
+    "e2e_latency_p95_s": 3.8763178953449815,
+    "gpu_hours": 0.13751606033907995,
+    "graph_hit_steps": 12252,
+    "graph_miss_rate": 0.045199501246882795,
+    "layer1_invariants": {
+      "cudagraph_identity": true,
+      "footer_balanced": true,
+      "footer_written_matches": true,
+      "schema_1": true,
+      "sidecar_agrees": true,
+      "sidecar_final": true,
+      "steps_unique_contiguous": true,
+      "token_composition": true,
+      "zero_drops": true
+    },
+    "layer1_records": 16491,
+    "model_step_duration_ms": 479367.749483,
+    "model_steps": 12832,
+    "padding_fraction": 0.08545632610514016,
+    "padding_tokens": 20331,
+    "schema": 1,
+    "token_efficiency_per_ms": 4.5540506267984115,
+    "useful_tokens": 2183065
+  },
+  "on": {
+    "arm": "ON",
+    "bucket_tokens": 227714,
+    "capture_sizes": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      16,
+      24,
+      32,
+      40,
+      48,
+      56,
+      64,
+      72,
+      80,
+      88,
+      96,
+      104,
+      112,
+      120,
+      128,
+      136,
+      144,
+      152,
+      160,
+      168,
+      176,
+      184,
+      192,
+      200,
+      208,
+      216,
+      224,
+      232,
+      240,
+      248,
+      256,
+      272,
+      288,
+      304,
+      320,
+      336,
+      352,
+      368,
+      384,
+      400,
+      416,
+      432,
+      448,
+      464,
+      480,
+      496,
+      512
+    ],
+    "clean_completed": 1190,
+    "clean_completed_throughput_rps": 4.958333333333333,
+    "clean_failed": 0,
+    "clean_offered_rps": 4.920833333333333,
+    "drain_seconds": 0.7876331160077825,
+    "e2e_latency_mean_s": 1.612280680370388,
+    "e2e_latency_p95_s": 3.9930475192406423,
+    "gpu_hours": 0.15887338949574364,
+    "graph_hit_steps": 14253,
+    "graph_miss_rate": 0.0389724226282786,
+    "layer1_invariants": {
+      "cudagraph_identity": true,
+      "footer_balanced": true,
+      "footer_written_matches": true,
+      "schema_1": true,
+      "sidecar_agrees": true,
+      "sidecar_final": true,
+      "steps_unique_contiguous": true,
+      "token_composition": true,
+      "zero_drops": true
+    },
+    "layer1_records": 17579,
+    "model_step_duration_ms": 478655.543996,
+    "model_steps": 14831,
+    "padding_fraction": 0.035658764941988635,
+    "padding_tokens": 8120,
+    "schema": 1,
+    "token_efficiency_per_ms": 4.562222306607711,
+    "useful_tokens": 2183733
+  },
+  "plan": {
+    "added_capture_sizes": [
+      3,
+      5,
+      6,
+      7,
+      9
+    ],
+    "clean_seconds": 240,
+    "gpu": 0,
+    "gpu_hour_limit": 16.0,
+    "load": "moderate",
+    "manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
+    "measured_padding_recovery_bound": 0.049776380388728225,
+    "on_capture_sizes": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      16,
+      24,
+      32,
+      40,
+      48,
+      56,
+      64,
+      72,
+      80,
+      88,
+      96,
+      104,
+      112,
+      120,
+      128,
+      136,
+      144,
+      152,
+      160,
+      168,
+      176,
+      184,
+      192,
+      200,
+      208,
+      216,
+      224,
+      232,
+      240,
+      248,
+      256,
+      272,
+      288,
+      304,
+      320,
+      336,
+      352,
+      368,
+      384,
+      400,
+      416,
+      432,
+      448,
+      464,
+      480,
+      496,
+      512
+    ],
+    "order": [
+      "ON",
+      "OFF"
+    ],
+    "pattern": "P09",
+    "primary_metric": "clean graph-hit padding_fraction",
+    "prior_gpu_hours": 14.025875418755744,
+    "profile": false,
+    "projected_increment_gpu_hours": 0.5,
+    "projected_total_gpu_hours": 14.525875418755744,
+    "rate_fraction": 0.6,
+    "saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json",
+    "schema": 1,
+    "secondary_metrics": [
+      "Layer-1 useful scheduled tokens/model-step millisecond",
+      "clean completed request throughput",
+      "clean request latency"
+    ],
+    "warmup_seconds": 60
+  },
+  "schema": 1
+}

runs/opprof-phase3/phase4/capture_validation.py

@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+"""One-pair P09 CUDAGraph capture-size validation for OpProf Phase 4."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import shlex
+import signal
+import subprocess
+import sys
+import time
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+import opprof_phase3_controller as common
+import opprof_phase3_matrix as matrix
+
+
+WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
+ROOT = WORKDIR / "runs/phase4-capture-p09"
+PHASE3 = WORKDIR / "runs/phase3"
+PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests")
+MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
+SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
+VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
+CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
+STATE = ROOT / "controller-state.json"
+GPU = 0
+PORT = 8200
+PRIOR_GPU_HOURS = 14.025875418755744
+GPU_HOUR_LIMIT = 16.0
+EXPECTED_INCREMENT_HOURS = 0.5
+ADDED_CAPTURE_SIZES = (3, 5, 6, 7, 9)
+DEFAULT_CAPTURE_SIZES = (
+    1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104,
+    112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200,
+    208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
+    352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
+)
+ON_CAPTURE_SIZES = tuple(sorted(set(DEFAULT_CAPTURE_SIZES + ADDED_CAPTURE_SIZES)))
+
+
+def plan() -> dict[str, Any]:
+    return {
+        "schema": 1,
+        "pattern": "P09",
+        "load": "moderate",
+        "order": ["ON", "OFF"],
+        "gpu": GPU,
+        "warmup_seconds": 60,
+        "clean_seconds": 240,
+        "profile": False,
+        "manifest": str(PRIVATE / "P09.jsonl"),
+        "saturation_result": str(
+            PHASE3 / "primary/P09-C00/saturation/client/result.json"
+        ),
+        "rate_fraction": 0.60,
+        "added_capture_sizes": list(ADDED_CAPTURE_SIZES),
+        "on_capture_sizes": list(ON_CAPTURE_SIZES),
+        "measured_padding_recovery_bound": 0.049776380388728225,
+        "primary_metric": "clean graph-hit padding_fraction",
+        "secondary_metrics": [
+            "Layer-1 useful scheduled tokens/model-step millisecond",
+            "clean completed request throughput",
+            "clean request latency",
+        ],
+        "prior_gpu_hours": PRIOR_GPU_HOURS,
+        "projected_increment_gpu_hours": EXPECTED_INCREMENT_HOURS,
+        "projected_total_gpu_hours": PRIOR_GPU_HOURS + EXPECTED_INCREMENT_HOURS,
+        "gpu_hour_limit": GPU_HOUR_LIMIT,
+    }
+
+
+def save_state(state: dict[str, Any]) -> None:
+    state["updated_at"] = time.time()
+    state["controller_pid"] = os.getpid()
+    common.atomic_json(STATE, state)
+
+
+def wait_ready(server: subprocess.Popen[Any]) -> None:
+    deadline = time.monotonic() + 300
+    while time.monotonic() < deadline:
+        if server.poll() is not None:
+            raise RuntimeError("server exited before readiness")
+        try:
+            with urllib.request.urlopen(
+                f"http://127.0.0.1:{PORT}/health", timeout=1
+            ) as response:
+                if response.status == 200:
+                    return
+        except Exception:
+            pass
+        time.sleep(1)
+    raise TimeoutError("server readiness timeout")
+
+
+def server_command(arm: str) -> list[str]:
+    command = [
+        "taskset", "-c", "0-19", str(VENV / "bin/vllm"), "serve", str(MODEL),
+        "--host", "127.0.0.1", "--port", str(PORT),
+        "--tensor-parallel-size", "1", "--enable-chunked-prefill",
+        "--enable-prefix-caching", "--shutdown-timeout", "120",
+    ]
+    if arm == "ON":
+        command.extend(("--cudagraph-capture-sizes", *map(str, ON_CAPTURE_SIZES)))
+    return command
+
+
+def client_command(run_dir: Path) -> list[str]:
+    return [
+        "taskset", "-c", "0-19", str(VENV / "bin/python"), str(CLIENT), "run",
+        "--manifest", str(PRIVATE / "P09.jsonl"),
+        "--base-url", f"http://127.0.0.1:{PORT}", "--model", str(MODEL),
+        "--load-point", "moderate", "--saturation-result",
+        str(PHASE3 / "primary/P09-C00/saturation/client/result.json"),
+        "--rate-fraction", "0.60", "--max-concurrency", "256", "--ignore-eos",
+        "--temperature", "0", "--warmup-seconds", "60",
+        "--clean-segment-seconds", "80", "--num-clean-segments", "3",
+        "--recovery-seconds", "30", "--drain-timeout-seconds", "120",
+        "--workload-seed", "20260712", "--server-seed", "20260712",
+        "--result-dir", str(run_dir / "client"),
+    ]
+
+
+def summarize(run_dir: Path, arm: str) -> dict[str, Any]:
+    result = json.loads((run_dir / "client/result.json").read_text())
+    requests = [
+        json.loads(line)
+        for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
+    ]
+    t0 = int(result["t0_mono_ns"])
+    start, end = t0 + int(60e9), t0 + int(300e9)
+    stream = next((run_dir / "opprof").glob("*.jsonl"))
+    records = []
+    for line in stream.read_text().splitlines():
+        record = json.loads(line)
+        if (
+            "step_index" in record
+            and start <= int(record["submit_mono_ns"]) < end
+        ):
+            records.append(record)
+    model = [record for record in records if record["model_executed"]]
+    hits = [
+        record
+        for record in model
+        if record["cudagraph"]["hit"]
+        and int(record["cudagraph"]["bucket_tokens"]) > 0
+    ]
+    pad = sum(int(record["cudagraph"]["padding_tokens"]) for record in hits)
+    bucket = sum(int(record["cudagraph"]["bucket_tokens"]) for record in hits)
+    useful = sum(
+        int(record["prefill_tokens"]) + int(record["decode_tokens"])
+        for record in records
+    )
+    duration_ms = sum(
+        (int(record["complete_mono_ns"]) - int(record["submit_mono_ns"])) / 1e6
+        for record in records
+    )
+    completed = [
+        request
+        for request in requests
+        if request["success"] and 60 <= float(request["completed_s"]) < 300
+    ]
+    e2e = np.asarray(
+        [float(request["completed_s"] - request["admitted_s"]) for request in completed]
+    )
+    layer1 = matrix.validate_layer1(run_dir)
+    return {
+        "schema": 1,
+        "arm": arm,
+        "capture_sizes": list(ON_CAPTURE_SIZES) if arm == "ON" else "default",
+        "clean_completed": len(completed),
+        "clean_failed": int(result["clean"]["failed"]),
+        "clean_completed_throughput_rps": float(
+            result["clean"]["completed_throughput_rps"]
+        ),
+        "clean_offered_rps": float(result["clean"]["offered_rps"]),
+        "e2e_latency_mean_s": float(e2e.mean()),
+        "e2e_latency_p95_s": float(np.quantile(e2e, 0.95)),
+        "model_steps": len(model),
+        "graph_hit_steps": len(hits),
+        "padding_tokens": pad,
+        "bucket_tokens": bucket,
+        "padding_fraction": pad / bucket,
+        "graph_miss_rate": sum(not record["cudagraph"]["hit"] for record in model)
+        / len(model),
+        "useful_tokens": useful,
+        "model_step_duration_ms": duration_ms,
+        "token_efficiency_per_ms": useful / duration_ms,
+        "layer1_records": layer1["records"],
+        "layer1_invariants": layer1["invariants"],
+        "drain_seconds": float(result["drain_seconds"]),
+    }
+
+
+def run_arm(state: dict[str, Any], arm: str) -> None:
+    if state["arms"].get(arm, {}).get("status") == "complete":
+        return
+    run_dir = ROOT / arm.lower()
+    if run_dir.exists():
+        run_dir.rename(run_dir.with_name(f"{run_dir.name}.interrupted-{int(time.time())}"))
+    run_dir.mkdir(parents=True)
+    common.preflight([GPU], run_dir)
+    server_cmd = server_command(arm)
+    client_cmd = client_command(run_dir)
+    commands_path = run_dir / "commands.log"
+    common.command_log(commands_path, f"P09-capture-{arm}-server", server_cmd, "6-9m")
+    common.command_log(commands_path, f"P09-capture-{arm}-client", client_cmd, "5-7m")
+    print(
+        f"GPU_COMMAND P09-capture-{arm}-server: {shlex.join(server_cmd)}; "
+        "expected=6-9m",
+        flush=True,
+    )
+    print(
+        f"GPU_COMMAND P09-capture-{arm}-client: {shlex.join(client_cmd)}; "
+        "expected=5-7m",
+        flush=True,
+    )
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "CUDA_VISIBLE_DEVICES": str(GPU),
+            "VLLM_OPPROF_DIR": str(run_dir / "opprof"),
+            "HF_HUB_OFFLINE": "1",
+            "TRANSFORMERS_OFFLINE": "1",
+            "PYTHONUNBUFFERED": "1",
+        }
+    )
+    server_log = (run_dir / "server.log").open("ab", buffering=0)
+    server_started = time.time()
+    server = subprocess.Popen(
+        server_cmd,
+        cwd=SOURCE,
+        env=environment,
+        stdout=server_log,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,
+    )
+    client = None
+    client_log = None
+    monitor = None
+    owned = {server.pid}
+    state["arms"][arm] = {
+        "status": "starting",
+        "server_pid": server.pid,
+        "started_at": server_started,
+    }
+    save_state(state)
+    failure = None
+    try:
+        wait_ready(server)
+        monitor = common.Monitor(run_dir / "monitor.jsonl", owned)
+        monitor.start()
+        client_log = (run_dir / "client.log").open("ab", buffering=0)
+        client = subprocess.Popen(
+            client_cmd,
+            cwd=WORKDIR,
+            stdout=client_log,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+        owned.add(client.pid)
+        state["arms"][arm].update(status="running", client_pid=client.pid)
+        save_state(state)
+        deadline = time.monotonic() + 900
+        while client.poll() is None and time.monotonic() < deadline:
+            if server.poll() is not None:
+                raise RuntimeError("server exited during client load")
+            if monitor.other_apps:
+                raise RuntimeError(f"other GPU process appeared: {monitor.other_apps}")
+            time.sleep(2)
+        if client.poll() is None:
+            raise TimeoutError("client exceeded 900 seconds")
+        if client.returncode:
+            raise RuntimeError(f"client failed with {client.returncode}")
+    except Exception as error:
+        failure = error
+    finally:
+        if client is not None and client.poll() is None:
+            try:
+                os.killpg(client.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+        common.stop_servers([server])
+        server_log.close()
+        if client_log is not None:
+            client_log.close()
+        if monitor is not None:
+            monitor.stop()
+        common.verify_idle([GPU], run_dir)
+    gpu_hours = (time.time() - server_started) / 3600
+    state["gpu_hours_increment"] += gpu_hours
+    if PRIOR_GPU_HOURS + state["gpu_hours_increment"] >= GPU_HOUR_LIMIT:
+        failure = failure or RuntimeError("GPU-hour limit reached")
+    if failure is not None:
+        state["arms"][arm].update(status="failed", failure=repr(failure))
+        state["status"] = "failed"
+        save_state(state)
+        raise failure
+    summary = summarize(run_dir, arm)
+    summary["gpu_hours"] = gpu_hours
+    common.atomic_json(run_dir / "summary.json", summary)
+    state["arms"][arm].update(status="complete", summary=summary)
+    save_state(state)
+
+
+def run() -> None:
+    ROOT.mkdir(parents=True, exist_ok=True)
+    if PRIOR_GPU_HOURS + EXPECTED_INCREMENT_HOURS >= GPU_HOUR_LIMIT:
+        raise RuntimeError("projected validation exceeds GPU-hour budget")
+    if STATE.exists():
+        state = json.loads(STATE.read_text())
+    else:
+        state = {
+            "schema": 1,
+            "status": "running",
+            "created_at": time.time(),
+            "plan": plan(),
+            "arms": {},
+            "gpu_hours_increment": 0.0,
+        }
+    for arm in ("ON", "OFF"):
+        run_arm(state, arm)
+    on, off = state["arms"]["ON"]["summary"], state["arms"]["OFF"]["summary"]
+    result = {
+        "schema": 1,
+        "plan": plan(),
+        "on": on,
+        "off": off,
+        "delta": {
+            "padding_fraction_points": on["padding_fraction"] - off["padding_fraction"],
+            "padding_reduction_fraction": 1 - on["padding_fraction"] / off["padding_fraction"],
+            "token_efficiency_relative": on["token_efficiency_per_ms"]
+            / off["token_efficiency_per_ms"]
+            - 1,
+            "completed_throughput_relative": on["clean_completed_throughput_rps"]
+            / off["clean_completed_throughput_rps"]
+            - 1,
+            "e2e_mean_latency_relative": on["e2e_latency_mean_s"]
+            / off["e2e_latency_mean_s"]
+            - 1,
+            "e2e_p95_latency_relative": on["e2e_latency_p95_s"]
+            / off["e2e_latency_p95_s"]
+            - 1,
+        },
+        "gpu_hours_increment": state["gpu_hours_increment"],
+        "gpu_hours_total": PRIOR_GPU_HOURS + state["gpu_hours_increment"],
+    }
+    common.atomic_json(ROOT / "result.json", result)
+    state["status"] = "complete"
+    state["result"] = result
+    save_state(state)
+    print(json.dumps(result, sort_keys=True))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("command", choices=("plan", "run"))
+    args = parser.parse_args()
+    if args.command == "plan":
+        print(json.dumps(plan(), indent=2, sort_keys=True))
+    else:
+        run()
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase3/provenance/opprof_phase3_client.py

@@ -0,0 +1,794 @@
+#!/usr/bin/env python3
+"""Token-exact fixed-duration client for the OpProf Phase-3 protocol."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import gzip
+import hashlib
+import json
+import math
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+SCHEMA = 1
+TOKEN_BASE = 1000
+TOKEN_SPAN = 100000
+
+
+class ManifestExhausted(RuntimeError):
+    pass
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, value: Any, mode: int = 0o640) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        json.dump(value, f, sort_keys=True, indent=2)
+        f.write("\n")
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)
+
+
+def atomic_jsonl(path: Path, rows: list[dict[str, Any]], mode: int = 0o640) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)
+
+
+def parse_range(value: str) -> tuple[int, int]:
+    lo_text, hi_text = value.split(":", 1)
+    lo, hi = int(lo_text), int(hi_text)
+    if lo <= 0 or hi < lo:
+        raise argparse.ArgumentTypeError(f"invalid positive range: {value}")
+    return lo, hi
+
+
+def _integer_counts(weights: list[float], total: int) -> list[int]:
+    raw = [w * total for w in weights]
+    counts = [math.floor(x) for x in raw]
+    order = sorted(
+        range(len(raw)), key=lambda i: raw[i] - counts[i], reverse=True
+    )
+    for idx in order[: total - sum(counts)]:
+        counts[idx] += 1
+    return counts
+
+
+def numeric_sanity(values: list[float | int]) -> dict[str, Any]:
+    finite = [float(x) for x in values if math.isfinite(float(x))]
+    return {
+        "n": len(values),
+        "finite_n": len(finite),
+        "missing_n": len(values) - len(finite),
+        "min": min(finite) if finite else None,
+        "max": max(finite) if finite else None,
+        "distinct_n": len(set(finite)),
+    }
+
+
+def manifest_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    return {
+        "schema": SCHEMA,
+        "rows": len(rows),
+        "input_tokens": numeric_sanity([int(r["input_tokens"]) for r in rows]),
+        "output_tokens": numeric_sanity([int(r["output_tokens"]) for r in rows]),
+        "arrival_values": sorted({str(r["arrival"]) for r in rows}),
+        "pattern_values": sorted({str(r["pattern_id"]) for r in rows}),
+    }
+
+
+def materialize(args: argparse.Namespace) -> dict[str, Any]:
+    import numpy as np
+
+    rng = np.random.default_rng(args.workload_seed)
+    n = args.num_requests
+    if args.kind == "prefix-pool":
+        if args.num_prefixes <= 0 or args.prefix_len <= 0 or args.suffix_fixed <= 0:
+            raise ValueError("prefix-pool requires positive pool/prefix/suffix")
+        lengths = np.full(n, args.prefix_len + args.suffix_fixed, dtype=np.int64)
+        prefix_ids = np.arange(n, dtype=np.int64) % args.num_prefixes
+        rng.shuffle(prefix_ids)
+    else:
+        prefix_ids = np.full(n, -1, dtype=np.int64)
+        if args.input_uniform:
+            lo, hi = parse_range(args.input_uniform)
+            lengths = rng.integers(lo, hi + 1, n, dtype=np.int64)
+        elif args.input_fixed:
+            lengths = np.full(n, args.input_fixed, dtype=np.int64)
+        elif args.input_mixture:
+            spec = json.loads(args.input_mixture)
+            if not isinstance(spec, dict) or not spec:
+                raise ValueError("input mixture must be a non-empty JSON object")
+            keys = list(spec)
+            weights = [float(spec[key]) for key in keys]
+            if any(w < 0 for w in weights) or not math.isclose(sum(weights), 1.0):
+                raise ValueError("mixture weights must be non-negative and sum to 1")
+            pieces = []
+            for key, count in zip(
+                keys, _integer_counts(weights, n), strict=True
+            ):
+                kind, lo_text, hi_text = key.split(":")
+                if kind != "uniform":
+                    raise ValueError(f"unsupported mixture component: {key}")
+                pieces.append(
+                    rng.integers(
+                        int(lo_text), int(hi_text) + 1, count, dtype=np.int64
+                    )
+                )
+            lengths = np.concatenate(pieces)
+            rng.shuffle(lengths)
+        else:
+            raise ValueError("exactly one input distribution is required")
+    if args.output_fixed <= 0 or args.arrival not in {"steady", "burst:8"}:
+        raise ValueError("invalid output length or arrival class")
+
+    rows = []
+    for i in range(n):
+        row = {
+            "schema": SCHEMA,
+            "request_id": f"{args.id}-{i:05d}",
+            "pattern_id": args.id,
+            "kind": args.kind,
+            "input_tokens": int(lengths[i]),
+            "output_tokens": args.output_fixed,
+            "arrival": args.arrival,
+            "token_seed": int(args.workload_seed * 1000003 + i),
+        }
+        if args.kind == "prefix-pool":
+            row.update(
+                {
+                    "prefix_id": int(prefix_ids[i]),
+                    "num_prefixes": args.num_prefixes,
+                    "prefix_tokens": args.prefix_len,
+                }
+            )
+        rows.append(row)
+
+    out = Path(args.out)
+    atomic_jsonl(out, rows, mode=0o600)
+    summary = manifest_summary(rows)
+    summary.update({"sha256": sha256_file(out), "path": str(out)})
+    atomic_json(out.with_suffix(out.suffix + ".summary.json"), summary, mode=0o600)
+    print(json.dumps(summary, sort_keys=True))
+    return summary
+
+
+def materialize_private(args: argparse.Namespace) -> dict[str, Any]:
+    from transformers import AutoTokenizer
+
+    source = Path(args.source)
+    selected: list[dict[str, Any]] = []
+    with source.open(encoding="utf-8") as f:
+        for source_index, line in enumerate(f):
+            row = json.loads(line)
+            if (
+                float(row["sampling_u"]) <= args.sampling_u_max
+                and int(row["input_length"]) <= args.max_input_tokens
+            ):
+                selected.append(
+                    {
+                        "schema": SCHEMA,
+                        "request_id": f"{args.id}-{len(selected):05d}",
+                        "pattern_id": args.id,
+                        "kind": "private-trace",
+                        "input_tokens": int(row["input_length"]),
+                        "output_tokens": min(
+                            int(row["output_length"]), args.output_cap
+                        ),
+                        "arrival": args.arrival,
+                        "source_index": source_index,
+                        "prompt": row["prompt"],
+                    }
+                )
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    diffs = [
+        len(tokenizer.encode(row["prompt"], add_special_tokens=False))
+        - row["input_tokens"]
+        for row in selected
+    ]
+    exact = sum(diff == 0 for diff in diffs)
+    exact_fraction = exact / len(diffs) if diffs else 0.0
+    max_abs = max((abs(diff) for diff in diffs), default=-1)
+    if exact_fraction < 0.99 or max_abs > 1:
+        raise RuntimeError(
+            "tokenizer parity gate failed: "
+            f"exact_fraction={exact_fraction:.6f} max_abs_error={max_abs}"
+        )
+
+    out = Path(args.out)
+    atomic_jsonl(out, selected, mode=0o600)
+    summary = manifest_summary(selected)
+    summary.update(
+        {
+            "sha256": sha256_file(out),
+            "source_sha256": sha256_file(source),
+            "tokenizer_exact_n": exact,
+            "tokenizer_exact_fraction": exact_fraction,
+            "tokenizer_max_abs_error": max_abs,
+            "path": str(out),
+        }
+    )
+    atomic_json(out.with_suffix(out.suffix + ".summary.json"), summary, mode=0o600)
+    print(json.dumps(summary, sort_keys=True))
+    return summary
+
+
+def load_manifest(path: Path) -> list[dict[str, Any]]:
+    rows = [json.loads(line) for line in path.read_text().splitlines() if line]
+    required = {
+        "request_id",
+        "pattern_id",
+        "input_tokens",
+        "output_tokens",
+        "arrival",
+    }
+    if not rows:
+        raise ValueError("empty manifest")
+    for row in rows:
+        if not required.issubset(row):
+            raise ValueError(f"manifest row lacks {sorted(required - set(row))}")
+    if len({row["request_id"] for row in rows}) != len(rows):
+        raise ValueError("duplicate request_id")
+    return rows
+
+
+def _token_stream(seed: int, count: int) -> list[int]:
+    state = seed & 0xFFFFFFFF
+    out = []
+    for _ in range(count):
+        state = (1664525 * state + 1013904223) & 0xFFFFFFFF
+        out.append(TOKEN_BASE + state % TOKEN_SPAN)
+    return out
+
+
+def synthetic_prompt(row: dict[str, Any]) -> list[int]:
+    length = int(row["input_tokens"])
+    seed = int(row.get("token_seed", 0))
+    if row.get("kind") == "prefix-pool":
+        prefix_n = int(row["prefix_tokens"])
+        tokens = _token_stream(0xA5A50000 + int(row["prefix_id"]), prefix_n)
+        tokens += _token_stream(seed, length - prefix_n)
+        offset = prefix_n
+    else:
+        tokens = _token_stream(seed, length)
+        offset = 0
+    if length - offset >= 3:
+        index = int(row["request_id"].rsplit("-", 1)[1])
+        tokens[offset : offset + 3] = [
+            TOKEN_BASE + index % 100,
+            TOKEN_BASE + (index // 100) % 100,
+            TOKEN_BASE + (index // 10000) % 100,
+        ]
+    return tokens
+
+
+@dataclass
+class RunContext:
+    args: argparse.Namespace
+    rows: list[dict[str, Any]]
+    t0: float
+    clean_end: float
+    stop_event: asyncio.Event
+    lock: asyncio.Lock
+    next_index: int = 0
+    in_flight: int = 0
+    max_in_flight: int = 0
+    exhausted: bool = False
+    admission_stop_s: float | None = None
+
+    async def next_row(self) -> dict[str, Any]:
+        async with self.lock:
+            if self.next_index >= len(self.rows):
+                self.exhausted = True
+                raise ManifestExhausted(
+                    f"manifest exhausted after {self.next_index} admissions"
+                )
+            row = self.rows[self.next_index]
+            self.next_index += 1
+            return row
+
+
+async def request_one(
+    ctx: RunContext,
+    session: aiohttp.ClientSession,
+    row: dict[str, Any],
+    scheduled: float,
+) -> dict[str, Any]:
+    loop = asyncio.get_running_loop()
+    admitted = loop.time()
+    ctx.in_flight += 1
+    ctx.max_in_flight = max(ctx.max_in_flight, ctx.in_flight)
+    status = 0
+    actual_output: int | None = None
+    first_token: float | None = None
+    error_kind: str | None = None
+    try:
+        prompt: str | list[int] = (
+            row["prompt"]
+            if row.get("kind") == "private-trace"
+            else synthetic_prompt(row)
+        )
+        if not isinstance(prompt, str) and len(prompt) != int(row["input_tokens"]):
+            raise AssertionError("synthetic prompt length drift")
+        payload = {
+            "model": ctx.args.model,
+            "prompt": prompt,
+            "max_tokens": int(row["output_tokens"]),
+            "temperature": ctx.args.temperature,
+            "ignore_eos": ctx.args.ignore_eos,
+            "stream": True,
+            "stream_options": {"include_usage": True},
+            "add_special_tokens": False,
+            "seed": ctx.args.server_seed,
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "x-request-id": str(row["request_id"]),
+        }
+        async with session.post(
+            ctx.args.base_url.rstrip("/") + "/v1/completions",
+            json=payload,
+            headers=headers,
+        ) as response:
+            status = response.status
+            if status != 200:
+                error_kind = f"http_{status}"
+            else:
+                buf = b""
+                async for chunk in response.content.iter_any():
+                    buf += chunk
+                    while b"\n" in buf:
+                        line, buf = buf.split(b"\n", 1)
+                        line = line.strip()
+                        if not line.startswith(b"data:"):
+                            continue
+                        data = line[5:].strip()
+                        if data == b"[DONE]":
+                            continue
+                        event = json.loads(data)
+                        if event.get("choices") and first_token is None:
+                            first_token = loop.time()
+                        if event.get("usage") is not None:
+                            actual_output = int(event["usage"]["completion_tokens"])
+                if actual_output is None:
+                    error_kind = "missing_usage"
+    except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
+        error_kind = type(exc).__name__
+    except Exception as exc:
+        error_kind = type(exc).__name__
+    finally:
+        completed = loop.time()
+        ctx.in_flight -= 1
+    success = (
+        status == 200
+        and error_kind is None
+        and actual_output == int(row["output_tokens"])
+    )
+    if status == 200 and actual_output is not None and not success:
+        error_kind = "output_token_mismatch"
+    return {
+        "schema": SCHEMA,
+        "request_id": row["request_id"],
+        "scheduled_s": scheduled - ctx.t0,
+        "admitted_s": admitted - ctx.t0,
+        "first_token_s": None if first_token is None else first_token - ctx.t0,
+        "completed_s": completed - ctx.t0,
+        "input_tokens": int(row["input_tokens"]),
+        "requested_output_tokens": int(row["output_tokens"]),
+        "actual_output_tokens": actual_output,
+        "http_status": status,
+        "success": success,
+        "error_kind": error_kind,
+    }
+
+
+async def saturation_load(
+    ctx: RunContext, session: aiohttp.ClientSession
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    async def worker() -> None:
+        while not ctx.stop_event.is_set():
+            try:
+                row = await ctx.next_row()
+            except ManifestExhausted:
+                ctx.stop_event.set()
+                return
+            results.append(
+                await request_one(ctx, session, row, asyncio.get_running_loop().time())
+            )
+
+    tasks = [
+        asyncio.create_task(worker()) for _ in range(ctx.args.max_concurrency)
+    ]
+    await asyncio.gather(*tasks)
+    return results
+
+
+async def finite_load(
+    ctx: RunContext, session: aiohttp.ClientSession, rate: float
+) -> list[dict[str, Any]]:
+    sem = asyncio.Semaphore(ctx.args.max_concurrency)
+    tasks: list[asyncio.Task[dict[str, Any]]] = []
+    batch = 8 if str(ctx.rows[0]["arrival"]) == "burst:8" else 1
+    period = batch / rate
+    event_index = 0
+
+    async def limited(row: dict[str, Any], scheduled: float) -> dict[str, Any]:
+        async with sem:
+            return await request_one(ctx, session, row, scheduled)
+
+    while not ctx.stop_event.is_set():
+        scheduled = ctx.t0 + event_index * period
+        delay = scheduled - asyncio.get_running_loop().time()
+        if delay > 0:
+            try:
+                await asyncio.wait_for(ctx.stop_event.wait(), timeout=delay)
+                break
+            except asyncio.TimeoutError:
+                pass
+        if ctx.stop_event.is_set():
+            break
+        try:
+            for _ in range(batch):
+                tasks.append(
+                    asyncio.create_task(limited(await ctx.next_row(), scheduled))
+                )
+        except ManifestExhausted:
+            ctx.stop_event.set()
+            break
+        event_index += 1
+    return await asyncio.gather(*tasks) if tasks else []
+
+
+async def post_profile(
+    session: aiohttp.ClientSession, base_url: str, endpoint: str
+) -> tuple[float, float, int]:
+    loop = asyncio.get_running_loop()
+    before = loop.time()
+    async with session.post(base_url.rstrip("/") + endpoint) as response:
+        status = response.status
+        await response.read()
+    return before, loop.time(), status
+
+
+def _trace_loadable(path: Path) -> bool:
+    try:
+        opener = gzip.open if path.suffix == ".gz" else open
+        with opener(path, "rt", encoding="utf-8") as f:
+            parsed = json.load(f)
+        return isinstance(parsed, dict) and isinstance(parsed.get("traceEvents"), list)
+    except (OSError, EOFError, json.JSONDecodeError):
+        return False
+
+
+async def wait_new_trace(
+    trace_dir: Path, before: set[Path], timeout: float
+) -> Path:
+    deadline = asyncio.get_running_loop().time() + timeout
+    while asyncio.get_running_loop().time() < deadline:
+        for path in sorted(set(trace_dir.glob("*.pt.trace.json*")) - before):
+            if _trace_loadable(path):
+                return path
+        await asyncio.sleep(0.25)
+    raise TimeoutError(f"no new loadable trace within {timeout}s")
+
+
+async def timeline(
+    ctx: RunContext, session: aiohttp.ClientSession
+) -> list[dict[str, Any]]:
+    args = ctx.args
+    profiles: list[dict[str, Any]] = []
+    await asyncio.sleep(max(0, ctx.clean_end - asyncio.get_running_loop().time()))
+    if args.profile_after_clean:
+        trace_dir = Path(args.profile_trace_dir)
+        for window in range(args.num_profile_windows):
+            prior = set(trace_dir.glob("*.pt.trace.json*"))
+            start_before, start_after, start_status = await post_profile(
+                session, args.base_url, "/start_profile"
+            )
+            trace = await wait_new_trace(
+                trace_dir, prior, args.profile_timeout_seconds
+            )
+            trace_ready = asyncio.get_running_loop().time()
+            stop_before, stop_after, stop_status = await post_profile(
+                session, args.base_url, "/stop_profile"
+            )
+            profiles.append(
+                {
+                    "window": window + 1,
+                    "start_call_s": start_before - ctx.t0,
+                    "start_return_s": start_after - ctx.t0,
+                    "trace_ready_s": trace_ready - ctx.t0,
+                    "stop_call_s": stop_before - ctx.t0,
+                    "stop_return_s": stop_after - ctx.t0,
+                    "start_status": start_status,
+                    "stop_status": stop_status,
+                    "trace_file": trace.name,
+                    "trace_sha256": sha256_file(trace),
+                }
+            )
+            if start_status != 200 or stop_status != 200:
+                raise RuntimeError("profile endpoint returned non-200")
+            await asyncio.sleep(args.recovery_seconds)
+    else:
+        await asyncio.sleep(args.post_clean_seconds)
+    ctx.admission_stop_s = asyncio.get_running_loop().time() - ctx.t0
+    ctx.stop_event.set()
+    return profiles
+
+
+def segment_summary(
+    records: list[dict[str, Any]], start: float, end: float
+) -> dict[str, Any]:
+    admitted = [r for r in records if start <= r["admitted_s"] < end]
+    completed = [r for r in records if start <= r["completed_s"] < end]
+    successes = [r for r in completed if r["success"]]
+    duration = end - start
+    return {
+        "start_s": start,
+        "end_s": end,
+        "duration_s": duration,
+        "admitted": len(admitted),
+        "completed": len(successes),
+        "failed": len(completed) - len(successes),
+        "offered_rps": len(admitted) / duration,
+        "completed_throughput_rps": len(successes) / duration,
+        "input_tokens": sum(r["input_tokens"] for r in successes),
+        "output_tokens": sum(r["actual_output_tokens"] or 0 for r in successes),
+    }
+
+
+async def run_load(args: argparse.Namespace) -> dict[str, Any]:
+    manifest = Path(args.manifest)
+    rows = load_manifest(manifest)
+    arrivals = {row["arrival"] for row in rows}
+    if len(arrivals) != 1:
+        raise ValueError("a manifest must have one arrival class")
+    if args.load_point == "saturation":
+        if args.request_rate != "inf":
+            raise ValueError("saturation requires --request-rate inf")
+        rate = math.inf
+    else:
+        if not args.saturation_result:
+            raise ValueError("moderate requires --saturation-result")
+        sat = json.loads(Path(args.saturation_result).read_text())
+        rate = args.rate_fraction * float(sat["clean"]["completed_throughput_rps"])
+        if not math.isfinite(rate) or rate <= 0:
+            raise ValueError("derived moderate rate must be positive and finite")
+
+    loop = asyncio.get_running_loop()
+    t0 = loop.time()
+    t0_mono_ns = int(t0 * 1e9)
+    t0_wall_ns = time.time_ns()
+    clean_seconds = args.clean_segment_seconds * args.num_clean_segments
+    ctx = RunContext(
+        args=args,
+        rows=rows,
+        t0=t0,
+        clean_end=t0 + args.warmup_seconds + clean_seconds,
+        stop_event=asyncio.Event(),
+        lock=asyncio.Lock(),
+    )
+    timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=600)
+    connector = aiohttp.TCPConnector(limit=args.max_concurrency)
+    control_connector = aiohttp.TCPConnector(limit=2)
+    async with (
+        aiohttp.ClientSession(timeout=timeout, connector=connector) as session,
+        aiohttp.ClientSession(
+            timeout=timeout, connector=control_connector
+        ) as control_session,
+    ):
+        profile_task = asyncio.create_task(timeline(ctx, control_session))
+        load_task = asyncio.create_task(
+            saturation_load(ctx, session)
+            if math.isinf(rate)
+            else finite_load(ctx, session, rate)
+        )
+        try:
+            profiles = await profile_task
+        except Exception:
+            ctx.stop_event.set()
+            await load_task
+            raise
+        records = await load_task
+
+    clean_start = args.warmup_seconds
+    clean_end = clean_start + clean_seconds
+    clean = segment_summary(records, clean_start, clean_end)
+    segments = []
+    for i in range(args.num_clean_segments):
+        start = clean_start + i * args.clean_segment_seconds
+        segments.append(
+            {
+                "name": chr(ord("A") + i),
+                **segment_summary(
+                    records, start, start + args.clean_segment_seconds
+                ),
+            }
+        )
+    successful = [r for r in records if r["success"]]
+    elapsed_seconds = loop.time() - t0
+    if ctx.admission_stop_s is None:
+        raise RuntimeError("admission stop timestamp was not recorded")
+    drain_seconds = elapsed_seconds - ctx.admission_stop_s
+    result = {
+        "schema": SCHEMA,
+        "manifest_sha256": sha256_file(manifest),
+        "manifest_rows": len(rows),
+        "manifest_admitted": ctx.next_index,
+        "manifest_wrapped": False,
+        "manifest_exhausted": ctx.exhausted,
+        "load_point": args.load_point,
+        "t0_mono_ns": t0_mono_ns,
+        "t0_wall_ns": t0_wall_ns,
+        "request_rate": "inf" if math.isinf(rate) else rate,
+        "rate_fraction": None if math.isinf(rate) else args.rate_fraction,
+        "arrival": next(iter(arrivals)),
+        "warmup_seconds": args.warmup_seconds,
+        "clean_segment_seconds": args.clean_segment_seconds,
+        "num_clean_segments": args.num_clean_segments,
+        "elapsed_seconds": elapsed_seconds,
+        "admission_stop_s": ctx.admission_stop_s,
+        "drain_seconds": drain_seconds,
+        "max_in_flight": ctx.max_in_flight,
+        "records": len(records),
+        "successful_records": len(successful),
+        "failed_records": len(records) - len(successful),
+        "clean": clean,
+        "segments": segments,
+        "profiles": profiles,
+    }
+    sanity = {
+        "schema": SCHEMA,
+        "numeric": {
+            "input_tokens": numeric_sanity([r["input_tokens"] for r in records]),
+            "requested_output_tokens": numeric_sanity(
+                [r["requested_output_tokens"] for r in records]
+            ),
+            "actual_output_tokens": numeric_sanity(
+                [
+                    r["actual_output_tokens"]
+                    for r in records
+                    if r["actual_output_tokens"] is not None
+                ]
+            ),
+            "scheduled_s": numeric_sanity([r["scheduled_s"] for r in records]),
+            "admitted_s": numeric_sanity([r["admitted_s"] for r in records]),
+            "completed_s": numeric_sanity([r["completed_s"] for r in records]),
+        },
+        "invariants": {
+            "clean_duration_exact": math.isclose(clean["duration_s"], clean_seconds),
+            "segment_count_exact": len(segments) == args.num_clean_segments,
+            "manifest_no_wrap": ctx.next_index <= len(rows),
+            "manifest_not_exhausted": not ctx.exhausted,
+            "concurrency_bounded": ctx.max_in_flight <= args.max_concurrency,
+            "drain_within_timeout": drain_seconds <= args.drain_timeout_seconds,
+            "output_tokens_exact": all(
+                r["actual_output_tokens"] == r["requested_output_tokens"]
+                for r in successful
+            ),
+            "clean_failures_zero": clean["failed"] == 0,
+            "profile_count_exact": len(profiles)
+            == (args.num_profile_windows if args.profile_after_clean else 0),
+            "profile_status_ok": all(
+                p["start_status"] == 200 and p["stop_status"] == 200
+                for p in profiles
+            ),
+        },
+    }
+    if not math.isinf(rate):
+        sanity["invariants"]["moderate_offered_within_5pct"] = (
+            abs(clean["offered_rps"] / rate - 1) <= 0.05
+        )
+    out = Path(args.result_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    atomic_jsonl(out / "requests.jsonl", sorted(records, key=lambda r: r["admitted_s"]))
+    atomic_jsonl(out / "segments.jsonl", segments)
+    atomic_json(out / "result.json", result)
+    atomic_json(out / "sanity.json", sanity)
+    if ctx.exhausted:
+        raise ManifestExhausted("manifest exhausted; result retained for diagnosis")
+    failed = [name for name, ok in sanity["invariants"].items() if not ok]
+    if failed:
+        raise RuntimeError(f"client sanity failure: {failed}")
+    return result
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="command", required=True)
+    mat = sub.add_parser("materialize")
+    mat.add_argument("--id", required=True)
+    mat.add_argument("--kind", choices=("synthetic", "prefix-pool"), required=True)
+    group = mat.add_mutually_exclusive_group()
+    group.add_argument("--input-uniform")
+    group.add_argument("--input-fixed", type=int)
+    group.add_argument("--input-mixture")
+    mat.add_argument("--output-fixed", type=int, required=True)
+    mat.add_argument("--prefix", default="none")
+    mat.add_argument("--arrival", required=True)
+    mat.add_argument("--num-requests", type=int, required=True)
+    mat.add_argument("--workload-seed", type=int, required=True)
+    mat.add_argument("--num-prefixes", type=int, default=0)
+    mat.add_argument("--prefix-len", type=int, default=0)
+    mat.add_argument("--suffix-fixed", type=int, default=0)
+    mat.add_argument("--out", required=True)
+    private = sub.add_parser("materialize-private")
+    private.add_argument("--id", required=True)
+    private.add_argument("--source", required=True)
+    private.add_argument("--sampling-u-max", type=float, required=True)
+    private.add_argument("--max-input-tokens", type=int, required=True)
+    private.add_argument("--output-cap", type=int, required=True)
+    private.add_argument("--preserve-prompts", action="store_true", required=True)
+    private.add_argument("--disable-shuffle", action="store_true", required=True)
+    private.add_argument("--arrival", required=True)
+    private.add_argument("--model", required=True)
+    private.add_argument("--out", required=True)
+    run = sub.add_parser("run")
+    run.add_argument("--manifest", required=True)
+    run.add_argument("--base-url", required=True)
+    run.add_argument("--model", required=True)
+    run.add_argument("--load-point", choices=("saturation", "moderate"), required=True)
+    run.add_argument("--request-rate")
+    run.add_argument("--saturation-result")
+    run.add_argument("--rate-fraction", type=float, default=0.60)
+    run.add_argument("--max-concurrency", type=int, default=256)
+    run.add_argument("--ignore-eos", action="store_true")
+    run.add_argument("--temperature", type=float, default=0.0)
+    run.add_argument("--warmup-seconds", type=float, default=60)
+    run.add_argument("--clean-segment-seconds", type=float, default=80)
+    run.add_argument("--num-clean-segments", type=int, default=3)
+    run.add_argument("--profile-after-clean", action="store_true")
+    run.add_argument("--num-profile-windows", type=int, default=0)
+    run.add_argument("--profile-warmup-iterations", type=int, default=2)
+    run.add_argument("--profile-active-iterations", type=int, default=8)
+    run.add_argument("--profile-trace-dir")
+    run.add_argument("--profile-timeout-seconds", type=float, default=120)
+    run.add_argument("--recovery-seconds", type=float, default=30)
+    run.add_argument("--post-clean-seconds", type=float, default=0)
+    run.add_argument("--drain-timeout-seconds", type=float, default=120)
+    run.add_argument("--workload-seed", type=int, default=20260712)
+    run.add_argument("--server-seed", type=int, default=20260712)
+    run.add_argument("--result-dir", required=True)
+    return parser
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.command == "materialize":
+        materialize(args)
+    elif args.command == "materialize-private":
+        materialize_private(args)
+    else:
+        if args.profile_after_clean and not args.profile_trace_dir:
+            raise ValueError("--profile-after-clean requires --profile-trace-dir")
+        print(json.dumps(asyncio.run(run_load(args)), sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase3/provenance/test_phase3_analysis.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+import analyze_phase3 as analysis
+
+
+class Phase3AnalysisTests(unittest.TestCase):
+    def test_ap36_stability_formula(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "client").mkdir()
+            (root / "opprof").mkdir()
+            (root / "client/result.json").write_text(
+                json.dumps({"t0_mono_ns": 0, "warmup_seconds": 60})
+            )
+            (root / "client/requests.jsonl").write_text(
+                "".join(
+                    json.dumps({"success": True, "completed_s": index + 1}) + "\n"
+                    for index in range(16)
+                )
+            )
+            records = []
+            for bin_index in range(3):
+                for step in range(16):
+                    records.append(
+                        {
+                            "step_index": len(records),
+                            "model_executed": True,
+                            "submit_mono_ns": int(
+                                (45 + 5 * bin_index + (step + 0.5) / 16 * 5)
+                                * 1e9
+                            ),
+                            "prefill_tokens": 100,
+                            "decode_tokens": 0,
+                        }
+                    )
+            (root / "opprof/test.jsonl").write_text(
+                "".join(json.dumps(item) + "\n" for item in records)
+            )
+            result = analysis.ap36_warmup_stability(root)
+            self.assertTrue(result["passes"])
+            self.assertEqual(result["normalized_drift"], 0)
+
+    def test_ap37_partial_verdict_can_confirm_but_not_refute(self):
+        self.assertEqual(analysis.partial_verdict(True), "PASS")
+        self.assertEqual(analysis.partial_verdict(False), "INCONCLUSIVE")
+
+    def test_accepted_markers_come_only_from_complete_stages(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            complete = root / "stages/primary-01-saturation"
+            complete.mkdir(parents=True)
+            (complete / "stage-complete.json").write_text(
+                json.dumps({"runs": ["P01-C00-saturation"]})
+            )
+            accepted = root / "primary/P01-C00/saturation"
+            accepted.mkdir(parents=True)
+            (accepted / "run-complete.json").write_text(
+                json.dumps({"run_id": "P01-C00-saturation"})
+            )
+            unaccepted = root / "primary/P05-C00/saturation"
+            unaccepted.mkdir(parents=True)
+            (unaccepted / "run-complete.json").write_text(
+                json.dumps({"run_id": "P05-C00-saturation"})
+            )
+            primary, confirmations, stages, excluded = analysis.accepted_marker_paths(
+                root
+            )
+            self.assertEqual(primary, [accepted / "run-complete.json"])
+            self.assertEqual(confirmations, [])
+            self.assertEqual(stages, [complete])
+            self.assertEqual(excluded, ["P05-C00-saturation"])
+
+    def test_r64_is_ratio_of_cohort_sums(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "manifest.jsonl"
+            rows = [{"input_tokens": value} for value in (1, 3, 2, 2)]
+            path.write_text("".join(json.dumps(row) + "\n" for row in rows))
+            value, pieces = analysis.manifest_raggedness(path, 2)
+            self.assertEqual(pieces, [(2.0, 6.0), (0.0, 4.0)])
+            self.assertAlmostEqual(value, 0.2)
+
+    def test_one_percentage_point_ranking_ties(self):
+        shares = dict.fromkeys(analysis.FAMILIES, 0.0)
+        shares.update(attention=0.40, moe_gemm=0.395, moe_router=0.20)
+        ranked = {item["family"]: item["rank"] for item in analysis.ranked_families(shares)}
+        self.assertEqual(ranked["attention"], ranked["moe_gemm"])
+        self.assertGreater(ranked["moe_router"], ranked["attention"])
+
+    def test_holm_uses_declared_total_test_family(self):
+        values = [{"p": 0.001}, {"p": 0.01}]
+        analysis.holm(values, total_tests=10)
+        self.assertAlmostEqual(values[0]["p_holm"], 0.01)
+        self.assertAlmostEqual(values[1]["p_holm"], 0.09)
+
+    def test_robust_spline_prediction_is_nonnegative(self):
+        rows = [(float(x), float(n), float(2 * x + n)) for x in range(1, 20) for n in (1, 4)]
+        predict, hull = analysis.fit_nonnegative_robust(rows)
+        self.assertGreaterEqual(predict(3, 2), 0)
+        self.assertTrue(analysis.inside_convex(hull, (3, 2)))
+        self.assertFalse(analysis.inside_convex(hull, (100, 2)))
+
+
+if __name__ == "__main__":
+    unittest.main()

runs/opprof-phase3/provenance/test_phase3_tools.py

@@ -0,0 +1,486 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from pathlib import Path
+from types import SimpleNamespace
+
+from aiohttp import web
+
+
+HERE = Path(__file__).parent
+
+
+def load_module(name: str, filename: str):
+    spec = importlib.util.spec_from_file_location(name, HERE / filename)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+client = load_module("phase3_client", "opprof_phase3_client.py")
+controller = load_module("phase3_controller", "opprof_phase3_controller.py")
+sys.modules["opprof_phase3_controller"] = controller
+matrix = load_module("phase3_matrix", "opprof_phase3_matrix.py")
+
+
+def rows(n: int, arrival: str = "steady") -> list[dict]:
+    return [
+        {
+            "schema": 1,
+            "request_id": f"T-{i:05d}",
+            "pattern_id": "T",
+            "kind": "synthetic",
+            "input_tokens": 8,
+            "output_tokens": 2,
+            "arrival": arrival,
+            "token_seed": i + 1,
+        }
+        for i in range(n)
+    ]
+
+
+class MockServer:
+    def __init__(self, delay: float = 0.01, trace_dir: Path | None = None) -> None:
+        self.active = 0
+        self.max_active = 0
+        self.payloads = []
+        self.runner = None
+        self.port = None
+        self.delay = delay
+        self.trace_dir = trace_dir
+        self.profile_count = 0
+
+    async def completion(self, request):
+        payload = await request.json()
+        self.payloads.append(payload)
+        self.active += 1
+        self.max_active = max(self.max_active, self.active)
+        await asyncio.sleep(self.delay)
+        response = web.StreamResponse(
+            status=200, headers={"Content-Type": "text/event-stream"}
+        )
+        await response.prepare(request)
+        await response.write(
+            b'data: {"choices":[{"text":"x"}],"usage":null}\n\n'
+        )
+        usage = json.dumps(
+            {
+                "choices": [],
+                "usage": {
+                    "prompt_tokens": len(payload["prompt"]),
+                    "completion_tokens": payload["max_tokens"],
+                },
+            }
+        ).encode()
+        await response.write(b"data: " + usage + b"\n\n")
+        await response.write(b"data: [DONE]\n\n")
+        await response.write_eof()
+        self.active -= 1
+        return response
+
+    async def start_profile(self, request):
+        self.profile_count += 1
+        path = self.trace_dir / f"window-{self.profile_count}.pt.trace.json"
+        path.write_text('{"traceEvents": []}')
+        return web.Response(status=200)
+
+    async def stop_profile(self, request):
+        return web.Response(status=200)
+
+    async def start(self):
+        app = web.Application()
+        app.router.add_post("/v1/completions", self.completion)
+        if self.trace_dir is not None:
+            app.router.add_post("/start_profile", self.start_profile)
+            app.router.add_post("/stop_profile", self.stop_profile)
+        self.runner = web.AppRunner(app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, "127.0.0.1", 0)
+        await site.start()
+        self.port = site._server.sockets[0].getsockname()[1]
+
+    async def stop(self):
+        await self.runner.cleanup()
+
+
+def run_args(
+    manifest: Path,
+    result_dir: Path,
+    port: int,
+    load_point: str = "saturation",
+    saturation_result: Path | None = None,
+):
+    return SimpleNamespace(
+        manifest=str(manifest),
+        base_url=f"http://127.0.0.1:{port}",
+        model="mock",
+        load_point=load_point,
+        request_rate="inf" if load_point == "saturation" else None,
+        saturation_result=None if saturation_result is None else str(saturation_result),
+        rate_fraction=0.60,
+        max_concurrency=3,
+        ignore_eos=True,
+        temperature=0,
+        warmup_seconds=0.04,
+        clean_segment_seconds=0.04,
+        num_clean_segments=3,
+        profile_after_clean=False,
+        num_profile_windows=0,
+        profile_warmup_iterations=2,
+        profile_active_iterations=8,
+        profile_trace_dir=None,
+        profile_timeout_seconds=1,
+        recovery_seconds=0,
+        post_clean_seconds=0,
+        drain_timeout_seconds=1,
+        workload_seed=20260712,
+        server_seed=20260712,
+        result_dir=str(result_dir),
+    )
+
+
+class Phase3ToolTests(unittest.TestCase):
+    def write_warmup_stream(self, root: Path, tokens: list[int]) -> None:
+        stream_dir = root / "opprof"
+        stream_dir.mkdir()
+        records = []
+        step = 0
+        for bin_index, token_count in enumerate(tokens):
+            per_step, remainder = divmod(token_count, 16)
+            for in_bin in range(16):
+                records.append(
+                    {
+                        "step_index": step,
+                        "model_executed": True,
+                        "submit_mono_ns": int(
+                            1e9
+                            + (45 + bin_index * 5 + (in_bin + 0.5) / 16 * 5)
+                            * 1e9
+                        ),
+                        "prefill_tokens": per_step + (in_bin < remainder),
+                        "decode_tokens": 0,
+                    }
+                )
+                step += 1
+        (stream_dir / "test.jsonl").write_text(
+            "".join(json.dumps(record) + "\n" for record in records)
+        )
+
+    def test_p10_warmup_stability_accepts_flat_trailing_quartile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            self.write_warmup_stream(root, [1600, 1600, 1600])
+            result = matrix.p10_warmup_stability(root, int(1e9))
+            self.assertTrue(result["passed"])
+            self.assertEqual(result["step_counts"], [16, 16, 16])
+            self.assertEqual(result["scheduled_token_throughput"], [320, 320, 320])
+            self.assertEqual(result["normalized_drift"], 0)
+
+    def test_p10_warmup_stability_rejects_large_drift(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            self.write_warmup_stream(root, [1600, 3200, 4800])
+            result = matrix.p10_warmup_stability(root, int(1e9))
+            self.assertFalse(result["passed"])
+            self.assertGreater(result["normalized_drift"], 0.10)
+
+    def test_only_clean_window_request_failures_are_hard_failures(self):
+        requests = [
+            {"success": False, "completed_s": 59.9, "error_kind": "warmup"},
+            {"success": False, "completed_s": 60.0, "error_kind": "clean"},
+            {"success": False, "completed_s": 299.9, "error_kind": "clean"},
+            {"success": False, "completed_s": 300.0, "error_kind": "recovery"},
+            {"success": True, "completed_s": 100.0, "error_kind": None},
+        ]
+        summary = matrix.summarize_request_failures(requests, 60.0, 300.0)
+        self.assertEqual(summary["failed"], 4)
+        self.assertEqual(summary["clean_failed"], 2)
+        self.assertEqual(summary["excluded"], 2)
+        self.assertEqual(summary["excluded_kinds"], {"warmup": 1, "recovery": 1})
+
+    def test_synthetic_prompt_exact_and_unique_early_prefix(self):
+        generated = [client.synthetic_prompt(row) for row in rows(200)]
+        self.assertTrue(all(len(value) == 8 for value in generated))
+        self.assertEqual(len({tuple(value[:3]) for value in generated}), 200)
+
+    def test_p05_manifest_has_exact_half_modes(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "P05.jsonl"
+            args = SimpleNamespace(
+                id="P05",
+                kind="synthetic",
+                input_uniform=None,
+                input_fixed=None,
+                input_mixture='{"uniform:128:512":0.5,"uniform:4096:8192":0.5}',
+                output_fixed=64,
+                prefix="none",
+                arrival="steady",
+                num_requests=100,
+                workload_seed=20260712,
+                num_prefixes=0,
+                prefix_len=0,
+                suffix_fixed=0,
+                out=str(out),
+            )
+            client.materialize(args)
+            manifest = client.load_manifest(out)
+            self.assertEqual(sum(r["input_tokens"] <= 512 for r in manifest), 50)
+            self.assertEqual(sum(r["input_tokens"] >= 4096 for r in manifest), 50)
+
+    def test_prefix_pool_balanced(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "P08.jsonl"
+            args = SimpleNamespace(
+                id="P08",
+                kind="prefix-pool",
+                input_uniform=None,
+                input_fixed=None,
+                input_mixture=None,
+                output_fixed=512,
+                prefix="none",
+                arrival="burst:8",
+                num_requests=80,
+                workload_seed=20260712,
+                num_prefixes=8,
+                prefix_len=1024,
+                suffix_fixed=256,
+                out=str(out),
+            )
+            client.materialize(args)
+            manifest = client.load_manifest(out)
+            counts = {i: 0 for i in range(8)}
+            for row in manifest:
+                counts[row["prefix_id"]] += 1
+                self.assertEqual(row["input_tokens"], 1280)
+            self.assertEqual(set(counts.values()), {10})
+
+    def test_fixed_duration_saturation_and_redaction(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    result = await client.run_load(
+                        run_args(manifest, root / "result", mock.port)
+                    )
+                finally:
+                    await mock.stop()
+                self.assertEqual(result["max_in_flight"], 3)
+                self.assertEqual(mock.max_active, 3)
+                self.assertAlmostEqual(result["clean"]["duration_s"], 0.12, places=9)
+                self.assertEqual(len(result["segments"]), 3)
+                self.assertLessEqual(result["drain_seconds"], 1)
+                self.assertTrue(
+                    all(p["max_tokens"] == 2 and p["ignore_eos"] for p in mock.payloads)
+                )
+                text = (root / "result/requests.jsonl").read_text()
+                self.assertNotIn('"prompt":', text)
+                self.assertNotIn('"text":', text)
+
+        asyncio.run(case())
+
+    def test_profile_control_plane_is_not_starved_by_data_connector(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                trace_dir = root / "traces"
+                trace_dir.mkdir()
+                client.atomic_jsonl(manifest, rows(1000))
+                server = MockServer(delay=0.5, trace_dir=trace_dir)
+                await server.start()
+                args = run_args(manifest, root / "result", server.port)
+                args.max_concurrency = 1
+                args.warmup_seconds = 0.01
+                args.clean_segment_seconds = 0.01
+                args.num_clean_segments = 1
+                args.profile_after_clean = True
+                args.num_profile_windows = 1
+                args.profile_trace_dir = str(trace_dir)
+                args.recovery_seconds = 0
+                try:
+                    result = await client.run_load(args)
+                finally:
+                    await server.stop()
+                self.assertEqual(server.max_active, 1)
+                self.assertEqual(server.profile_count, 1)
+                self.assertLess(result["profiles"][0]["start_return_s"], 0.25)
+
+        asyncio.run(case())
+
+    def test_drain_timeout_is_a_hard_sanity_failure(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    args = run_args(manifest, root / "result", mock.port)
+                    args.drain_timeout_seconds = 0
+                    with self.assertRaisesRegex(RuntimeError, "drain_within_timeout"):
+                        await client.run_load(args)
+                finally:
+                    await mock.stop()
+                sanity = json.loads((root / "result/sanity.json").read_text())
+                self.assertFalse(sanity["invariants"]["drain_within_timeout"])
+
+        asyncio.run(case())
+
+    def test_burst_schedule_is_eight_at_once(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000, "burst:8"))
+                sat = root / "sat.json"
+                client.atomic_json(
+                    sat, {"clean": {"completed_throughput_rps": 100.0}}
+                )
+                mock = MockServer()
+                await mock.start()
+                try:
+                    args = run_args(
+                        manifest, root / "result", mock.port, "moderate", sat
+                    )
+                    args.warmup_seconds = 0
+                    args.clean_segment_seconds = 0.4 / 3
+                    result = await client.run_load(args)
+                finally:
+                    await mock.stop()
+                records = [
+                    json.loads(line)
+                    for line in (root / "result/requests.jsonl").read_text().splitlines()
+                ]
+                groups = {}
+                for record in records:
+                    groups.setdefault(round(record["scheduled_s"], 6), 0)
+                    groups[round(record["scheduled_s"], 6)] += 1
+                self.assertTrue(all(value == 8 for value in groups.values()))
+                starts = sorted(groups)
+                if len(starts) > 1:
+                    self.assertAlmostEqual(starts[1] - starts[0], 8 / 60, places=5)
+                self.assertAlmostEqual(result["request_rate"], 60.0)
+
+        asyncio.run(case())
+
+    def test_manifest_exhaustion_stops_instead_of_wrapping(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(2))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    with self.assertRaises(client.ManifestExhausted):
+                        await client.run_load(
+                            run_args(manifest, root / "result", mock.port)
+                        )
+                finally:
+                    await mock.stop()
+
+        asyncio.run(case())
+
+    def test_cpu_affinity_sets_are_disjoint_and_cover_host(self):
+        values = []
+        for spec in controller.CPU_MAP.values():
+            lo, hi = [int(x) for x in spec.split("-")]
+            values.extend(range(lo, hi + 1))
+        self.assertEqual(sorted(values), list(range(160)))
+        self.assertEqual(len(values), len(set(values)))
+
+    def test_preflight_waits_for_three_stable_zero_samples(self):
+        def sample(memory):
+            return [
+                {
+                    "index": 0,
+                    "uuid": "GPU-test",
+                    "memory_used_mib": memory,
+                    "utilization_pct": 0,
+                }
+            ]
+
+        with tempfile.TemporaryDirectory() as tmp:
+            with (
+                mock.patch.object(
+                    controller,
+                    "gpu_query",
+                    side_effect=[sample(4), sample(0), sample(0), sample(0)],
+                ),
+                mock.patch.object(controller, "compute_apps", return_value=[]),
+                mock.patch.object(controller, "run_text", return_value=""),
+                mock.patch.object(controller.time, "sleep"),
+            ):
+                root = Path(tmp)
+                controller.preflight([0], root)
+                samples = json.loads((root / "gpu-before-samples.json").read_text())
+                self.assertEqual(len(samples), 4)
+                self.assertEqual(samples[0]["gpus"][0]["memory_used_mib"], 4)
+                self.assertEqual(samples[-1]["gpus"][0]["memory_used_mib"], 0)
+
+    def test_kernel_mapping_priority(self):
+        cases = {
+            "void vllm::moe::topkGating": "moe_router",
+            "ncclDevKernel_AllReduce": "collective",
+            "flash_fwd_kernel": "attention",
+            "nvjet_sm90_tst": "moe_gemm",
+            "argmax_kernel": "sampler",
+            "cutlass_gemm": "dense_gemm",
+            "triton_red_fused_add_rms_norm": "norm_elementwise",
+            "cache_swap_kernel": "kv_memory",
+            "unknown": "other",
+        }
+        self.assertEqual(
+            {name: controller.classify_kernel(name) for name in cases}, cases
+        )
+
+    def test_atomic_state_replacement(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "state.json"
+            controller.atomic_json(path, {"schema": 1, "value": 1})
+            controller.atomic_json(path, {"schema": 1, "value": 2})
+            self.assertEqual(json.loads(path.read_text())["value"], 2)
+            self.assertFalse(list(path.parent.glob("*.tmp.*")))
+
+    def test_fingerprint_survives_json_roundtrip(self):
+        original_run_text = controller.run_text
+        original_hash = controller.sha256_file
+        try:
+            controller.run_text = lambda *args, **kwargs: "deadbeef\n"
+            controller.sha256_file = lambda path: "a" * 64
+            fingerprint = controller.make_fingerprint()
+        finally:
+            controller.run_text = original_run_text
+            controller.sha256_file = original_hash
+        self.assertEqual(json.loads(json.dumps(fingerprint)), fingerprint)
+        self.assertEqual(set(fingerprint["cpu_map"]), {str(i) for i in range(8)})
+
+    def test_server_shutdown_signals_parent_before_group(self):
+        process = SimpleNamespace(pid=12345, poll=lambda: None)
+        with (
+            mock.patch.object(controller.os, "kill") as kill,
+            mock.patch.object(controller.os, "killpg") as killpg,
+            mock.patch.object(controller, "_process_group_alive", return_value=False),
+        ):
+            controller.stop_servers([process])
+        kill.assert_called_once_with(12345, controller.signal.SIGINT)
+        killpg.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

runs/opprof-phase3/provenance/test_phase3_tools_ea.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from pathlib import Path
+from types import SimpleNamespace
+
+from aiohttp import web
+
+
+HERE = Path(__file__).parent
+
+
+def load_module(name: str, filename: str):
+    spec = importlib.util.spec_from_file_location(name, HERE / filename)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+client = load_module("phase3_client", "opprof_phase3_client.py")
+controller = load_module("phase3_controller", "opprof_phase3_controller.py")
+
+
+def rows(n: int, arrival: str = "steady") -> list[dict]:
+    return [
+        {
+            "schema": 1,
+            "request_id": f"T-{i:05d}",
+            "pattern_id": "T",
+            "kind": "synthetic",
+            "input_tokens": 8,
+            "output_tokens": 2,
+            "arrival": arrival,
+            "token_seed": i + 1,
+        }
+        for i in range(n)
+    ]
+
+
+class MockServer:
+    def __init__(self) -> None:
+        self.active = 0
+        self.max_active = 0
+        self.payloads = []
+        self.runner = None
+        self.port = None
+
+    async def completion(self, request):
+        payload = await request.json()
+        self.payloads.append(payload)
+        self.active += 1
+        self.max_active = max(self.max_active, self.active)
+        await asyncio.sleep(0.01)
+        response = web.StreamResponse(
+            status=200, headers={"Content-Type": "text/event-stream"}
+        )
+        await response.prepare(request)
+        await response.write(
+            b'data: {"choices":[{"text":"x"}],"usage":null}\n\n'
+        )
+        usage = json.dumps(
+            {
+                "choices": [],
+                "usage": {
+                    "prompt_tokens": len(payload["prompt"]),
+                    "completion_tokens": payload["max_tokens"],
+                },
+            }
+        ).encode()
+        await response.write(b"data: " + usage + b"\n\n")
+        await response.write(b"data: [DONE]\n\n")
+        await response.write_eof()
+        self.active -= 1
+        return response
+
+    async def start(self):
+        app = web.Application()
+        app.router.add_post("/v1/completions", self.completion)
+        self.runner = web.AppRunner(app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, "127.0.0.1", 0)
+        await site.start()
+        self.port = site._server.sockets[0].getsockname()[1]
+
+    async def stop(self):
+        await self.runner.cleanup()
+
+
+def run_args(
+    manifest: Path,
+    result_dir: Path,
+    port: int,
+    load_point: str = "saturation",
+    saturation_result: Path | None = None,
+):
+    return SimpleNamespace(
+        manifest=str(manifest),
+        base_url=f"http://127.0.0.1:{port}",
+        model="mock",
+        load_point=load_point,
+        request_rate="inf" if load_point == "saturation" else None,
+        saturation_result=None if saturation_result is None else str(saturation_result),
+        rate_fraction=0.60,
+        max_concurrency=3,
+        ignore_eos=True,
+        temperature=0,
+        warmup_seconds=0.04,
+        clean_segment_seconds=0.04,
+        num_clean_segments=3,
+        profile_after_clean=False,
+        num_profile_windows=0,
+        profile_warmup_iterations=2,
+        profile_active_iterations=8,
+        profile_trace_dir=None,
+        profile_timeout_seconds=1,
+        recovery_seconds=0,
+        post_clean_seconds=0,
+        drain_timeout_seconds=1,
+        workload_seed=20260712,
+        server_seed=20260712,
+        result_dir=str(result_dir),
+    )
+
+
+class Phase3ToolTests(unittest.TestCase):
+    def test_synthetic_prompt_exact_and_unique_early_prefix(self):
+        generated = [client.synthetic_prompt(row) for row in rows(200)]
+        self.assertTrue(all(len(value) == 8 for value in generated))
+        self.assertEqual(len({tuple(value[:3]) for value in generated}), 200)
+
+    def test_p05_manifest_has_exact_half_modes(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "P05.jsonl"
+            args = SimpleNamespace(
+                id="P05",
+                kind="synthetic",
+                input_uniform=None,
+                input_fixed=None,
+                input_mixture='{"uniform:128:512":0.5,"uniform:4096:8192":0.5}',
+                output_fixed=64,
+                prefix="none",
+                arrival="steady",
+                num_requests=100,
+                workload_seed=20260712,
+                num_prefixes=0,
+                prefix_len=0,
+                suffix_fixed=0,
+                out=str(out),
+            )
+            client.materialize(args)
+            manifest = client.load_manifest(out)
+            self.assertEqual(sum(r["input_tokens"] <= 512 for r in manifest), 50)
+            self.assertEqual(sum(r["input_tokens"] >= 4096 for r in manifest), 50)
+
+    def test_prefix_pool_balanced(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "P08.jsonl"
+            args = SimpleNamespace(
+                id="P08",
+                kind="prefix-pool",
+                input_uniform=None,
+                input_fixed=None,
+                input_mixture=None,
+                output_fixed=512,
+                prefix="none",
+                arrival="burst:8",
+                num_requests=80,
+                workload_seed=20260712,
+                num_prefixes=8,
+                prefix_len=1024,
+                suffix_fixed=256,
+                out=str(out),
+            )
+            client.materialize(args)
+            manifest = client.load_manifest(out)
+            counts = {i: 0 for i in range(8)}
+            for row in manifest:
+                counts[row["prefix_id"]] += 1
+                self.assertEqual(row["input_tokens"], 1280)
+            self.assertEqual(set(counts.values()), {10})
+
+    def test_fixed_duration_saturation_and_redaction(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    result = await client.run_load(
+                        run_args(manifest, root / "result", mock.port)
+                    )
+                finally:
+                    await mock.stop()
+                self.assertEqual(result["max_in_flight"], 3)
+                self.assertEqual(mock.max_active, 3)
+                self.assertAlmostEqual(result["clean"]["duration_s"], 0.12, places=9)
+                self.assertEqual(len(result["segments"]), 3)
+                self.assertLessEqual(result["drain_seconds"], 1)
+                self.assertTrue(
+                    all(p["max_tokens"] == 2 and p["ignore_eos"] for p in mock.payloads)
+                )
+                text = (root / "result/requests.jsonl").read_text()
+                self.assertNotIn('"prompt":', text)
+                self.assertNotIn('"text":', text)
+
+        asyncio.run(case())
+
+    def test_drain_timeout_is_a_hard_sanity_failure(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    args = run_args(manifest, root / "result", mock.port)
+                    args.drain_timeout_seconds = 0
+                    with self.assertRaisesRegex(RuntimeError, "drain_within_timeout"):
+                        await client.run_load(args)
+                finally:
+                    await mock.stop()
+                sanity = json.loads((root / "result/sanity.json").read_text())
+                self.assertFalse(sanity["invariants"]["drain_within_timeout"])
+
+        asyncio.run(case())
+
+    def test_burst_schedule_is_eight_at_once(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(1000, "burst:8"))
+                sat = root / "sat.json"
+                client.atomic_json(
+                    sat, {"clean": {"completed_throughput_rps": 100.0}}
+                )
+                mock = MockServer()
+                await mock.start()
+                try:
+                    args = run_args(
+                        manifest, root / "result", mock.port, "moderate", sat
+                    )
+                    args.warmup_seconds = 0
+                    args.clean_segment_seconds = 0.4 / 3
+                    result = await client.run_load(args)
+                finally:
+                    await mock.stop()
+                records = [
+                    json.loads(line)
+                    for line in (root / "result/requests.jsonl").read_text().splitlines()
+                ]
+                groups = {}
+                for record in records:
+                    groups.setdefault(round(record["scheduled_s"], 6), 0)
+                    groups[round(record["scheduled_s"], 6)] += 1
+                self.assertTrue(all(value == 8 for value in groups.values()))
+                starts = sorted(groups)
+                if len(starts) > 1:
+                    self.assertAlmostEqual(starts[1] - starts[0], 8 / 60, places=5)
+                self.assertAlmostEqual(result["request_rate"], 60.0)
+
+        asyncio.run(case())
+
+    def test_manifest_exhaustion_stops_instead_of_wrapping(self):
+        async def case():
+            with tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                manifest = root / "m.jsonl"
+                client.atomic_jsonl(manifest, rows(2))
+                mock = MockServer()
+                await mock.start()
+                try:
+                    with self.assertRaises(client.ManifestExhausted):
+                        await client.run_load(
+                            run_args(manifest, root / "result", mock.port)
+                        )
+                finally:
+                    await mock.stop()
+
+        asyncio.run(case())
+
+    def test_cpu_affinity_sets_are_disjoint_and_cover_host(self):
+        values = []
+        for spec in controller.CPU_MAP.values():
+            lo, hi = [int(x) for x in spec.split("-")]
+            values.extend(range(lo, hi + 1))
+        self.assertEqual(sorted(values), list(range(160)))
+        self.assertEqual(len(values), len(set(values)))
+
+    def test_kernel_mapping_priority(self):
+        cases = {
+            "void vllm::moe::topkGating": "moe_router",
+            "ncclDevKernel_AllReduce": "collective",
+            "flash_fwd_kernel": "attention",
+            "nvjet_sm90_tst": "moe_gemm",
+            "argmax_kernel": "sampler",
+            "cutlass_gemm": "dense_gemm",
+            "triton_red_fused_add_rms_norm": "norm_elementwise",
+            "cache_swap_kernel": "kv_memory",
+            "unknown": "other",
+        }
+        self.assertEqual(
+            {name: controller.classify_kernel(name) for name in cases}, cases
+        )
+
+    def test_atomic_state_replacement(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "state.json"
+            controller.atomic_json(path, {"schema": 1, "value": 1})
+            controller.atomic_json(path, {"schema": 1, "value": 2})
+            self.assertEqual(json.loads(path.read_text())["value"], 2)
+            self.assertFalse(list(path.parent.glob("*.tmp.*")))
+
+    def test_fingerprint_survives_json_roundtrip(self):
+        original_run_text = controller.run_text
+        original_hash = controller.sha256_file
+        try:
+            controller.run_text = lambda *args, **kwargs: "deadbeef\n"
+            controller.sha256_file = lambda path: "a" * 64
+            fingerprint = controller.make_fingerprint()
+        finally:
+            controller.run_text = original_run_text
+            controller.sha256_file = original_hash
+        self.assertEqual(json.loads(json.dumps(fingerprint)), fingerprint)
+        self.assertEqual(set(fingerprint["cpu_map"]), {str(i) for i in range(8)})
+
+    def test_server_shutdown_signals_parent_before_group(self):
+        process = SimpleNamespace(pid=12345, poll=lambda: None)
+        with (
+            mock.patch.object(controller.os, "kill") as kill,
+            mock.patch.object(controller.os, "killpg") as killpg,
+            mock.patch.object(controller, "_process_group_alive", return_value=False),
+        ):
+            controller.stop_servers([process])
+        kill.assert_called_once_with(12345, controller.signal.SIGINT)
+        killpg.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

runs/opprof-phase3/provenance/verify_shutdown_footer.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+"""One-GPU verification of API-parent-first OpProf shutdown."""
+
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import shutil
+import signal
+import subprocess
+import time
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
+RUN_DIR = WORKDIR / "runs/e-b-shutdown-verification"
+SOURCE = Path(
+    "/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0"
+)
+VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
+MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
+CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
+MANIFEST = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl")
+PORT = 8010
+
+
+def atomic_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
+    with tmp.open("w", encoding="utf-8") as f:
+        json.dump(value, f, sort_keys=True, indent=2)
+        f.write("\n")
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)
+
+
+def run_text(command: list[str], check: bool = True) -> str:
+    result = subprocess.run(
+        command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    if check and result.returncode:
+        raise RuntimeError(
+            f"command failed ({result.returncode}): {shlex.join(command)}\n"
+            f"{result.stdout}"
+        )
+    return result.stdout
+
+
+def gpu_rows() -> list[dict[str, int]]:
+    output = run_text(
+        [
+            "nvidia-smi",
+            "--query-gpu=index,memory.used,utilization.gpu",
+            "--format=csv,noheader,nounits",
+        ]
+    )
+    rows = []
+    for line in output.strip().splitlines():
+        index, memory, utilization = [int(part.strip()) for part in line.split(",")]
+        rows.append(
+            {"index": index, "memory_mib": memory, "utilization_pct": utilization}
+        )
+    return rows
+
+
+def compute_apps() -> str:
+    return run_text(
+        [
+            "nvidia-smi",
+            "--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
+            "--format=csv,noheader,nounits",
+        ],
+        check=False,
+    ).strip()
+
+
+def group_alive(pgid: int, process: subprocess.Popen[Any] | None = None) -> bool:
+    if process is not None:
+        process.poll()  # reap an exited API parent before probing its group
+    try:
+        os.killpg(pgid, 0)
+        return True
+    except ProcessLookupError:
+        return False
+
+
+def wait_ready(process: subprocess.Popen[Any], timeout: float = 300) -> None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError("server exited before readiness")
+        try:
+            with urllib.request.urlopen(
+                f"http://127.0.0.1:{PORT}/health", timeout=1
+            ) as response:
+                if response.status == 200:
+                    return
+        except Exception:
+            pass
+        time.sleep(1)
+    raise TimeoutError("server readiness timeout")
+
+
+def shutdown_parent_first(process: subprocess.Popen[Any]) -> dict[str, Any]:
+    called_at = time.time()
+    os.kill(process.pid, signal.SIGINT)
+    deadline = time.monotonic() + 120
+    while time.monotonic() < deadline and group_alive(process.pid, process):
+        time.sleep(1)
+    graceful = not group_alive(process.pid, process)
+    if not graceful:
+        for sig in (signal.SIGTERM, signal.SIGKILL):
+            try:
+                os.killpg(process.pid, sig)
+            except ProcessLookupError:
+                break
+            time.sleep(5)
+    return {
+        "api_parent_pid": process.pid,
+        "sigint_wall_time": called_at,
+        "group_gone_wall_time": time.time(),
+        "graceful": graceful,
+        "returncode": process.poll(),
+    }
+
+
+def wait_stable_zero() -> list[list[dict[str, int]]]:
+    samples = []
+    consecutive = 0
+    deadline = time.monotonic() + 120
+    while time.monotonic() < deadline and consecutive < 3:
+        rows = gpu_rows()
+        samples.append(rows)
+        zero = rows[0]["memory_mib"] == 0 and not compute_apps()
+        consecutive = consecutive + 1 if zero else 0
+        if consecutive < 3:
+            time.sleep(5)
+    if consecutive < 3:
+        raise RuntimeError("GPU0 did not reach three stable zero samples")
+    return samples
+
+
+def validate_footer() -> dict[str, Any]:
+    files = sorted((RUN_DIR / "opprof").glob("*.jsonl"))
+    if len(files) != 1:
+        raise RuntimeError(f"expected one OpProf JSONL, found {len(files)}")
+    lines = files[0].read_text().splitlines()
+    decoded = [json.loads(line) for line in lines]
+    if not decoded or decoded[-1].get("record_type") != "footer":
+        raise RuntimeError("Layer-1 footer missing")
+    records, footer = decoded[:-1], decoded[-1]
+    indices = [record["step_index"] for record in records]
+    invariants = {
+        "all_schema_1": all(item.get("schema") == 1 for item in decoded),
+        "one_footer_last": sum(
+            item.get("record_type") == "footer" for item in decoded
+        )
+        == 1,
+        "steps_contiguous": indices == list(range(len(indices))),
+        "written_matches_records": footer["written_records"] == len(records),
+        "encoded_balanced": footer["encoded_records"]
+        == footer["written_records"] + footer["dropped_records"],
+        "zero_drops": footer["dropped_records"] == 0
+        and all(record["dropped_records_before"] == 0 for record in records),
+    }
+    if not all(invariants.values()):
+        raise RuntimeError(f"footer accounting invalid: {invariants}, {footer}")
+    return {
+        "path": str(files[0]),
+        "bytes": files[0].stat().st_size,
+        "records": len(records),
+        "footer": footer,
+        "invariants": invariants,
+    }
+
+
+def main() -> None:
+    if RUN_DIR.exists():
+        raise RuntimeError(f"refusing to overwrite {RUN_DIR}")
+    RUN_DIR.mkdir(parents=True)
+    state: dict[str, Any] = {
+        "schema": 1,
+        "status": "preflight",
+        "started_at": time.time(),
+        "gpu": 0,
+    }
+    atomic_json(RUN_DIR / "state.json", state)
+    before = gpu_rows()
+    if before[0]["memory_mib"] != 0 or before[0]["utilization_pct"] != 0:
+        raise RuntimeError(f"GPU0 is not idle: {before[0]}")
+    if compute_apps():
+        raise RuntimeError("a compute process exists before verification")
+    (RUN_DIR / "gpu-before.json").write_text(json.dumps(before, indent=2) + "\n")
+    (RUN_DIR / "clocks-before.txt").write_text(
+        run_text(["nvidia-smi", "-q", "-d", "CLOCK"])
+    )
+    profile_config = {
+        "profiler": "torch",
+        "torch_profiler_dir": "/tmp/wjh-opprof-p3-footer-verify",
+        "ignore_frontend": True,
+        "wait_iterations": 0,
+        "warmup_iterations": 2,
+        "active_iterations": 8,
+    }
+    trace_dir = Path(profile_config["torch_profiler_dir"])
+    if trace_dir.exists():
+        shutil.rmtree(trace_dir)
+    trace_dir.mkdir()
+    server_command = [
+        "taskset",
+        "-c",
+        "0-19",
+        str(VENV / "bin/vllm"),
+        "serve",
+        str(MODEL),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(PORT),
+        "--tensor-parallel-size",
+        "1",
+        "--enable-chunked-prefill",
+        "--enable-prefix-caching",
+        "--profiler-config",
+        json.dumps(profile_config, separators=(",", ":")),
+    ]
+    client_command = [
+        "taskset",
+        "-c",
+        "0-19",
+        str(VENV / "bin/python"),
+        str(CLIENT),
+        "run",
+        "--manifest",
+        str(MANIFEST),
+        "--base-url",
+        f"http://127.0.0.1:{PORT}",
+        "--model",
+        str(MODEL),
+        "--load-point",
+        "saturation",
+        "--request-rate",
+        "inf",
+        "--max-concurrency",
+        "256",
+        "--ignore-eos",
+        "--temperature",
+        "0",
+        "--warmup-seconds",
+        "30",
+        "--clean-segment-seconds",
+        "40",
+        "--num-clean-segments",
+        "3",
+        "--drain-timeout-seconds",
+        "120",
+        "--workload-seed",
+        "20260712",
+        "--result-dir",
+        str(RUN_DIR / "client"),
+    ]
+    with (RUN_DIR / "commands.log").open("w") as f:
+        f.write(
+            "GPU_COMMAND shutdown footer server: "
+            + shlex.join(server_command)
+            + " ; expected=60-180s startup + 150-180s load\n"
+        )
+        f.write(
+            "GPU_COMMAND shutdown footer client: "
+            + shlex.join(client_command)
+            + " ; expected=30s warmup + 120s clean + drain\n"
+        )
+    server_log = (RUN_DIR / "server.log").open("ab", buffering=0)
+    env = os.environ.copy()
+    env.update(
+        {
+            "CUDA_VISIBLE_DEVICES": "0",
+            "VLLM_OPPROF_DIR": str(RUN_DIR / "opprof"),
+            "HF_HUB_OFFLINE": "1",
+            "TRANSFORMERS_OFFLINE": "1",
+            "PYTHONUNBUFFERED": "1",
+        }
+    )
+    server: subprocess.Popen[Any] | None = None
+    try:
+        print(
+            "GPU_COMMAND shutdown-fix verification: P01/C00 GPU0, "
+            "30s warmup + 120s clean, expected 4-6 wall-min",
+            flush=True,
+        )
+        server = subprocess.Popen(
+            server_command,
+            cwd=SOURCE,
+            env=env,
+            stdout=server_log,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+        state.update({"status": "server_starting", "server_pid": server.pid})
+        atomic_json(RUN_DIR / "state.json", state)
+        wait_ready(server)
+        state.update({"status": "client_running", "server_ready_at": time.time()})
+        atomic_json(RUN_DIR / "state.json", state)
+        with (RUN_DIR / "client.log").open("ab", buffering=0) as client_log:
+            result = subprocess.run(
+                client_command,
+                cwd=WORKDIR,
+                stdout=client_log,
+                stderr=subprocess.STDOUT,
+            )
+        if result.returncode:
+            raise RuntimeError(f"client exited {result.returncode}")
+        state["status"] = "parent_first_shutdown"
+        atomic_json(RUN_DIR / "state.json", state)
+        shutdown = shutdown_parent_first(server)
+        if not shutdown["graceful"]:
+            raise RuntimeError(f"server required escalation: {shutdown}")
+        footer = validate_footer()
+        zero_samples = wait_stable_zero()
+        client_result = json.loads((RUN_DIR / "client/result.json").read_text())
+        result_json = {
+            "schema": 1,
+            "status": "pass",
+            "gpu": 0,
+            "shutdown": shutdown,
+            "footer": footer,
+            "clean": client_result["clean"],
+            "failed_records": client_result["failed_records"],
+            "gpu_zero_samples": zero_samples,
+            "gpu_seconds": zero_samples and time.time() - state["started_at"],
+        }
+        atomic_json(RUN_DIR / "result.json", result_json)
+        state.update({"status": "complete", "completed_at": time.time()})
+        atomic_json(RUN_DIR / "state.json", state)
+        print(json.dumps(result_json, sort_keys=True), flush=True)
+    except Exception as error:
+        state.update({"status": "failed", "failure": repr(error)})
+        atomic_json(RUN_DIR / "state.json", state)
+        if server is not None and group_alive(server.pid, server):
+            try:
+                os.killpg(server.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+        raise
+    finally:
+        server_log.close()
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase3/provenance/verify_sidecar_shutdown.py

@@ -0,0 +1,392 @@
+#!/usr/bin/env python3
+"""GPU verification of graceful and hard-kill OpProf accounting."""
+
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import signal
+import subprocess
+import time
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
+RUN_ROOT = WORKDIR / "runs/e-b-sidecar-verification"
+SOURCE = Path(
+    "/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0"
+)
+VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
+MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
+CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
+MANIFEST = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl")
+FLUSH_INTERVAL_SECONDS = 1.0
+CHECKPOINT_TOLERANCE_SECONDS = 0.1
+
+
+def atomic_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f"{path.name}.tmp-{os.getpid()}")
+    with temporary.open("w", encoding="utf-8") as output:
+        json.dump(value, output, indent=2, sort_keys=True)
+        output.write("\n")
+        output.flush()
+        os.fsync(output.fileno())
+    os.replace(temporary, path)
+
+
+def run_text(command: list[str], check: bool = True) -> str:
+    result = subprocess.run(
+        command,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    if check and result.returncode:
+        raise RuntimeError(
+            f"command failed ({result.returncode}): {shlex.join(command)}\n"
+            f"{result.stdout}"
+        )
+    return result.stdout
+
+
+def gpu_rows() -> list[dict[str, int]]:
+    output = run_text(
+        [
+            "nvidia-smi",
+            "--query-gpu=index,memory.used,utilization.gpu",
+            "--format=csv,noheader,nounits",
+        ]
+    )
+    rows = []
+    for line in output.strip().splitlines():
+        index, memory, utilization = (int(value.strip()) for value in line.split(","))
+        rows.append(
+            {
+                "index": index,
+                "memory_mib": memory,
+                "utilization_pct": utilization,
+            }
+        )
+    return rows
+
+
+def compute_apps() -> str:
+    return run_text(
+        [
+            "nvidia-smi",
+            "--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
+            "--format=csv,noheader,nounits",
+        ],
+        check=False,
+    ).strip()
+
+
+def assert_idle() -> list[dict[str, int]]:
+    rows = gpu_rows()
+    if any(row["memory_mib"] or row["utilization_pct"] for row in rows):
+        raise RuntimeError(f"dash0 is not GPU-idle: {rows}")
+    applications = compute_apps()
+    if applications:
+        raise RuntimeError(f"compute applications present: {applications}")
+    return rows
+
+
+def wait_ready(process: subprocess.Popen[Any], port: int) -> None:
+    deadline = time.monotonic() + 300
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError("server exited before readiness")
+        try:
+            with urllib.request.urlopen(
+                f"http://127.0.0.1:{port}/health", timeout=1
+            ) as response:
+                if response.status == 200:
+                    return
+        except Exception:
+            pass
+        time.sleep(1)
+    raise TimeoutError("server readiness timeout")
+
+
+def wait_zero() -> tuple[list[list[dict[str, int]]], float]:
+    samples: list[list[dict[str, int]]] = []
+    consecutive = 0
+    deadline = time.monotonic() + 180
+    while time.monotonic() < deadline and consecutive < 3:
+        rows = gpu_rows()
+        samples.append(rows)
+        zero = all(row["memory_mib"] == 0 for row in rows) and not compute_apps()
+        consecutive = consecutive + 1 if zero else 0
+        if consecutive < 3:
+            time.sleep(2)
+    if consecutive < 3:
+        raise RuntimeError("GPUs did not reach three stable zero samples")
+    return samples, time.time()
+
+
+def wait_fresh_sidecar(run_dir: Path) -> dict[str, Any]:
+    deadline = time.monotonic() + 10
+    while time.monotonic() < deadline:
+        files = sorted((run_dir / "opprof").glob("*.jsonl.footer.json"))
+        if len(files) == 1:
+            sidecar = json.loads(files[0].read_text())
+            age = time.time_ns() - sidecar["checkpoint_wall_ns"]
+            if 0 <= age <= 250_000_000:
+                return sidecar
+        time.sleep(0.02)
+    raise TimeoutError("could not observe a fresh OpProf sidecar")
+
+
+def validate_accounting(
+    run_dir: Path, mode: str, termination_wall_ns: int
+) -> dict[str, Any]:
+    streams = sorted((run_dir / "opprof").glob("*.jsonl"))
+    sidecars = sorted((run_dir / "opprof").glob("*.jsonl.footer.json"))
+    if len(streams) != 1 or len(sidecars) != 1:
+        raise RuntimeError(
+            f"expected one stream/sidecar, got {len(streams)}/{len(sidecars)}"
+        )
+    raw = streams[0].read_bytes()
+    if not raw.endswith(b"\n"):
+        raise RuntimeError("partial final JSONL line")
+    decoded = [json.loads(line) for line in raw.splitlines()]
+    footers = [row for row in decoded if row.get("record_type") == "footer"]
+    records = [row for row in decoded if row.get("record_type") != "footer"]
+    sidecar = json.loads(sidecars[0].read_text())
+    indices = [row["step_index"] for row in records]
+    checkpoint_age = (
+        termination_wall_ns - sidecar["checkpoint_wall_ns"]
+    ) / 1e9
+    common = {
+        "all_schema_1": all(row.get("schema") == 1 for row in decoded)
+        and sidecar.get("schema") == 1,
+        "steps_contiguous": indices == list(range(len(indices))),
+        "written_matches_records": sidecar["written_records"] == len(records),
+        "encoded_balanced": sidecar["encoded_records"]
+        == sidecar["written_records"] + sidecar["dropped_records"],
+        "last_step_matches": bool(records)
+        and sidecar["last_step_index"] == records[-1]["step_index"],
+        "zero_drops": sidecar["dropped_records"] == 0
+        and all(row["dropped_records_before"] == 0 for row in records),
+    }
+    if mode == "graceful":
+        footer_ok = len(footers) == 1 and decoded[-1] is footers[0]
+        agreement = footer_ok and all(
+            footers[0][counter] == sidecar[counter]
+            for counter in ("encoded_records", "written_records", "dropped_records")
+        )
+        specific = {
+            "one_footer_last": footer_ok,
+            "final_sidecar": sidecar["final"] is True,
+            "footer_sidecar_agree": agreement,
+        }
+    else:
+        specific = {
+            "no_in_stream_footer": not footers,
+            "checkpoint_sidecar": sidecar["final"] is False,
+            "checkpoint_within_bound": checkpoint_age
+            <= FLUSH_INTERVAL_SECONDS + CHECKPOINT_TOLERANCE_SECONDS,
+        }
+    invariants = {**common, **specific}
+    if not all(invariants.values()):
+        raise RuntimeError(
+            f"{mode} accounting invalid: {invariants}; sidecar={sidecar}"
+        )
+    return {
+        "stream": str(streams[0]),
+        "sidecar": str(sidecars[0]),
+        "bytes": len(raw),
+        "records": len(records),
+        "footer_count": len(footers),
+        "checkpoint_age_seconds": checkpoint_age,
+        "counters": {
+            key: sidecar[key]
+            for key in ("encoded_records", "written_records", "dropped_records")
+        },
+        "last_step_index": sidecar["last_step_index"],
+        "invariants": invariants,
+    }
+
+
+def run_trial(mode: str, port: int) -> dict[str, Any]:
+    run_dir = RUN_ROOT / mode
+    if run_dir.exists():
+        raise RuntimeError(f"refusing to overwrite {run_dir}")
+    run_dir.mkdir(parents=True)
+    before = assert_idle()
+    atomic_json(run_dir / "gpu-before.json", before)
+    (run_dir / "clocks-before.txt").write_text(
+        run_text(["nvidia-smi", "-q", "-d", "CLOCK"])
+    )
+    server_command = [
+        "taskset",
+        "-c",
+        "0-19",
+        str(VENV / "bin/vllm"),
+        "serve",
+        str(MODEL),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(port),
+        "--tensor-parallel-size",
+        "1",
+        "--enable-chunked-prefill",
+        "--enable-prefix-caching",
+        "--shutdown-timeout",
+        "120",
+    ]
+    client_command = [
+        "taskset",
+        "-c",
+        "0-19",
+        str(VENV / "bin/python"),
+        str(CLIENT),
+        "run",
+        "--manifest",
+        str(MANIFEST),
+        "--base-url",
+        f"http://127.0.0.1:{port}",
+        "--model",
+        str(MODEL),
+        "--load-point",
+        "saturation",
+        "--request-rate",
+        "inf",
+        "--max-concurrency",
+        "256",
+        "--ignore-eos",
+        "--temperature",
+        "0",
+        "--warmup-seconds",
+        "20",
+        "--clean-segment-seconds",
+        "40",
+        "--num-clean-segments",
+        "3",
+        "--drain-timeout-seconds",
+        "120",
+        "--workload-seed",
+        "20260712",
+        "--result-dir",
+        str(run_dir / "client"),
+    ]
+    with (run_dir / "commands.log").open("w", encoding="utf-8") as output:
+        output.write(
+            f"GPU_COMMAND {mode} server: {shlex.join(server_command)}; "
+            "expected=60-180s startup + 140-180s load\n"
+        )
+        output.write(
+            f"GPU_COMMAND {mode} client: {shlex.join(client_command)}; "
+            "expected=20s warmup + 120s clean + drain\n"
+        )
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "CUDA_VISIBLE_DEVICES": "0",
+            "VLLM_OPPROF_DIR": str(run_dir / "opprof"),
+            "HF_HUB_OFFLINE": "1",
+            "TRANSFORMERS_OFFLINE": "1",
+            "PYTHONUNBUFFERED": "1",
+        }
+    )
+    started = time.time()
+    server_log = (run_dir / "server.log").open("ab", buffering=0)
+    server: subprocess.Popen[Any] | None = None
+    try:
+        print(
+            f"GPU_COMMAND sidecar-{mode}: P01/C00 GPU0, 20s warmup + "
+            "120s clean, expected 4-6 wall-min",
+            flush=True,
+        )
+        server = subprocess.Popen(
+            server_command,
+            cwd=SOURCE,
+            env=environment,
+            stdout=server_log,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+        atomic_json(
+            run_dir / "state.json",
+            {"status": "server_starting", "server_pid": server.pid},
+        )
+        wait_ready(server, port)
+        with (run_dir / "client.log").open("ab", buffering=0) as client_log:
+            client = subprocess.run(
+                client_command,
+                cwd=WORKDIR,
+                stdout=client_log,
+                stderr=subprocess.STDOUT,
+            )
+        if client.returncode:
+            raise RuntimeError(f"client exited {client.returncode}")
+        if mode == "graceful":
+            termination_wall_ns = time.time_ns()
+            os.kill(server.pid, signal.SIGINT)
+            server.wait(timeout=150)
+            log_text = (run_dir / "server.log").read_text(errors="replace")
+            if "mode=drain timeout=120s" not in log_text:
+                raise RuntimeError("official drain-mode log not observed")
+        else:
+            wait_fresh_sidecar(run_dir)
+            termination_wall_ns = time.time_ns()
+            os.killpg(server.pid, signal.SIGKILL)
+            server.wait(timeout=30)
+        zero_samples, zero_at = wait_zero()
+        accounting = validate_accounting(run_dir, mode, termination_wall_ns)
+        client_result = json.loads((run_dir / "client/result.json").read_text())
+        result = {
+            "schema": 1,
+            "status": "pass",
+            "mode": mode,
+            "server_returncode": server.returncode,
+            "termination_wall_ns": termination_wall_ns,
+            "accounting": accounting,
+            "clean": client_result["clean"],
+            "failed_records": client_result["failed_records"],
+            "gpu_zero_samples": zero_samples,
+            "gpu_seconds": zero_at - started,
+        }
+        atomic_json(run_dir / "result.json", result)
+        atomic_json(run_dir / "state.json", {"status": "complete"})
+        return result
+    except Exception as error:
+        atomic_json(
+            run_dir / "state.json",
+            {"status": "failed", "failure": repr(error)},
+        )
+        if server is not None and server.poll() is None:
+            try:
+                os.killpg(server.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+        wait_zero()
+        raise
+    finally:
+        server_log.close()
+
+
+def main() -> None:
+    if RUN_ROOT.exists():
+        raise RuntimeError(f"refusing to overwrite {RUN_ROOT}")
+    RUN_ROOT.mkdir(parents=True)
+    state: dict[str, Any] = {"schema": 1, "status": "running", "results": {}}
+    atomic_json(RUN_ROOT / "state.json", state)
+    for offset, mode in enumerate(("graceful", "hard-kill")):
+        result = run_trial(mode, 8010 + offset)
+        state["results"][mode] = result
+        atomic_json(RUN_ROOT / "state.json", state)
+    state["status"] = "complete"
+    state["gpu_seconds"] = sum(
+        result["gpu_seconds"] for result in state["results"].values()
+    )
+    atomic_json(RUN_ROOT / "state.json", state)
+    print(json.dumps(state, sort_keys=True), flush=True)
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase5/analyze_phase5.py

@@ -0,0 +1,537 @@
+#!/usr/bin/env python3
+"""Frozen Phase-5 bridge/control ledger analysis."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+SEED = 20260716
+RESAMPLES = 100_000
+RATE = 0.4725
+ARMS = ("base", "A1", "A2", "A3", "A4")
+MECHANISMS = ("A1", "A2", "A3", "A4")
+CAPTURE_SIZES = {
+    1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88,
+    96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192,
+    200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
+    352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
+}
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def numeric(values: list[float | int | None]) -> dict[str, Any]:
+    finite = [float(value) for value in values if value is not None and math.isfinite(float(value))]
+    return {
+        "n": len(values),
+        "finite_n": len(finite),
+        "missing_n": len(values) - len(finite),
+        "min": min(finite) if finite else None,
+        "max": max(finite) if finite else None,
+        "distinct_n": len(set(finite)),
+    }
+
+
+def ci(draws: np.ndarray) -> list[float]:
+    low, high = np.quantile(draws, [0.025, 0.975])
+    return [float(low), float(high)]
+
+
+def cv(values: list[float]) -> float:
+    array = np.asarray(values, dtype=np.float64)
+    mean = float(array.mean())
+    if mean == 0:
+        return 0.0 if np.all(array == 0) else math.inf
+    return float(array.std(ddof=0) / mean)
+
+
+def parse_run(run_dir: Path) -> dict[str, Any]:
+    result = json.loads((run_dir / "client/result.json").read_text())
+    complete = json.loads((run_dir / "run-complete.json").read_text())
+    t0 = int(result["t0_mono_ns"])
+    clean_start = float(result["clean"]["start_s"])
+    clean_end = float(result["clean"]["end_s"])
+    stream = next((run_dir / "opprof").glob("*.jsonl"))
+    records = []
+    for line in stream.read_text().splitlines():
+        item = json.loads(line)
+        if "step_index" not in item:
+            continue
+        relative = (int(item["submit_mono_ns"]) - t0) / 1e9
+        if clean_start <= relative < clean_end:
+            item["relative_s"] = relative
+            records.append(item)
+    blocks = []
+    decode_means = []
+    waiting_means = []
+    for block_index in range(48):
+        start = clean_start + 5 * block_index
+        selected = [item for item in records if start <= item["relative_s"] < start + 5]
+        if not selected:
+            # Fixed-time blocks are the resampling unit.  A rate-following trace
+            # can legitimately have an idle block, which contributes no tokens
+            # and no model-step time but must remain in the arrival-variance
+            # distribution.
+            blocks.append([0, 0.0])
+            decode_means.append(0.0)
+            waiting_means.append(0.0)
+            continue
+        tokens = sum(int(item["prefill_tokens"]) + int(item["decode_tokens"]) for item in selected)
+        duration = sum(
+            (int(item["complete_mono_ns"]) - int(item["submit_mono_ns"])) / 1e6
+            for item in selected
+        )
+        blocks.append([tokens, duration])
+        decode_means.append(float(np.mean([int(item["decode_batch_size"]) for item in selected])))
+        waiting_means.append(float(np.mean([int(item["queues"]["waiting"]) for item in selected])))
+    pure_decode = [
+        item for item in records
+        if int(item["prefill_tokens"]) == 0 and int(item["decode_batch_size"]) > 0
+    ]
+    pure_bucket = sum(int(item["cudagraph"]["bucket_tokens"]) for item in pure_decode)
+    pure_padding = sum(int(item["cudagraph"]["padding_tokens"]) for item in pure_decode)
+    support = sorted({int(item["decode_batch_size"]) for item in pure_decode})
+    covered = sum(int(item["decode_batch_size"]) in CAPTURE_SIZES for item in pure_decode)
+    prefix_queries = 0
+    prefix_hits = 0
+    prefix_present = 0
+    for item in records:
+        local = item.get("prefix", {}).get("local")
+        if local is None:
+            continue
+        prefix_present += 1
+        prefix_queries += int(local.get("queries", 0))
+        prefix_hits += int(local.get("hits", 0))
+    block_array = np.asarray(blocks, dtype=np.float64)
+    return {
+        "run_id": complete["run_id"],
+        "run_dir": str(run_dir),
+        "stream_sha256": sha256_file(stream),
+        "blocks": block_array,
+        "token_efficiency_per_ms": float(block_array[:, 0].sum() / block_array[:, 1].sum()),
+        "clean_steps": len(records),
+        "clean_duration_s": clean_end - clean_start,
+        "clean_failed": int(result["clean"]["failed"]),
+        "offered_rps": float(result["clean"]["offered_rps"]),
+        "drain_seconds": float(result["drain_seconds"]),
+        "warmup_completions": int(complete["client"]["warmup_completions"]),
+        "warmup_gate_branch": complete["client"].get("warmup_gate_branch", "P3-pre-A-P5-1"),
+        "warmup_stability": complete["client"].get("warmup_stability"),
+        "cold_start_gate": complete["client"].get("cold_start_gate"),
+        "decode_B_block_cv": cv(decode_means),
+        "waiting_block_cv": cv(waiting_means),
+        "pure_decode_steps": len(pure_decode),
+        "pure_decode_support": support,
+        "pure_decode_support_coverage": covered / len(pure_decode),
+        "pure_decode_padding_tokens": pure_padding,
+        "pure_decode_bucket_tokens": pure_bucket,
+        "pure_decode_padding_fraction": pure_padding / pure_bucket,
+        "prefix_present_steps": prefix_present,
+        "prefix_queries": prefix_queries,
+        "prefix_hits": prefix_hits,
+        "prefix_hit_ratio": prefix_hits / prefix_queries if prefix_queries else 0.0,
+        "layer1_invariants": complete["layer1"]["invariants"],
+        "client_invariants": complete["client"]["invariants"],
+        "server_invariants": complete["server_invariants"],
+        "drain_quarantined": bool(complete["drain_quarantined"]),
+    }
+
+
+def hierarchical_draws(runs: list[dict[str, Any]], rng: np.random.Generator) -> np.ndarray:
+    count = len(runs)
+    output = np.empty(RESAMPLES, dtype=np.float64)
+    for start in range(0, RESAMPLES, 5000):
+        size = min(5000, RESAMPLES - start)
+        token_sum = np.zeros(size, dtype=np.float64)
+        duration_sum = np.zeros(size, dtype=np.float64)
+        selected_runs = rng.integers(0, count, size=(size, count))
+        for position in range(count):
+            block_indices = rng.integers(0, 48, size=(size, 48))
+            for run_index, run in enumerate(runs):
+                mask = selected_runs[:, position] == run_index
+                if not np.any(mask):
+                    continue
+                blocks = run["blocks"]
+                choices = block_indices[mask]
+                token_sum[mask] += blocks[choices, 0].sum(axis=1)
+                duration_sum[mask] += blocks[choices, 1].sum(axis=1)
+        output[start : start + size] = token_sum / duration_sum
+    return output
+
+
+def point_efficiency(runs: list[dict[str, Any]]) -> float:
+    tokens = sum(float(run["blocks"][:, 0].sum()) for run in runs)
+    duration = sum(float(run["blocks"][:, 1].sum()) for run in runs)
+    return tokens / duration
+
+
+def two_sided_p(draws: np.ndarray) -> float:
+    return float(min(1.0, 2 * min(np.mean(draws <= 0), np.mean(draws >= 0))))
+
+
+def holm(pvalues: dict[str, float]) -> dict[str, float]:
+    ordered = sorted(pvalues, key=pvalues.get)
+    adjusted: dict[str, float] = {}
+    running = 0.0
+    total = len(ordered)
+    for rank, key in enumerate(ordered):
+        running = max(running, (total - rank) * pvalues[key])
+        adjusted[key] = min(1.0, running)
+    return adjusted
+
+
+def discover_primary(root: Path) -> dict[str, list[dict[str, Any]]]:
+    result = {arm: [] for arm in ARMS}
+    for marker in sorted((root / "primary").glob("*/moderate/run-complete.json")):
+        if "background" in str(marker):
+            continue
+        parsed = parse_run(marker.parent)
+        arm = parsed["run_id"].split("-r", 1)[0]
+        if arm in result:
+            result[arm].append(parsed)
+    if any(len(result[arm]) != 3 for arm in ARMS):
+        raise RuntimeError(f"primary replication mismatch: { {key:len(value) for key,value in result.items()} }")
+    return result
+
+
+def control_runs(root: Path, p3root: Path, pattern: str) -> tuple[list[dict[str, Any]], str]:
+    fresh = sorted((root / "primary").glob(f"control-{pattern}-r*-C00/moderate/run-complete.json"))
+    if fresh:
+        if len(fresh) != 3:
+            raise RuntimeError(f"partial fresh control set: {pattern}: {len(fresh)}")
+        return [parse_run(path.parent) for path in fresh], "P5-rerun"
+    return [parse_run(p3root / f"primary/{pattern}-C00/moderate")], "P3-reused"
+
+
+def aggregate_aux(runs: list[dict[str, Any]], key: str) -> float:
+    return float(np.mean([float(run[key]) for run in runs]))
+
+
+def analyze(root: Path, p3root: Path, private: Path) -> dict[str, Any]:
+    primary = discover_primary(root)
+    p3_base = [parse_run(p3root / "primary/P10-C00/moderate")]
+    controls = {}
+    control_source = {}
+    for pattern in ("P03", "P04"):
+        controls[pattern], control_source[pattern] = control_runs(root, p3root, pattern)
+    rng = np.random.default_rng(SEED)
+    draws = {arm: hierarchical_draws(primary[arm], rng) for arm in ARMS}
+    p3_base_draws = hierarchical_draws(p3_base, rng)
+    control_draws = {pattern: hierarchical_draws(controls[pattern], rng) for pattern in controls}
+    points = {arm: point_efficiency(primary[arm]) for arm in ARMS}
+    p3_base_point = point_efficiency(p3_base)
+    control_points = {pattern: point_efficiency(controls[pattern]) for pattern in controls}
+
+    bridge_draws = draws["A3"] - p3_base_draws
+    bridge_point = points["A3"] - p3_base_point
+    bridge_ci = ci(bridge_draws)
+    bridge = {
+        "point_delta": bridge_point,
+        "relative_abs_delta": abs(bridge_point) / p3_base_point,
+        "ci95": bridge_ci,
+        "within_3pct": abs(bridge_point) / p3_base_point <= 0.03,
+        "ci_contains_zero": bridge_ci[0] <= 0 <= bridge_ci[1],
+    }
+    bridge["reuse_passed"] = bridge["within_3pct"] and bridge["ci_contains_zero"]
+
+    deltas = {arm: draws[arm] - draws["base"] for arm in MECHANISMS}
+    raw_p = {arm: two_sided_p(deltas[arm]) for arm in MECHANISMS}
+    holm_p = holm(raw_p)
+    manifests = {
+        name: json.loads((private / f"P10-{name}.jsonl.summary.json").read_text())
+        for name in ("base", "A1", "A3")
+    }
+    base_prefix = aggregate_aux(primary["base"], "prefix_hit_ratio")
+    a1_prefix = aggregate_aux(primary["A1"], "prefix_hit_ratio")
+    base_padding = aggregate_aux(primary["base"], "pure_decode_padding_fraction")
+    a2_padding = aggregate_aux(primary["A2"], "pure_decode_padding_fraction")
+    padding_reduction = (base_padding - a2_padding) / base_padding if base_padding > 0 else 0.0
+    manipulations = {
+        "A1": {
+            "passed": (
+                manifests["base"]["r16"] - manifests["A1"]["r16"] >= 0.15
+                and (manifests["base"]["r16"] - manifests["A1"]["r16"]) / manifests["base"]["r16"] >= 0.20
+                and manifests["A1"]["max_added_delay_seconds"] <= 64
+                and abs(a1_prefix - base_prefix) <= 0.01
+            ),
+            "base_R16": manifests["base"]["r16"],
+            "ablated_R16": manifests["A1"]["r16"],
+            "prefix_hit_ratio_delta": a1_prefix - base_prefix,
+            "max_added_delay_seconds": manifests["A1"]["max_added_delay_seconds"],
+        },
+        "A2": {
+            "passed": aggregate_aux(primary["A2"], "pure_decode_support_coverage") >= 0.99 and padding_reduction >= 0.90,
+            "support_coverage": aggregate_aux(primary["A2"], "pure_decode_support_coverage"),
+            "base_padding_fraction": base_padding,
+            "ablated_padding_fraction": a2_padding,
+            "padding_reduction": padding_reduction,
+            "observed_support": sorted({value for run in primary["A2"] for value in run["pure_decode_support"]}),
+        },
+        "A3": {
+            "passed": (
+                aggregate_aux(primary["A3"], "decode_B_block_cv") < aggregate_aux(primary["base"], "decode_B_block_cv")
+                and aggregate_aux(primary["A3"], "waiting_block_cv") < aggregate_aux(primary["base"], "waiting_block_cv")
+            ),
+            "base_decode_B_cv": aggregate_aux(primary["base"], "decode_B_block_cv"),
+            "ablated_decode_B_cv": aggregate_aux(primary["A3"], "decode_B_block_cv"),
+            "base_waiting_cv": aggregate_aux(primary["base"], "waiting_block_cv"),
+            "ablated_waiting_cv": aggregate_aux(primary["A3"], "waiting_block_cv"),
+        },
+        "A4": {
+            "passed": sum(run["prefix_queries"] for run in primary["A4"]) == 0 and sum(run["prefix_hits"] for run in primary["A4"]) == 0,
+            "prefix_queries": sum(run["prefix_queries"] for run in primary["A4"]),
+            "prefix_hits": sum(run["prefix_hits"] for run in primary["A4"]),
+        },
+    }
+
+    ledgers = {}
+    for control in ("P03", "P04"):
+        denominator = control_draws[control] - draws["base"]
+        point_denominator = control_points[control] - points["base"]
+        denominator_ci = ci(denominator)
+        stable_denominator = bool(
+            point_denominator > 0
+            and denominator_ci[0] > 0
+            and np.mean(denominator <= 0) <= 0.05
+        )
+        rows = {}
+        share_draws = {}
+        for arm in MECHANISMS:
+            share_draws[arm] = deltas[arm] / denominator
+            point = (points[arm] - points["base"]) / point_denominator
+            interval = ci(share_draws[arm])
+            reportable = stable_denominator and manipulations[arm]["passed"]
+            rows[arm] = {
+                "delta_E": points[arm] - points["base"],
+                "delta_E_ci95": ci(deltas[arm]),
+                "share": point if reportable else None,
+                "share_ci95": interval if reportable else None,
+                "diagnostic_share": point,
+                "diagnostic_share_ci95": interval,
+                "share_status": (
+                    "REPORTABLE"
+                    if reportable
+                    else (
+                        "N/A—unstable control denominator"
+                        if not stable_denominator
+                        else "N/A—manipulation failed"
+                    )
+                ),
+                "p_two_sided": raw_p[arm],
+                "p_holm": holm_p[arm],
+                "manipulation_passed": manipulations[arm]["passed"],
+            }
+        residual_draws = 1.0 - sum(share_draws.values())
+        residual_point = 1.0 - sum(row["diagnostic_share"] for row in rows.values())
+        ledgers[control] = {
+            "status": "EVALUABLE" if stable_denominator else "INCONCLUSIVE—unstable denominator",
+            "control_source": control_source[control],
+            "control_E": control_points[control],
+            "base_E": points["base"],
+            "gap_E": point_denominator,
+            "gap_ci95": denominator_ci,
+            "denominator_nonpositive_fraction": float(np.mean(denominator <= 0)),
+            "stable_denominator": stable_denominator,
+            "mechanisms": rows,
+            "diagnostic_share_sum": sum(row["diagnostic_share"] for row in rows.values()),
+            "residual_interaction": residual_point,
+            "residual_interaction_ci95": ci(residual_draws),
+            "residual_status": (
+                "REPORTABLE"
+                if stable_denominator and all(item["passed"] for item in manipulations.values())
+                else "DIAGNOSTIC_ONLY—incomplete official share ledger"
+            ),
+        }
+
+    dominance = {}
+    for arm in MECHANISMS:
+        per_control = {}
+        for control in ("P03", "P04"):
+            row = ledgers[control]["mechanisms"][arm]
+            if not ledgers[control]["stable_denominator"] or not row["manipulation_passed"]:
+                per_control[control] = "NOT_EVALUABLE"
+                continue
+            direction_ok = row["delta_E"] > 0 if arm != "A4" else True
+            per_control[control] = (
+                row["share"] >= 0.30
+                and row["share_ci95"][0] > 0.15
+                and row["p_holm"] < 0.05
+                and direction_ok
+                and row["manipulation_passed"]
+            )
+        dominance[arm] = {
+            "per_control": per_control,
+            "verdict": (
+                "NOT EVALUABLE"
+                if any(value == "NOT_EVALUABLE" for value in per_control.values())
+                else (
+                    "DOMINANT"
+                    if all(per_control.values())
+                    else ("CONTROL-SENSITIVE" if any(per_control.values()) else "NOT DOMINANT")
+                )
+            ),
+        }
+
+    all_runs = [run for arm in ARMS for run in primary[arm]]
+    share_widths = [
+        ledgers[control]["mechanisms"][arm]["diagnostic_share_ci95"][1]
+        - ledgers[control]["mechanisms"][arm]["diagnostic_share_ci95"][0]
+        for control in ("P03", "P04") for arm in MECHANISMS
+    ]
+    state = json.loads((root / "controller-state.json").read_text())
+    amendment_evidence_path = root / "a-p5-1-retained-audit.jsonl"
+    amendment_evidence = []
+    if amendment_evidence_path.exists():
+        amendment_evidence = [
+            json.loads(line) for line in amendment_evidence_path.read_text().splitlines()
+        ]
+    invariants = {
+        "primary_runs_15": len(all_runs) == 15,
+        "three_replicates_per_arm": all(len(primary[arm]) == 3 for arm in ARMS),
+        "clean_duration_240": all(math.isclose(run["clean_duration_s"], 240.0) for run in all_runs),
+        "clean_failures_zero": all(run["clean_failed"] == 0 for run in all_runs),
+        "offered_rate_within_5pct": all(abs(run["offered_rps"] / RATE - 1) <= 0.05 for run in all_runs),
+        "layer1_accounting": all(all(run["layer1_invariants"].values()) for run in all_runs),
+        "client_invariants": all(all(value for key, value in run["client_invariants"].items() if key != "drain_re_adjudicated") for run in all_runs),
+        "server_invariants": all(all(run["server_invariants"].values()) for run in all_runs),
+        "a_p5_1_cold_start_gates": all(
+            run["cold_start_gate"] is not None
+            and run["cold_start_gate"]["passed"]
+            for run in all_runs
+        ),
+        "drain_quarantine_under_20pct": sum(run["drain_quarantined"] for run in all_runs) / 15 <= 0.20,
+        "gpu_budget_below_6": float(state["gpu_hours_total"]) < 6.0,
+        "manifests_same_ids": len({manifests[name]["request_id_set_sha256"] for name in manifests}) == 1,
+        "manifests_same_token_sums": len({manifests[name]["input_tokens"]["sum"] for name in manifests}) == 1 and len({manifests[name]["output_tokens"]["sum"] for name in manifests}) == 1,
+        "control_denominators_stable": all(ledgers[control]["stable_denominator"] for control in ledgers),
+        "bridge_decision_resolved": bridge["reuse_passed"] or all(source == "P5-rerun" for source in control_source.values()),
+        "ratios_finite": all(math.isfinite(ledgers[c]["mechanisms"][a]["diagnostic_share"]) for c in ledgers for a in MECHANISMS),
+        "per_arm_results_not_all_identical": len({round(points[arm], 12) for arm in ARMS}) > 1,
+    }
+    red_flags = [key for key, value in invariants.items() if not value]
+    publishable = (
+        not red_flags
+        and all(item["passed"] for item in manipulations.values())
+        and sum(width > 0.50 for width in share_widths) < 2
+    )
+    return {
+        "schema": 1,
+        "status": "PUBLISHABLE" if publishable else "INCONCLUSIVE_OR_PARTIAL",
+        "limitation": "Recorded-arrival P5 bridge ledger anchored to P3 controls; not a literal decomposition of P3's already-uniform P10 gap.",
+        "analysis_seed": SEED,
+        "bootstrap_resamples": RESAMPLES,
+        "efficiency": {
+            arm: {
+                "point": points[arm],
+                "ci95": ci(draws[arm]),
+                "runs": [
+                    {key: value for key, value in run.items() if key not in {"blocks", "warmup_stability"}}
+                    for run in primary[arm]
+                ],
+            }
+            for arm in ARMS
+        },
+        "p3_base_E": p3_base_point,
+        "bridge": bridge,
+        "control_sources": control_source,
+        "manipulations": manipulations,
+        "holm": {"family": list(MECHANISMS), "raw_p": raw_p, "adjusted_p": holm_p},
+        "ledgers": ledgers,
+        "dominance": dominance,
+        "config_tier_A2": {
+            "delta_E": points["A2"] - points["base"],
+            "relative_E_delta": points["A2"] / points["base"] - 1,
+            "delta_E_ci95": ci(deltas["A2"]),
+            "base_padding_fraction": base_padding,
+            "A2_padding_fraction": a2_padding,
+            "padding_reduction": padding_reduction,
+        },
+        "amendment_A_P5_1": {
+            "reason": "Rate-following throughput drift tracks arrival shape and is not a cold-start stationarity test.",
+            "retained_failed_run_evidence": amendment_evidence,
+            "recorded_drift_range": (
+                [
+                    min(
+                        item["superseded_drift_evidence"]["normalized_drift"]
+                        for item in amendment_evidence
+                        if item["run"] != "A3-r1-C00"
+                    ),
+                    max(
+                        item["superseded_drift_evidence"]["normalized_drift"]
+                        for item in amendment_evidence
+                        if item["run"] != "A3-r1-C00"
+                    ),
+                ]
+                if amendment_evidence else None
+            ),
+            "uniform_A3_drift": (
+                next(
+                    item["superseded_drift_evidence"]["normalized_drift"]
+                    for item in amendment_evidence
+                    if item["run"] == "A3-r1-C00"
+                )
+                if amendment_evidence else None
+            ),
+        },
+        "gpu": {
+            "new_h20_hours": float(state["gpu_hours_total"]),
+            "hard_cap": 6.0,
+            "controller_status": state["status"],
+            "completed_measured_runs_including_background": state["completed_measured_runs"],
+            "completed_burnins": state["completed_burnins"],
+        },
+        "sanity": {
+            "red_flags": red_flags,
+            "invariants": invariants,
+            "numeric": {
+                "primary_E": numeric([run["token_efficiency_per_ms"] for run in all_runs]),
+                "clean_steps": numeric([run["clean_steps"] for run in all_runs]),
+                "offered_rps": numeric([run["offered_rps"] for run in all_runs]),
+                "drain_seconds": numeric([run["drain_seconds"] for run in all_runs]),
+                "diagnostic_share": numeric([ledgers[c]["mechanisms"][a]["diagnostic_share"] for c in ledgers for a in MECHANISMS]),
+                "residual_interaction": numeric([ledgers[c]["residual_interaction"] for c in ledgers]),
+                "share_ci_width": numeric(share_widths),
+            },
+            "declared": {
+                "manipulation_failures": [arm for arm, item in manipulations.items() if not item["passed"]],
+                "control_sources": control_source,
+                "bridge_reuse_passed": bridge["reuse_passed"],
+            },
+        },
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--p3-root", type=Path, required=True)
+    parser.add_argument("--private", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args()
+    result = analyze(args.root, args.p3_root, args.private)
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
+    print(json.dumps({
+        "status": result["status"],
+        "bridge": result["bridge"],
+        "red_flags": result["sanity"]["red_flags"],
+        "gpu": result["gpu"],
+    }, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase5/opprof_phase5_client.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""Phase-5 private-manifest transforms and timestamp-scheduled P3 client wrapper."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import math
+import sys
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+import opprof_phase3_client as p3
+
+
+def _numeric(values: list[float | int]) -> dict[str, Any]:
+    finite = [float(value) for value in values if math.isfinite(float(value))]
+    return {
+        "n": len(values),
+        "finite_n": len(finite),
+        "missing_n": len(values) - len(finite),
+        "min": min(finite) if finite else None,
+        "max": max(finite) if finite else None,
+        "distinct_n": len(set(finite)),
+        "sum": sum(finite),
+    }
+
+
+def _r16(rows: list[dict[str, Any]]) -> float:
+    groups = [rows[index : index + 16] for index in range(0, len(rows) - 15, 16)]
+    useful = sum(sum(int(row["input_tokens"]) for row in group) for group in groups)
+    rectangular = sum(16 * max(int(row["input_tokens"]) for row in group) for group in groups)
+    return 1.0 - useful / rectangular
+
+
+def _source_timestamps(path: Path, indices: set[int], field: str) -> dict[int, float]:
+    result: dict[int, float] = {}
+    maximum = max(indices)
+    with path.open(encoding="utf-8") as source:
+        for index, line in enumerate(source):
+            if index in indices:
+                value = float(json.loads(line)[field])
+                if not math.isfinite(value):
+                    raise ValueError(f"non-finite source timestamp at {index}")
+                result[index] = value
+            if index >= maximum:
+                break
+    if set(result) != indices:
+        raise ValueError("timestamp source did not cover all source_index values")
+    return result
+
+
+def transform(args: argparse.Namespace) -> dict[str, Any]:
+    rows = p3.load_manifest(Path(args.input))[: args.take_first]
+    if len(rows) != args.take_first:
+        raise ValueError(f"requested {args.take_first} rows, found {len(rows)}")
+    indices = [int(row[args.join_key]) for row in rows]
+    timestamps = _source_timestamps(
+        Path(args.timestamp_source), set(indices), args.timestamp_field
+    )
+    source_times = [timestamps[index] for index in indices]
+    if any(right < left for left, right in zip(source_times, source_times[1:])):
+        raise ValueError("selected timestamps are not nondecreasing")
+    if source_times[-1] <= source_times[0]:
+        raise ValueError("selected timestamp span is not positive")
+    end_s = (len(rows) - 1) / args.target_rate
+    scale = end_s / (source_times[-1] - source_times[0])
+    recorded_slots = [(value - source_times[0]) * scale for value in source_times]
+    uniform_slots = [index / args.target_rate for index in range(len(rows))]
+    slots = recorded_slots if args.arrival == "recorded-scaled" else uniform_slots
+
+    for index, row in enumerate(rows):
+        row["arrival"] = args.arrival
+        row["arrival_s"] = slots[index]
+        row["original_index"] = index
+        row["source_timestamp"] = source_times[index]
+
+    original = list(rows)
+    max_added_delay = 0.0
+    if args.service_order == "length-binned":
+        edges = [int(item) for item in args.length_bin_edges.split(",")]
+
+        def bin_id(row: dict[str, Any]) -> int:
+            length = int(row["input_tokens"])
+            for index, edge in enumerate(edges):
+                if length <= edge:
+                    return index
+            raise ValueError(f"input length {length} exceeds final edge")
+
+        reordered: list[dict[str, Any]] = []
+        for offset in range(0, len(rows), args.reorder_block_size):
+            block = rows[offset : offset + args.reorder_block_size]
+            ordered = sorted(
+                block,
+                key=lambda row: (
+                    bin_id(row),
+                    int(row["input_tokens"]),
+                    int(row["original_index"]),
+                ),
+            )
+            block_slots = slots[offset : offset + len(block)]
+            for position, row in enumerate(ordered):
+                added = max(0.0, block_slots[position] - float(row["arrival_s"]))
+                max_added_delay = max(max_added_delay, added)
+                row["arrival_s"] = block_slots[position]
+            reordered.extend(ordered)
+        rows = reordered
+        if max_added_delay > args.max_added_delay_seconds + 1e-9:
+            raise ValueError(
+                f"fairness cap exceeded: {max_added_delay} > {args.max_added_delay_seconds}"
+            )
+    elif args.service_order != "original":
+        raise ValueError(f"unsupported service order: {args.service_order}")
+
+    if sorted(row["request_id"] for row in rows) != sorted(
+        row["request_id"] for row in original
+    ):
+        raise AssertionError("request identity changed")
+    for key in ("input_tokens", "output_tokens"):
+        if sum(int(row[key]) for row in rows) != sum(int(row[key]) for row in original):
+            raise AssertionError(f"{key} total changed")
+    arrival_values = [float(row["arrival_s"]) for row in rows]
+    if any(right < left for left, right in zip(arrival_values, arrival_values[1:])):
+        raise AssertionError("assigned arrival slots are not nondecreasing")
+
+    output = Path(args.out)
+    p3.atomic_jsonl(output, rows, mode=0o600)
+    summary = {
+        "schema": 1,
+        "path": str(output),
+        "sha256": p3.sha256_file(output),
+        "rows": len(rows),
+        "arrival": args.arrival,
+        "service_order": args.service_order,
+        "target_rate": args.target_rate,
+        "input_tokens": _numeric([int(row["input_tokens"]) for row in rows]),
+        "output_tokens": _numeric([int(row["output_tokens"]) for row in rows]),
+        "arrival_s": _numeric(arrival_values),
+        "source_timestamp": _numeric(source_times),
+        "r16": _r16(rows),
+        "max_added_delay_seconds": max_added_delay,
+        "request_id_set_sha256": p3.hashlib.sha256(
+            "\n".join(sorted(str(row["request_id"]) for row in rows)).encode()
+        ).hexdigest(),
+        "invariants": {
+            "same_request_ids": True,
+            "same_input_tokens": True,
+            "same_output_tokens": True,
+            "arrival_nondecreasing": True,
+            "fairness_cap": max_added_delay <= args.max_added_delay_seconds + 1e-9,
+            "no_prompt_in_summary": True,
+        },
+    }
+    p3.atomic_json(output.with_suffix(output.suffix + ".summary.json"), summary, mode=0o600)
+    print(json.dumps(summary, sort_keys=True))
+    return summary
+
+
+async def finite_timestamp_load(
+    ctx: p3.RunContext, session: aiohttp.ClientSession, rate: float
+) -> list[dict[str, Any]]:
+    if "arrival_s" not in ctx.rows[0]:
+        return await _ORIGINAL_FINITE_LOAD(ctx, session, rate)
+    sem = asyncio.Semaphore(ctx.args.max_concurrency)
+    tasks: list[asyncio.Task[dict[str, Any]]] = []
+
+    async def limited(row: dict[str, Any], scheduled: float) -> dict[str, Any]:
+        async with sem:
+            return await p3.request_one(ctx, session, row, scheduled)
+
+    for expected in ctx.rows:
+        scheduled = ctx.t0 + float(expected["arrival_s"])
+        delay = scheduled - asyncio.get_running_loop().time()
+        if delay > 0:
+            try:
+                await asyncio.wait_for(ctx.stop_event.wait(), timeout=delay)
+                break
+            except asyncio.TimeoutError:
+                pass
+        if ctx.stop_event.is_set():
+            break
+        row = await ctx.next_row()
+        if row["request_id"] != expected["request_id"]:
+            raise AssertionError("timestamp scheduler row drift")
+        tasks.append(asyncio.create_task(limited(row, scheduled)))
+    return await asyncio.gather(*tasks) if tasks else []
+
+
+_ORIGINAL_FINITE_LOAD = p3.finite_load
+p3.finite_load = finite_timestamp_load
+
+
+def build_transform_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in", dest="input", required=True)
+    parser.add_argument("--take-first", type=int, required=True)
+    parser.add_argument("--timestamp-source", required=True)
+    parser.add_argument("--join-key", default="source_index")
+    parser.add_argument("--timestamp-field", default="timestamp")
+    parser.add_argument("--arrival", choices=("recorded-scaled", "uniform"), required=True)
+    parser.add_argument("--target-rate", type=float, required=True)
+    parser.add_argument("--service-order", choices=("original", "length-binned"), required=True)
+    parser.add_argument("--reorder-block-size", type=int, default=32)
+    parser.add_argument("--analysis-cohort-size", type=int, default=16)
+    parser.add_argument("--length-bin-edges", default="512,1024,2048,4096,8192,16384,32768")
+    parser.add_argument("--max-added-delay-seconds", type=float, default=64)
+    parser.add_argument("--out", required=True)
+    return parser
+
+
+def main() -> None:
+    if len(sys.argv) > 1 and sys.argv[1] == "transform":
+        transform(build_transform_parser().parse_args(sys.argv[2:]))
+        return
+    fixed_rate = None
+    if "--fixed-request-rate" in sys.argv:
+        index = sys.argv.index("--fixed-request-rate")
+        fixed_rate = float(sys.argv[index + 1])
+        del sys.argv[index : index + 2]
+    args = p3.build_parser().parse_args()
+    if args.command != "run":
+        p3.main()
+        return
+    if fixed_rate is not None:
+        if args.load_point != "moderate" or fixed_rate <= 0 or not math.isfinite(fixed_rate):
+            raise ValueError("--fixed-request-rate requires positive finite moderate rate")
+        result_dir = Path(args.result_dir)
+        result_dir.mkdir(parents=True, exist_ok=True)
+        source = result_dir / "fixed-rate-source.json"
+        p3.atomic_json(source, {"clean": {"completed_throughput_rps": fixed_rate}})
+        args.saturation_result = str(source)
+        args.rate_fraction = 1.0
+    if args.profile_after_clean and not args.profile_trace_dir:
+        raise ValueError("--profile-after-clean requires --profile-trace-dir")
+    print(json.dumps(asyncio.run(p3.run_load(args)), sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase5/opprof_phase5_controller.py

@@ -0,0 +1,670 @@
+#!/usr/bin/env python3
+"""Detached, resumable Phase-5 four-way primary/control controller."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import json
+import math
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import time
+from pathlib import Path
+from typing import Any
+
+import opprof_phase3_matrix as m
+
+
+WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
+RUN_ROOT = WORKDIR / "runs/phase5"
+PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase5-private/manifests")
+P3_PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests")
+MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
+SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
+VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
+CLIENT = WORKDIR / "scripts/opprof_phase5_client.py"
+P3_CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
+STATE = RUN_ROOT / "controller-state.json"
+RATE = 0.4725
+CAPTURE_SIZES = (
+    1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88,
+    96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192,
+    200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
+    352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
+)
+
+
+m.WORKDIR = WORKDIR
+m.RUN_ROOT = RUN_ROOT
+m.PRIVATE = PRIVATE
+m.MODEL = MODEL
+m.SOURCE = SOURCE
+m.VENV = VENV
+m.CLIENT = CLIENT
+m.STATE = STATE
+m.GPU_HOUR_LIMIT = 6.0
+m.PRIOR_GPU_HOURS = 0.0
+m.CONFIGS = {
+    "C00": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
+    "A2": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
+    "A4": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
+}
+
+
+def sha256_file(path: Path) -> str:
+    return m.sha256_file(path)
+
+
+def arm_name(pattern: str) -> str:
+    return pattern.split("-r", 1)[0]
+
+
+def manifest_for(pattern: str, burnin: bool) -> Path:
+    if burnin or pattern.startswith("background"):
+        return P3_PRIVATE / "P06.jsonl"
+    if pattern.startswith("control-P03"):
+        return P3_PRIVATE / "P03.jsonl"
+    if pattern.startswith("control-P04"):
+        return P3_PRIVATE / "P04.jsonl"
+    arm = arm_name(pattern)
+    return PRIVATE / ("P10-base.jsonl" if arm in {"base", "A2", "A4"} else f"P10-{arm}.jsonl")
+
+
+def saturation_result_for(pattern: str) -> Path:
+    if pattern.startswith("control-P03"):
+        cell = "P03-C00"
+    elif pattern.startswith("control-P04"):
+        cell = "P04-C00"
+    else:
+        cell = "P10-C00"
+    return WORKDIR / f"runs/phase3/primary/{cell}/saturation/client/result.json"
+
+
+def drain_budget(_pattern: str) -> int:
+    return 600
+
+
+m.drain_budget = drain_budget
+
+
+def server_command(assignment: m.Assignment, port: int, _trace_dir: Path) -> list[str]:
+    arm = arm_name(assignment.cell.pattern)
+    command = [
+        "taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/vllm"),
+        "serve", str(MODEL), "--host", "127.0.0.1", "--port", str(port),
+        "--tensor-parallel-size", "1", "--enable-chunked-prefill",
+    ]
+    if arm != "A4" and assignment.cell.config != "A4":
+        command.append("--enable-prefix-caching")
+    command.extend(("--shutdown-timeout", "600"))
+    if arm == "A2" or assignment.cell.config == "A2":
+        command.extend(("--cudagraph-capture-sizes", *map(str, CAPTURE_SIZES)))
+    return command
+
+
+def client_command(
+    assignment: m.Assignment,
+    port: int,
+    run_dir: Path,
+    _load_point: str,
+    _profile: bool,
+    burnin: bool,
+    _saturation_result: Path | None,
+) -> list[str]:
+    pattern = assignment.cell.pattern
+    common = [
+        "taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/python"),
+        str(CLIENT), "run", "--manifest", str(manifest_for(pattern, burnin)),
+        "--base-url", f"http://127.0.0.1:{port}", "--model", str(MODEL),
+        "--max-concurrency", "256", "--ignore-eos", "--temperature", "0",
+        "--workload-seed", "20260712", "--server-seed", "20260712",
+        "--result-dir", str(run_dir / "client"),
+    ]
+    if burnin:
+        return common + [
+            "--load-point", "saturation", "--request-rate", "inf",
+            "--warmup-seconds", "0", "--clean-segment-seconds", "20",
+            "--num-clean-segments", "3", "--post-clean-seconds", "0",
+            "--drain-timeout-seconds", "600",
+        ]
+    if pattern.startswith("background"):
+        return common + [
+            "--load-point", "saturation", "--request-rate", "inf",
+            "--warmup-seconds", "60", "--clean-segment-seconds", "80",
+            "--num-clean-segments", "3", "--post-clean-seconds", "0",
+            "--drain-timeout-seconds", "600",
+        ]
+    if pattern.startswith("control-"):
+        return common + [
+            "--load-point", "moderate", "--saturation-result",
+            str(saturation_result_for(pattern)), "--rate-fraction", "0.60",
+            "--warmup-seconds", "60", "--clean-segment-seconds", "80",
+            "--num-clean-segments", "3", "--post-clean-seconds", "0",
+            "--drain-timeout-seconds", "600",
+        ]
+    return common + [
+        "--load-point", "moderate", "--fixed-request-rate", str(RATE),
+        "--warmup-seconds", "60", "--clean-segment-seconds", "80",
+        "--num-clean-segments", "3", "--post-clean-seconds", "0",
+        "--drain-timeout-seconds", "600",
+    ]
+
+
+m.server_command = server_command
+m.client_command = client_command
+
+
+_ORIGINAL_VALIDATE_CLIENT = m.validate_client
+
+
+def _log_event_time(line: str, t0_wall_ns: int) -> float | None:
+    match = re.search(r"INFO\s+(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})", line)
+    if match is None:
+        return None
+    month, day, hour, minute, second = map(int, match.groups())
+    year = dt.datetime.fromtimestamp(t0_wall_ns / 1e9, tz=dt.timezone.utc).year
+    stamp = dt.datetime(year, month, day, hour, minute, second, tzinfo=dt.timezone.utc)
+    return stamp.timestamp()
+
+
+def cold_start_gate(run_dir: Path) -> dict[str, Any]:
+    result = json.loads((run_dir / "client/result.json").read_text())
+    requests = [
+        json.loads(line)
+        for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
+    ]
+    warm = [
+        request for request in requests
+        if request["success"] and 0 <= float(request["completed_s"]) < 60
+    ]
+    warm_long = [request for request in warm if int(request["input_tokens"]) >= 8192]
+    t0_mono_ns = int(result["t0_mono_ns"])
+    stream = next((run_dir / "opprof").glob("*.jsonl"))
+    warm_descriptors: set[tuple[str, int]] = set()
+    clean_descriptors: set[tuple[str, int]] = set()
+    for line in stream.read_text().splitlines():
+        item = json.loads(line)
+        if "step_index" not in item or not item.get("model_executed"):
+            continue
+        graph = item["cudagraph"]
+        if not graph.get("hit"):
+            continue
+        relative_s = (int(item["submit_mono_ns"]) - t0_mono_ns) / 1e9
+        descriptor = (str(graph["runtime_mode"]), int(graph["bucket_tokens"]))
+        if 0 <= relative_s < 60:
+            warm_descriptors.add(descriptor)
+        elif 60 <= relative_s < 300:
+            clean_descriptors.add(descriptor)
+
+    log_lines = (run_dir / "server.log").read_text(errors="replace").splitlines()
+    ready_indices = [
+        index for index, line in enumerate(log_lines)
+        if "Application startup complete" in line
+    ]
+    if len(ready_indices) != 1:
+        raise RuntimeError(f"expected one server ready marker: {run_dir}: {ready_indices}")
+    ready_index = ready_indices[0]
+    event_pattern = re.compile(
+        r"torch\.compile took|Directly load AOT compilation|\bCompiling\b|Capturing CUDA graphs",
+        re.IGNORECASE,
+    )
+    events = []
+    clean_boundary_wall_s = int(result["t0_wall_ns"]) / 1e9 + 60
+    clean_end_wall_s = int(result["t0_wall_ns"]) / 1e9 + 300
+    event_gate = True
+    clean_events = 0
+    for index, line in enumerate(log_lines):
+        if not event_pattern.search(line):
+            continue
+        timestamp = _log_event_time(line, int(result["t0_wall_ns"]))
+        phase = "pre-ready" if index < ready_index else "post-ready"
+        if index >= ready_index:
+            if timestamp is None:
+                event_gate = False
+                phase = "post-ready-unparseable"
+            elif timestamp >= clean_boundary_wall_s:
+                event_gate = False
+                phase = "clean" if timestamp < clean_end_wall_s else "post-clean"
+                if timestamp < clean_end_wall_s:
+                    clean_events += 1
+            else:
+                phase = "warmup"
+        events.append(
+            {
+                "line": index + 1,
+                "phase": phase,
+                "timestamp_s": timestamp,
+                "message_prefix": line[:200],
+            }
+        )
+
+    config_match = re.search(
+        r"cudagraph_capture_sizes['\"]?:\s*\[([^\]]+)\]",
+        "\n".join(log_lines[:ready_index]),
+    )
+    startup_sizes = (
+        {int(value.strip()) for value in config_match.group(1).split(",")}
+        if config_match else set()
+    )
+    startup_modes = set()
+    for line in log_lines[:ready_index]:
+        if "Capturing CUDA graphs" not in line:
+            continue
+        if "PIECEWISE" in line:
+            startup_modes.add("PIECEWISE")
+        if "FULL" in line:
+            startup_modes.add("FULL")
+    uncovered = sorted(
+        descriptor for descriptor in clean_descriptors
+        if descriptor[0] not in startup_modes or descriptor[1] not in startup_sizes
+    )
+    passed = (
+        event_gate
+        and len(warm) >= 16
+        and len(warm_long) >= 1
+        and startup_modes == {"FULL", "PIECEWISE"}
+        and bool(startup_sizes)
+        and not uncovered
+        and clean_events == 0
+    )
+    return {
+        "amendment": "A-P5-1",
+        "passed": passed,
+        "warmup_completions": len(warm),
+        "warmup_long_completions": len(warm_long),
+        "warmup_long_min_tokens": min(
+            (int(request["input_tokens"]) for request in warm_long), default=None
+        ),
+        "server_ready_line": ready_index + 1,
+        "compile_capture_events": events,
+        "event_gate_passed": event_gate,
+        "clean_capture_events": clean_events,
+        "startup_capture_sizes": sorted(startup_sizes),
+        "startup_capture_modes": sorted(startup_modes),
+        "warmup_descriptors": [list(item) for item in sorted(warm_descriptors)],
+        "clean_descriptors": [list(item) for item in sorted(clean_descriptors)],
+        "uncovered_clean_descriptors": [list(item) for item in uncovered],
+        "invariants": {
+            "events_preclean": event_gate,
+            "warmup_completions_ge_16": len(warm) >= 16,
+            "warmup_long_completion": len(warm_long) >= 1,
+            "startup_modes_complete": startup_modes == {"FULL", "PIECEWISE"},
+            "startup_sizes_present": bool(startup_sizes),
+            "clean_descriptors_covered": not uncovered,
+            "zero_clean_capture_events": clean_events == 0,
+        },
+    }
+
+
+def validate_rate_client(run_dir: Path) -> dict[str, Any]:
+    result = json.loads((run_dir / "client/result.json").read_text())
+    sanity = json.loads((run_dir / "client/sanity.json").read_text())
+    requests = [
+        json.loads(line)
+        for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
+    ]
+    failed_sanity = [
+        key for key, value in sanity["invariants"].items()
+        if not value and key != "drain_within_timeout"
+    ]
+    failure_summary = m.summarize_request_failures(
+        requests, float(result["clean"]["start_s"]), float(result["clean"]["end_s"])
+    )
+    cold = cold_start_gate(run_dir)
+    quarantined = float(result["drain_seconds"]) > 600
+    invariants = {
+        "client_sanity": not failed_sanity,
+        "clean_duration": math.isclose(float(result["clean"]["duration_s"]), 240.0),
+        "clean_failures_zero": result["clean"]["failed"] == 0
+        and failure_summary["clean_failed"] == 0,
+        "failed_records_accounted": result["failed_records"] == failure_summary["failed"],
+        "manifest_no_wrap": not result["manifest_wrapped"]
+        and not result["manifest_exhausted"],
+        "warmup_cold_start_gate": cold["passed"],
+        "profile_count": len(result["profiles"]) == 0,
+        "drain_re_adjudicated": not quarantined,
+    }
+    non_drain = {key: value for key, value in invariants.items() if key != "drain_re_adjudicated"}
+    if not all(non_drain.values()):
+        raise RuntimeError(
+            f"A-P5-1 rate-client invariant failure: {run_dir}: "
+            f"invariants={invariants}; failed_sanity={failed_sanity}; cold={cold}"
+        )
+    return {
+        "result": result,
+        "sanity": sanity,
+        "request_count": len(requests),
+        "warmup_completions": cold["warmup_completions"],
+        "warmup_required": 16,
+        "warmup_gate_branch": "A-P5-1-cold-start",
+        "warmup_stability": None,
+        "cold_start_gate": cold,
+        "drain_budget_seconds": 600,
+        "drain_quarantined": quarantined,
+        "excluded_window_failures": failure_summary["excluded"],
+        "excluded_window_failure_kinds": failure_summary["excluded_kinds"],
+        "invariants": invariants,
+    }
+
+
+def validate_run(
+    entry: dict[str, Any], profile: bool, burnin: bool, allow_missing_traces: bool = False
+) -> dict[str, Any]:
+    del profile, allow_missing_traces
+    pattern = entry["assignment"].cell.pattern
+    if burnin or pattern.startswith("background"):
+        validation_pattern = "P06"
+    elif pattern.startswith("control-P03"):
+        validation_pattern = "P03"
+    elif pattern.startswith("control-P04"):
+        validation_pattern = "P04"
+    else:
+        validation_pattern = "P10"
+    if burnin or pattern.startswith("background"):
+        client = _ORIGINAL_VALIDATE_CLIENT(
+            entry["run_dir"], validation_pattern, False, burnin
+        )
+    else:
+        client = validate_rate_client(entry["run_dir"])
+    layer1 = m.validate_layer1(entry["run_dir"])
+    log = (entry["run_dir"] / "server.log").read_text(errors="replace")
+    invariants = {
+        "triton_moe": "Using TRITON Unquantized MoE backend" in log,
+        "chunked_mbt": "Chunked prefill is enabled with max_num_batched_tokens=8192" in log,
+        "tp1": "tensor_parallel_size=1" in log,
+        "drain_shutdown": "mode=drain timeout=600s" in log,
+        "a2_sizes": entry["assignment"].cell.config != "A2"
+        or all(str(size) in log for size in (3, 5, 6, 7)),
+    }
+    if not all(invariants.values()):
+        raise RuntimeError(f"server invariant failure: {entry['run_id']}: {invariants}")
+    forbidden = re.compile(r'"(?:prompt|messages|content|text)"\s*:')
+    for path in (
+        entry["run_dir"] / "client/requests.jsonl",
+        entry["run_dir"] / "client/result.json",
+        Path(layer1["stream"]),
+    ):
+        if forbidden.search(path.read_text(errors="replace")):
+            raise RuntimeError(f"private text leaked: {path}")
+    summary = {
+        "schema": 1,
+        "run_id": entry["run_id"],
+        "pattern": pattern,
+        "config": entry["assignment"].cell.config,
+        "gpus": entry["assignment"].gpus,
+        "client": client,
+        "layer1": layer1,
+        "traces": [],
+        "missing_trace_files": 0,
+        "layer2_missing_after_controller_cleanup": False,
+        "drain_quarantined": client["drain_quarantined"],
+        "server_invariants": invariants,
+    }
+    m.atomic_json(entry["run_dir"] / "run-complete.json", summary)
+    return summary
+
+
+m.validate_run = validate_run
+
+
+def manifests() -> dict[str, Any]:
+    result = {}
+    for name in ("base", "A1", "A3"):
+        path = PRIVATE / f"P10-{name}.jsonl"
+        summary = json.loads(path.with_suffix(path.suffix + ".summary.json").read_text())
+        if summary["rows"] != 142 or summary["sha256"] != sha256_file(path):
+            raise RuntimeError(f"manifest verification failed: {name}")
+        result[name] = {"path": str(path), "sha256": summary["sha256"]}
+    return result
+
+
+def fingerprint() -> dict[str, Any]:
+    return {
+        "source_commit": subprocess.check_output(
+            ["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True
+        ).strip(),
+        "source_tree": subprocess.check_output(
+            ["git", "-C", str(SOURCE), "rev-parse", "HEAD^{tree}"], text=True
+        ).strip(),
+        "client_sha256": sha256_file(CLIENT),
+        "controller_sha256": sha256_file(Path(__file__)),
+        "p3_client_sha256": sha256_file(P3_CLIENT),
+        "p3_matrix_sha256": sha256_file(Path(m.__file__).resolve()),
+        "manifests": manifests(),
+        "capture_sizes": list(CAPTURE_SIZES),
+        "rate": RATE,
+    }
+
+
+def load_state(resume: bool) -> dict[str, Any]:
+    if STATE.exists():
+        if not resume:
+            raise RuntimeError("controller state exists; use --resume")
+        return json.loads(STATE.read_text())
+    return {
+        "schema": 1,
+        "status": "created",
+        "created_at": time.time(),
+        "controller_pid": os.getpid(),
+        "gpu_hours_total": 0.0,
+        "gpu_hours_this_stage": 0.0,
+        "completed_measured_runs": 0,
+        "completed_burnins": 0,
+        "drain_quarantined_runs": 0,
+        "clean_window_failures": 0,
+        "missing_trace_files": 0,
+        "stages": {},
+        "fingerprint": {},
+    }
+
+
+def save_state(state: dict[str, Any]) -> None:
+    state["controller_pid"] = os.getpid()
+    state["updated_at"] = time.time()
+    m.atomic_json(STATE, state)
+
+
+m.save_state = save_state
+
+
+def ensure_provenance() -> None:
+    destination = RUN_ROOT / "provenance"
+    destination.mkdir(parents=True, exist_ok=True)
+    sources = [CLIENT, Path(__file__).resolve(), Path(m.__file__).resolve(), Path(m.common.__file__).resolve()]
+    hashes = {}
+    for source in sources:
+        target = destination / source.name
+        digest = sha256_file(source)
+        if target.exists() and sha256_file(target) != digest:
+            target = destination / f"{source.stem}.{digest[:12]}{source.suffix}"
+        if target.exists() and sha256_file(target) != digest:
+            raise RuntimeError(f"content-addressed provenance mismatch: {target}")
+        if not target.exists():
+            shutil.copy2(source, target)
+        hashes[target.name] = digest
+    m.atomic_json(destination / "sha256.json", hashes)
+
+
+def primary_cells() -> list[m.Cell]:
+    config = {"base": "C00", "A1": "C00", "A2": "A2", "A3": "C00", "A4": "A4"}
+    items = [m.Cell(f"{arm}-r{replicate}", config[arm]) for arm in config for replicate in range(1, 4)]
+    return sorted(
+        items,
+        key=lambda cell: hashlib.sha256(
+            f"20260715:{cell.pattern.rsplit('-r',1)[1]}:{arm_name(cell.pattern)}".encode()
+        ).hexdigest(),
+    )
+
+
+def pack_unique(items: list[m.Cell], prefix: str) -> list[list[m.Assignment]]:
+    remaining = list(items)
+    waves: list[list[m.Assignment]] = []
+    wave_index = 0
+    while remaining:
+        selected: list[m.Cell] = []
+        used: set[str] = set()
+        for cell in list(remaining):
+            key = arm_name(cell.pattern)
+            if key in used:
+                continue
+            selected.append(cell)
+            used.add(key)
+            remaining.remove(cell)
+            if len(selected) == 4:
+                break
+        while len(selected) < 4:
+            selected.append(m.Cell(f"background-{prefix}-{wave_index}-{len(selected)}", "C00"))
+        assignments = []
+        for slot, cell in enumerate(selected):
+            gpu = (slot + wave_index) % 4
+            assignments.append(m.Assignment(cell, (gpu,)))
+        waves.append(assignments)
+        wave_index += 1
+    return waves
+
+
+def pack_primary(items: list[m.Cell]) -> list[list[m.Assignment]]:
+    """Pack SHA-ordered cells as 4/4/4/3 without duplicate arms per wave."""
+    capacities = (4, 4, 4, 3)
+    waves: list[list[m.Cell]] = [[] for _ in capacities]
+
+    def place(index: int) -> bool:
+        if index == len(items):
+            return all(len(wave) == capacity for wave, capacity in zip(waves, capacities, strict=True))
+        cell = items[index]
+        arm = arm_name(cell.pattern)
+        for wave_index, capacity in enumerate(capacities):
+            if len(waves[wave_index]) >= capacity:
+                continue
+            if any(arm_name(existing.pattern) == arm for existing in waves[wave_index]):
+                continue
+            waves[wave_index].append(cell)
+            if place(index + 1):
+                return True
+            waves[wave_index].pop()
+        return False
+
+    if not place(0):
+        raise RuntimeError("cannot pack frozen primary assignments into 4/4/4/3")
+    result: list[list[m.Assignment]] = []
+    for wave_index, cells in enumerate(waves):
+        if len(cells) == 3:
+            cells.append(m.Cell("background-primary-final", "C00"))
+        result.append(
+            [
+                m.Assignment(cell, ((slot + wave_index) % 4,))
+                for slot, cell in enumerate(cells)
+            ]
+        )
+    return result
+
+
+def execute_primary(resume: bool, amendment_a_p5_1: bool = False) -> None:
+    RUN_ROOT.mkdir(parents=True, exist_ok=True)
+    state = load_state(resume)
+    if resume:
+        m.cleanup_recorded(state)
+    current = fingerprint()
+    if state["fingerprint"] and state["fingerprint"] != current:
+        failure = str(state.get("stages", {}).get("primary-01", {}).get("failure", ""))
+        if not (
+            amendment_a_p5_1
+            and state.get("status") == "failed"
+            and "warmup" in failure.lower()
+        ):
+            raise RuntimeError("resume fingerprint differs from frozen Phase-5 plan")
+        state.setdefault("amendments", {})["A-P5-1"] = {
+            "approved": True,
+            "applied_at": time.time(),
+            "reason": "replace rate-following drift gate with cold-start gates",
+            "prior_fingerprint": state["fingerprint"],
+            "replacement_fingerprint": current,
+            "retained_gpu_hours": state["gpu_hours_total"],
+            "burnins_reused": state["completed_burnins"],
+        }
+        state["fingerprint"] = current
+        state["status"] = "amended_resume_A-P5-1"
+        save_state(state)
+    state["fingerprint"] = current
+    state["status"] = "running_primary"
+    save_state(state)
+    ensure_provenance()
+    burnins = [
+        m.Assignment(m.Cell("burnin-C00", "C00"), (0,)),
+        m.Assignment(m.Cell("burnin-A2", "A2"), (1,)),
+        m.Assignment(m.Cell("burnin-A4", "A4"), (2,)),
+    ]
+    m.run_stage(state, "burnins", burnins, "saturation", profile=False, burnin=True)
+    waves = pack_primary(primary_cells())
+    for index, wave in enumerate(waves, 1):
+        m.run_stage(state, f"primary-{index:02d}", wave, "moderate", profile=False)
+    primary = [
+        path for path in (RUN_ROOT / "primary").glob("*-r*-*/moderate/run-complete.json")
+        if "background" not in str(path)
+    ]
+    if len(primary) != 15:
+        raise RuntimeError(f"primary completion mismatch: {len(primary)} != 15")
+    state["primary_runs"] = 15
+    state["background_runs"] = state["completed_measured_runs"] - 15
+    state["status"] = "primary_complete"
+    state["completed_at"] = time.time()
+    save_state(state)
+
+
+def execute_controls(resume: bool) -> None:
+    state = load_state(resume)
+    m.cleanup_recorded(state)
+    if state.get("fingerprint") != fingerprint():
+        raise RuntimeError("control resume fingerprint mismatch")
+    cells = [
+        m.Cell(f"control-{pattern}-r{replicate}", "C00")
+        for pattern in ("P03", "P04") for replicate in range(1, 4)
+    ]
+    for index, wave in enumerate(pack_unique(cells, "controls"), 1):
+        m.run_stage(state, f"controls-{index:02d}", wave, "moderate", profile=False)
+    state["status"] = "controls_complete"
+    state["controls_completed_at"] = time.time()
+    save_state(state)
+
+
+def plan() -> dict[str, Any]:
+    return {
+        "schema": 1,
+        "primary_runs": 15,
+        "burnins": 3,
+        "waves": [[{"cell": a.cell.cell_id, "gpus": a.gpus} for a in wave] for wave in pack_primary(primary_cells())],
+        "rate": RATE,
+        "clean_seconds": 240,
+        "drain_seconds": 600,
+        "gpu_hour_limit": 6.0,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="command", required=True)
+    for name in ("primary", "controls"):
+        item = sub.add_parser(name)
+        item.add_argument("--resume", action="store_true")
+        if name == "primary":
+            item.add_argument("--amendment-a-p5-1", action="store_true")
+    sub.add_parser("plan")
+    sub.add_parser("status")
+    args = parser.parse_args()
+    if args.command == "primary":
+        execute_primary(args.resume, args.amendment_a_p5_1)
+    elif args.command == "controls":
+        execute_controls(args.resume)
+    elif args.command == "plan":
+        print(json.dumps(plan(), sort_keys=True, indent=2))
+    else:
+        print(STATE.read_text() if STATE.exists() else '{"status":"absent"}')
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase5/phase5/controller-state.json

@@ -0,0 +1,212 @@
+{
+  "clean_window_failures": 0,
+  "completed_burnins": 3,
+  "completed_measured_runs": 0,
+  "controller_pid": 2505724,
+  "created_at": 1783858074.2830184,
+  "drain_quarantined_runs": 0,
+  "fingerprint": {
+    "capture_sizes": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      16,
+      24,
+      32,
+      40,
+      48,
+      56,
+      64,
+      72,
+      80,
+      88,
+      96,
+      104,
+      112,
+      120,
+      128,
+      136,
+      144,
+      152,
+      160,
+      168,
+      176,
+      184,
+      192,
+      200,
+      208,
+      216,
+      224,
+      232,
+      240,
+      248,
+      256,
+      272,
+      288,
+      304,
+      320,
+      336,
+      352,
+      368,
+      384,
+      400,
+      416,
+      432,
+      448,
+      464,
+      480,
+      496,
+      512
+    ],
+    "client_sha256": "f935486ab2588c1fca514be29b59361f53118eb000ea037863de48ce4fc76b16",
+    "controller_sha256": "ea2e2066a8b6d9427c59fe80db44cf87a86776570ca9066bfdff0c0b0e7f4a46",
+    "manifests": {
+      "A1": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase5-private/manifests/P10-A1.jsonl",
+        "sha256": "cab8468983fb7397ec88eb5e88a44c8b1b53d8021b7867e7bec6f58d3d903806"
+      },
+      "A3": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase5-private/manifests/P10-A3.jsonl",
+        "sha256": "ec5eaa29fcd3ec421e4f68a902c7e7fea9c0bc6b16f1013cef30ccc1f97bae24"
+      },
+      "base": {
+        "path": "/home/admin/cpfs/wjh/opprof-phase5-private/manifests/P10-base.jsonl",
+        "sha256": "d3b4f540ddd629dd0e34fff55c3b3837f359bdd6e5947309a1ea2a358f811ffc"
+      }
+    },
+    "p3_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
+    "p3_matrix_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
+    "rate": 0.4725,
+    "source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
+    "source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
+  },
+  "gpu_hours_this_stage": 0.4224752351972792,
+  "gpu_hours_total": 0.6476566231913037,
+  "missing_trace_files": 0,
+  "schema": 1,
+  "stages": {
+    "burnins": {
+      "assignments": [
+        {
+          "cell": "burnin-C00-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "burnin-A2-A2",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "burnin-A4-A4",
+          "gpus": [
+            2
+          ]
+        }
+      ],
+      "burnin": true,
+      "clients": {},
+      "completed_at": 1783858361.2828288,
+      "confirmation": false,
+      "gpu_hours": 0.22518138799402448,
+      "load_point": "saturation",
+      "profile": false,
+      "servers": {},
+      "started_at": 1783858074.4911764,
+      "status": "complete"
+    },
+    "primary-01": {
+      "assignments": [
+        {
+          "cell": "base-r2-C00",
+          "gpus": [
+            0
+          ]
+        },
+        {
+          "cell": "A3-r1-C00",
+          "gpus": [
+            1
+          ]
+        },
+        {
+          "cell": "A1-r2-C00",
+          "gpus": [
+            2
+          ]
+        },
+        {
+          "cell": "A4-r1-A4",
+          "gpus": [
+            3
+          ]
+        }
+      ],
+      "burnin": false,
+      "clients": {
+        "A1-r2-C00-moderate": {
+          "pgid": 2512122,
+          "pid": 2512122
+        },
+        "A3-r1-C00-moderate": {
+          "pgid": 2512121,
+          "pid": 2512121
+        },
+        "A4-r1-A4-moderate": {
+          "pgid": 2512123,
+          "pid": 2512123
+        },
+        "base-r2-C00-moderate": {
+          "pgid": 2512119,
+          "pid": 2512119
+        }
+      },
+      "confirmation": false,
+      "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase5/primary/base-r2-C00/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=25; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [380, 201, 187], 'scheduled_tokens': [26927, 27616, 463], 'scheduled_token_throughput': [5385.4, 5523.2, 92.6], 'mean_scheduled_token_throughput': 3667.066666666666, 'slope_tokens_per_second_squared': -529.28, 'normalized_drift': 2.1650001817983497, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")",
+      "gpu_hours": 0.4224752351972792,
+      "load_point": "moderate",
+      "profile": false,
+      "servers": {
+        "A1-r2-C00-moderate": {
+          "gpus": [
+            2
+          ],
+          "pgid": 2510424,
+          "pid": 2510424
+        },
+        "A3-r1-C00-moderate": {
+          "gpus": [
+            1
+          ],
+          "pgid": 2510423,
+          "pid": 2510423
+        },
+        "A4-r1-A4-moderate": {
+          "gpus": [
+            3
+          ],
+          "pgid": 2510425,
+          "pid": 2510425
+        },
+        "base-r2-C00-moderate": {
+          "gpus": [
+            0
+          ],
+          "pgid": 2510422,
+          "pid": 2510422
+        }
+      },
+      "started_at": 1783858361.3184204,
+      "status": "failed"
+    }
+  },
+  "status": "failed",
+  "updated_at": 1783858758.067235
+}

runs/opprof-phase5/phase5/launch-echo.log

@@ -0,0 +1 @@
+LAUNCH_ECHO utc=2026-07-12T12:07:54Z host=dash0 gpus=0-3 cpus=0-79 source=/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0@4b253fd manifests=/home/admin/cpfs/wjh/opprof-phase5-private/manifests outputs=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase5 runs=3burnin+15primary+1background rate=0.4725 warmup=60s clean=240s drain=600s est_wall=35-60min est_gpu=1.7-2.1_H20h hard_cap=6.0_H20h conditional_controls=6_if_bridge_fails

runs/opprof-phase5/phase5/plan.json

@@ -0,0 +1,115 @@
+{
+  "burnins": 3,
+  "clean_seconds": 240,
+  "drain_seconds": 600,
+  "gpu_hour_limit": 6.0,
+  "primary_runs": 15,
+  "rate": 0.4725,
+  "schema": 1,
+  "waves": [
+    [
+      {
+        "cell": "base-r2-C00",
+        "gpus": [
+          0
+        ]
+      },
+      {
+        "cell": "A3-r1-C00",
+        "gpus": [
+          1
+        ]
+      },
+      {
+        "cell": "A1-r2-C00",
+        "gpus": [
+          2
+        ]
+      },
+      {
+        "cell": "A4-r1-A4",
+        "gpus": [
+          3
+        ]
+      }
+    ],
+    [
+      {
+        "cell": "A1-r3-C00",
+        "gpus": [
+          1
+        ]
+      },
+      {
+        "cell": "base-r1-C00",
+        "gpus": [
+          2
+        ]
+      },
+      {
+        "cell": "A3-r3-C00",
+        "gpus": [
+          3
+        ]
+      },
+      {
+        "cell": "A2-r1-A2",
+        "gpus": [
+          0
+        ]
+      }
+    ],
+    [
+      {
+        "cell": "base-r3-C00",
+        "gpus": [
+          2
+        ]
+      },
+      {
+        "cell": "A3-r2-C00",
+        "gpus": [
+          3
+        ]
+      },
+      {
+        "cell": "A4-r2-A4",
+        "gpus": [
+          0
+        ]
+      },
+      {
+        "cell": "A2-r3-A2",
+        "gpus": [
+          1
+        ]
+      }
+    ],
+    [
+      {
+        "cell": "A4-r3-A4",
+        "gpus": [
+          3
+        ]
+      },
+      {
+        "cell": "A1-r1-C00",
+        "gpus": [
+          0
+        ]
+      },
+      {
+        "cell": "A2-r2-A2",
+        "gpus": [
+          1
+        ]
+      },
+      {
+        "cell": "background-primary-final-C00",
+        "gpus": [
+          2
+        ]
+      }
+    ]
+  ]
+}

runs/opprof-phase5/test_phase5_analysis.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import numpy as np
+
+import analyze_phase5 as a
+
+
+def main() -> None:
+    adjusted = a.holm({"A1": 0.001, "A2": 0.02, "A3": 0.04, "A4": 0.5})
+    assert adjusted == {"A1": 0.004, "A2": 0.06, "A3": 0.08, "A4": 0.5}
+    assert a.ci(np.arange(100, dtype=np.float64)) == [2.475, 96.52499999999999]
+    runs = [
+        {"blocks": np.asarray([[10.0, 2.0]] * 48)},
+        {"blocks": np.asarray([[20.0, 4.0]] * 48)},
+        {"blocks": np.asarray([[30.0, 6.0]] * 48)},
+    ]
+    draws = a.hierarchical_draws(runs, np.random.default_rng(a.SEED))
+    assert draws.shape == (a.RESAMPLES,)
+    assert np.allclose(draws, 5.0)
+    assert a.point_efficiency(runs) == 5.0
+    idle_blocks = np.asarray([[0.0, 0.0]] + [[10.0, 2.0]] * 47)
+    idle_draws = a.hierarchical_draws(
+        [{"blocks": idle_blocks}], np.random.default_rng(a.SEED)
+    )
+    assert np.all(np.isfinite(idle_draws))
+    assert np.allclose(idle_draws, 5.0)
+    assert a.point_efficiency([{"blocks": idle_blocks}]) == 5.0
+    assert a.two_sided_p(np.asarray([-1.0, 1.0])) == 1.0
+    print("phase5 analysis: PASS")
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase5/test_phase5_tools.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+HERE = Path(__file__).resolve().parent
+CLIENT = HERE / "opprof_phase5_client.py"
+
+
+def write_jsonl(path: Path, rows: list[dict]) -> None:
+    path.write_text("".join(json.dumps(row) + "\n" for row in rows))
+
+
+def run(command: list[str]) -> None:
+    completed = subprocess.run(command, text=True, capture_output=True)
+    if completed.returncode:
+        raise RuntimeError(f"command failed: {completed.stderr}")
+
+
+def main() -> None:
+    with tempfile.TemporaryDirectory() as tmp_text:
+        tmp = Path(tmp_text)
+        manifest = tmp / "p3.jsonl"
+        source = tmp / "source.jsonl"
+        base = tmp / "base.jsonl"
+        a1 = tmp / "a1.jsonl"
+        rows = []
+        source_rows = []
+        lengths = [128, 8192, 256, 4096] * 8
+        for index, length in enumerate(lengths):
+            rows.append({
+                "request_id": f"P10-{index}", "pattern_id": "P10",
+                "input_tokens": length, "output_tokens": 8,
+                "arrival": "steady", "kind": "private-trace",
+                "source_index": index, "prompt": f"private-{index}",
+            })
+            source_rows.append({"timestamp": index * 0.1, "prompt": f"private-{index}"})
+        write_jsonl(manifest, rows)
+        write_jsonl(source, source_rows)
+        common = [
+            sys.executable, str(CLIENT), "transform", "--in", str(manifest),
+            "--take-first", "32", "--timestamp-source", str(source),
+            "--join-key", "source_index", "--timestamp-field", "timestamp",
+            "--arrival", "recorded-scaled", "--target-rate", "0.5",
+        ]
+        run(common + ["--service-order", "original", "--out", str(base)])
+        run(common + [
+            "--service-order", "length-binned", "--reorder-block-size", "32",
+            "--analysis-cohort-size", "16", "--max-added-delay-seconds", "64",
+            "--out", str(a1),
+        ])
+        base_summary = json.loads((base.with_suffix(".jsonl.summary.json")).read_text())
+        a1_summary = json.loads((a1.with_suffix(".jsonl.summary.json")).read_text())
+        assert base_summary["rows"] == a1_summary["rows"] == 32
+        assert base_summary["request_id_set_sha256"] == a1_summary["request_id_set_sha256"]
+        assert a1_summary["r16"] < base_summary["r16"]
+        assert a1_summary["max_added_delay_seconds"] <= 64
+        assert "private-" not in json.dumps(a1_summary)
+    print("phase5 tools: PASS")
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase6/analyze_phase6.py

@@ -0,0 +1,466 @@
+#!/usr/bin/env python3
+"""Frozen Phase-6 paired-anchor, frontier, rank, and Layer-1 analysis."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+OLD_BEST = "tp2_mns32"
+TOP20 = {"tp2_mns32", "tp2_mns64", "tp4_mns16", "tp4_mns32", "tp4_mns64"}
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def numeric(values: list[float | int | None]) -> dict[str, Any]:
+    finite = [float(x) for x in values if x is not None and math.isfinite(float(x))]
+    return {
+        "n": len(values), "finite_n": len(finite), "missing_n": len(values) - len(finite),
+        "min": min(finite) if finite else None, "max": max(finite) if finite else None,
+        "distinct_n": len(set(finite)),
+    }
+
+
+def cv(values: list[float]) -> float:
+    if not values:
+        return 0.0
+    array = np.asarray(values, dtype=np.float64)
+    mean = float(array.mean())
+    return float(array.std(ddof=0) / mean) if mean else 0.0
+
+
+def floor_buckets(scores: dict[str, float]) -> tuple[float, dict[str, int]]:
+    tol = max(1e-9, 1e-6 * max(abs(x) for x in scores.values()))
+    return tol, {key: math.floor(value / tol) for key, value in scores.items()}
+
+
+def kendall_tau_b(x: dict[str, int], y: dict[str, int]) -> dict[str, Any]:
+    keys = sorted(x)
+    c = d = tx = ty = joint = 0
+    for i, left in enumerate(keys):
+        for right in keys[i + 1:]:
+            dx = (x[left] > x[right]) - (x[left] < x[right])
+            dy = (y[left] > y[right]) - (y[left] < y[right])
+            if dx == 0 and dy == 0:
+                joint += 1
+            elif dx == 0:
+                tx += 1
+            elif dy == 0:
+                ty += 1
+            elif dx == dy:
+                c += 1
+            else:
+                d += 1
+    denom = math.sqrt((c + d + tx) * (c + d + ty))
+    return {
+        "tau_b": (c - d) / denom if denom else None, "concordant": c,
+        "discordant": d, "v20_only_ties": tx, "v24_only_ties": ty,
+        "joint_ties": joint, "pairs": len(keys) * (len(keys) - 1) // 2,
+    }
+
+
+def layer_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:
+    model = [x for x in records if x.get("model_executed")]
+    mode = Counter(str(x["cudagraph"]["runtime_mode"]) for x in model)
+    total_bucket = sum(int(x["cudagraph"]["bucket_tokens"]) for x in model)
+    total_padding = sum(int(x["cudagraph"]["padding_tokens"]) for x in model)
+    kv = [float(x["kv"]["usage"]) for x in model]
+    waiting = [float(x["queues"]["waiting"]) for x in model]
+    decode_b = [float(x["decode_batch_size"]) for x in model]
+    return {
+        "steps": len(records), "model_steps": len(model),
+        "prefill_only_steps": sum(int(x["prefill_tokens"]) > 0 and int(x["decode_tokens"]) == 0 for x in model),
+        "decode_only_steps": sum(int(x["prefill_tokens"]) == 0 and int(x["decode_tokens"]) > 0 for x in model),
+        "mixed_steps": sum(int(x["prefill_tokens"]) > 0 and int(x["decode_tokens"]) > 0 for x in model),
+        "prefill_tokens": sum(int(x["prefill_tokens"]) for x in model),
+        "decode_tokens": sum(int(x["decode_tokens"]) for x in model),
+        "preemptions": sum(int(x["preemptions"]) for x in model),
+        "kv_usage_mean": float(np.mean(kv)) if kv else None,
+        "kv_usage_max": max(kv) if kv else None,
+        "waiting_mean": float(np.mean(waiting)) if waiting else None,
+        "waiting_max": max(waiting) if waiting else None,
+        "waiting_cv": cv(waiting), "decode_B_mean": float(np.mean(decode_b)) if decode_b else None,
+        "decode_B_cv": cv(decode_b), "graph_mode_counts": dict(sorted(mode.items())),
+        "graph_mode_shares": {key: value / len(model) for key, value in sorted(mode.items())} if model else {},
+        "padding_fraction": total_padding / total_bucket if total_bucket else 0.0,
+    }
+
+
+def load_stream(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text().splitlines() if "step_index" in json.loads(line)]
+
+
+def p3_reference(p3_root: Path) -> dict[str, Any]:
+    run = p3_root / "primary/P10-C00/moderate"
+    result = json.loads((run / "client/result.json").read_text())
+    t0 = int(result["t0_mono_ns"])
+    lo = t0 + int(float(result["clean"]["start_s"]) * 1e9)
+    hi = t0 + int(float(result["clean"]["end_s"]) * 1e9)
+    stream = next((run / "opprof").glob("*.jsonl"))
+    records = [x for x in load_stream(stream) if lo <= int(x["submit_mono_ns"]) < hi]
+    return {
+        "limitation": "P3 P10 reference is TP1/MNS-default, steady arrival, sampling_u<=0.125, output<=256; it is contextual rather than a matched v0.20 composition baseline.",
+        "stream_sha256": sha256_file(stream), "metrics": layer_metrics(records),
+    }
+
+
+def accepted_anchor(anchor_dir: Path) -> dict[str, Any]:
+    primary = json.loads((anchor_dir / "result.json").read_text())
+    cell_dir = anchor_dir.parent
+    anchor = float(primary["anchor"])
+    confirms = []
+    for path in sorted(cell_dir.glob("confirm-*-anchor-*/result.json")):
+        item = json.loads(path.read_text())
+        if math.isclose(float(item["anchor"]), anchor, rel_tol=0, abs_tol=1e-15):
+            confirms.append(item)
+    trials = [primary, *confirms]
+    votes = [bool(x["feasible"]) for x in trials]
+    if len(votes) == 1 or len(set(votes)) == 1:
+        verdict = votes[0]
+        resolved = True
+    elif len(votes) >= 3:
+        verdict = sum(votes) >= 2
+        resolved = True
+    else:
+        verdict = None
+        resolved = False
+    return {
+        "anchor": anchor, "primary": primary, "confirmations": confirms,
+        "trial_count": len(trials), "pass_rates": [x["pass_rate"] for x in trials],
+        "feasible_votes": votes, "accepted_feasible": verdict, "resolved": resolved,
+        "accepted_pass_rate": float(np.median([x["pass_rate"] for x in trials])),
+    }
+
+
+def matching_anchor_dir(cell_dir: Path, anchor: float) -> Path | None:
+    for path in cell_dir.glob("anchor-*/result.json"):
+        item = json.loads(path.read_text())
+        if math.isclose(float(item["anchor"]), anchor, rel_tol=0, abs_tol=1e-15):
+            return path.parent
+    return None
+
+
+def analyze(root: Path, ground_path: Path, p3_root: Path) -> dict[str, Any]:
+    ground = json.loads(ground_path.read_text())
+    old_cells = {x["cell_id"]: x for x in ground["cells"]}
+    old_probe = {
+        (x["cell_id"], float(p["sampling_u"])): p
+        for x in ground["cells"] for p in x["probe_history"]
+    }
+    cells: dict[str, Any] = {}
+    all_primary = []
+    all_client_invariants = []
+    selection_matches = []
+    for cell, old in old_cells.items():
+        cell_dir = root / "solo-authoritative/cells" / cell
+        colocated_dir = root / "cells" / cell
+        f20 = float(old["measured_objective"]["slo_feasible_req_s_per_gpu"])
+        if not (cell_dir / "cell-valid.json").exists():
+            cells[cell] = {
+                "tp": old["tensor_parallel_size"], "mns": old["max_num_seqs"],
+                "measurement_status": "UNMEASURED_SOLO",
+                "f20": f20, "f24": None, "drift": None,
+                "observed_max_feasible_rate": None,
+                "material_frontier_moved": None, "boundary": "UNMEASURED",
+                "bounded": False, "censor": "UNMEASURED_SOLO",
+                "anchors": [], "cell_valid": None,
+            }
+            continue
+        valid = json.loads((cell_dir / "cell-valid.json").read_text())
+        stream = next((cell_dir / "opprof").glob("*.jsonl"))
+        stream_records = load_stream(stream)
+        anchors = []
+        for path in sorted(cell_dir.glob("anchor-*/result.json")):
+            accepted = accepted_anchor(path.parent)
+            primary = accepted["primary"]
+            key = (cell, float(primary["anchor"]))
+            historical = old_probe[key]
+            lo = int(primary["interval"]["start_mono_ns"])
+            hi = int(primary["interval"]["end_mono_ns"])
+            records = [x for x in stream_records if lo <= int(x["submit_mono_ns"]) <= hi]
+            accepted["layer1"] = layer_metrics(records)
+            for confirmation in accepted["confirmations"]:
+                confirm_lo = int(confirmation["interval"]["start_mono_ns"])
+                confirm_hi = int(confirmation["interval"]["end_mono_ns"])
+                confirmation["layer1"] = layer_metrics([
+                    x for x in stream_records
+                    if confirm_lo <= int(x["submit_mono_ns"]) <= confirm_hi
+                ])
+            accepted["v20"] = {
+                "pass_rate": historical["pass_rate"], "feasible": historical["feasible"],
+                "request_count": historical["request_count"],
+                "rate_per_gpu": historical["request_rate_per_gpu_req_s_gpu"],
+            }
+            accepted["v24"] = {
+                "pass_rate": accepted["accepted_pass_rate"],
+                "feasible": accepted["accepted_feasible"],
+                "request_count": primary["selection"]["count"],
+                "rate_per_gpu": primary["selection"]["offered_req_s_per_gpu"],
+            }
+            colocated_anchor_dir = matching_anchor_dir(colocated_dir, float(primary["anchor"]))
+            if colocated_anchor_dir is not None:
+                colocated = accepted_anchor(colocated_anchor_dir)
+                accepted["colocated"] = {
+                    "primary_pass_rate": colocated["primary"]["pass_rate"],
+                    "primary_feasible": colocated["primary"]["feasible"],
+                    "trial_pass_rates": colocated["pass_rates"],
+                    "accepted_feasible": colocated["accepted_feasible"],
+                    "solo_minus_colocated_primary_pass_rate": (
+                        accepted["accepted_pass_rate"] - colocated["primary"]["pass_rate"]
+                    ),
+                }
+            else:
+                accepted["colocated"] = None
+            accepted["feasibility_flip"] = (
+                accepted["accepted_feasible"] is not None
+                and accepted["accepted_feasible"] != historical["feasible"]
+            )
+            anchors.append(accepted)
+            all_primary.append(primary)
+            all_client_invariants.extend(primary["invariants"].values())
+            selection_matches.append(primary["selection"]["count"] == historical["request_count"])
+        peak_u = float(old["measured_objective"]["best_sampling_u"])
+        peak = next(x for x in anchors if math.isclose(x["anchor"], peak_u, abs_tol=1e-15))
+        feasible = [x for x in anchors if x["accepted_feasible"] is True]
+        infeasible = [x for x in anchors if x["accepted_feasible"] is False]
+        unresolved = [x for x in anchors if x["accepted_feasible"] is None]
+        observed_frontier = max((x["v24"]["rate_per_gpu"] for x in feasible), default=None)
+        lower = [x for x in anchors if x["anchor"] < peak_u]
+        upper = [x for x in anchors if x["anchor"] > peak_u]
+        if unresolved:
+            bounded, censor = False, "UNRESOLVED_SOLO_ANCHOR"
+        elif feasible and infeasible:
+            monotonic = max(x["anchor"] for x in feasible) < min(x["anchor"] for x in infeasible)
+            bounded = monotonic
+            censor = None if bounded else "NONMONOTONIC_SOLO_ANCHORS"
+        elif feasible:
+            bounded, censor = False, "RIGHT_CENSORED_HISTORY_EDGE"
+        elif infeasible:
+            bounded, censor = False, "LEFT_CENSORED_HISTORY_EDGE"
+        else:
+            bounded, censor = False, "NO_RESOLVED_SOLO_ANCHOR"
+        frontier = (
+            None if censor in {
+                "UNRESOLVED_SOLO_ANCHOR", "NONMONOTONIC_SOLO_ANCHORS",
+                "NO_RESOLVED_SOLO_ANCHOR",
+            } else observed_frontier
+        )
+        drift = (frontier / f20 - 1) if frontier is not None else None
+        if censor == "NONMONOTONIC_SOLO_ANCHORS":
+            boundary = "NONMONOTONIC"
+        elif peak["accepted_feasible"] is False:
+            boundary = "DOWN"
+        elif peak["accepted_feasible"] is None:
+            boundary = "UNRESOLVED"
+        elif any(x["accepted_feasible"] is True for x in upper):
+            boundary = "UP"
+        else:
+            boundary = "STABLE"
+        cells[cell] = {
+            "tp": old["tensor_parallel_size"], "mns": old["max_num_seqs"],
+            "measurement_status": "SOLO_AUTHORITATIVE",
+            "f20": f20, "f24": frontier, "drift": drift,
+            "observed_max_feasible_rate": observed_frontier,
+            "material_frontier_moved": bounded and drift is not None and abs(drift) > .05,
+            "boundary": boundary, "bounded": bounded, "censor": censor,
+            "anchors": anchors, "cell_valid": valid,
+        }
+
+    scores20 = {key: value["f20"] for key, value in cells.items()}
+    scores24 = {key: value["f24"] for key, value in cells.items() if value["f24"] is not None}
+    tol20, buckets20 = floor_buckets(scores20)
+    tol24, buckets24 = floor_buckets(scores24)
+    full_bounded = len(scores24) == 12 and all(x["bounded"] for x in cells.values())
+    max24 = max(buckets24.values())
+    if not full_bounded:
+        argmax = "INCONCLUSIVE"
+    else:
+        argmax = "SURVIVED" if buckets24[OLD_BEST] == max24 else "MOVED"
+    tau = kendall_tau_b(buckets20, buckets24) if full_bounded else None
+    reversals = []
+    for left in sorted(TOP20):
+        for right in sorted(TOP20):
+            if left >= right:
+                continue
+            if not cells[left]["bounded"] or not cells[right]["bounded"]:
+                continue
+            if left not in scores24 or right not in scores24:
+                continue
+            old_delta = scores20[left] - scores20[right]
+            if abs(old_delta) / max(scores20[left], scores20[right]) <= .05:
+                continue
+            new_delta = scores24[left] - scores24[right]
+            if old_delta * new_delta < 0:
+                reversals.append([left, right])
+    if not full_bounded or argmax == "INCONCLUSIVE" or tau is None:
+        ranking = "INCONCLUSIVE"
+    elif argmax == "MOVED" or tau["tau_b"] < .8 or reversals:
+        ranking = "MOVED"
+    else:
+        ranking = "SURVIVED"
+    trap_inputs = ["tp4_mns8", "tp4_mns16", "tp4_mns32"]
+    if not all(cells[x]["bounded"] for x in trap_inputs):
+        trap = "INCONCLUSIVE"
+    elif buckets24["tp4_mns16"] == max24:
+        trap = "CEASES_TO_BE_A_TRAP"
+    elif buckets24["tp4_mns16"] >= max(buckets24["tp4_mns8"], buckets24["tp4_mns32"]):
+        trap = "PERSISTS"
+    else:
+        trap = "ESCAPES"
+
+    colocated_state = json.loads((root / "controller-state.json").read_text())
+    solo_state_path = root / "solo-authoritative/controller-state.json"
+    solo_state = json.loads(solo_state_path.read_text()) if solo_state_path.exists() else None
+    state = solo_state or colocated_state
+    measured_cells = [
+        x for x in cells.values() if x["measurement_status"] == "SOLO_AUTHORITATIVE"
+    ]
+    coverage = {
+        "solo_primary_anchors_at_least_25": len(all_primary) >= 25,
+        "solo_measured_cells_12": len(measured_cells) == 12,
+    }
+    invariants = {
+        "surface_rows_12": len(cells) == 12,
+        "selection_counts_match": all(selection_matches),
+        "client_invariants": all(all_client_invariants),
+        "cell_validity": all(
+            all(x["cell_valid"]["invariants"].values()) for x in measured_cells
+        ),
+        "gpu_below_6": float(state["gpu_hours_total"]) < 6.0,
+        "rates_nonnegative": all(x["f24"] is None or x["f24"] >= 0 for x in cells.values()),
+        "surface_not_identical": len(set(scores24.values())) > 1,
+    }
+    red_flags = [key for key, value in {**coverage, **invariants}.items() if not value]
+    drifted = [key for key, value in cells.items() if value["material_frontier_moved"] is True]
+    flip_cells = [
+        key for key, value in cells.items()
+        if any(anchor["feasibility_flip"] for anchor in value["anchors"])
+    ]
+    partial = bool([key for key, value in coverage.items() if not value])
+    w1_audit_path = root / "w1-readjudication-A-P6-1.json"
+    w1_audit = json.loads(w1_audit_path.read_text()) if w1_audit_path.exists() else None
+    censored = {key: value["censor"] for key, value in cells.items() if not value["bounded"]}
+    colocated_deltas = [
+        {
+            "cell": cell, "anchor": anchor["anchor"],
+            "solo_pass_rate": anchor["accepted_pass_rate"],
+            "solo_feasible": anchor["accepted_feasible"],
+            **anchor["colocated"],
+        }
+        for cell, value in cells.items() for anchor in value["anchors"]
+        if anchor.get("colocated") is not None
+    ]
+    return {
+        "schema": 1,
+        "status": "BUDGET_STOP_PARTIAL" if partial else ("VALID" if not red_flags else "INVALID"),
+        "authoritative_tier": "A-P6-2 solo host placement",
+        "limitation": "Upgrade-path churn includes dash1->dash0 and resolved-default changes. Co-located W1-W3 values are indicative only; P3 composition is contextual, not a matched v0.20 Layer-1 baseline.",
+        "ground_truth_sha256": sha256_file(ground_path), "cells": cells,
+        "floor_buckets": {"v20_tol": tol20, "v20": buckets20, "v24_tol": tol24, "v24": buckets24},
+        "verdicts": {
+            "argmax": argmax, "ranking": ranking, "trap": trap,
+            "full_surface_bounded": full_bounded, "tau_b": tau,
+            "top_pair_reversals_gt5pct": reversals,
+        },
+        "materially_drifted_cells": drifted,
+        "feasibility_flip_cells": flip_cells,
+        "decision_blockers": {
+            "coverage": [key for key, value in coverage.items() if not value],
+            "unbounded_or_unresolved_cells": censored,
+        },
+        "solo_vs_colocated": colocated_deltas,
+        "w1_readjudication": w1_audit,
+        "run_stats": {
+            "measured_cells": len(measured_cells),
+            "surface_cells": len(cells),
+            "primary_anchor_runs": len(all_primary),
+            "confirmation_runs": sum(
+                len(anchor["confirmations"])
+                for cell in cells.values() for anchor in cell["anchors"]
+            ),
+            "accepted_anchor_trials": sum(
+                anchor["trial_count"]
+                for cell in cells.values() for anchor in cell["anchors"]
+            ),
+            "warmup_runs": len(measured_cells),
+            "solo_cell_gpu_hours": {
+                key: value.get("gpu_hours") for key, value in (solo_state or {}).get("cells", {}).items()
+            },
+            "colocated_wave_gpu_hours": {
+                key: value.get("gpu_hours") for key, value in colocated_state["waves"].items()
+            },
+            "launch_echo": (
+                (root / "launch-echo.log").read_text().splitlines()
+                + ((root / "solo-authoritative/launch-echo.log").read_text().splitlines()
+                   if (root / "solo-authoritative/launch-echo.log").exists() else [])
+            ),
+        },
+        "attempt_history": {
+            "colocated_status": colocated_state["status"],
+            "colocated_h20_hours": colocated_state["gpu_hours_total"],
+            "colocated_primary_anchors": colocated_state["completed_primary_anchors"],
+            "colocated_confirmations": colocated_state["completed_confirmations"],
+            "colocated_budget_stop": colocated_state.get("budget_stop"),
+            "solo_status": (solo_state or {}).get("status"),
+            "solo_repairs": (solo_state or {}).get("repairs", []),
+            "solo_failures": (solo_state or {}).get("failures", []),
+            "raw_roots": {
+                "colocated": str(root / "cells"),
+                "solo_authoritative": str(root / "solo-authoritative/cells"),
+            },
+        },
+        "p3_composition_reference": p3_reference(p3_root),
+        "gpu": {
+            "new_h20_hours": state["gpu_hours_total"], "hard_cap": 6.0,
+            "prior_colocated_h20_hours": colocated_state["gpu_hours_total"],
+            "solo_h20_hours": (solo_state or {}).get("solo_gpu_hours", 0.0),
+            "completed_primary_anchors": (solo_state or {}).get("primary_anchors", 0),
+            "confirmations": (solo_state or {}).get("confirmations", 0),
+            "controller_status": state["status"],
+            "budget_stop": state.get("budget_stop"),
+        },
+        "sanity": {
+            "red_flags": red_flags, "coverage": coverage, "invariants": invariants,
+            "numeric": {
+                "v20_score": numeric(list(scores20.values())),
+                "v24_score": numeric(list(scores24.values())),
+                "drift": numeric([x["drift"] for x in cells.values()]),
+                "primary_pass_rate": numeric([x["pass_rate"] for x in all_primary]),
+                "selected_count": numeric([x["selection"]["count"] for x in all_primary]),
+                "layer1_steps": numeric([
+                    anchor["layer1"]["steps"] for cell in cells.values() for anchor in cell["anchors"]
+                ]),
+            },
+        },
+    }
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--root", type=Path, required=True)
+    p.add_argument("--ground-truth", type=Path, required=True)
+    p.add_argument("--p3-root", type=Path, required=True)
+    p.add_argument("--out", type=Path, required=True)
+    args = p.parse_args()
+    result = analyze(args.root, args.ground_truth, args.p3_root)
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
+    print(json.dumps({"status": result["status"], "verdicts": result["verdicts"], "red_flags": result["sanity"]["red_flags"], "gpu": result["gpu"]}, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase6/opprof_phase6_client.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""Exact C1 anchor replay using the pinned AITuner trace/worker/SLO paths."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import hashlib
+import json
+import math
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2]))
+sys.path.insert(0, str(AITUNER_ROOT / "src"))
+os.environ.setdefault("AITUNER_CODEX_BASE_URL", "http://127.0.0.1:1")
+
+from aituner.slo import evaluate_request, summarize_evaluations  # noqa: E402
+from aituner.spec import load_study_spec  # noqa: E402
+from aituner.trace import load_trace_requests, select_requests_for_threshold  # noqa: E402
+from aituner.worker import _probe_drain_deadline, _replay_requests  # noqa: E402
+
+
+def atomic_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n")
+    os.replace(tmp, path)
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def numeric(values: list[float | int | None]) -> dict[str, Any]:
+    finite = [float(x) for x in values if x is not None and math.isfinite(float(x))]
+    return {
+        "n": len(values), "finite_n": len(finite), "missing_n": len(values) - len(finite),
+        "min": min(finite) if finite else None, "max": max(finite) if finite else None,
+        "distinct_n": len(set(finite)),
+    }
+
+
+def load_selected(study_path: Path, anchor: float):
+    study = load_study_spec(study_path)
+    window, requests = load_trace_requests(study, study_spec_path=study_path)
+    selected = select_requests_for_threshold(requests, threshold=anchor)
+    return study, window, requests, selected
+
+
+def selected_summary(selected, duration_s: float, tp: int) -> dict[str, Any]:
+    ids = "\n".join(item.row_id for item in selected)
+    arrival = "\n".join(f"{item.arrival_s:.12f}" for item in selected)
+    lengths = "\n".join(str(item.prompt_tokens_hint) for item in selected)
+    return {
+        "count": len(selected),
+        "offered_req_s": len(selected) / duration_s,
+        "offered_req_s_per_gpu": len(selected) / duration_s / tp,
+        "request_id_order_sha256": hashlib.sha256(ids.encode()).hexdigest(),
+        "arrival_order_sha256": hashlib.sha256(arrival.encode()).hexdigest(),
+        "raw_length_order_sha256": hashlib.sha256(lengths.encode()).hexdigest(),
+        "arrival_s": numeric([item.arrival_s for item in selected]),
+        "raw_input_tokens": numeric([item.prompt_tokens_hint for item in selected]),
+        "long_gt4096": sum(int(item.prompt_tokens_hint or 0) > 4096 for item in selected),
+    }
+
+
+def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]:
+    study_path = Path(args.study)
+    study, window, _requests, selected = load_selected(study_path, args.anchor)
+    if warmup:
+        first = selected[:16]
+        if not any(int(item.prompt_tokens_hint or 0) > 4096 for item in first):
+            long_item = next(item for item in selected if int(item.prompt_tokens_hint or 0) > 4096)
+            first = [*selected[:15], long_item]
+        first = sorted({item.row_id: item for item in first}.values(), key=lambda item: item.arrival_s)
+        if len(first) < 16:
+            raise RuntimeError("warmup set has fewer than 16 unique requests")
+        start = first[0].arrival_s
+        selected = [dataclasses.replace(item, arrival_s=item.arrival_s - start) for item in first]
+
+    duration_s = float(window.window_end - window.window_start)
+    interval_start_mono_ns = time.monotonic_ns()
+    interval_start_wall_ns = time.time_ns()
+    outcomes, early_stopped, early_stop_reason = _replay_requests(
+        selected,
+        base_url=args.base_url,
+        timeout_s=study.engine.request_timeout_s,
+        max_concurrency=study.trace.max_concurrency,
+        target_pass_rate=(0.0 if warmup else study.slo.target_pass_rate),
+        max_lag_s=study.trace.early_stop_max_lag_s,
+        max_elapsed_s=(
+            120.0 if warmup else _probe_drain_deadline(
+                selected, study.slo, ceiling=study.trace.early_stop_max_elapsed_s
+            )
+        ),
+        evaluate_outcome=lambda outcome: evaluate_request(outcome, study.slo),
+        drain_inflight_on_early_stop=True,
+    )
+    interval_end_mono_ns = time.monotonic_ns()
+    interval_end_wall_ns = time.time_ns()
+    evaluations, slo_summary = summarize_evaluations(outcomes, study.slo)
+    by_id = {item.row_id: item for item in selected}
+    details = []
+    for outcome, evaluation in zip(outcomes, evaluations):
+        request = by_id[outcome.request_id]
+        details.append({
+            "request_id": outcome.request_id,
+            "sampling_u": request.sampling_u,
+            "arrival_s": request.arrival_s,
+            "raw_input_tokens": request.prompt_tokens_hint,
+            "success": outcome.success,
+            "ttft_ms": outcome.ttft_ms,
+            "tpot_ms": outcome.tpot_ms,
+            "completion_tokens": outcome.completion_tokens,
+            "completion_tokens_source": outcome.completion_tokens_source,
+            "slo_pass": evaluation.passed,
+            "reasons": evaluation.reasons,
+            "error": outcome.error,
+        })
+    out = Path(args.result_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    with (out / "requests.jsonl").open("w") as f:
+        for item in details:
+            f.write(json.dumps(item, sort_keys=True) + "\n")
+    summary = selected_summary(selected, duration_s, args.tp)
+    exact = sum(
+        item.success and item.completion_tokens_source == "usage" and item.completion_tokens == 128
+        for item in outcomes
+    )
+    result = {
+        "schema": 1,
+        "kind": "warmup" if warmup else "anchor",
+        "cell": args.cell,
+        "anchor": args.anchor,
+        "tp": args.tp,
+        "mns": args.mns,
+        "study_sha256": sha256_file(study_path),
+        "interval": {
+            "start_mono_ns": interval_start_mono_ns, "end_mono_ns": interval_end_mono_ns,
+            "start_wall_ns": interval_start_wall_ns, "end_wall_ns": interval_end_wall_ns,
+            "elapsed_s": (interval_end_mono_ns - interval_start_mono_ns) / 1e9,
+        },
+        "selection": summary,
+        "observed_count": len(outcomes),
+        "exact_output_count": exact,
+        "slo_pass_count": slo_summary["slo_pass_count"],
+        "pass_rate": slo_summary["slo_pass_rate"],
+        "feasible": bool(slo_summary["feasible"]),
+        "early_stopped": early_stopped,
+        "early_stop_reason": early_stop_reason,
+        "ttft_ms": numeric([item.ttft_ms for item in outcomes]),
+        "tpot_ms": numeric([item.tpot_ms for item in outcomes]),
+        "invariants": {
+            "selected_nonempty": bool(selected),
+            "outcomes_cover_selected": len(outcomes) == len(selected),
+            "exact_output_or_failed": all(
+                (not item.success) or (
+                    item.completion_tokens_source == "usage" and item.completion_tokens == 128
+                ) for item in outcomes
+            ),
+            "raw_lengths_present": all(item.prompt_tokens_hint is not None for item in selected),
+            "arrival_nondecreasing": all(
+                b.arrival_s >= a.arrival_s for a, b in zip(selected, selected[1:])
+            ),
+            "warmup_16": (len(outcomes) >= 16 if warmup else True),
+            "warmup_exact_16": (exact >= 16 if warmup else True),
+            "warmup_long": (
+                any(int(item.prompt_tokens_hint or 0) > 4096 for item in selected)
+                if warmup else True
+            ),
+        },
+    }
+    atomic_json(out / "result.json", result)
+    print(json.dumps({k: result[k] for k in ("cell", "anchor", "kind", "pass_rate", "feasible")}))
+    if not all(result["invariants"].values()):
+        raise RuntimeError(f"client invariants failed: {result['invariants']}")
+    return result
+
+
+def preflight(args: argparse.Namespace) -> None:
+    ground = json.loads(Path(args.ground_truth).read_text())
+    studies = {1: Path(args.primary_study), 2: Path(args.primary_study), 4: Path(args.tp4_study)}
+    loaded = {}
+    mismatches = []
+    values = []
+    for cell in ground["cells"]:
+        tp = int(cell["tensor_parallel_size"])
+        if tp not in loaded:
+            _study, _window, requests, _selected = load_selected(studies[tp], 0.0)
+            loaded[tp] = requests
+        for historical in cell["probe_history"]:
+            selected = select_requests_for_threshold(
+                loaded[tp], threshold=float(historical["sampling_u"])
+            )
+            values.append(len(selected))
+            if len(selected) != int(historical["request_count"]):
+                mismatches.append({
+                    "cell": cell["cell_id"], "anchor": historical["sampling_u"],
+                    "expected": historical["request_count"], "actual": len(selected),
+                })
+    result = {
+        "schema": 1, "observations": len(values), "mismatches": mismatches,
+        "request_counts": numeric(values),
+        "invariants": {"observations_92": len(values) == 92, "counts_match": not mismatches},
+    }
+    atomic_json(Path(args.out), result)
+    print(json.dumps(result, sort_keys=True))
+    if not all(result["invariants"].values()):
+        raise RuntimeError("preflight count reconstruction failed")
+
+
+def parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser()
+    sub = p.add_subparsers(dest="command", required=True)
+    pf = sub.add_parser("preflight")
+    pf.add_argument("--ground-truth", required=True)
+    pf.add_argument("--primary-study", required=True)
+    pf.add_argument("--tp4-study", required=True)
+    pf.add_argument("--out", required=True)
+    for name in ("warmup", "run-anchor"):
+        q = sub.add_parser(name)
+        q.add_argument("--study", required=True)
+        q.add_argument("--cell", required=True)
+        q.add_argument("--anchor", type=float, required=True)
+        q.add_argument("--tp", type=int, required=True)
+        q.add_argument("--mns", type=int, required=True)
+        q.add_argument("--base-url", required=True)
+        q.add_argument("--result-dir", required=True)
+    return p
+
+
+def main() -> None:
+    args = parser().parse_args()
+    if args.command == "preflight":
+        preflight(args)
+    else:
+        run_replay(args, warmup=args.command == "warmup")
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase6/opprof_phase6_controller.py

@@ -0,0 +1,492 @@
+#!/usr/bin/env python3
+"""Detached adaptive Phase-6 controller for the frozen 25-anchor surface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shlex
+import signal
+import subprocess
+import time
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712")
+RUN_ROOT = WORKDIR / "runs/phase6"
+STATE = RUN_ROOT / "controller-state.json"
+SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
+VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
+AITUNER = Path("/home/admin/cpfs/wjh/aituner/aituner")
+MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
+CLIENT = WORKDIR / "scripts/opprof_phase6_client.py"
+PRIMARY_STUDY = WORKDIR / "provenance/study-primary.json"
+TP4_STUDY = WORKDIR / "provenance/study-tp4.json"
+GROUND = WORKDIR / "provenance/ground_truth.json"
+GPU_LIMIT = 3.0
+CPU_MAP = {i: f"{20*i}-{20*i+19}" for i in range(8)}
+MARKER = "opprof-phase6-20260712"
+OWNED_PGIDS: set[int] = set()
+
+
+CELLS = {
+    "tp1_mns8": {"tp": 1, "mns": 8, "lower": .21875, "peak": .2265625, "upper": .23046875},
+    "tp1_mns16": {"tp": 1, "mns": 16, "lower": .2421875, "peak": .24609375, "upper": .25},
+    "tp1_mns32": {"tp": 1, "mns": 32, "lower": .234375, "peak": .2421875, "upper": .24609375},
+    "tp1_mns64": {"tp": 1, "mns": 64, "lower": .234375, "peak": .2421875, "upper": .24609375},
+    "tp2_mns8": {"tp": 2, "mns": 8, "lower": .4921875, "peak": .49609375, "upper": .5},
+    "tp2_mns16": {"tp": 2, "mns": 16, "lower": .4921875, "peak": .49609375, "upper": .5},
+    "tp2_mns32": {"tp": 2, "mns": 32, "lower": .75, "peak": .75390625, "upper": .7578125},
+    "tp2_mns64": {"tp": 2, "mns": 64, "lower": .5, "peak": .75, "upper": .75390625},
+    "tp4_mns8": {"tp": 4, "mns": 8, "lower": .016055910008, "peak": .016591107009, "upper": .017126304009},
+    "tp4_mns16": {"tp": 4, "mns": 16, "lower": .033182214016, "peak": .033717411016, "upper": .034252608017, "trap": True},
+    "tp4_mns32": {"tp": 4, "mns": 32, "lower": .033182214016, "peak": .033717411016, "upper": .034252608017},
+    "tp4_mns64": {"tp": 4, "mns": 64, "lower": .033182214016, "peak": .033717411016, "upper": .034252608017},
+}
+WAVES = [
+    ("W1-tp1", [("tp1_mns8", (0,)), ("tp1_mns16", (1,)), ("tp1_mns32", (2,)), ("tp1_mns64", (3,))], .35),
+    ("W2-tp2", [("tp2_mns8", (0,1)), ("tp2_mns16", (2,3)), ("tp2_mns32", (4,5)), ("tp2_mns64", (6,7))], .65),
+    ("W3-tp4-trap", [("tp4_mns8", (0,1,2,3)), ("tp4_mns16", (4,5,6,7))], .85),
+    ("W4-tp4", [("tp4_mns32", (0,1,2,3)), ("tp4_mns64", (4,5,6,7))], .65),
+]
+
+
+def atomic_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n")
+    os.replace(tmp, path)
+
+
+def load_state() -> dict[str, Any]:
+    if STATE.exists():
+        return json.loads(STATE.read_text())
+    return {
+        "schema": 1, "status": "initialized", "gpu_hours_total": 0.0,
+        "completed_primary_anchors": 0, "completed_confirmations": 0,
+        "waves": {}, "failures": [], "started_at": time.time(),
+    }
+
+
+def save_state(state: dict[str, Any]) -> None:
+    atomic_json(STATE, state)
+
+
+def cpu_mask(gpus: tuple[int, ...]) -> str:
+    return ",".join(CPU_MAP[g] for g in gpus)
+
+
+def study_for(tp: int) -> Path:
+    return TP4_STUDY if tp == 4 else PRIMARY_STUDY
+
+
+def run_text(command: list[str], check: bool = True) -> str:
+    result = subprocess.run(command, text=True, capture_output=True)
+    if check and result.returncode:
+        raise RuntimeError(f"command failed {command}: {result.stderr}")
+    return result.stdout
+
+
+def compute_pids() -> list[int]:
+    text = run_text([
+        "nvidia-smi", "--query-compute-apps=pid", "--format=csv,noheader,nounits"
+    ], check=False)
+    return sorted({int(x.strip()) for x in text.splitlines() if x.strip().isdigit()})
+
+
+def pid_owned(pid: int) -> bool:
+    try:
+        if os.getpgid(pid) in OWNED_PGIDS:
+            return True
+    except ProcessLookupError:
+        return True
+    try:
+        env = Path(f"/proc/{pid}/environ").read_bytes().split(b"\0")
+    except (FileNotFoundError, PermissionError):
+        return False
+    return f"OPPROF_PHASE6_MARKER={MARKER}".encode() in env
+
+
+def assert_no_other_compute() -> None:
+    other = [pid for pid in compute_pids() if not pid_owned(pid)]
+    if other:
+        raise RuntimeError(f"outside GPU processes detected: {other}")
+
+
+def assert_all_idle() -> None:
+    if compute_pids():
+        raise RuntimeError(f"GPU compute processes remain: {compute_pids()}")
+    rows = run_text([
+        "nvidia-smi", "--query-gpu=index,memory.used,utilization.gpu",
+        "--format=csv,noheader,nounits",
+    ])
+    bad = []
+    for line in rows.splitlines():
+        index, memory, util = [int(x.strip()) for x in line.split(",")]
+        if memory != 0 or util != 0:
+            bad.append((index, memory, util))
+    if bad:
+        raise RuntimeError(f"GPU cleanup failure: {bad}")
+
+
+def wait_ready(entry: dict[str, Any], timeout: float = 300.0) -> None:
+    deadline = time.monotonic() + timeout
+    url = f"http://127.0.0.1:{entry['port']}/v1/models"
+    while time.monotonic() < deadline:
+        if entry["server"].poll() is not None:
+            raise RuntimeError(f"server exited before ready: {entry['cell']}")
+        try:
+            with urllib.request.urlopen(url, timeout=2) as response:
+                if response.status < 500:
+                    return
+        except Exception:
+            pass
+        assert_no_other_compute()
+        time.sleep(1)
+    raise TimeoutError(f"server ready timeout: {entry['cell']}")
+
+
+def server_command(cell: str, gpus: tuple[int, ...], port: int) -> list[str]:
+    cfg = CELLS[cell]
+    return [
+        "taskset", "-c", cpu_mask(gpus), str(VENV / "bin/vllm"), "serve", str(MODEL),
+        "--host", "127.0.0.1", "--port", str(port),
+        "--served-model-name", "qwen3-30b-a3b-community",
+        "--max-num-batched-tokens", "8192", "--max-num-seqs", str(cfg["mns"]),
+        "--tensor-parallel-size", str(cfg["tp"]), "--shutdown-timeout", "120",
+    ]
+
+
+def client_command(entry: dict[str, Any], anchor: float, out: Path, warmup: bool) -> list[str]:
+    cfg = CELLS[entry["cell"]]
+    return [
+        "taskset", "-c", cpu_mask(entry["gpus"]), str(VENV / "bin/python"), str(CLIENT),
+        "warmup" if warmup else "run-anchor", "--study", str(study_for(cfg["tp"])),
+        "--cell", entry["cell"], "--anchor", str(anchor), "--tp", str(cfg["tp"]),
+        "--mns", str(cfg["mns"]), "--base-url", f"http://127.0.0.1:{entry['port']}",
+        "--result-dir", str(out),
+    ]
+
+
+def live_gpu_hours(entries: list[dict[str, Any]]) -> float:
+    now = time.time()
+    return sum(
+        len(e["gpus"]) * ((e.get("stopped_at") or now) - e["spawned_at"])
+        for e in entries
+    ) / 3600
+
+
+def run_clients(
+    entries: list[dict[str, Any]], assignments: list[tuple[dict[str, Any], float, Path]],
+    state: dict[str, Any], wave_name: str, warmup: bool = False,
+) -> list[dict[str, Any]]:
+    processes = []
+    handles = []
+    for entry, anchor, out in assignments:
+        command = client_command(entry, anchor, out, warmup)
+        with (entry["dir"] / "commands.log").open("a") as f:
+            f.write(f"CLIENT {shlex.join(command)}\n")
+        handle = (out.parent / f"{out.name}.log").open("ab", buffering=0)
+        handles.append(handle)
+        client_env = os.environ.copy()
+        client_env.update({"AITUNER_ROOT": str(AITUNER), "PYTHONUNBUFFERED": "1"})
+        p = subprocess.Popen(
+            command, cwd=WORKDIR, env=client_env, stdout=handle,
+            stderr=subprocess.STDOUT, start_new_session=True,
+        )
+        processes.append((entry, anchor, out, p))
+    deadline = time.monotonic() + 180
+    while any(p.poll() is None for *_rest, p in processes):
+        if time.monotonic() > deadline:
+            raise TimeoutError(f"client batch timeout: {wave_name}")
+        for entry in entries:
+            if entry.get("stopped_at") is None and entry["server"].poll() is not None:
+                raise RuntimeError(f"server exited during client: {entry['cell']}")
+        assert_no_other_compute()
+        if state["gpu_hours_total"] + live_gpu_hours(entries) >= GPU_LIMIT:
+            raise RuntimeError("3.0 H20-hour hard stop reached")
+        time.sleep(1)
+    for handle in handles:
+        handle.close()
+    bad = [(e["cell"], a, p.returncode) for e, a, _o, p in processes if p.returncode]
+    if bad:
+        raise RuntimeError(f"client failures: {bad}")
+    results = []
+    for entry, anchor, out, _p in processes:
+        result = json.loads((out / "result.json").read_text())
+        results.append(result)
+        entry.setdefault("results", []).append({"anchor": anchor, "dir": str(out), "kind": result["kind"]})
+    return results
+
+
+def stop_entry(entry: dict[str, Any]) -> None:
+    if entry.get("stopped_at") is not None:
+        return
+    process = entry["server"]
+    try:
+        # Official vLLM shutdown: signal the API parent so EngineCore drains and
+        # emits the in-stream footer/final sidecar. Process-group signals are
+        # fallback only.
+        os.kill(process.pid, signal.SIGINT)
+    except ProcessLookupError:
+        pass
+    try:
+        process.wait(timeout=150)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(process.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+        try:
+            process.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+            process.wait(timeout=30)
+    entry["stopped_at"] = time.time()
+    entry["server_handle"].close()
+
+
+def validate_cell(entry: dict[str, Any]) -> dict[str, Any]:
+    log = (entry["dir"] / "server.log").read_text(errors="replace").splitlines()
+    ready = [i for i, line in enumerate(log) if "Application startup complete" in line]
+    event = re.compile(r"torch\.compile took|Directly load AOT compilation|\bCompiling\b|Capturing CUDA graphs", re.I)
+    post_ready_events = [i + 1 for i, line in enumerate(log) if event.search(line) and ready and i > ready[0]]
+    streams = sorted((entry["dir"] / "opprof").glob("*.jsonl"))
+    sidecars = sorted((entry["dir"] / "opprof").glob("*.jsonl.footer.json"))
+    if len(streams) != 1 or len(sidecars) != 1:
+        raise RuntimeError(f"Layer1 stream/sidecar mismatch: {entry['cell']}")
+    decoded = [json.loads(line) for line in streams[0].read_text().splitlines()]
+    footers = [item for item in decoded if item.get("record_type") == "footer"]
+    records = [item for item in decoded if "step_index" in item]
+    sidecar = json.loads(sidecars[0].read_text())
+    indices = [int(item["step_index"]) for item in records]
+    warm = json.loads((entry["dir"] / "warmup/result.json").read_text())
+    intervals_ok = True
+    for item in entry.get("results", []):
+        if item["kind"] != "anchor":
+            continue
+        result = json.loads((Path(item["dir"]) / "result.json").read_text())
+        lo, hi = result["interval"]["start_mono_ns"], result["interval"]["end_mono_ns"]
+        intervals_ok &= any(lo <= int(r["submit_mono_ns"]) <= hi for r in records)
+    common = {
+        "one_ready_marker": len(ready) == 1,
+        "compile_capture_pre_ready": not post_ready_events,
+        "warmup_exact_16": warm["exact_output_count"] >= 16,
+        "warmup_long": warm["selection"]["long_gt4096"] >= 1,
+        "layer1_contiguous": indices == list(range(len(indices))),
+        "written_matches_records": sidecar.get("written_records") == len(records),
+        "encoded_balanced": sidecar.get("encoded_records") == sidecar.get("written_records") + sidecar.get("dropped_records"),
+        "last_step_matches": bool(records) and sidecar.get("last_step_index") == records[-1]["step_index"],
+        "layer1_zero_drops": sidecar.get("dropped_records") == 0,
+        "anchor_intervals_present": intervals_ok,
+    }
+    if footers:
+        accounting = {
+            "one_footer_last": len(footers) == 1 and decoded[-1] is footers[0],
+            "sidecar_final": sidecar.get("final") is True,
+            "footer_sidecar_agrees": all(
+                footers[0].get(key) == sidecar.get(key)
+                for key in ("encoded_records", "written_records", "dropped_records")
+            ),
+        }
+        accounting_mode = "graceful-footer"
+    else:
+        delta = abs(streams[0].stat().st_mtime_ns - int(sidecar["checkpoint_wall_ns"])) / 1e9
+        accounting = {
+            "checkpoint_sidecar": sidecar.get("final") is False,
+            "checkpoint_within_flush_of_stream": delta <= float(sidecar["flush_interval_seconds"]) + .1,
+        }
+        accounting_mode = "checkpoint-sidecar-fallback"
+    invariants = {**common, **accounting}
+    if not all(invariants.values()):
+        raise RuntimeError(f"cell validity failure {entry['cell']}: {invariants}")
+    result = {
+        "cell": entry["cell"], "invariants": invariants, "layer1_records": len(records),
+        "stream": str(streams[0]), "post_ready_capture_events": post_ready_events,
+        "accounting_mode": accounting_mode,
+    }
+    atomic_json(entry["dir"] / "cell-valid.json", result)
+    return result
+
+
+def historical_expected() -> dict[tuple[str, float], dict[str, Any]]:
+    ground = json.loads(GROUND.read_text())
+    result = {}
+    for cell in ground["cells"]:
+        for probe in cell["probe_history"]:
+            result[(cell["cell_id"], float(probe["sampling_u"]))] = probe
+    return result
+
+
+def execute_wave(index: int, state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]]) -> None:
+    wave_name, assignments, estimate = WAVES[index]
+    if state["waves"].get(wave_name, {}).get("status") == "complete":
+        return
+    future = sum(w[2] for w in WAVES[index:]) + .10
+    if state["gpu_hours_total"] + future >= GPU_LIMIT:
+        raise RuntimeError(f"projected budget exceeds cap before {wave_name}: {state['gpu_hours_total']+future}")
+    echo = (
+        f"WAVE_ECHO wave={wave_name} assignments="
+        + ",".join(f"{cell}:gpu{'+'.join(map(str, gpus))}" for cell, gpus in assignments)
+        + f" spent_h20h={state['gpu_hours_total']:.6f} wave_est_h20h={estimate:.3f} "
+        + f"remaining_projection_h20h={future:.3f} cap_h20h={GPU_LIMIT:.1f} "
+        + f"ground_truth={GROUND} workload=chat_w20260311_1000.jsonl"
+    )
+    with (RUN_ROOT / "launch-echo.log").open("a") as handle:
+        handle.write(echo + "\n")
+    print(echo, flush=True)
+    assert_all_idle()
+    wave_dir = RUN_ROOT / "waves" / wave_name
+    wave_dir.mkdir(parents=True, exist_ok=True)
+    entries = []
+    state["status"] = "running"
+    state["waves"][wave_name] = {"status": "starting", "estimate_h20_hours": estimate, "started_at": time.time()}
+    save_state(state)
+    failure = None
+    try:
+        for offset, (cell, gpus) in enumerate(assignments):
+            cell_dir = RUN_ROOT / "cells" / cell
+            cell_dir.mkdir(parents=True, exist_ok=True)
+            port = 8500 + index * 10 + offset
+            command = server_command(cell, gpus, port)
+            with (cell_dir / "commands.log").open("a") as f:
+                f.write(f"SERVER {shlex.join(command)}\n")
+            handle = (cell_dir / "server.log").open("ab", buffering=0)
+            env = os.environ.copy()
+            env.update({
+                "CUDA_VISIBLE_DEVICES": ",".join(map(str, gpus)),
+                "VLLM_OPPROF_DIR": str(cell_dir / "opprof"),
+                "OPPROF_PHASE6_MARKER": MARKER, "AITUNER_ROOT": str(AITUNER),
+                "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
+            })
+            server = subprocess.Popen(command, cwd=SOURCE, env=env, stdout=handle, stderr=subprocess.STDOUT, start_new_session=True)
+            OWNED_PGIDS.add(server.pid)
+            entries.append({
+                "cell": cell, "gpus": gpus, "port": port, "dir": cell_dir,
+                "server": server, "server_handle": handle, "spawned_at": time.time(),
+            })
+        for entry in entries:
+            wait_ready(entry)
+        state["waves"][wave_name]["status"] = "warmup"
+        save_state(state)
+        run_clients(entries, [
+            (e, CELLS[e["cell"]]["peak"], e["dir"] / "warmup") for e in entries
+        ], state, wave_name, warmup=True)
+        state["waves"][wave_name]["status"] = "peaks"
+        save_state(state)
+        peaks = run_clients(entries, [
+            (e, CELLS[e["cell"]]["peak"], e["dir"] / f"anchor-{CELLS[e['cell']]['peak']}")
+            for e in entries
+        ], state, wave_name)
+        for result in peaks:
+            old = expected[(result["cell"], float(result["anchor"]))]
+            if result["selection"]["count"] != old["request_count"]:
+                raise RuntimeError(f"selection mismatch {result['cell']} peak")
+        neighbor_jobs = []
+        for entry, result in zip(entries, peaks, strict=True):
+            key = "upper" if result["feasible"] else "lower"
+            anchor = CELLS[entry["cell"]][key]
+            neighbor_jobs.append((entry, anchor, entry["dir"] / f"anchor-{anchor}"))
+        neighbors = run_clients(entries, neighbor_jobs, state, wave_name)
+        for result in neighbors:
+            old = expected[(result["cell"], float(result["anchor"]))]
+            if result["selection"]["count"] != old["request_count"]:
+                raise RuntimeError(f"selection mismatch {result['cell']} neighbor")
+        trap_entry = next((e for e in entries if CELLS[e["cell"]].get("trap")), None)
+        if trap_entry is not None:
+            used = {float(item["anchor"]) for item in trap_entry["results"] if item["kind"] == "anchor"}
+            extra = next(CELLS[trap_entry["cell"]][k] for k in ("lower", "upper") if CELLS[trap_entry["cell"]][k] not in used)
+            extra_result = run_clients(entries, [(trap_entry, extra, trap_entry["dir"] / f"anchor-{extra}")], state, wave_name)[0]
+            if extra_result["selection"]["count"] != expected[(extra_result["cell"], float(extra))]["request_count"]:
+                raise RuntimeError("trap extra selection mismatch")
+
+        # Confirm only protocol-triggered anchors while the relevant server is hot.
+        triggers = []
+        for entry in entries:
+            for item in entry.get("results", []):
+                if item["kind"] != "anchor":
+                    continue
+                result = json.loads((Path(item["dir"]) / "result.json").read_text())
+                old = expected[(entry["cell"], float(result["anchor"]))]
+                flip = bool(result["feasible"]) != bool(old["feasible"])
+                if .93 <= float(result["pass_rate"]) <= .97 or (entry["cell"] in {"tp2_mns32", "tp4_mns16"} and flip):
+                    priority = 0 if entry["cell"] == "tp2_mns32" else (1 if entry["cell"] == "tp4_mns16" else 2)
+                    triggers.append((priority, entry, float(result["anchor"])))
+        triggers.sort(key=lambda x: x[0])
+        for _priority, entry, anchor in triggers:
+            projected_extra = len(entry["gpus"]) * 80 / 3600
+            future_primary = sum(w[2] for w in WAVES[index + 1:])
+            if state["gpu_hours_total"] + live_gpu_hours(entries) + future_primary + projected_extra + .03 >= GPU_LIMIT:
+                state.setdefault("unconfirmed_triggers", []).append({"cell": entry["cell"], "anchor": anchor})
+                continue
+            confirm_index = 1 + sum(1 for item in entry.get("results", []) if item["kind"] == "anchor" and Path(item["dir"]).name.startswith("confirm"))
+            out = entry["dir"] / f"confirm-{confirm_index}-anchor-{anchor}"
+            run_clients(entries, [(entry, anchor, out)], state, wave_name)
+            state["completed_confirmations"] += 1
+        state["waves"][wave_name]["status"] = "stopping"
+        save_state(state)
+    except Exception as error:
+        failure = error
+    finally:
+        for entry in entries:
+            try:
+                stop_entry(entry)
+            except Exception as error:
+                failure = failure or error
+        time.sleep(2)
+        try:
+            assert_all_idle()
+        except Exception as error:
+            failure = failure or error
+    wave_hours = live_gpu_hours(entries)
+    state["gpu_hours_total"] += wave_hours
+    state["waves"][wave_name]["gpu_hours"] = wave_hours
+    if failure is not None:
+        state["waves"][wave_name]["status"] = "failed"
+        state["waves"][wave_name]["failure"] = repr(failure)
+        state["status"] = "failed"
+        state["failures"].append({"wave": wave_name, "failure": repr(failure)})
+        save_state(state)
+        raise failure
+    validations = [validate_cell(entry) for entry in entries]
+    primary_count = sum(
+        1 for entry in entries for item in entry.get("results", [])
+        if item["kind"] == "anchor" and not Path(item["dir"]).name.startswith("confirm")
+    )
+    state["completed_primary_anchors"] += primary_count
+    state["waves"][wave_name].update({
+        "status": "complete", "completed_at": time.time(), "primary_anchors": primary_count,
+        "validations": validations,
+    })
+    save_state(state)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--resume", action="store_true")
+    args = parser.parse_args()
+    RUN_ROOT.mkdir(parents=True, exist_ok=True)
+    state = load_state()
+    expected = historical_expected()
+    state["status"] = "running"
+    save_state(state)
+    for index in range(len(WAVES)):
+        execute_wave(index, state, expected)
+    state["status"] = "primary_complete"
+    state["completed_at"] = time.time()
+    save_state(state)
+    print(json.dumps({
+        "status": state["status"], "primary_anchors": state["completed_primary_anchors"],
+        "confirmations": state["completed_confirmations"], "gpu_hours": state["gpu_hours_total"],
+    }, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase6/opprof_phase6_solo_controller.py

@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+"""A-P6-2 serialized solo controller for authoritative Phase-6 frontiers."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import shlex
+import subprocess
+import time
+from pathlib import Path
+from typing import Any
+
+import opprof_phase6_controller as base
+
+
+SOLO_ROOT = base.RUN_ROOT / "solo-authoritative"
+STATE = SOLO_ROOT / "controller-state.json"
+CAMPAIGN_STATE = base.RUN_ROOT / "controller-state.json"
+GPU_LIMIT = 6.0
+SAFETY_HOURS = 0.20
+MARKER = "opprof-phase6-solo-A-P6-2"
+TRACE = base.AITUNER / "trace_windows/traces/chat_w20260311_1000.jsonl"
+
+ORDER = [
+    "tp4_mns32", "tp4_mns64", "tp2_mns32", "tp2_mns64",
+    "tp4_mns16", "tp2_mns8", "tp2_mns16", "tp4_mns8",
+    "tp1_mns8", "tp1_mns16", "tp1_mns32", "tp1_mns64",
+]
+
+# Exact co-located primaries are remeasured before adaptive crawl. W4 had no
+# prior primary, so it starts at P and selects L/U from the solo result.
+CORE = {
+    "tp1_mns8": [base.CELLS["tp1_mns8"]["peak"], base.CELLS["tp1_mns8"]["lower"]],
+    "tp1_mns16": [base.CELLS["tp1_mns16"]["peak"], base.CELLS["tp1_mns16"]["upper"]],
+    "tp1_mns32": [base.CELLS["tp1_mns32"]["peak"], base.CELLS["tp1_mns32"]["upper"]],
+    "tp1_mns64": [base.CELLS["tp1_mns64"]["peak"], base.CELLS["tp1_mns64"]["upper"]],
+    "tp2_mns8": [base.CELLS["tp2_mns8"]["peak"], base.CELLS["tp2_mns8"]["lower"]],
+    "tp2_mns16": [base.CELLS["tp2_mns16"]["peak"], base.CELLS["tp2_mns16"]["lower"]],
+    "tp2_mns32": [base.CELLS["tp2_mns32"]["peak"], base.CELLS["tp2_mns32"]["lower"]],
+    "tp2_mns64": [base.CELLS["tp2_mns64"]["peak"], base.CELLS["tp2_mns64"]["lower"]],
+    "tp4_mns8": [base.CELLS["tp4_mns8"]["peak"], base.CELLS["tp4_mns8"]["lower"]],
+    "tp4_mns16": [
+        base.CELLS["tp4_mns16"]["peak"], base.CELLS["tp4_mns16"]["lower"],
+        base.CELLS["tp4_mns16"]["upper"],
+    ],
+    "tp4_mns32": [base.CELLS["tp4_mns32"]["peak"]],
+    "tp4_mns64": [base.CELLS["tp4_mns64"]["peak"]],
+}
+
+CELL_ESTIMATE = {cell: {1: .11, 2: .22, 4: .48}[cfg["tp"]] for cell, cfg in base.CELLS.items()}
+
+
+def atomic_json(path: Path, value: Any) -> None:
+    base.atomic_json(path, value)
+
+
+def load_state() -> dict[str, Any]:
+    if STATE.exists():
+        return json.loads(STATE.read_text())
+    campaign = json.loads(CAMPAIGN_STATE.read_text())
+    return {
+        "schema": 1, "amendment": "A-P6-2", "status": "initialized",
+        "hard_cap_h20_hours": GPU_LIMIT,
+        "prior_h20_hours": float(campaign["gpu_hours_total"]),
+        "gpu_hours_total": float(campaign["gpu_hours_total"]),
+        "solo_gpu_hours": 0.0, "completed_cells": 0,
+        "primary_anchors": 0, "confirmations": 0,
+        "cells": {}, "failures": [], "started_at": time.time(),
+    }
+
+
+def save_state(state: dict[str, Any]) -> None:
+    atomic_json(STATE, state)
+
+
+def historical() -> tuple[dict[tuple[str, float], dict[str, Any]], dict[str, list[float]]]:
+    ground = json.loads(base.GROUND.read_text())
+    expected = {}
+    histories = {}
+    for cell in ground["cells"]:
+        anchors = []
+        for probe in cell["probe_history"]:
+            anchor = float(probe["sampling_u"])
+            expected[(cell["cell_id"], anchor)] = probe
+            anchors.append(anchor)
+        histories[cell["cell_id"]] = sorted(anchors)
+    return expected, histories
+
+
+def same_anchor(left: float, right: float) -> bool:
+    return math.isclose(left, right, rel_tol=0, abs_tol=1e-15)
+
+
+def colocated_primary(cell: str, anchor: float) -> dict[str, Any] | None:
+    cell_dir = base.RUN_ROOT / "cells" / cell
+    for path in cell_dir.glob("anchor-*/result.json"):
+        item = json.loads(path.read_text())
+        if same_anchor(float(item["anchor"]), anchor):
+            return item
+    return None
+
+
+def append_echo(line: str) -> None:
+    SOLO_ROOT.mkdir(parents=True, exist_ok=True)
+    with (SOLO_ROOT / "launch-echo.log").open("a") as handle:
+        handle.write(line + "\n")
+    print(line, flush=True)
+
+
+def wait_all_idle(timeout: float = 30.0) -> None:
+    deadline = time.monotonic() + timeout
+    error = None
+    while time.monotonic() < deadline:
+        try:
+            base.assert_all_idle()
+            return
+        except RuntimeError as current:
+            error = current
+            time.sleep(1)
+    raise error or RuntimeError("GPU cleanup did not reach idle")
+
+
+def remaining_projection(index: int) -> float:
+    return sum(CELL_ESTIMATE[cell] for cell in ORDER[index:]) + SAFETY_HOURS
+
+
+def start_entry(cell: str, index: int) -> dict[str, Any]:
+    cfg = base.CELLS[cell]
+    gpus = tuple(range(int(cfg["tp"])))
+    cell_dir = SOLO_ROOT / "cells" / cell
+    cell_dir.mkdir(parents=True, exist_ok=True)
+    port = 8700 + index
+    command = base.server_command(cell, gpus, port)
+    with (cell_dir / "commands.log").open("a") as handle:
+        handle.write(f"SERVER {shlex.join(command)}\n")
+    server_handle = (cell_dir / "server.log").open("ab", buffering=0)
+    env = os.environ.copy()
+    env.update({
+        "CUDA_VISIBLE_DEVICES": ",".join(map(str, gpus)),
+        "VLLM_OPPROF_DIR": str(cell_dir / "opprof"),
+        "OPPROF_PHASE6_MARKER": MARKER, "AITUNER_ROOT": str(base.AITUNER),
+        "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
+    })
+    server = subprocess.Popen(
+        command, cwd=base.SOURCE, env=env, stdout=server_handle,
+        stderr=subprocess.STDOUT, start_new_session=True,
+    )
+    base.OWNED_PGIDS.add(server.pid)
+    return {
+        "cell": cell, "gpus": gpus, "port": port, "dir": cell_dir,
+        "server": server, "server_handle": server_handle,
+        "spawned_at": time.time(), "results": [],
+    }
+
+
+def run_one(
+    entry: dict[str, Any], anchor: float, out: Path, state: dict[str, Any],
+    cell_state: dict[str, Any], role: str,
+) -> dict[str, Any]:
+    result = base.run_clients(
+        [entry], [(entry, anchor, out)], state, f"solo-{entry['cell']}"
+    )[0]
+    expected_count = cell_state["expected_counts"][str(anchor)]
+    if int(result["selection"]["count"]) != int(expected_count):
+        raise RuntimeError(
+            f"selection mismatch {entry['cell']} {anchor}: "
+            f"{result['selection']['count']} != {expected_count}"
+        )
+    cell_state.setdefault("runs", []).append({
+        "anchor": anchor, "role": role, "dir": str(out),
+        "pass_rate": result["pass_rate"], "feasible": result["feasible"],
+    })
+    save_state(state)
+    return result
+
+
+def anchor_trials(cell_state: dict[str, Any], anchor: float) -> list[dict[str, Any]]:
+    return [
+        item for item in cell_state.get("runs", [])
+        if same_anchor(float(item["anchor"]), anchor)
+    ]
+
+
+def accepted_feasible(cell_state: dict[str, Any], anchor: float) -> bool | None:
+    trials = anchor_trials(cell_state, anchor)
+    votes = [bool(item["feasible"]) for item in trials]
+    if not votes:
+        return None
+    if len(votes) == 1 or len(set(votes)) == 1:
+        return votes[0]
+    if len(votes) >= 3:
+        return sum(votes) >= 2
+    return None
+
+
+def optional_fits(
+    state: dict[str, Any], entry: dict[str, Any], future_after: float,
+) -> bool:
+    replay = len(entry["gpus"]) * 80 / 3600
+    projected = (
+        float(state["gpu_hours_total"]) + base.live_gpu_hours([entry])
+        + future_after + replay + SAFETY_HOURS
+    )
+    return projected < GPU_LIMIT
+
+
+def maybe_confirm(
+    entry: dict[str, Any], anchor: float, primary: dict[str, Any],
+    state: dict[str, Any], cell_state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]],
+    future_after: float,
+) -> None:
+    old = expected[(entry["cell"], anchor)]
+    coloc = colocated_primary(entry["cell"], anchor)
+    disagreement = (
+        bool(primary["feasible"]) != bool(old["feasible"])
+        or (coloc is not None and bool(primary["feasible"]) != bool(coloc["feasible"]))
+    )
+    boundary = .93 <= float(primary["pass_rate"]) <= .97
+    if not (disagreement or boundary):
+        return
+    while len(anchor_trials(cell_state, anchor)) < 3:
+        trials = anchor_trials(cell_state, anchor)
+        if len(trials) >= 2 and len({bool(item["feasible"]) for item in trials}) == 1:
+            return
+        if not optional_fits(state, entry, future_after):
+            cell_state.setdefault("deferred_confirmations", []).append(anchor)
+            return
+        trial = len(trials) + 1
+        out = entry["dir"] / f"confirm-{trial - 1}-anchor-{anchor}"
+        run_one(entry, anchor, out, state, cell_state, f"confirmation-{trial}")
+        state["confirmations"] += 1
+
+
+def run_primary(
+    entry: dict[str, Any], anchor: float, state: dict[str, Any],
+    cell_state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]],
+    future_after: float, role: str,
+) -> dict[str, Any]:
+    existing = [item for item in cell_state.get("runs", []) if item["role"].startswith("primary")]
+    if any(same_anchor(float(item["anchor"]), anchor) for item in existing):
+        path = next(
+            Path(item["dir"]) for item in existing
+            if same_anchor(float(item["anchor"]), anchor)
+        )
+        return json.loads((path / "result.json").read_text())
+    out = entry["dir"] / f"anchor-{anchor}"
+    result = run_one(entry, anchor, out, state, cell_state, role)
+    state["primary_anchors"] += 1
+    maybe_confirm(entry, anchor, result, state, cell_state, expected, future_after)
+    return result
+
+
+def next_below(history: list[float], tested: set[float]) -> float | None:
+    if not tested:
+        return None
+    candidates = [x for x in history if x < min(tested) and x not in tested]
+    return max(candidates) if candidates else None
+
+
+def next_above(history: list[float], tested: set[float]) -> float | None:
+    if not tested:
+        return None
+    candidates = [x for x in history if x > max(tested) and x not in tested]
+    return min(candidates) if candidates else None
+
+
+def execute_cell(
+    index: int, cell: str, state: dict[str, Any],
+    expected: dict[tuple[str, float], dict[str, Any]], histories: dict[str, list[float]],
+) -> None:
+    if state["cells"].get(cell, {}).get("status") == "complete":
+        return
+    future = remaining_projection(index)
+    if float(state["gpu_hours_total"]) + future >= GPU_LIMIT:
+        state["status"] = "budget_projection_stop"
+        state["budget_stop"] = {
+            "before_cell": cell, "spent_h20_hours": state["gpu_hours_total"],
+            "remaining_projection_h20_hours": future,
+            "projected_total_h20_hours": state["gpu_hours_total"] + future,
+            "hard_cap_h20_hours": GPU_LIMIT,
+        }
+        save_state(state)
+        raise RuntimeError(f"projected budget exceeds cap before {cell}")
+    cfg = base.CELLS[cell]
+    echo = (
+        f"SOLO_WAVE_ECHO cell={cell} tp={cfg['tp']} mns={cfg['mns']} "
+        f"gpus=0-{int(cfg['tp'])-1} mandatory={','.join(map(str, CORE[cell]))} "
+        f"spent_h20h={state['gpu_hours_total']:.6f} cell_est_h20h={CELL_ESTIMATE[cell]:.3f} "
+        f"remaining_projection_h20h={future:.3f} cap_h20h={GPU_LIMIT:.1f} "
+        f"ground_truth={base.GROUND} trace={TRACE}"
+    )
+    append_echo(echo)
+    wait_all_idle()
+    cell_state = {
+        "status": "starting", "started_at": time.time(), "tp": cfg["tp"], "mns": cfg["mns"],
+        "mandatory": CORE[cell],
+        "expected_counts": {
+            str(anchor): expected[(cell, anchor)]["request_count"] for anchor in histories[cell]
+        },
+        "runs": [],
+    }
+    state["status"] = "running"
+    state["cells"][cell] = cell_state
+    save_state(state)
+    entry = start_entry(cell, index)
+    failure = None
+    future_after = sum(CELL_ESTIMATE[item] for item in ORDER[index + 1:])
+    try:
+        base.wait_ready(entry)
+        cell_state["status"] = "warmup"
+        save_state(state)
+        warm = base.run_clients(
+            [entry], [(entry, cfg["peak"], entry["dir"] / "warmup")],
+            state, f"solo-{cell}", warmup=True,
+        )[0]
+        cell_state["warmup"] = {
+            "exact_output_count": warm["exact_output_count"],
+            "long_gt4096": warm["selection"]["long_gt4096"],
+        }
+        cell_state["status"] = "mandatory"
+        save_state(state)
+        for anchor in CORE[cell]:
+            run_primary(entry, anchor, state, cell_state, expected, future_after, "primary-mandatory")
+
+        peak = float(cfg["peak"])
+        peak_vote = accepted_feasible(cell_state, peak)
+        if cell in {"tp4_mns32", "tp4_mns64"} and peak_vote is not None:
+            direction = float(cfg["upper"] if peak_vote else cfg["lower"])
+            run_primary(entry, direction, state, cell_state, expected, future_after, "primary-direction")
+
+        cell_state["status"] = "crawl"
+        save_state(state)
+        while True:
+            primary_anchors = {
+                float(item["anchor"]) for item in cell_state["runs"]
+                if item["role"].startswith("primary")
+            }
+            votes = {anchor: accepted_feasible(cell_state, anchor) for anchor in primary_anchors}
+            pass_anchors = [anchor for anchor, vote in votes.items() if vote is True]
+            fail_anchors = [anchor for anchor, vote in votes.items() if vote is False]
+            if pass_anchors and fail_anchors and max(pass_anchors) < min(fail_anchors):
+                break
+            if any(vote is None for vote in votes.values()):
+                cell_state["censor"] = "UNRESOLVED_SOLO_ANCHOR"
+                break
+            if pass_anchors and not fail_anchors:
+                candidate = next_above(histories[cell], primary_anchors)
+            elif fail_anchors and not pass_anchors:
+                candidate = next_below(histories[cell], primary_anchors)
+            else:
+                # Non-monotonic anchors already contain both states but no valid bracket.
+                cell_state["censor"] = "NONMONOTONIC_SOLO_ANCHORS"
+                break
+            if candidate is None:
+                cell_state["censor"] = "HISTORY_EDGE"
+                break
+            if not optional_fits(state, entry, future_after):
+                cell_state["censor"] = "BUDGET_CENSORED"
+                break
+            run_primary(entry, candidate, state, cell_state, expected, future_after, "primary-crawl")
+        cell_state["status"] = "stopping"
+        save_state(state)
+    except Exception as error:
+        failure = error
+    finally:
+        try:
+            base.stop_entry(entry)
+        except Exception as error:
+            failure = failure or error
+        time.sleep(2)
+        try:
+            wait_all_idle()
+        except Exception as error:
+            failure = failure or error
+    hours = base.live_gpu_hours([entry])
+    state["gpu_hours_total"] += hours
+    state["solo_gpu_hours"] += hours
+    cell_state["gpu_hours"] = hours
+    if failure is not None:
+        cell_state["status"] = "failed"
+        cell_state["failure"] = repr(failure)
+        state["status"] = "failed"
+        state["failures"].append({"cell": cell, "failure": repr(failure)})
+        save_state(state)
+        raise failure
+    validation = base.validate_cell(entry)
+    cell_state["validation"] = validation
+    cell_state["status"] = "complete"
+    cell_state["completed_at"] = time.time()
+    state["completed_cells"] += 1
+    save_state(state)
+
+
+def main() -> None:
+    SOLO_ROOT.mkdir(parents=True, exist_ok=True)
+    base.GPU_LIMIT = GPU_LIMIT
+    base.MARKER = MARKER
+    expected, histories = historical()
+    state = load_state()
+    state["status"] = "running"
+    save_state(state)
+    for index, cell in enumerate(ORDER):
+        execute_cell(index, cell, state, expected, histories)
+    state["status"] = "complete"
+    state["completed_at"] = time.time()
+    save_state(state)
+    print(json.dumps({
+        "status": state["status"], "cells": state["completed_cells"],
+        "primary_anchors": state["primary_anchors"],
+        "confirmations": state["confirmations"],
+        "solo_gpu_hours": state["solo_gpu_hours"],
+        "campaign_gpu_hours": state["gpu_hours_total"],
+    }, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/opprof-phase6/p3-reference/primary/P10-C00/moderate/client/result.json

@@ -0,0 +1,103 @@
+{
+  "admission_stop_s": 363.15318555399426,
+  "arrival": "steady",
+  "clean": {
+    "admitted": 113,
+    "completed": 111,
+    "completed_throughput_rps": 0.4625,
+    "duration_s": 240.0,
+    "end_s": 300.0,
+    "failed": 0,
+    "input_tokens": 968714,
+    "offered_rps": 0.4708333333333333,
+    "output_tokens": 25409,
+    "start_s": 60.0
+  },
+  "clean_segment_seconds": 80.0,
+  "drain_seconds": 1.3658369119802956,
+  "elapsed_seconds": 364.51902246597456,
+  "failed_records": 0,
+  "load_point": "moderate",
+  "manifest_admitted": 172,
+  "manifest_exhausted": false,
+  "manifest_rows": 4011,
+  "manifest_sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33",
+  "manifest_wrapped": false,
+  "max_in_flight": 7,
+  "num_clean_segments": 3,
+  "profiles": [
+    {
+      "start_call_s": 300.00394913199125,
+      "start_return_s": 300.03973603399936,
+      "start_status": 200,
+      "stop_call_s": 300.89519165997626,
+      "stop_return_s": 301.6286224719952,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783849466906401944.pt.trace.json.gz",
+      "trace_ready_s": 300.8951868820004,
+      "trace_sha256": "a15670f86d843d40e7c4b6f354dd0193ea1259824a96440392d923f5270ba095",
+      "window": 1
+    },
+    {
+      "start_call_s": 331.62960390897933,
+      "start_return_s": 331.63382844498847,
+      "start_status": 200,
+      "stop_call_s": 332.4971265429922,
+      "stop_return_s": 333.15040952397976,
+      "stop_status": 200,
+      "trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783849498465727706.pt.trace.json.gz",
+      "trace_ready_s": 332.4971234249824,
+      "trace_sha256": "0f5962e193e09a9b2f1df6cea23f92419dcf2b322ee4ef2135a4fc3e1fe0617c",
+      "window": 2
+    }
+  ],
+  "rate_fraction": 0.6,
+  "records": 172,
+  "request_rate": 0.4725,
+  "schema": 1,
+  "segments": [
+    {
+      "admitted": 38,
+      "completed": 37,
+      "completed_throughput_rps": 0.4625,
+      "duration_s": 80.0,
+      "end_s": 140.0,
+      "failed": 0,
+      "input_tokens": 404192,
+      "name": "A",
+      "offered_rps": 0.475,
+      "output_tokens": 8525,
+      "start_s": 60.0
+    },
+    {
+      "admitted": 37,
+      "completed": 38,
+      "completed_throughput_rps": 0.475,
+      "duration_s": 80.0,
+      "end_s": 220.0,
+      "failed": 0,
+      "input_tokens": 327301,
+      "name": "B",
+      "offered_rps": 0.4625,
+      "output_tokens": 8617,
+      "start_s": 140.0
+    },
+    {
+      "admitted": 38,
+      "completed": 36,
+      "completed_throughput_rps": 0.45,
+      "duration_s": 80.0,
+      "end_s": 300.0,
+      "failed": 0,
+      "input_tokens": 237221,
+      "name": "C",
+      "offered_rps": 0.475,
+      "output_tokens": 8267,
+      "start_s": 220.0
+    }
+  ],
+  "successful_records": 172,
+  "t0_mono_ns": 180739547167834,
+  "t0_wall_ns": 1783849166673595809,
+  "warmup_seconds": 60.0
+}

runs/opprof-phase6/phase6/cells/tp1_mns16/anchor-0.24609375.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns16/anchor-0.24609375/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.24609375,
+  "cell": "tp1_mns16",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 141,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.328174978,
+    "end_mono_ns": 199601637648726,
+    "end_wall_ns": 1783868028764076133,
+    "start_mono_ns": 199540309473748,
+    "start_wall_ns": 1783867967435901551
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 16,
+  "observed_count": 141,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
+    "arrival_s": {
+      "distinct_n": 141,
+      "finite_n": 141,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 141
+    },
+    "count": 141,
+    "long_gt4096": 50,
+    "offered_req_s": 2.35,
+    "offered_req_s_per_gpu": 2.35,
+    "raw_input_tokens": {
+      "distinct_n": 133,
+      "finite_n": 141,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 141
+    },
+    "raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
+    "request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
+  },
+  "slo_pass_count": 141,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 141,
+    "finite_n": 141,
+    "max": 39.29021051171873,
+    "min": 4.445230456776771,
+    "missing_n": 0,
+    "n": 141
+  },
+  "ttft_ms": {
+    "distinct_n": 141,
+    "finite_n": 141,
+    "max": 1787.3226249939762,
+    "min": 37.47074800776318,
+    "missing_n": 0,
+    "n": 141
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns16/anchor-0.25.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns16/anchor-0.25/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.25,
+  "cell": "tp1_mns16",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 143,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.471211524,
+    "end_mono_ns": 199668842271892,
+    "end_wall_ns": 1783868095968699176,
+    "start_mono_ns": 199607371060368,
+    "start_wall_ns": 1783868034497487431
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 16,
+  "observed_count": 143,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
+    "arrival_s": {
+      "distinct_n": 143,
+      "finite_n": 143,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 143
+    },
+    "count": 143,
+    "long_gt4096": 51,
+    "offered_req_s": 2.3833333333333333,
+    "offered_req_s_per_gpu": 2.3833333333333333,
+    "raw_input_tokens": {
+      "distinct_n": 135,
+      "finite_n": 143,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 143
+    },
+    "raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
+    "request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
+  },
+  "slo_pass_count": 143,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 143,
+    "finite_n": 143,
+    "max": 44.43295162989699,
+    "min": 4.492281684945301,
+    "missing_n": 0,
+    "n": 143
+  },
+  "ttft_ms": {
+    "distinct_n": 143,
+    "finite_n": 143,
+    "max": 1784.063341008732,
+    "min": 57.29520702152513,
+    "missing_n": 0,
+    "n": 143
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns16/cell-valid.json

@@ -0,0 +1,21 @@
+{
+  "accounting_mode": "A-P6-1-checkpoint-sidecar",
+  "cell": "tp1_mns16",
+  "invariants": {
+    "all_anchor_intervals_covered": true,
+    "all_schema_1": true,
+    "checkpoint_after_all_anchor_intervals": true,
+    "checkpoint_sidecar": true,
+    "checkpoint_within_flush_of_stream": true,
+    "complete_final_newline": true,
+    "encoded_balanced": true,
+    "last_step_matches": true,
+    "no_in_stream_footer": true,
+    "steps_contiguous": true,
+    "two_anchor_intervals": true,
+    "written_matches_records": true,
+    "zero_drops": true
+  },
+  "layer1_records": 9212,
+  "stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl"
+}

runs/opprof-phase6/phase6/cells/tp1_mns16/commands.log

@@ -0,0 +1,4 @@
+SERVER taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8501 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 1 --shutdown-timeout 120
+CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/warmup
+CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/anchor-0.24609375
+CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/anchor-0.25

runs/opprof-phase6/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl.footer.json

@@ -0,0 +1 @@
+{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638886-1783867898065648658.jsonl","encoded_records":9212,"written_records":9212,"dropped_records":0,"last_step_index":9211,"checkpoint_wall_ns":1783868096969027153,"flush_interval_seconds":1.0,"final":false}

runs/opprof-phase6/phase6/cells/tp1_mns16/server.log

@@ -0,0 +1,442 @@
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]        █     █     █▄   ▄█
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]  ▄▄ ▄█ █     █     █ ▀▄▀ █  version 0.24.1.dev3+g668cfb7e2
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]   █▄█▀ █     █     █     █  model   /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]    ▀▀  ▀▀▀▀▀ ▀▀▀▀▀ ▀     ▀
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8501, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
+(APIServer pid=2638187) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
+(APIServer pid=2638187) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
+(APIServer pid=2638187) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
+(APIServer pid=2638187) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
+(APIServer pid=2638187) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(EngineCore pid=2638886) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
+(EngineCore pid=2638886) INFO 07-12 14:51:05 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:37021 backend=nccl
+(EngineCore pid=2638886) INFO 07-12 14:51:05 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
+(EngineCore pid=2638886) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
+(EngineCore pid=2638886) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
+(EngineCore pid=2638886) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore pid=2638886) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
+(EngineCore pid=2638886) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
+(EngineCore pid=2638886) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
+(EngineCore pid=2638886) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:   0% Completed | 0/16 [00:00<?, ?it/s]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:   6% Completed | 1/16 [00:01<00:15,  1.06s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  12% Completed | 2/16 [00:02<00:14,  1.07s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  19% Completed | 3/16 [00:03<00:13,  1.06s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  25% Completed | 4/16 [00:04<00:13,  1.10s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  31% Completed | 5/16 [00:05<00:11,  1.09s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  38% Completed | 6/16 [00:06<00:10,  1.09s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  44% Completed | 7/16 [00:07<00:09,  1.10s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  50% Completed | 8/16 [00:08<00:08,  1.09s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  56% Completed | 9/16 [00:09<00:07,  1.08s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  62% Completed | 10/16 [00:10<00:06,  1.06s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  69% Completed | 11/16 [00:11<00:05,  1.06s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  75% Completed | 12/16 [00:12<00:04,  1.04s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  81% Completed | 13/16 [00:13<00:03,  1.06s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  88% Completed | 14/16 [00:15<00:02,  1.09s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards:  94% Completed | 15/16 [00:16<00:01,  1.09s/it]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.17it/s]
+(EngineCore pid=2638886) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.03s/it]
+(EngineCore pid=2638886) 
+(EngineCore pid=2638886) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.52 seconds
+(EngineCore pid=2638886) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
+(EngineCore pid=2638886) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.268242 seconds
+(EngineCore pid=2638886) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/f4a50989f8/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore pid=2638886) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.31 s
+(EngineCore pid=2638886) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.252 s
+(EngineCore pid=2638886) INFO 07-12 14:51:31 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/fe5a82dbe929018b7aea7dee05b5cd31a21fe2682aca78ee4cbf2b37c8a086d6/rank_0_0/model
+(EngineCore pid=2638886) INFO 07-12 14:51:31 [monitor.py:53] torch.compile took 5.96 s in total
+(EngineCore pid=2638886) INFO 07-12 14:51:32 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
+(EngineCore pid=2638886) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.20 s
+(EngineCore pid=2638886) INFO 07-12 14:51:32 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
+(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
+(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.43 GiB
+(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
+(EngineCore pid=2638886) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 321,456 tokens
+(EngineCore pid=2638886) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
+(EngineCore pid=2638886) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
+(EngineCore pid=2638886) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
+(EngineCore pid=2638886) 2026-07-12 14:51:35,155 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
+(EngineCore pid=2638886) 2026-07-12 14:51:35,199 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
+(EngineCore pid=2638886) 
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/7 [00:00<?, ?it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  14%|█▍        | 1/7 [00:00<00:00,  9.98it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  43%|████▎     | 3/7 [00:00<00:00, 10.33it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  71%|███████▏  | 5/7 [00:00<00:00,  9.18it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  86%|████████▌ | 6/7 [00:00<00:00,  8.44it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00,  7.90it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00,  8.49it/s]
+(EngineCore pid=2638886) 
+Capturing CUDA graphs (decode, FULL):   0%|          | 0/5 [00:00<?, ?it/s]
+Capturing CUDA graphs (decode, FULL):  40%|████      | 2/5 [00:00<00:00, 10.74it/s]
+Capturing CUDA graphs (decode, FULL):  80%|████████  | 4/5 [00:00<00:00, 11.19it/s]
+Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 11.25it/s]
+(EngineCore pid=2638886) INFO 07-12 14:51:37 [gpu_model_runner.py:6660] Graph capturing finished in 2 secs, took 0.10 GiB
+(EngineCore pid=2638886) INFO 07-12 14:51:37 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.02 GiB (15.7%).
+(EngineCore pid=2638886) INFO 07-12 14:51:37 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
+(EngineCore pid=2638886) INFO 07-12 14:51:37 [core.py:337] init engine (profile, create kv cache, warmup model) took 11.83 s (compilation: 5.96 s)
+(EngineCore pid=2638886) INFO 07-12 14:51:38 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl
+(EngineCore pid=2638886) INFO 07-12 14:51:38 [vllm.py:1006] Asynchronous scheduling is enabled.
+(EngineCore pid=2638886) INFO 07-12 14:51:38 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(APIServer pid=2638187) INFO 07-12 14:51:38 [api_server.py:577] Supported tasks: ['generate']
+(APIServer pid=2638187) WARNING 07-12 14:51:38 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+(APIServer pid=2638187) INFO 07-12 14:51:38 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
+(APIServer pid=2638187) INFO 07-12 14:51:38 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8501
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:37] Available routes are:
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /docs, Methods: GET, HEAD
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /load, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /version, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /health, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /metrics, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /tokenize, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /detokenize, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/models, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /ping, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /ping, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /invocations, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/messages, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /generative_scoring, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions/render, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
+(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
+(APIServer pid=2638187) INFO:     Started server process [2638187]
+(APIServer pid=2638187) INFO:     Waiting for application startup.
+(APIServer pid=2638187) INFO:     Application startup complete.
+(APIServer pid=2638187) INFO:     127.0.0.1:52764 - "GET /v1/models HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:52778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638886) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638187) INFO:     127.0.0.1:52788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:52802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638886) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638187) INFO:     127.0.0.1:52808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:52:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5299.4 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
+(APIServer pid=2638187) INFO:     127.0.0.1:44164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:52:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8.5 tokens/s, Avg generation throughput: 67.6 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 35.1%
+(APIServer pid=2638187) INFO:     127.0.0.1:44244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:44254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:52:59 [loggers.py:273] Engine 000: Avg prompt throughput: 2139.7 tokens/s, Avg generation throughput: 267.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 43.4%
+(APIServer pid=2638187) INFO:     127.0.0.1:38776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:38806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:45828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:53:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5435.1 tokens/s, Avg generation throughput: 320.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 33.3%
+(APIServer pid=2638187) INFO:     127.0.0.1:48040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:53:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4881.2 tokens/s, Avg generation throughput: 164.9 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.6%, Prefix cache hit rate: 29.1%
+(APIServer pid=2638187) INFO:     127.0.0.1:48158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:48204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39864 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:53:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6255.1 tokens/s, Avg generation throughput: 265.4 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.7%, Prefix cache hit rate: 24.2%
+(APIServer pid=2638187) INFO:     127.0.0.1:39928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:40072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:53:39 [loggers.py:273] Engine 000: Avg prompt throughput: 9198.7 tokens/s, Avg generation throughput: 388.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.2%, Prefix cache hit rate: 20.8%
+(APIServer pid=2638187) INFO:     127.0.0.1:40078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:42996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:43012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:43016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:43018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:43030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:53:49 [loggers.py:273] Engine 000: Avg prompt throughput: 6382.3 tokens/s, Avg generation throughput: 331.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.1%
+(APIServer pid=2638187) INFO:     127.0.0.1:53576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:53:59 [loggers.py:273] Engine 000: Avg prompt throughput: 3970.7 tokens/s, Avg generation throughput: 126.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.3%, Prefix cache hit rate: 17.8%
+(APIServer pid=2638187) INFO:     127.0.0.1:53718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:53988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:54:09 [loggers.py:273] Engine 000: Avg prompt throughput: 6063.3 tokens/s, Avg generation throughput: 290.7 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.9%, Prefix cache hit rate: 17.1%
+(APIServer pid=2638187) INFO:     127.0.0.1:54092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54104 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:54120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:54:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4203.4 tokens/s, Avg generation throughput: 273.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 16.7%
+(APIServer pid=2638187) INFO:     127.0.0.1:56492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:56502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:34994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:35024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:54:29 [loggers.py:273] Engine 000: Avg prompt throughput: 4720.8 tokens/s, Avg generation throughput: 256.7 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.3%
+(APIServer pid=2638187) INFO:     127.0.0.1:35028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:54:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8295.5 tokens/s, Avg generation throughput: 280.7 tokens/s, Running: 16 reqs, Waiting: 1 reqs, GPU KV cache usage: 15.5%, Prefix cache hit rate: 15.5%
+(APIServer pid=2638187) INFO:     127.0.0.1:57270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:57276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO 07-12 14:54:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8540.1 tokens/s, Avg generation throughput: 333.9 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.1%, Prefix cache hit rate: 14.6%
+(APIServer pid=2638187) INFO:     127.0.0.1:36952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638187) INFO:     127.0.0.1:36962 - "POST /v1/chat/completions HTTP/1.1" 200 OK

runs/opprof-phase6/phase6/cells/tp1_mns16/warmup.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns16/warmup/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.24609375,
+  "cell": "tp1_mns16",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 16,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 8.031388113,
+    "end_mono_ns": 199530259142500,
+    "end_wall_ns": 1783867957385569525,
+    "start_mono_ns": 199522227754387,
+    "start_wall_ns": 1783867949354181463
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "warmup",
+  "mns": 16,
+  "observed_count": 16,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
+    "arrival_s": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 6.901699999999983,
+      "min": 0.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "count": 16,
+    "long_gt4096": 8,
+    "offered_req_s": 0.26666666666666666,
+    "offered_req_s_per_gpu": 0.26666666666666666,
+    "raw_input_tokens": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 7270.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
+    "request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
+  },
+  "slo_pass_count": 16,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 28.370322086701652,
+    "min": 4.984751267873821,
+    "missing_n": 0,
+    "n": 16
+  },
+  "ttft_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 696.6323160158936,
+    "min": 74.85444800113328,
+    "missing_n": 0,
+    "n": 16
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns32/anchor-0.2421875.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns32/anchor-0.2421875/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.2421875,
+  "cell": "tp1_mns32",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 137,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.294849452,
+    "end_mono_ns": 199601604247251,
+    "end_wall_ns": 1783868028730674502,
+    "start_mono_ns": 199540309397799,
+    "start_wall_ns": 1783867967435824912
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 32,
+  "observed_count": 137,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
+    "arrival_s": {
+      "distinct_n": 137,
+      "finite_n": 137,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 137
+    },
+    "count": 137,
+    "long_gt4096": 47,
+    "offered_req_s": 2.283333333333333,
+    "offered_req_s_per_gpu": 2.283333333333333,
+    "raw_input_tokens": {
+      "distinct_n": 129,
+      "finite_n": 137,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 137
+    },
+    "raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
+    "request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
+  },
+  "slo_pass_count": 137,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 137,
+    "finite_n": 137,
+    "max": 35.17025839386134,
+    "min": 4.435019960625127,
+    "missing_n": 0,
+    "n": 137
+  },
+  "ttft_ms": {
+    "distinct_n": 137,
+    "finite_n": 137,
+    "max": 1485.6346309825312,
+    "min": 34.36101900297217,
+    "missing_n": 0,
+    "n": 137
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns32/anchor-0.24609375.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns32", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns32/anchor-0.24609375/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.24609375,
+  "cell": "tp1_mns32",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 141,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.288241408,
+    "end_mono_ns": 199668653603418,
+    "end_wall_ns": 1783868095780030817,
+    "start_mono_ns": 199607365362010,
+    "start_wall_ns": 1783868034491789246
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 32,
+  "observed_count": 141,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
+    "arrival_s": {
+      "distinct_n": 141,
+      "finite_n": 141,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 141
+    },
+    "count": 141,
+    "long_gt4096": 50,
+    "offered_req_s": 2.35,
+    "offered_req_s_per_gpu": 2.35,
+    "raw_input_tokens": {
+      "distinct_n": 133,
+      "finite_n": 141,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 141
+    },
+    "raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
+    "request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
+  },
+  "slo_pass_count": 141,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 141,
+    "finite_n": 141,
+    "max": 45.32469625197036,
+    "min": 4.495332385807511,
+    "missing_n": 0,
+    "n": 141
+  },
+  "ttft_ms": {
+    "distinct_n": 141,
+    "finite_n": 141,
+    "max": 1799.771893012803,
+    "min": 49.012041999958456,
+    "missing_n": 0,
+    "n": 141
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns32/cell-valid.json

@@ -0,0 +1,21 @@
+{
+  "accounting_mode": "A-P6-1-checkpoint-sidecar",
+  "cell": "tp1_mns32",
+  "invariants": {
+    "all_anchor_intervals_covered": true,
+    "all_schema_1": true,
+    "checkpoint_after_all_anchor_intervals": true,
+    "checkpoint_sidecar": true,
+    "checkpoint_within_flush_of_stream": true,
+    "complete_final_newline": true,
+    "encoded_balanced": true,
+    "last_step_matches": true,
+    "no_in_stream_footer": true,
+    "steps_contiguous": true,
+    "two_anchor_intervals": true,
+    "written_matches_records": true,
+    "zero_drops": true
+  },
+  "layer1_records": 9633,
+  "stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl"
+}

runs/opprof-phase6/phase6/cells/tp1_mns32/commands.log

@@ -0,0 +1,4 @@
+SERVER taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8502 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 1 --shutdown-timeout 120
+CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/warmup
+CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/anchor-0.2421875
+CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.24609375 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/anchor-0.24609375

runs/opprof-phase6/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl.footer.json

@@ -0,0 +1 @@
+{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638900-1783867898685556703.jsonl","encoded_records":9633,"written_records":9633,"dropped_records":0,"last_step_index":9632,"checkpoint_wall_ns":1783868096780742186,"flush_interval_seconds":1.0,"final":false}

runs/opprof-phase6/phase6/cells/tp1_mns32/server.log

@@ -0,0 +1,436 @@
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]        █     █     █▄   ▄█
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]  ▄▄ ▄█ █     █     █ ▀▄▀ █  version 0.24.1.dev3+g668cfb7e2
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]   █▄█▀ █     █     █     █  model   /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]    ▀▀  ▀▀▀▀▀ ▀▀▀▀▀ ▀     ▀
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8502, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 32, 'shutdown_timeout': 120}
+(APIServer pid=2638188) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
+(APIServer pid=2638188) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
+(APIServer pid=2638188) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
+(APIServer pid=2638188) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
+(APIServer pid=2638188) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(EngineCore pid=2638900) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
+(EngineCore pid=2638900) INFO 07-12 14:51:05 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:50247 backend=nccl
+(EngineCore pid=2638900) INFO 07-12 14:51:05 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
+(EngineCore pid=2638900) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
+(EngineCore pid=2638900) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
+(EngineCore pid=2638900) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore pid=2638900) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
+(EngineCore pid=2638900) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
+(EngineCore pid=2638900) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
+(EngineCore pid=2638900) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:   0% Completed | 0/16 [00:00<?, ?it/s]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:   6% Completed | 1/16 [00:01<00:16,  1.07s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  12% Completed | 2/16 [00:02<00:14,  1.07s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  19% Completed | 3/16 [00:03<00:13,  1.06s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  25% Completed | 4/16 [00:04<00:13,  1.10s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  31% Completed | 5/16 [00:05<00:11,  1.09s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  38% Completed | 6/16 [00:06<00:10,  1.09s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  44% Completed | 7/16 [00:07<00:09,  1.10s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  50% Completed | 8/16 [00:08<00:08,  1.09s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  56% Completed | 9/16 [00:09<00:07,  1.08s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  62% Completed | 10/16 [00:10<00:06,  1.06s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  69% Completed | 11/16 [00:11<00:05,  1.06s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  75% Completed | 12/16 [00:12<00:04,  1.04s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  81% Completed | 13/16 [00:13<00:03,  1.06s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  88% Completed | 14/16 [00:15<00:02,  1.09s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards:  94% Completed | 15/16 [00:16<00:01,  1.09s/it]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.18it/s]
+(EngineCore pid=2638900) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.03s/it]
+(EngineCore pid=2638900) 
+(EngineCore pid=2638900) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.49 seconds
+(EngineCore pid=2638900) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
+(EngineCore pid=2638900) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.231743 seconds
+(EngineCore pid=2638900) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/30ef41b5f5/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore pid=2638900) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.33 s
+(EngineCore pid=2638900) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.266 s
+(EngineCore pid=2638900) INFO 07-12 14:51:32 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/738c624149a63d13eaf115eec4d2189ece948ac500e524d3e06e801d9915352d/rank_0_0/model
+(EngineCore pid=2638900) INFO 07-12 14:51:32 [monitor.py:53] torch.compile took 6.02 s in total
+(EngineCore pid=2638900) INFO 07-12 14:51:32 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
+(EngineCore pid=2638900) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.20 s
+(EngineCore pid=2638900) INFO 07-12 14:51:33 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=11 (largest=64), FULL=7 (largest=32)
+(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.11 GiB total
+(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.4 GiB
+(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9188 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9212. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
+(EngineCore pid=2638900) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 321,136 tokens
+(EngineCore pid=2638900) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.84x
+(EngineCore pid=2638900) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
+(EngineCore pid=2638900) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
+(EngineCore pid=2638900) 2026-07-12 14:51:35,118 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
+(EngineCore pid=2638900) 2026-07-12 14:51:35,162 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
+(EngineCore pid=2638900) 
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/11 [00:00<?, ?it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   9%|▉         | 1/11 [00:00<00:01,  9.36it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  18%|█▊        | 2/11 [00:00<00:00,  9.41it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  27%|██▋       | 3/11 [00:00<00:00,  9.30it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  36%|███▋      | 4/11 [00:00<00:00,  9.23it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  45%|████▌     | 5/11 [00:00<00:00,  9.16it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  55%|█████▍    | 6/11 [00:00<00:00,  9.24it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  64%|██████▎   | 7/11 [00:00<00:00,  8.56it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  73%|███████▎  | 8/11 [00:00<00:00,  8.19it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  82%|████████▏ | 9/11 [00:01<00:00,  7.88it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████ | 10/11 [00:01<00:00,  7.84it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00,  7.60it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00,  8.31it/s]
+(EngineCore pid=2638900) 
+Capturing CUDA graphs (decode, FULL):   0%|          | 0/7 [00:00<?, ?it/s]
+Capturing CUDA graphs (decode, FULL):  29%|██▊       | 2/7 [00:00<00:00, 10.42it/s]
+Capturing CUDA graphs (decode, FULL):  57%|█████▋    | 4/7 [00:00<00:00, 10.82it/s]
+Capturing CUDA graphs (decode, FULL):  86%|████████▌ | 6/7 [00:00<00:00, 10.80it/s]
+Capturing CUDA graphs (decode, FULL): 100%|██████████| 7/7 [00:00<00:00, 10.91it/s]
+(EngineCore pid=2638900) INFO 07-12 14:51:37 [gpu_model_runner.py:6660] Graph capturing finished in 3 secs, took 0.13 GiB
+(EngineCore pid=2638900) INFO 07-12 14:51:37 [gpu_worker.py:667] CUDA graph pool memory: 0.13 GiB (actual), 0.11 GiB (estimated), difference: 0.02 GiB (12.1%).
+(EngineCore pid=2638900) INFO 07-12 14:51:37 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
+(EngineCore pid=2638900) INFO 07-12 14:51:38 [core.py:337] init engine (profile, create kv cache, warmup model) took 12.52 s (compilation: 6.02 s)
+(EngineCore pid=2638900) INFO 07-12 14:51:38 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl
+(EngineCore pid=2638900) INFO 07-12 14:51:38 [vllm.py:1006] Asynchronous scheduling is enabled.
+(EngineCore pid=2638900) INFO 07-12 14:51:38 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(APIServer pid=2638188) INFO 07-12 14:51:38 [api_server.py:577] Supported tasks: ['generate']
+(APIServer pid=2638188) WARNING 07-12 14:51:38 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+(APIServer pid=2638188) INFO 07-12 14:51:39 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
+(APIServer pid=2638188) INFO 07-12 14:51:39 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8502
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:37] Available routes are:
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /docs, Methods: HEAD, GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /load, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /version, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /health, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /metrics, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /tokenize, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /detokenize, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/models, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /ping, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /ping, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /invocations, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/messages, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /generative_scoring, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions/render, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
+(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
+(APIServer pid=2638188) INFO:     Started server process [2638188]
+(APIServer pid=2638188) INFO:     Waiting for application startup.
+(APIServer pid=2638188) INFO:     Application startup complete.
+(APIServer pid=2638188) INFO:     127.0.0.1:43588 - "GET /v1/models HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:43598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638900) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638188) INFO:     127.0.0.1:43614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:43624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638900) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638188) INFO:     127.0.0.1:43626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:52:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5299.4 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
+(APIServer pid=2638188) INFO:     127.0.0.1:33708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:52:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8.5 tokens/s, Avg generation throughput: 95.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 35.1%
+(APIServer pid=2638188) INFO:     127.0.0.1:33798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:52:59 [loggers.py:273] Engine 000: Avg prompt throughput: 3058.0 tokens/s, Avg generation throughput: 240.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 40.3%
+(APIServer pid=2638188) INFO:     127.0.0.1:42332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:42340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:53:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5130.2 tokens/s, Avg generation throughput: 333.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 33.3%
+(APIServer pid=2638188) INFO:     127.0.0.1:38132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:53:19 [loggers.py:273] Engine 000: Avg prompt throughput: 3991.7 tokens/s, Avg generation throughput: 168.0 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.7%, Prefix cache hit rate: 29.4%
+(APIServer pid=2638188) INFO:     127.0.0.1:38242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:38250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:55992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:53:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6239.1 tokens/s, Avg generation throughput: 226.6 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.3%, Prefix cache hit rate: 24.2%
+(APIServer pid=2638188) INFO:     127.0.0.1:56116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:56140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33508 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:53:39 [loggers.py:273] Engine 000: Avg prompt throughput: 7873.3 tokens/s, Avg generation throughput: 414.1 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.4%, Prefix cache hit rate: 21.7%
+(APIServer pid=2638188) INFO:     127.0.0.1:59514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:59666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:53:49 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.7 tokens/s, Avg generation throughput: 274.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
+(APIServer pid=2638188) INFO:     127.0.0.1:45016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:45124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:53:59 [loggers.py:273] Engine 000: Avg prompt throughput: 4474.5 tokens/s, Avg generation throughput: 142.1 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 18.4%
+(APIServer pid=2638188) INFO:     127.0.0.1:45128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:36594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:54:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5897.7 tokens/s, Avg generation throughput: 295.1 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.9%, Prefix cache hit rate: 17.5%
+(APIServer pid=2638188) INFO:     127.0.0.1:45994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:46140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:54:19 [loggers.py:273] Engine 000: Avg prompt throughput: 3893.6 tokens/s, Avg generation throughput: 269.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 17.2%
+(APIServer pid=2638188) INFO:     127.0.0.1:46150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:48914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:54:29 [loggers.py:273] Engine 000: Avg prompt throughput: 4692.9 tokens/s, Avg generation throughput: 265.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.7%
+(APIServer pid=2638188) INFO:     127.0.0.1:48924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:40112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO 07-12 14:54:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8338.1 tokens/s, Avg generation throughput: 294.5 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.7%, Prefix cache hit rate: 16.0%
+(APIServer pid=2638188) INFO:     127.0.0.1:40114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638188) INFO:     127.0.0.1:33384 - "POST /v1/chat/completions HTTP/1.1" 200 OK

runs/opprof-phase6/phase6/cells/tp1_mns32/warmup.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns32/warmup/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.2421875,
+  "cell": "tp1_mns32",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 16,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 8.025285552,
+    "end_mono_ns": 199530264041528,
+    "end_wall_ns": 1783867957390468387,
+    "start_mono_ns": 199522238755976,
+    "start_wall_ns": 1783867949365182961
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "warmup",
+  "mns": 32,
+  "observed_count": 16,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
+    "arrival_s": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 6.901699999999983,
+      "min": 0.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "count": 16,
+    "long_gt4096": 8,
+    "offered_req_s": 0.26666666666666666,
+    "offered_req_s_per_gpu": 0.26666666666666666,
+    "raw_input_tokens": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 7270.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
+    "request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
+  },
+  "slo_pass_count": 16,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 28.123532992047,
+    "min": 4.953902173286861,
+    "missing_n": 0,
+    "n": 16
+  },
+  "ttft_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 682.8775229805615,
+    "min": 74.39436702406965,
+    "missing_n": 0,
+    "n": 16
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns64/anchor-0.2421875.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns64/anchor-0.2421875/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.2421875,
+  "cell": "tp1_mns64",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 137,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.27871102,
+    "end_mono_ns": 199601593594258,
+    "end_wall_ns": 1783868028720022637,
+    "start_mono_ns": 199540314883238,
+    "start_wall_ns": 1783867967441310423
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 64,
+  "observed_count": 137,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
+    "arrival_s": {
+      "distinct_n": 137,
+      "finite_n": 137,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 137
+    },
+    "count": 137,
+    "long_gt4096": 47,
+    "offered_req_s": 2.283333333333333,
+    "offered_req_s_per_gpu": 2.283333333333333,
+    "raw_input_tokens": {
+      "distinct_n": 129,
+      "finite_n": 137,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 137
+    },
+    "raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
+    "request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
+  },
+  "slo_pass_count": 137,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 137,
+    "finite_n": 137,
+    "max": 34.5918034883051,
+    "min": 4.451991834606257,
+    "missing_n": 0,
+    "n": 137
+  },
+  "ttft_ms": {
+    "distinct_n": 137,
+    "finite_n": 137,
+    "max": 1472.6779730117414,
+    "min": 33.561496005859226,
+    "missing_n": 0,
+    "n": 137
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns64/anchor-0.24609375.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns64", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns64/anchor-0.24609375/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.24609375,
+  "cell": "tp1_mns64",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 141,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.283849775,
+    "end_mono_ns": 199668751241940,
+    "end_wall_ns": 1783868095877669237,
+    "start_mono_ns": 199607467392165,
+    "start_wall_ns": 1783868034593819130
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 64,
+  "observed_count": 141,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
+    "arrival_s": {
+      "distinct_n": 141,
+      "finite_n": 141,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 141
+    },
+    "count": 141,
+    "long_gt4096": 50,
+    "offered_req_s": 2.35,
+    "offered_req_s_per_gpu": 2.35,
+    "raw_input_tokens": {
+      "distinct_n": 133,
+      "finite_n": 141,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 141
+    },
+    "raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
+    "request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
+  },
+  "slo_pass_count": 141,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 141,
+    "finite_n": 141,
+    "max": 48.397922928960206,
+    "min": 4.49719877962177,
+    "missing_n": 0,
+    "n": 141
+  },
+  "ttft_ms": {
+    "distinct_n": 141,
+    "finite_n": 141,
+    "max": 1777.0718620158732,
+    "min": 49.70270599005744,
+    "missing_n": 0,
+    "n": 141
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns64/cell-valid.json

@@ -0,0 +1,21 @@
+{
+  "accounting_mode": "A-P6-1-checkpoint-sidecar",
+  "cell": "tp1_mns64",
+  "invariants": {
+    "all_anchor_intervals_covered": true,
+    "all_schema_1": true,
+    "checkpoint_after_all_anchor_intervals": true,
+    "checkpoint_sidecar": true,
+    "checkpoint_within_flush_of_stream": true,
+    "complete_final_newline": true,
+    "encoded_balanced": true,
+    "last_step_matches": true,
+    "no_in_stream_footer": true,
+    "steps_contiguous": true,
+    "two_anchor_intervals": true,
+    "written_matches_records": true,
+    "zero_drops": true
+  },
+  "layer1_records": 9654,
+  "stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl"
+}

runs/opprof-phase6/phase6/cells/tp1_mns64/commands.log

@@ -0,0 +1,4 @@
+SERVER taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8503 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 64 --tensor-parallel-size 1 --shutdown-timeout 120
+CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/warmup
+CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/anchor-0.2421875
+CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.24609375 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/anchor-0.24609375

runs/opprof-phase6/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl.footer.json

@@ -0,0 +1 @@
+{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638898-1783867899798067669.jsonl","encoded_records":9654,"written_records":9654,"dropped_records":0,"last_step_index":9653,"checkpoint_wall_ns":1783868097891402992,"flush_interval_seconds":1.0,"final":false}

runs/opprof-phase6/phase6/cells/tp1_mns64/server.log

@@ -0,0 +1,437 @@
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]        █     █     █▄   ▄█
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]  ▄▄ ▄█ █     █     █ ▀▄▀ █  version 0.24.1.dev3+g668cfb7e2
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]   █▄█▀ █     █     █     █  model   /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]    ▀▀  ▀▀▀▀▀ ▀▀▀▀▀ ▀     ▀
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8503, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 64, 'shutdown_timeout': 120}
+(APIServer pid=2638189) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
+(APIServer pid=2638189) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
+(APIServer pid=2638189) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
+(APIServer pid=2638189) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
+(APIServer pid=2638189) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(EngineCore pid=2638898) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
+(EngineCore pid=2638898) INFO 07-12 14:51:06 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:39471 backend=nccl
+(EngineCore pid=2638898) INFO 07-12 14:51:06 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
+(EngineCore pid=2638898) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
+(EngineCore pid=2638898) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
+(EngineCore pid=2638898) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore pid=2638898) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
+(EngineCore pid=2638898) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
+(EngineCore pid=2638898) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
+(EngineCore pid=2638898) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:   0% Completed | 0/16 [00:00<?, ?it/s]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:   6% Completed | 1/16 [00:01<00:15,  1.06s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  12% Completed | 2/16 [00:02<00:14,  1.07s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  19% Completed | 3/16 [00:03<00:13,  1.06s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  25% Completed | 4/16 [00:04<00:13,  1.10s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  31% Completed | 5/16 [00:05<00:11,  1.08s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  38% Completed | 6/16 [00:06<00:10,  1.09s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  44% Completed | 7/16 [00:07<00:09,  1.11s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  50% Completed | 8/16 [00:08<00:08,  1.09s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  56% Completed | 9/16 [00:09<00:07,  1.08s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  62% Completed | 10/16 [00:10<00:06,  1.06s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  69% Completed | 11/16 [00:11<00:05,  1.06s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  75% Completed | 12/16 [00:12<00:04,  1.04s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  81% Completed | 13/16 [00:13<00:03,  1.06s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  88% Completed | 14/16 [00:15<00:02,  1.09s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards:  94% Completed | 15/16 [00:16<00:01,  1.09s/it]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.17it/s]
+(EngineCore pid=2638898) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.03s/it]
+(EngineCore pid=2638898) 
+(EngineCore pid=2638898) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.52 seconds
+(EngineCore pid=2638898) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
+(EngineCore pid=2638898) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.262335 seconds
+(EngineCore pid=2638898) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/65b50dadd2/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore pid=2638898) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.30 s
+(EngineCore pid=2638898) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.131 s
+(EngineCore pid=2638898) INFO 07-12 14:51:31 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/ab53f49ec98f407fe24fecb037cb59739264f283939c70f41986d8369686d472/rank_0_0/model
+(EngineCore pid=2638898) INFO 07-12 14:51:31 [monitor.py:53] torch.compile took 5.83 s in total
+(EngineCore pid=2638898) INFO 07-12 14:51:31 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
+(EngineCore pid=2638898) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.21 s
+(EngineCore pid=2638898) INFO 07-12 14:51:32 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
+(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.19 GiB total
+(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.33 GiB
+(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9180 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9220. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
+(EngineCore pid=2638898) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 320,304 tokens
+(EngineCore pid=2638898) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.82x
+(EngineCore pid=2638898) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
+(EngineCore pid=2638898) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
+(EngineCore pid=2638898) 2026-07-12 14:51:35,044 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
+(EngineCore pid=2638898) 2026-07-12 14:51:35,116 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
+(EngineCore pid=2638898) 
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/19 [00:00<?, ?it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   5%|▌         | 1/19 [00:00<00:01,  9.91it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  16%|█▌        | 3/19 [00:00<00:01, 10.16it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  26%|██▋       | 5/19 [00:00<00:01, 10.01it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  37%|███▋      | 7/19 [00:00<00:01,  9.67it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  42%|████▏     | 8/19 [00:00<00:01,  9.52it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  53%|█████▎    | 10/19 [00:01<00:00,  9.72it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  58%|█████▊    | 11/19 [00:01<00:00,  9.69it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  63%|██████▎   | 12/19 [00:01<00:00,  9.19it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  68%|██████▊   | 13/19 [00:01<00:00,  9.33it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  74%|███████▎  | 14/19 [00:01<00:00,  8.79it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  79%|███████▉  | 15/19 [00:01<00:00,  8.44it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  84%|████████▍ | 16/19 [00:01<00:00,  8.20it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  89%|████████▉ | 17/19 [00:01<00:00,  7.99it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  95%|█████████▍| 18/19 [00:02<00:00,  8.05it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00,  7.91it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00,  8.89it/s]
+(EngineCore pid=2638898) 
+Capturing CUDA graphs (decode, FULL):   0%|          | 0/11 [00:00<?, ?it/s]
+Capturing CUDA graphs (decode, FULL):  18%|█▊        | 2/11 [00:00<00:00, 10.99it/s]
+Capturing CUDA graphs (decode, FULL):  36%|███▋      | 4/11 [00:00<00:00, 11.14it/s]
+Capturing CUDA graphs (decode, FULL):  55%|█████▍    | 6/11 [00:00<00:00, 11.17it/s]
+Capturing CUDA graphs (decode, FULL):  73%|███████▎  | 8/11 [00:00<00:00, 11.23it/s]
+Capturing CUDA graphs (decode, FULL):  91%|█████████ | 10/11 [00:00<00:00, 11.31it/s]
+Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 11.28it/s]
+(EngineCore pid=2638898) INFO 07-12 14:51:38 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.24 GiB
+(EngineCore pid=2638898) INFO 07-12 14:51:38 [gpu_worker.py:667] CUDA graph pool memory: 0.24 GiB (actual), 0.19 GiB (estimated), difference: 0.05 GiB (19.8%).
+(EngineCore pid=2638898) INFO 07-12 14:51:38 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
+(EngineCore pid=2638898) INFO 07-12 14:51:39 [core.py:337] init engine (profile, create kv cache, warmup model) took 13.60 s (compilation: 5.83 s)
+(EngineCore pid=2638898) INFO 07-12 14:51:39 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl
+(EngineCore pid=2638898) INFO 07-12 14:51:39 [vllm.py:1006] Asynchronous scheduling is enabled.
+(EngineCore pid=2638898) INFO 07-12 14:51:39 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(APIServer pid=2638189) INFO 07-12 14:51:39 [api_server.py:577] Supported tasks: ['generate']
+(APIServer pid=2638189) WARNING 07-12 14:51:40 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+(APIServer pid=2638189) INFO 07-12 14:51:40 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
+(APIServer pid=2638189) INFO 07-12 14:51:40 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8503
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:37] Available routes are:
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /docs, Methods: GET, HEAD
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /load, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /version, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /health, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /metrics, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /tokenize, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /detokenize, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/models, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /ping, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /ping, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /invocations, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/messages, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /generative_scoring, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions/render, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
+(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
+(APIServer pid=2638189) INFO:     Started server process [2638189]
+(APIServer pid=2638189) INFO:     Waiting for application startup.
+(APIServer pid=2638189) INFO:     Application startup complete.
+(APIServer pid=2638189) INFO:     127.0.0.1:46204 - "GET /v1/models HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638898) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638189) INFO:     127.0.0.1:46214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638898) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638189) INFO:     127.0.0.1:46234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:52:30 [loggers.py:273] Engine 000: Avg prompt throughput: 1668.3 tokens/s, Avg generation throughput: 1.5 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 4.0%
+(APIServer pid=2638189) INFO:     127.0.0.1:54168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:54256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:52:40 [loggers.py:273] Engine 000: Avg prompt throughput: 3630.8 tokens/s, Avg generation throughput: 203.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
+(APIServer pid=2638189) INFO:     127.0.0.1:39372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:39452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:52:50 [loggers.py:273] Engine 000: Avg prompt throughput: 11.3 tokens/s, Avg generation throughput: 117.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 42.7%
+(APIServer pid=2638189) INFO:     127.0.0.1:46416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:46636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:53:00 [loggers.py:273] Engine 000: Avg prompt throughput: 3820.4 tokens/s, Avg generation throughput: 268.8 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 40.3%
+(APIServer pid=2638189) INFO:     127.0.0.1:33660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:33836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:53:10 [loggers.py:273] Engine 000: Avg prompt throughput: 4364.6 tokens/s, Avg generation throughput: 291.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.3%
+(APIServer pid=2638189) INFO:     127.0.0.1:51654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:51834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:53:20 [loggers.py:273] Engine 000: Avg prompt throughput: 4686.8 tokens/s, Avg generation throughput: 194.6 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 28.9%
+(APIServer pid=2638189) INFO:     127.0.0.1:55448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:55658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:53:30 [loggers.py:273] Engine 000: Avg prompt throughput: 7377.9 tokens/s, Avg generation throughput: 242.3 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.3%, Prefix cache hit rate: 24.2%
+(APIServer pid=2638189) INFO:     127.0.0.1:37564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:53:40 [loggers.py:273] Engine 000: Avg prompt throughput: 6039.0 tokens/s, Avg generation throughput: 407.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 21.3%
+(APIServer pid=2638189) INFO:     127.0.0.1:52718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:52840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:53:50 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.5 tokens/s, Avg generation throughput: 230.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
+(APIServer pid=2638189) INFO:     127.0.0.1:53304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:53446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:54:00 [loggers.py:273] Engine 000: Avg prompt throughput: 4481.1 tokens/s, Avg generation throughput: 175.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 18.4%
+(APIServer pid=2638189) INFO:     127.0.0.1:42240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:42428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:54:10 [loggers.py:273] Engine 000: Avg prompt throughput: 6653.1 tokens/s, Avg generation throughput: 325.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 17.6%
+(APIServer pid=2638189) INFO:     127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:43288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:54:20 [loggers.py:273] Engine 000: Avg prompt throughput: 3169.4 tokens/s, Avg generation throughput: 228.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.3%
+(APIServer pid=2638189) INFO:     127.0.0.1:37150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:37330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:54:30 [loggers.py:273] Engine 000: Avg prompt throughput: 4896.1 tokens/s, Avg generation throughput: 259.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 16.9%
+(APIServer pid=2638189) INFO:     127.0.0.1:59884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:59996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:60124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO 07-12 14:54:40 [loggers.py:273] Engine 000: Avg prompt throughput: 8725.1 tokens/s, Avg generation throughput: 309.3 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.8%, Prefix cache hit rate: 15.8%
+(APIServer pid=2638189) INFO:     127.0.0.1:48398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638189) INFO:     127.0.0.1:48462 - "POST /v1/chat/completions HTTP/1.1" 200 OK

runs/opprof-phase6/phase6/cells/tp1_mns64/warmup.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns64/warmup/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.2421875,
+  "cell": "tp1_mns64",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 16,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 8.029548257,
+    "end_mono_ns": 199530257119882,
+    "end_wall_ns": 1783867957383546922,
+    "start_mono_ns": 199522227571625,
+    "start_wall_ns": 1783867949353998820
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "warmup",
+  "mns": 64,
+  "observed_count": 16,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
+    "arrival_s": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 6.901699999999983,
+      "min": 0.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "count": 16,
+    "long_gt4096": 8,
+    "offered_req_s": 0.26666666666666666,
+    "offered_req_s_per_gpu": 0.26666666666666666,
+    "raw_input_tokens": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 7270.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
+    "request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
+  },
+  "slo_pass_count": 16,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 28.450962196848554,
+    "min": 4.971639937024534,
+    "missing_n": 0,
+    "n": 16
+  },
+  "ttft_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 697.7603349951096,
+    "min": 52.274041023338214,
+    "missing_n": 0,
+    "n": 16
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns8/anchor-0.21875.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns8", "anchor": 0.21875, "kind": "anchor", "pass_rate": 0.9917355371900827, "feasible": true}

runs/opprof-phase6/phase6/cells/tp1_mns8/anchor-0.21875/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.21875,
+  "cell": "tp1_mns8",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 121,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.294286331,
+    "end_mono_ns": 199668659416799,
+    "end_wall_ns": 1783868095785843711,
+    "start_mono_ns": 199607365130468,
+    "start_wall_ns": 1783868034491558052
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 8,
+  "observed_count": 121,
+  "pass_rate": 0.9917355371900827,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "f585e4793a8a6a621dbfe622514743ac316e34e3e93aaa1c6ba5f1eb58e8ff2d",
+    "arrival_s": {
+      "distinct_n": 121,
+      "finite_n": 121,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 121
+    },
+    "count": 121,
+    "long_gt4096": 42,
+    "offered_req_s": 2.0166666666666666,
+    "offered_req_s_per_gpu": 2.0166666666666666,
+    "raw_input_tokens": {
+      "distinct_n": 114,
+      "finite_n": 121,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 121
+    },
+    "raw_length_order_sha256": "061592e591871368954f4f80ee75cde2cfc399bd719f3f5deaad275e9af72e8b",
+    "request_id_order_sha256": "9b8c177ea00e5e42bd3294328201cb57addfd823913a5721714ce69646d715f4"
+  },
+  "slo_pass_count": 120,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 121,
+    "finite_n": 121,
+    "max": 30.344350779553743,
+    "min": 4.4550072992088525,
+    "missing_n": 0,
+    "n": 121
+  },
+  "ttft_ms": {
+    "distinct_n": 121,
+    "finite_n": 121,
+    "max": 2038.4164949937258,
+    "min": 37.96417900593951,
+    "missing_n": 0,
+    "n": 121
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns8/anchor-0.2265625.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "anchor", "pass_rate": 0.20634920634920634, "feasible": false}

runs/opprof-phase6/phase6/cells/tp1_mns8/anchor-0.2265625/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.2265625,
+  "cell": "tp1_mns8",
+  "early_stop_reason": "slo_pass_rate_unrecoverable",
+  "early_stopped": true,
+  "exact_output_count": 48,
+  "feasible": false,
+  "interval": {
+    "elapsed_s": 28.662121521,
+    "end_mono_ns": 199568971776020,
+    "end_wall_ns": 1783867996098203014,
+    "start_mono_ns": 199540309654499,
+    "start_wall_ns": 1783867967436081499
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 8,
+  "observed_count": 126,
+  "pass_rate": 0.20634920634920634,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "1b67241583bd6f75b1ad3f6d8c86bf0c815306a28f18ea49b1511fed6c091abe",
+    "arrival_s": {
+      "distinct_n": 126,
+      "finite_n": 126,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 126
+    },
+    "count": 126,
+    "long_gt4096": 44,
+    "offered_req_s": 2.1,
+    "offered_req_s_per_gpu": 2.1,
+    "raw_input_tokens": {
+      "distinct_n": 119,
+      "finite_n": 126,
+      "max": 8149.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 126
+    },
+    "raw_length_order_sha256": "29e3240c20df0cc68172d39aeae84d79908acf564c1b1e2edd447a984e7bdb0e",
+    "request_id_order_sha256": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90"
+  },
+  "slo_pass_count": 26,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 48,
+    "finite_n": 48,
+    "max": 48.57467018882744,
+    "min": 4.850139889790948,
+    "missing_n": 78,
+    "n": 126
+  },
+  "ttft_ms": {
+    "distinct_n": 48,
+    "finite_n": 48,
+    "max": 7013.455057982355,
+    "min": 39.39470701152459,
+    "missing_n": 78,
+    "n": 126
+  }
+}

runs/opprof-phase6/phase6/cells/tp1_mns8/cell-valid.json

@@ -0,0 +1,21 @@
+{
+  "accounting_mode": "A-P6-1-checkpoint-sidecar",
+  "cell": "tp1_mns8",
+  "invariants": {
+    "all_anchor_intervals_covered": true,
+    "all_schema_1": true,
+    "checkpoint_after_all_anchor_intervals": true,
+    "checkpoint_sidecar": true,
+    "checkpoint_within_flush_of_stream": true,
+    "complete_final_newline": true,
+    "encoded_balanced": true,
+    "last_step_matches": true,
+    "no_in_stream_footer": true,
+    "steps_contiguous": true,
+    "two_anchor_intervals": true,
+    "written_matches_records": true,
+    "zero_drops": true
+  },
+  "layer1_records": 7683,
+  "stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl"
+}

runs/opprof-phase6/phase6/cells/tp1_mns8/commands.log

@@ -0,0 +1,4 @@
+SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8500 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 1 --shutdown-timeout 120
+CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/warmup
+CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/anchor-0.2265625
+CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.21875 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/anchor-0.21875

runs/opprof-phase6/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl.footer.json

@@ -0,0 +1 @@
+{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638884-1783867942846550713.jsonl","encoded_records":7683,"written_records":7683,"dropped_records":0,"last_step_index":7682,"checkpoint_wall_ns":1783868096786407319,"flush_interval_seconds":1.0,"final":false}

runs/opprof-phase6/phase6/cells/tp1_mns8/server.log

@@ -0,0 +1,327 @@
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]        █     █     █▄   ▄█
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]  ▄▄ ▄█ █     █     █ ▀▄▀ █  version 0.24.1.dev3+g668cfb7e2
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]   █▄█▀ █     █     █     █  model   /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]    ▀▀  ▀▀▀▀▀ ▀▀▀▀▀ ▀     ▀
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] 
+(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8500, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 8, 'shutdown_timeout': 120}
+(APIServer pid=2638186) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
+(APIServer pid=2638186) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
+(APIServer pid=2638186) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
+(APIServer pid=2638186) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
+(APIServer pid=2638186) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(EngineCore pid=2638884) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
+(EngineCore pid=2638884) INFO 07-12 14:51:06 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:48769 backend=nccl
+(EngineCore pid=2638884) INFO 07-12 14:51:06 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
+(EngineCore pid=2638884) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
+(EngineCore pid=2638884) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
+(EngineCore pid=2638884) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore pid=2638884) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
+(EngineCore pid=2638884) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
+(EngineCore pid=2638884) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
+(EngineCore pid=2638884) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:   0% Completed | 0/16 [00:00<?, ?it/s]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:   6% Completed | 1/16 [00:01<00:15,  1.06s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  12% Completed | 2/16 [00:02<00:14,  1.07s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  19% Completed | 3/16 [00:03<00:13,  1.06s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  25% Completed | 4/16 [00:04<00:13,  1.10s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  31% Completed | 5/16 [00:05<00:11,  1.07s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  38% Completed | 6/16 [00:06<00:10,  1.09s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  44% Completed | 7/16 [00:07<00:09,  1.11s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  50% Completed | 8/16 [00:08<00:08,  1.10s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  56% Completed | 9/16 [00:09<00:07,  1.08s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  62% Completed | 10/16 [00:10<00:06,  1.07s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  69% Completed | 11/16 [00:11<00:05,  1.06s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  75% Completed | 12/16 [00:12<00:04,  1.04s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  81% Completed | 13/16 [00:13<00:03,  1.06s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  88% Completed | 14/16 [00:15<00:02,  1.09s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards:  94% Completed | 15/16 [00:16<00:01,  1.09s/it]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.18it/s]
+(EngineCore pid=2638884) 
+Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00,  1.03s/it]
+(EngineCore pid=2638884) 
+(EngineCore pid=2638884) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.49 seconds
+(EngineCore pid=2638884) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
+(EngineCore pid=2638884) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.240093 seconds
+(EngineCore pid=2638884) INFO 07-12 14:51:35 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/b95eb69a3d/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore pid=2638884) INFO 07-12 14:51:35 [backends.py:1148] Dynamo bytecode transform time: 9.16 s
+(EngineCore pid=2638884) INFO 07-12 14:51:50 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
+(EngineCore pid=2638884) INFO 07-12 14:51:58 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 23.00 s
+(EngineCore pid=2638884) INFO 07-12 14:52:03 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/802e9eed10a34f87d59e9a1acfc3b883a63a5eaeb40312e4832ac27dfbf354e5/rank_0_0/model
+(EngineCore pid=2638884) INFO 07-12 14:52:03 [monitor.py:53] torch.compile took 37.35 s in total
+(EngineCore pid=2638884) INFO 07-12 14:52:03 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
+(EngineCore pid=2638884) INFO 07-12 14:52:05 [monitor.py:81] Initial profiling/warmup run took 2.19 s
+(EngineCore pid=2638884) INFO 07-12 14:52:12 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
+(EngineCore pid=2638884) INFO 07-12 14:52:17 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
+(EngineCore pid=2638884) INFO 07-12 14:52:18 [gpu_worker.py:508] Available KV cache memory: 29.44 GiB
+(EngineCore pid=2638884) INFO 07-12 14:52:18 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
+(EngineCore pid=2638884) INFO 07-12 14:52:18 [kv_cache_utils.py:2146] GPU KV cache size: 321,600 tokens
+(EngineCore pid=2638884) INFO 07-12 14:52:18 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
+(EngineCore pid=2638884) INFO 07-12 14:52:18 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
+(EngineCore pid=2638884) INFO 07-12 14:52:18 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
+(EngineCore pid=2638884) 2026-07-12 14:52:18,390 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
+(EngineCore pid=2638884) 2026-07-12 14:52:18,429 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
+(EngineCore pid=2638884) 
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/5 [00:00<?, ?it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  40%|████      | 2/5 [00:00<00:00, 10.98it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 4/5 [00:01<00:00,  2.57it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00,  1.77it/s]
+Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00,  2.12it/s]
+(EngineCore pid=2638884) 
+Capturing CUDA graphs (decode, FULL):   0%|          | 0/4 [00:00<?, ?it/s]
+Capturing CUDA graphs (decode, FULL):  50%|█████     | 2/4 [00:00<00:00, 11.61it/s]
+Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 11.88it/s]
+Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 11.84it/s]
+(EngineCore pid=2638884) INFO 07-12 14:52:21 [gpu_model_runner.py:6660] Graph capturing finished in 3 secs, took 0.08 GiB
+(EngineCore pid=2638884) INFO 07-12 14:52:21 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (12.2%).
+(EngineCore pid=2638884) INFO 07-12 14:52:21 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
+(EngineCore pid=2638884) INFO 07-12 14:52:22 [core.py:337] init engine (profile, create kv cache, warmup model) took 56.63 s (compilation: 37.35 s)
+(EngineCore pid=2638884) INFO 07-12 14:52:22 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl
+(EngineCore pid=2638884) INFO 07-12 14:52:22 [vllm.py:1006] Asynchronous scheduling is enabled.
+(EngineCore pid=2638884) INFO 07-12 14:52:22 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(APIServer pid=2638186) INFO 07-12 14:52:22 [api_server.py:577] Supported tasks: ['generate']
+(APIServer pid=2638186) WARNING 07-12 14:52:23 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+(APIServer pid=2638186) INFO 07-12 14:52:23 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
+(APIServer pid=2638186) INFO 07-12 14:52:23 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8500
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:37] Available routes are:
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /docs, Methods: GET, HEAD
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /load, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /version, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /health, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /metrics, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /tokenize, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /detokenize, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/models, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /ping, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /ping, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /invocations, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/messages, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /generative_scoring, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions/render, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
+(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
+(APIServer pid=2638186) INFO:     Started server process [2638186]
+(APIServer pid=2638186) INFO:     Waiting for application startup.
+(APIServer pid=2638186) INFO:     Application startup complete.
+(APIServer pid=2638186) INFO:     127.0.0.1:50438 - "GET /v1/models HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:50454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:50460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638884) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638186) INFO:     127.0.0.1:50472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(EngineCore pid=2638884) WARNING 07-12 14:52:31 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
+(APIServer pid=2638186) INFO:     127.0.0.1:51354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:52:33 [loggers.py:273] Engine 000: Avg prompt throughput: 1338.9 tokens/s, Avg generation throughput: 0.9 tokens/s, Running: 8 reqs, Waiting: 1 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 4.2%
+(APIServer pid=2638186) INFO:     127.0.0.1:51380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:52:43 [loggers.py:273] Engine 000: Avg prompt throughput: 2863.4 tokens/s, Avg generation throughput: 203.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 5.0%
+(APIServer pid=2638186) INFO:     127.0.0.1:34496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:34572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:52:53 [loggers.py:273] Engine 000: Avg prompt throughput: 14.3 tokens/s, Avg generation throughput: 153.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 47.0%
+(APIServer pid=2638186) INFO:     127.0.0.1:51288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:51438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:53:03 [loggers.py:273] Engine 000: Avg prompt throughput: 3326.2 tokens/s, Avg generation throughput: 188.1 tokens/s, Running: 7 reqs, Waiting: 8 reqs, GPU KV cache usage: 7.0%, Prefix cache hit rate: 39.8%
+(APIServer pid=2638186) INFO:     127.0.0.1:42448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42476 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:42522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:53:13 [loggers.py:273] Engine 000: Avg prompt throughput: 3070.5 tokens/s, Avg generation throughput: 160.1 tokens/s, Running: 7 reqs, Waiting: 3 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 34.1%
+(APIServer pid=2638186) INFO 07-12 14:53:23 [loggers.py:273] Engine 000: Avg prompt throughput: 674.1 tokens/s, Avg generation throughput: 112.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.4%
+(APIServer pid=2638186) INFO 07-12 14:53:33 [loggers.py:273] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.4%
+(APIServer pid=2638186) INFO:     127.0.0.1:35688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35656 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:54:03 [loggers.py:273] Engine 000: Avg prompt throughput: 20.3 tokens/s, Avg generation throughput: 219.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 47.8%
+(APIServer pid=2638186) INFO:     127.0.0.1:35706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:35836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:54:13 [loggers.py:273] Engine 000: Avg prompt throughput: 21.0 tokens/s, Avg generation throughput: 311.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 58.8%
+(APIServer pid=2638186) INFO:     127.0.0.1:59770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:59822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:54:23 [loggers.py:273] Engine 000: Avg prompt throughput: 1393.0 tokens/s, Avg generation throughput: 110.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 56.5%
+(APIServer pid=2638186) INFO:     127.0.0.1:36450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:36558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:54:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5941.7 tokens/s, Avg generation throughput: 269.5 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 48.4%
+(APIServer pid=2638186) INFO:     127.0.0.1:58076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58206 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:58228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO 07-12 14:54:43 [loggers.py:273] Engine 000: Avg prompt throughput: 8113.3 tokens/s, Avg generation throughput: 349.9 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 41.3%
+(APIServer pid=2638186) INFO:     127.0.0.1:47602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:47708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+(APIServer pid=2638186) INFO:     127.0.0.1:39082 - "POST /v1/chat/completions HTTP/1.1" 200 OK

runs/opprof-phase6/phase6/cells/tp1_mns8/warmup.log

@@ -0,0 +1 @@
+{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "warmup", "pass_rate": 0.3125, "feasible": false}

runs/opprof-phase6/phase6/cells/tp1_mns8/warmup/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.2265625,
+  "cell": "tp1_mns8",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 16,
+  "feasible": false,
+  "interval": {
+    "elapsed_s": 11.581101349,
+    "end_mono_ns": 199533809459936,
+    "end_wall_ns": 1783867960935887048,
+    "start_mono_ns": 199522228358587,
+    "start_wall_ns": 1783867949354785531
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "warmup",
+  "mns": 8,
+  "observed_count": 16,
+  "pass_rate": 0.3125,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "899b928ee7acd2abc5bf86f6641591111d09f06ec3dbf35a687067b5dfe9ad57",
+    "arrival_s": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 7.4525000000000095,
+      "min": 0.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "count": 16,
+    "long_gt4096": 6,
+    "offered_req_s": 0.26666666666666666,
+    "offered_req_s_per_gpu": 0.26666666666666666,
+    "raw_input_tokens": {
+      "distinct_n": 16,
+      "finite_n": 16,
+      "max": 7270.0,
+      "min": 72.0,
+      "missing_n": 0,
+      "n": 16
+    },
+    "raw_length_order_sha256": "c99abf7970483c5386d5c34fad5a1b2664c29b8363b5b6e096338ed8e9773d27",
+    "request_id_order_sha256": "9a7cbf2821c1a7b2a498858815a6f31b0d307f2c8a4900d8be107e6a15858964"
+  },
+  "slo_pass_count": 5,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 1,
+  "tpot_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 37.95462277145895,
+    "min": 10.364419787317354,
+    "missing_n": 0,
+    "n": 16
+  },
+  "ttft_ms": {
+    "distinct_n": 16,
+    "finite_n": 16,
+    "max": 4013.1819479865953,
+    "min": 2348.2964480062947,
+    "missing_n": 0,
+    "n": 16
+  }
+}

runs/opprof-phase6/phase6/cells/tp2_mns16/anchor-0.4921875.log

@@ -0,0 +1 @@
+{"cell": "tp2_mns16", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

runs/opprof-phase6/phase6/cells/tp2_mns16/anchor-0.4921875/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.4921875,
+  "cell": "tp2_mns16",
+  "early_stop_reason": "",
+  "early_stopped": false,
+  "exact_output_count": 269,
+  "feasible": true,
+  "interval": {
+    "elapsed_s": 61.339108631,
+    "end_mono_ns": 201223962200316,
+    "end_wall_ns": 1783869651088627469,
+    "start_mono_ns": 201162623091685,
+    "start_wall_ns": 1783869589749518962
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 16,
+  "observed_count": 269,
+  "pass_rate": 1.0,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
+    "arrival_s": {
+      "distinct_n": 269,
+      "finite_n": 269,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 269
+    },
+    "count": 269,
+    "long_gt4096": 95,
+    "offered_req_s": 4.483333333333333,
+    "offered_req_s_per_gpu": 2.2416666666666667,
+    "raw_input_tokens": {
+      "distinct_n": 250,
+      "finite_n": 269,
+      "max": 8149.0,
+      "min": 71.0,
+      "missing_n": 0,
+      "n": 269
+    },
+    "raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
+    "request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
+  },
+  "slo_pass_count": 269,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 2,
+  "tpot_ms": {
+    "distinct_n": 269,
+    "finite_n": 269,
+    "max": 37.621230566938316,
+    "min": 5.02581280306913,
+    "missing_n": 0,
+    "n": 269
+  },
+  "ttft_ms": {
+    "distinct_n": 269,
+    "finite_n": 269,
+    "max": 1802.1751450141892,
+    "min": 38.09416797594167,
+    "missing_n": 0,
+    "n": 269
+  }
+}

runs/opprof-phase6/phase6/cells/tp2_mns16/anchor-0.49609375.log

@@ -0,0 +1 @@
+{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 0.08791208791208792, "feasible": false}

runs/opprof-phase6/phase6/cells/tp2_mns16/anchor-0.49609375/result.json

@@ -0,0 +1,74 @@
+{
+  "anchor": 0.49609375,
+  "cell": "tp2_mns16",
+  "early_stop_reason": "slo_pass_rate_unrecoverable",
+  "early_stopped": true,
+  "exact_output_count": 85,
+  "feasible": false,
+  "interval": {
+    "elapsed_s": 29.202033933,
+    "end_mono_ns": 201151639670622,
+    "end_wall_ns": 1783869578766097690,
+    "start_mono_ns": 201122437636689,
+    "start_wall_ns": 1783869549564063872
+  },
+  "invariants": {
+    "arrival_nondecreasing": true,
+    "exact_output_or_failed": true,
+    "outcomes_cover_selected": true,
+    "raw_lengths_present": true,
+    "selected_nonempty": true,
+    "warmup_16": true,
+    "warmup_exact_16": true,
+    "warmup_long": true
+  },
+  "kind": "anchor",
+  "mns": 16,
+  "observed_count": 273,
+  "pass_rate": 0.08791208791208792,
+  "schema": 1,
+  "selection": {
+    "arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
+    "arrival_s": {
+      "distinct_n": 273,
+      "finite_n": 273,
+      "max": 59.78950000000005,
+      "min": 0.008300000000008368,
+      "missing_n": 0,
+      "n": 273
+    },
+    "count": 273,
+    "long_gt4096": 97,
+    "offered_req_s": 4.55,
+    "offered_req_s_per_gpu": 2.275,
+    "raw_input_tokens": {
+      "distinct_n": 253,
+      "finite_n": 273,
+      "max": 8149.0,
+      "min": 71.0,
+      "missing_n": 0,
+      "n": 273
+    },
+    "raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
+    "request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
+  },
+  "slo_pass_count": 24,
+  "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
+  "tp": 2,
+  "tpot_ms": {
+    "distinct_n": 85,
+    "finite_n": 85,
+    "max": 56.565095488335864,
+    "min": 5.2603521261925215,
+    "missing_n": 188,
+    "n": 273
+  },
+  "ttft_ms": {
+    "distinct_n": 85,
+    "finite_n": 85,
+    "max": 11585.765292984433,
+    "min": 40.21007599658333,
+    "missing_n": 188,
+    "n": 273
+  }
+}