gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

537
runs/opprof-phase5/analyze_phase5.py Normal file
View File

@@ -0,0 +1,537 @@
#!/usr/bin/env python3
"""Frozen Phase-5 bridge/control ledger analysis."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from pathlib import Path
from typing import Any
import numpy as np
SEED = 20260716
RESAMPLES = 100_000
RATE = 0.4725
ARMS = ("base", "A1", "A2", "A3", "A4")
MECHANISMS = ("A1", "A2", "A3", "A4")
CAPTURE_SIZES = {
1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88,
96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192,
200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
}
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1 << 20), b""):
digest.update(chunk)
return digest.hexdigest()
def numeric(values: list[float | int | None]) -> dict[str, Any]:
finite = [float(value) for value in values if value is not None and math.isfinite(float(value))]
return {
"n": len(values),
"finite_n": len(finite),
"missing_n": len(values) - len(finite),
"min": min(finite) if finite else None,
"max": max(finite) if finite else None,
"distinct_n": len(set(finite)),
}
def ci(draws: np.ndarray) -> list[float]:
low, high = np.quantile(draws, [0.025, 0.975])
return [float(low), float(high)]
def cv(values: list[float]) -> float:
array = np.asarray(values, dtype=np.float64)
mean = float(array.mean())
if mean == 0:
return 0.0 if np.all(array == 0) else math.inf
return float(array.std(ddof=0) / mean)
def parse_run(run_dir: Path) -> dict[str, Any]:
result = json.loads((run_dir / "client/result.json").read_text())
complete = json.loads((run_dir / "run-complete.json").read_text())
t0 = int(result["t0_mono_ns"])
clean_start = float(result["clean"]["start_s"])
clean_end = float(result["clean"]["end_s"])
stream = next((run_dir / "opprof").glob("*.jsonl"))
records = []
for line in stream.read_text().splitlines():
item = json.loads(line)
if "step_index" not in item:
continue
relative = (int(item["submit_mono_ns"]) - t0) / 1e9
if clean_start <= relative < clean_end:
item["relative_s"] = relative
records.append(item)
blocks = []
decode_means = []
waiting_means = []
for block_index in range(48):
start = clean_start + 5 * block_index
selected = [item for item in records if start <= item["relative_s"] < start + 5]
if not selected:
# Fixed-time blocks are the resampling unit. A rate-following trace
# can legitimately have an idle block, which contributes no tokens
# and no model-step time but must remain in the arrival-variance
# distribution.
blocks.append([0, 0.0])
decode_means.append(0.0)
waiting_means.append(0.0)
continue
tokens = sum(int(item["prefill_tokens"]) + int(item["decode_tokens"]) for item in selected)
duration = sum(
(int(item["complete_mono_ns"]) - int(item["submit_mono_ns"])) / 1e6
for item in selected
)
blocks.append([tokens, duration])
decode_means.append(float(np.mean([int(item["decode_batch_size"]) for item in selected])))
waiting_means.append(float(np.mean([int(item["queues"]["waiting"]) for item in selected])))
pure_decode = [
item for item in records
if int(item["prefill_tokens"]) == 0 and int(item["decode_batch_size"]) > 0
]
pure_bucket = sum(int(item["cudagraph"]["bucket_tokens"]) for item in pure_decode)
pure_padding = sum(int(item["cudagraph"]["padding_tokens"]) for item in pure_decode)
support = sorted({int(item["decode_batch_size"]) for item in pure_decode})
covered = sum(int(item["decode_batch_size"]) in CAPTURE_SIZES for item in pure_decode)
prefix_queries = 0
prefix_hits = 0
prefix_present = 0
for item in records:
local = item.get("prefix", {}).get("local")
if local is None:
continue
prefix_present += 1
prefix_queries += int(local.get("queries", 0))
prefix_hits += int(local.get("hits", 0))
block_array = np.asarray(blocks, dtype=np.float64)
return {
"run_id": complete["run_id"],
"run_dir": str(run_dir),
"stream_sha256": sha256_file(stream),
"blocks": block_array,
"token_efficiency_per_ms": float(block_array[:, 0].sum() / block_array[:, 1].sum()),
"clean_steps": len(records),
"clean_duration_s": clean_end - clean_start,
"clean_failed": int(result["clean"]["failed"]),
"offered_rps": float(result["clean"]["offered_rps"]),
"drain_seconds": float(result["drain_seconds"]),
"warmup_completions": int(complete["client"]["warmup_completions"]),
"warmup_gate_branch": complete["client"].get("warmup_gate_branch", "P3-pre-A-P5-1"),
"warmup_stability": complete["client"].get("warmup_stability"),
"cold_start_gate": complete["client"].get("cold_start_gate"),
"decode_B_block_cv": cv(decode_means),
"waiting_block_cv": cv(waiting_means),
"pure_decode_steps": len(pure_decode),
"pure_decode_support": support,
"pure_decode_support_coverage": covered / len(pure_decode),
"pure_decode_padding_tokens": pure_padding,
"pure_decode_bucket_tokens": pure_bucket,
"pure_decode_padding_fraction": pure_padding / pure_bucket,
"prefix_present_steps": prefix_present,
"prefix_queries": prefix_queries,
"prefix_hits": prefix_hits,
"prefix_hit_ratio": prefix_hits / prefix_queries if prefix_queries else 0.0,
"layer1_invariants": complete["layer1"]["invariants"],
"client_invariants": complete["client"]["invariants"],
"server_invariants": complete["server_invariants"],
"drain_quarantined": bool(complete["drain_quarantined"]),
}
def hierarchical_draws(runs: list[dict[str, Any]], rng: np.random.Generator) -> np.ndarray:
count = len(runs)
output = np.empty(RESAMPLES, dtype=np.float64)
for start in range(0, RESAMPLES, 5000):
size = min(5000, RESAMPLES - start)
token_sum = np.zeros(size, dtype=np.float64)
duration_sum = np.zeros(size, dtype=np.float64)
selected_runs = rng.integers(0, count, size=(size, count))
for position in range(count):
block_indices = rng.integers(0, 48, size=(size, 48))
for run_index, run in enumerate(runs):
mask = selected_runs[:, position] == run_index
if not np.any(mask):
continue
blocks = run["blocks"]
choices = block_indices[mask]
token_sum[mask] += blocks[choices, 0].sum(axis=1)
duration_sum[mask] += blocks[choices, 1].sum(axis=1)
output[start : start + size] = token_sum / duration_sum
return output
def point_efficiency(runs: list[dict[str, Any]]) -> float:
tokens = sum(float(run["blocks"][:, 0].sum()) for run in runs)
duration = sum(float(run["blocks"][:, 1].sum()) for run in runs)
return tokens / duration
def two_sided_p(draws: np.ndarray) -> float:
return float(min(1.0, 2 * min(np.mean(draws <= 0), np.mean(draws >= 0))))
def holm(pvalues: dict[str, float]) -> dict[str, float]:
ordered = sorted(pvalues, key=pvalues.get)
adjusted: dict[str, float] = {}
running = 0.0
total = len(ordered)
for rank, key in enumerate(ordered):
running = max(running, (total - rank) * pvalues[key])
adjusted[key] = min(1.0, running)
return adjusted
def discover_primary(root: Path) -> dict[str, list[dict[str, Any]]]:
result = {arm: [] for arm in ARMS}
for marker in sorted((root / "primary").glob("*/moderate/run-complete.json")):
if "background" in str(marker):
continue
parsed = parse_run(marker.parent)
arm = parsed["run_id"].split("-r", 1)[0]
if arm in result:
result[arm].append(parsed)
if any(len(result[arm]) != 3 for arm in ARMS):
raise RuntimeError(f"primary replication mismatch: { {key:len(value) for key,value in result.items()} }")
return result
def control_runs(root: Path, p3root: Path, pattern: str) -> tuple[list[dict[str, Any]], str]:
fresh = sorted((root / "primary").glob(f"control-{pattern}-r*-C00/moderate/run-complete.json"))
if fresh:
if len(fresh) != 3:
raise RuntimeError(f"partial fresh control set: {pattern}: {len(fresh)}")
return [parse_run(path.parent) for path in fresh], "P5-rerun"
return [parse_run(p3root / f"primary/{pattern}-C00/moderate")], "P3-reused"
def aggregate_aux(runs: list[dict[str, Any]], key: str) -> float:
return float(np.mean([float(run[key]) for run in runs]))
def analyze(root: Path, p3root: Path, private: Path) -> dict[str, Any]:
primary = discover_primary(root)
p3_base = [parse_run(p3root / "primary/P10-C00/moderate")]
controls = {}
control_source = {}
for pattern in ("P03", "P04"):
controls[pattern], control_source[pattern] = control_runs(root, p3root, pattern)
rng = np.random.default_rng(SEED)
draws = {arm: hierarchical_draws(primary[arm], rng) for arm in ARMS}
p3_base_draws = hierarchical_draws(p3_base, rng)
control_draws = {pattern: hierarchical_draws(controls[pattern], rng) for pattern in controls}
points = {arm: point_efficiency(primary[arm]) for arm in ARMS}
p3_base_point = point_efficiency(p3_base)
control_points = {pattern: point_efficiency(controls[pattern]) for pattern in controls}
bridge_draws = draws["A3"] - p3_base_draws
bridge_point = points["A3"] - p3_base_point
bridge_ci = ci(bridge_draws)
bridge = {
"point_delta": bridge_point,
"relative_abs_delta": abs(bridge_point) / p3_base_point,
"ci95": bridge_ci,
"within_3pct": abs(bridge_point) / p3_base_point <= 0.03,
"ci_contains_zero": bridge_ci[0] <= 0 <= bridge_ci[1],
}
bridge["reuse_passed"] = bridge["within_3pct"] and bridge["ci_contains_zero"]
deltas = {arm: draws[arm] - draws["base"] for arm in MECHANISMS}
raw_p = {arm: two_sided_p(deltas[arm]) for arm in MECHANISMS}
holm_p = holm(raw_p)
manifests = {
name: json.loads((private / f"P10-{name}.jsonl.summary.json").read_text())
for name in ("base", "A1", "A3")
}
base_prefix = aggregate_aux(primary["base"], "prefix_hit_ratio")
a1_prefix = aggregate_aux(primary["A1"], "prefix_hit_ratio")
base_padding = aggregate_aux(primary["base"], "pure_decode_padding_fraction")
a2_padding = aggregate_aux(primary["A2"], "pure_decode_padding_fraction")
padding_reduction = (base_padding - a2_padding) / base_padding if base_padding > 0 else 0.0
manipulations = {
"A1": {
"passed": (
manifests["base"]["r16"] - manifests["A1"]["r16"] >= 0.15
and (manifests["base"]["r16"] - manifests["A1"]["r16"]) / manifests["base"]["r16"] >= 0.20
and manifests["A1"]["max_added_delay_seconds"] <= 64
and abs(a1_prefix - base_prefix) <= 0.01
),
"base_R16": manifests["base"]["r16"],
"ablated_R16": manifests["A1"]["r16"],
"prefix_hit_ratio_delta": a1_prefix - base_prefix,
"max_added_delay_seconds": manifests["A1"]["max_added_delay_seconds"],
},
"A2": {
"passed": aggregate_aux(primary["A2"], "pure_decode_support_coverage") >= 0.99 and padding_reduction >= 0.90,
"support_coverage": aggregate_aux(primary["A2"], "pure_decode_support_coverage"),
"base_padding_fraction": base_padding,
"ablated_padding_fraction": a2_padding,
"padding_reduction": padding_reduction,
"observed_support": sorted({value for run in primary["A2"] for value in run["pure_decode_support"]}),
},
"A3": {
"passed": (
aggregate_aux(primary["A3"], "decode_B_block_cv") < aggregate_aux(primary["base"], "decode_B_block_cv")
and aggregate_aux(primary["A3"], "waiting_block_cv") < aggregate_aux(primary["base"], "waiting_block_cv")
),
"base_decode_B_cv": aggregate_aux(primary["base"], "decode_B_block_cv"),
"ablated_decode_B_cv": aggregate_aux(primary["A3"], "decode_B_block_cv"),
"base_waiting_cv": aggregate_aux(primary["base"], "waiting_block_cv"),
"ablated_waiting_cv": aggregate_aux(primary["A3"], "waiting_block_cv"),
},
"A4": {
"passed": sum(run["prefix_queries"] for run in primary["A4"]) == 0 and sum(run["prefix_hits"] for run in primary["A4"]) == 0,
"prefix_queries": sum(run["prefix_queries"] for run in primary["A4"]),
"prefix_hits": sum(run["prefix_hits"] for run in primary["A4"]),
},
}
ledgers = {}
for control in ("P03", "P04"):
denominator = control_draws[control] - draws["base"]
point_denominator = control_points[control] - points["base"]
denominator_ci = ci(denominator)
stable_denominator = bool(
point_denominator > 0
and denominator_ci[0] > 0
and np.mean(denominator <= 0) <= 0.05
)
rows = {}
share_draws = {}
for arm in MECHANISMS:
share_draws[arm] = deltas[arm] / denominator
point = (points[arm] - points["base"]) / point_denominator
interval = ci(share_draws[arm])
reportable = stable_denominator and manipulations[arm]["passed"]
rows[arm] = {
"delta_E": points[arm] - points["base"],
"delta_E_ci95": ci(deltas[arm]),
"share": point if reportable else None,
"share_ci95": interval if reportable else None,
"diagnostic_share": point,
"diagnostic_share_ci95": interval,
"share_status": (
"REPORTABLE"
if reportable
else (
"N/A—unstable control denominator"
if not stable_denominator
else "N/A—manipulation failed"
)
),
"p_two_sided": raw_p[arm],
"p_holm": holm_p[arm],
"manipulation_passed": manipulations[arm]["passed"],
}
residual_draws = 1.0 - sum(share_draws.values())
residual_point = 1.0 - sum(row["diagnostic_share"] for row in rows.values())
ledgers[control] = {
"status": "EVALUABLE" if stable_denominator else "INCONCLUSIVE—unstable denominator",
"control_source": control_source[control],
"control_E": control_points[control],
"base_E": points["base"],
"gap_E": point_denominator,
"gap_ci95": denominator_ci,
"denominator_nonpositive_fraction": float(np.mean(denominator <= 0)),
"stable_denominator": stable_denominator,
"mechanisms": rows,
"diagnostic_share_sum": sum(row["diagnostic_share"] for row in rows.values()),
"residual_interaction": residual_point,
"residual_interaction_ci95": ci(residual_draws),
"residual_status": (
"REPORTABLE"
if stable_denominator and all(item["passed"] for item in manipulations.values())
else "DIAGNOSTIC_ONLY—incomplete official share ledger"
),
}
dominance = {}
for arm in MECHANISMS:
per_control = {}
for control in ("P03", "P04"):
row = ledgers[control]["mechanisms"][arm]
if not ledgers[control]["stable_denominator"] or not row["manipulation_passed"]:
per_control[control] = "NOT_EVALUABLE"
continue
direction_ok = row["delta_E"] > 0 if arm != "A4" else True
per_control[control] = (
row["share"] >= 0.30
and row["share_ci95"][0] > 0.15
and row["p_holm"] < 0.05
and direction_ok
and row["manipulation_passed"]
)
dominance[arm] = {
"per_control": per_control,
"verdict": (
"NOT EVALUABLE"
if any(value == "NOT_EVALUABLE" for value in per_control.values())
else (
"DOMINANT"
if all(per_control.values())
else ("CONTROL-SENSITIVE" if any(per_control.values()) else "NOT DOMINANT")
)
),
}
all_runs = [run for arm in ARMS for run in primary[arm]]
share_widths = [
ledgers[control]["mechanisms"][arm]["diagnostic_share_ci95"][1]
- ledgers[control]["mechanisms"][arm]["diagnostic_share_ci95"][0]
for control in ("P03", "P04") for arm in MECHANISMS
]
state = json.loads((root / "controller-state.json").read_text())
amendment_evidence_path = root / "a-p5-1-retained-audit.jsonl"
amendment_evidence = []
if amendment_evidence_path.exists():
amendment_evidence = [
json.loads(line) for line in amendment_evidence_path.read_text().splitlines()
]
invariants = {
"primary_runs_15": len(all_runs) == 15,
"three_replicates_per_arm": all(len(primary[arm]) == 3 for arm in ARMS),
"clean_duration_240": all(math.isclose(run["clean_duration_s"], 240.0) for run in all_runs),
"clean_failures_zero": all(run["clean_failed"] == 0 for run in all_runs),
"offered_rate_within_5pct": all(abs(run["offered_rps"] / RATE - 1) <= 0.05 for run in all_runs),
"layer1_accounting": all(all(run["layer1_invariants"].values()) for run in all_runs),
"client_invariants": all(all(value for key, value in run["client_invariants"].items() if key != "drain_re_adjudicated") for run in all_runs),
"server_invariants": all(all(run["server_invariants"].values()) for run in all_runs),
"a_p5_1_cold_start_gates": all(
run["cold_start_gate"] is not None
and run["cold_start_gate"]["passed"]
for run in all_runs
),
"drain_quarantine_under_20pct": sum(run["drain_quarantined"] for run in all_runs) / 15 <= 0.20,
"gpu_budget_below_6": float(state["gpu_hours_total"]) < 6.0,
"manifests_same_ids": len({manifests[name]["request_id_set_sha256"] for name in manifests}) == 1,
"manifests_same_token_sums": len({manifests[name]["input_tokens"]["sum"] for name in manifests}) == 1 and len({manifests[name]["output_tokens"]["sum"] for name in manifests}) == 1,
"control_denominators_stable": all(ledgers[control]["stable_denominator"] for control in ledgers),
"bridge_decision_resolved": bridge["reuse_passed"] or all(source == "P5-rerun" for source in control_source.values()),
"ratios_finite": all(math.isfinite(ledgers[c]["mechanisms"][a]["diagnostic_share"]) for c in ledgers for a in MECHANISMS),
"per_arm_results_not_all_identical": len({round(points[arm], 12) for arm in ARMS}) > 1,
}
red_flags = [key for key, value in invariants.items() if not value]
publishable = (
not red_flags
and all(item["passed"] for item in manipulations.values())
and sum(width > 0.50 for width in share_widths) < 2
)
return {
"schema": 1,
"status": "PUBLISHABLE" if publishable else "INCONCLUSIVE_OR_PARTIAL",
"limitation": "Recorded-arrival P5 bridge ledger anchored to P3 controls; not a literal decomposition of P3's already-uniform P10 gap.",
"analysis_seed": SEED,
"bootstrap_resamples": RESAMPLES,
"efficiency": {
arm: {
"point": points[arm],
"ci95": ci(draws[arm]),
"runs": [
{key: value for key, value in run.items() if key not in {"blocks", "warmup_stability"}}
for run in primary[arm]
],
}
for arm in ARMS
},
"p3_base_E": p3_base_point,
"bridge": bridge,
"control_sources": control_source,
"manipulations": manipulations,
"holm": {"family": list(MECHANISMS), "raw_p": raw_p, "adjusted_p": holm_p},
"ledgers": ledgers,
"dominance": dominance,
"config_tier_A2": {
"delta_E": points["A2"] - points["base"],
"relative_E_delta": points["A2"] / points["base"] - 1,
"delta_E_ci95": ci(deltas["A2"]),
"base_padding_fraction": base_padding,
"A2_padding_fraction": a2_padding,
"padding_reduction": padding_reduction,
},
"amendment_A_P5_1": {
"reason": "Rate-following throughput drift tracks arrival shape and is not a cold-start stationarity test.",
"retained_failed_run_evidence": amendment_evidence,
"recorded_drift_range": (
[
min(
item["superseded_drift_evidence"]["normalized_drift"]
for item in amendment_evidence
if item["run"] != "A3-r1-C00"
),
max(
item["superseded_drift_evidence"]["normalized_drift"]
for item in amendment_evidence
if item["run"] != "A3-r1-C00"
),
]
if amendment_evidence else None
),
"uniform_A3_drift": (
next(
item["superseded_drift_evidence"]["normalized_drift"]
for item in amendment_evidence
if item["run"] == "A3-r1-C00"
)
if amendment_evidence else None
),
},
"gpu": {
"new_h20_hours": float(state["gpu_hours_total"]),
"hard_cap": 6.0,
"controller_status": state["status"],
"completed_measured_runs_including_background": state["completed_measured_runs"],
"completed_burnins": state["completed_burnins"],
},
"sanity": {
"red_flags": red_flags,
"invariants": invariants,
"numeric": {
"primary_E": numeric([run["token_efficiency_per_ms"] for run in all_runs]),
"clean_steps": numeric([run["clean_steps"] for run in all_runs]),
"offered_rps": numeric([run["offered_rps"] for run in all_runs]),
"drain_seconds": numeric([run["drain_seconds"] for run in all_runs]),
"diagnostic_share": numeric([ledgers[c]["mechanisms"][a]["diagnostic_share"] for c in ledgers for a in MECHANISMS]),
"residual_interaction": numeric([ledgers[c]["residual_interaction"] for c in ledgers]),
"share_ci_width": numeric(share_widths),
},
"declared": {
"manipulation_failures": [arm for arm, item in manipulations.items() if not item["passed"]],
"control_sources": control_source,
"bridge_reuse_passed": bridge["reuse_passed"],
},
},
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, required=True)
parser.add_argument("--p3-root", type=Path, required=True)
parser.add_argument("--private", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
result = analyze(args.root, args.p3_root, args.private)
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
print(json.dumps({
"status": result["status"],
"bridge": result["bridge"],
"red_flags": result["sanity"]["red_flags"],
"gpu": result["gpu"],
}, sort_keys=True))
if __name__ == "__main__":
main()

242
runs/opprof-phase5/opprof_phase5_client.py Normal file
View File

@@ -0,0 +1,242 @@
#!/usr/bin/env python3
"""Phase-5 private-manifest transforms and timestamp-scheduled P3 client wrapper."""
from __future__ import annotations
import argparse
import asyncio
import json
import math
import sys
from pathlib import Path
from typing import Any
import aiohttp
import opprof_phase3_client as p3
def _numeric(values: list[float | int]) -> dict[str, Any]:
finite = [float(value) for value in values if math.isfinite(float(value))]
return {
"n": len(values),
"finite_n": len(finite),
"missing_n": len(values) - len(finite),
"min": min(finite) if finite else None,
"max": max(finite) if finite else None,
"distinct_n": len(set(finite)),
"sum": sum(finite),
}
def _r16(rows: list[dict[str, Any]]) -> float:
groups = [rows[index : index + 16] for index in range(0, len(rows) - 15, 16)]
useful = sum(sum(int(row["input_tokens"]) for row in group) for group in groups)
rectangular = sum(16 * max(int(row["input_tokens"]) for row in group) for group in groups)
return 1.0 - useful / rectangular
def _source_timestamps(path: Path, indices: set[int], field: str) -> dict[int, float]:
result: dict[int, float] = {}
maximum = max(indices)
with path.open(encoding="utf-8") as source:
for index, line in enumerate(source):
if index in indices:
value = float(json.loads(line)[field])
if not math.isfinite(value):
raise ValueError(f"non-finite source timestamp at {index}")
result[index] = value
if index >= maximum:
break
if set(result) != indices:
raise ValueError("timestamp source did not cover all source_index values")
return result
def transform(args: argparse.Namespace) -> dict[str, Any]:
rows = p3.load_manifest(Path(args.input))[: args.take_first]
if len(rows) != args.take_first:
raise ValueError(f"requested {args.take_first} rows, found {len(rows)}")
indices = [int(row[args.join_key]) for row in rows]
timestamps = _source_timestamps(
Path(args.timestamp_source), set(indices), args.timestamp_field
)
source_times = [timestamps[index] for index in indices]
if any(right < left for left, right in zip(source_times, source_times[1:])):
raise ValueError("selected timestamps are not nondecreasing")
if source_times[-1] <= source_times[0]:
raise ValueError("selected timestamp span is not positive")
end_s = (len(rows) - 1) / args.target_rate
scale = end_s / (source_times[-1] - source_times[0])
recorded_slots = [(value - source_times[0]) * scale for value in source_times]
uniform_slots = [index / args.target_rate for index in range(len(rows))]
slots = recorded_slots if args.arrival == "recorded-scaled" else uniform_slots
for index, row in enumerate(rows):
row["arrival"] = args.arrival
row["arrival_s"] = slots[index]
row["original_index"] = index
row["source_timestamp"] = source_times[index]
original = list(rows)
max_added_delay = 0.0
if args.service_order == "length-binned":
edges = [int(item) for item in args.length_bin_edges.split(",")]
def bin_id(row: dict[str, Any]) -> int:
length = int(row["input_tokens"])
for index, edge in enumerate(edges):
if length <= edge:
return index
raise ValueError(f"input length {length} exceeds final edge")
reordered: list[dict[str, Any]] = []
for offset in range(0, len(rows), args.reorder_block_size):
block = rows[offset : offset + args.reorder_block_size]
ordered = sorted(
block,
key=lambda row: (
bin_id(row),
int(row["input_tokens"]),
int(row["original_index"]),
),
)
block_slots = slots[offset : offset + len(block)]
for position, row in enumerate(ordered):
added = max(0.0, block_slots[position] - float(row["arrival_s"]))
max_added_delay = max(max_added_delay, added)
row["arrival_s"] = block_slots[position]
reordered.extend(ordered)
rows = reordered
if max_added_delay > args.max_added_delay_seconds + 1e-9:
raise ValueError(
f"fairness cap exceeded: {max_added_delay} > {args.max_added_delay_seconds}"
)
elif args.service_order != "original":
raise ValueError(f"unsupported service order: {args.service_order}")
if sorted(row["request_id"] for row in rows) != sorted(
row["request_id"] for row in original
):
raise AssertionError("request identity changed")
for key in ("input_tokens", "output_tokens"):
if sum(int(row[key]) for row in rows) != sum(int(row[key]) for row in original):
raise AssertionError(f"{key} total changed")
arrival_values = [float(row["arrival_s"]) for row in rows]
if any(right < left for left, right in zip(arrival_values, arrival_values[1:])):
raise AssertionError("assigned arrival slots are not nondecreasing")
output = Path(args.out)
p3.atomic_jsonl(output, rows, mode=0o600)
summary = {
"schema": 1,
"path": str(output),
"sha256": p3.sha256_file(output),
"rows": len(rows),
"arrival": args.arrival,
"service_order": args.service_order,
"target_rate": args.target_rate,
"input_tokens": _numeric([int(row["input_tokens"]) for row in rows]),
"output_tokens": _numeric([int(row["output_tokens"]) for row in rows]),
"arrival_s": _numeric(arrival_values),
"source_timestamp": _numeric(source_times),
"r16": _r16(rows),
"max_added_delay_seconds": max_added_delay,
"request_id_set_sha256": p3.hashlib.sha256(
"\n".join(sorted(str(row["request_id"]) for row in rows)).encode()
).hexdigest(),
"invariants": {
"same_request_ids": True,
"same_input_tokens": True,
"same_output_tokens": True,
"arrival_nondecreasing": True,
"fairness_cap": max_added_delay <= args.max_added_delay_seconds + 1e-9,
"no_prompt_in_summary": True,
},
}
p3.atomic_json(output.with_suffix(output.suffix + ".summary.json"), summary, mode=0o600)
print(json.dumps(summary, sort_keys=True))
return summary
async def finite_timestamp_load(
ctx: p3.RunContext, session: aiohttp.ClientSession, rate: float
) -> list[dict[str, Any]]:
if "arrival_s" not in ctx.rows[0]:
return await _ORIGINAL_FINITE_LOAD(ctx, session, rate)
sem = asyncio.Semaphore(ctx.args.max_concurrency)
tasks: list[asyncio.Task[dict[str, Any]]] = []
async def limited(row: dict[str, Any], scheduled: float) -> dict[str, Any]:
async with sem:
return await p3.request_one(ctx, session, row, scheduled)
for expected in ctx.rows:
scheduled = ctx.t0 + float(expected["arrival_s"])
delay = scheduled - asyncio.get_running_loop().time()
if delay > 0:
try:
await asyncio.wait_for(ctx.stop_event.wait(), timeout=delay)
break
except asyncio.TimeoutError:
pass
if ctx.stop_event.is_set():
break
row = await ctx.next_row()
if row["request_id"] != expected["request_id"]:
raise AssertionError("timestamp scheduler row drift")
tasks.append(asyncio.create_task(limited(row, scheduled)))
return await asyncio.gather(*tasks) if tasks else []
_ORIGINAL_FINITE_LOAD = p3.finite_load
p3.finite_load = finite_timestamp_load
def build_transform_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
parser.add_argument("--in", dest="input", required=True)
parser.add_argument("--take-first", type=int, required=True)
parser.add_argument("--timestamp-source", required=True)
parser.add_argument("--join-key", default="source_index")
parser.add_argument("--timestamp-field", default="timestamp")
parser.add_argument("--arrival", choices=("recorded-scaled", "uniform"), required=True)
parser.add_argument("--target-rate", type=float, required=True)
parser.add_argument("--service-order", choices=("original", "length-binned"), required=True)
parser.add_argument("--reorder-block-size", type=int, default=32)
parser.add_argument("--analysis-cohort-size", type=int, default=16)
parser.add_argument("--length-bin-edges", default="512,1024,2048,4096,8192,16384,32768")
parser.add_argument("--max-added-delay-seconds", type=float, default=64)
parser.add_argument("--out", required=True)
return parser
def main() -> None:
if len(sys.argv) > 1 and sys.argv[1] == "transform":
transform(build_transform_parser().parse_args(sys.argv[2:]))
return
fixed_rate = None
if "--fixed-request-rate" in sys.argv:
index = sys.argv.index("--fixed-request-rate")
fixed_rate = float(sys.argv[index + 1])
del sys.argv[index : index + 2]
args = p3.build_parser().parse_args()
if args.command != "run":
p3.main()
return
if fixed_rate is not None:
if args.load_point != "moderate" or fixed_rate <= 0 or not math.isfinite(fixed_rate):
raise ValueError("--fixed-request-rate requires positive finite moderate rate")
result_dir = Path(args.result_dir)
result_dir.mkdir(parents=True, exist_ok=True)
source = result_dir / "fixed-rate-source.json"
p3.atomic_json(source, {"clean": {"completed_throughput_rps": fixed_rate}})
args.saturation_result = str(source)
args.rate_fraction = 1.0
if args.profile_after_clean and not args.profile_trace_dir:
raise ValueError("--profile-after-clean requires --profile-trace-dir")
print(json.dumps(asyncio.run(p3.run_load(args)), sort_keys=True))
if __name__ == "__main__":
main()

670
runs/opprof-phase5/opprof_phase5_controller.py Normal file
View File

@@ -0,0 +1,670 @@
#!/usr/bin/env python3
"""Detached, resumable Phase-5 four-way primary/control controller."""
from __future__ import annotations
import argparse
import datetime as dt
import hashlib
import json
import math
import os
import re
import shlex
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any
import opprof_phase3_matrix as m
WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
RUN_ROOT = WORKDIR / "runs/phase5"
PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase5-private/manifests")
P3_PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests")
MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
CLIENT = WORKDIR / "scripts/opprof_phase5_client.py"
P3_CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
STATE = RUN_ROOT / "controller-state.json"
RATE = 0.4725
CAPTURE_SIZES = (
1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88,
96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192,
200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
)
m.WORKDIR = WORKDIR
m.RUN_ROOT = RUN_ROOT
m.PRIVATE = PRIVATE
m.MODEL = MODEL
m.SOURCE = SOURCE
m.VENV = VENV
m.CLIENT = CLIENT
m.STATE = STATE
m.GPU_HOUR_LIMIT = 6.0
m.PRIOR_GPU_HOURS = 0.0
m.CONFIGS = {
"C00": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
"A2": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
"A4": {"tp": 1, "mns": 1024, "mbt": 8192, "flags": []},
}
def sha256_file(path: Path) -> str:
return m.sha256_file(path)
def arm_name(pattern: str) -> str:
return pattern.split("-r", 1)[0]
def manifest_for(pattern: str, burnin: bool) -> Path:
if burnin or pattern.startswith("background"):
return P3_PRIVATE / "P06.jsonl"
if pattern.startswith("control-P03"):
return P3_PRIVATE / "P03.jsonl"
if pattern.startswith("control-P04"):
return P3_PRIVATE / "P04.jsonl"
arm = arm_name(pattern)
return PRIVATE / ("P10-base.jsonl" if arm in {"base", "A2", "A4"} else f"P10-{arm}.jsonl")
def saturation_result_for(pattern: str) -> Path:
if pattern.startswith("control-P03"):
cell = "P03-C00"
elif pattern.startswith("control-P04"):
cell = "P04-C00"
else:
cell = "P10-C00"
return WORKDIR / f"runs/phase3/primary/{cell}/saturation/client/result.json"
def drain_budget(_pattern: str) -> int:
return 600
m.drain_budget = drain_budget
def server_command(assignment: m.Assignment, port: int, _trace_dir: Path) -> list[str]:
arm = arm_name(assignment.cell.pattern)
command = [
"taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/vllm"),
"serve", str(MODEL), "--host", "127.0.0.1", "--port", str(port),
"--tensor-parallel-size", "1", "--enable-chunked-prefill",
]
if arm != "A4" and assignment.cell.config != "A4":
command.append("--enable-prefix-caching")
command.extend(("--shutdown-timeout", "600"))
if arm == "A2" or assignment.cell.config == "A2":
command.extend(("--cudagraph-capture-sizes", *map(str, CAPTURE_SIZES)))
return command
def client_command(
assignment: m.Assignment,
port: int,
run_dir: Path,
_load_point: str,
_profile: bool,
burnin: bool,
_saturation_result: Path | None,
) -> list[str]:
pattern = assignment.cell.pattern
common = [
"taskset", "-c", m.cpu_mask(assignment.gpus), str(VENV / "bin/python"),
str(CLIENT), "run", "--manifest", str(manifest_for(pattern, burnin)),
"--base-url", f"http://127.0.0.1:{port}", "--model", str(MODEL),
"--max-concurrency", "256", "--ignore-eos", "--temperature", "0",
"--workload-seed", "20260712", "--server-seed", "20260712",
"--result-dir", str(run_dir / "client"),
]
if burnin:
return common + [
"--load-point", "saturation", "--request-rate", "inf",
"--warmup-seconds", "0", "--clean-segment-seconds", "20",
"--num-clean-segments", "3", "--post-clean-seconds", "0",
"--drain-timeout-seconds", "600",
]
if pattern.startswith("background"):
return common + [
"--load-point", "saturation", "--request-rate", "inf",
"--warmup-seconds", "60", "--clean-segment-seconds", "80",
"--num-clean-segments", "3", "--post-clean-seconds", "0",
"--drain-timeout-seconds", "600",
]
if pattern.startswith("control-"):
return common + [
"--load-point", "moderate", "--saturation-result",
str(saturation_result_for(pattern)), "--rate-fraction", "0.60",
"--warmup-seconds", "60", "--clean-segment-seconds", "80",
"--num-clean-segments", "3", "--post-clean-seconds", "0",
"--drain-timeout-seconds", "600",
]
return common + [
"--load-point", "moderate", "--fixed-request-rate", str(RATE),
"--warmup-seconds", "60", "--clean-segment-seconds", "80",
"--num-clean-segments", "3", "--post-clean-seconds", "0",
"--drain-timeout-seconds", "600",
]
m.server_command = server_command
m.client_command = client_command
_ORIGINAL_VALIDATE_CLIENT = m.validate_client
def _log_event_time(line: str, t0_wall_ns: int) -> float | None:
match = re.search(r"INFO\s+(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})", line)
if match is None:
return None
month, day, hour, minute, second = map(int, match.groups())
year = dt.datetime.fromtimestamp(t0_wall_ns / 1e9, tz=dt.timezone.utc).year
stamp = dt.datetime(year, month, day, hour, minute, second, tzinfo=dt.timezone.utc)
return stamp.timestamp()
def cold_start_gate(run_dir: Path) -> dict[str, Any]:
result = json.loads((run_dir / "client/result.json").read_text())
requests = [
json.loads(line)
for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
]
warm = [
request for request in requests
if request["success"] and 0 <= float(request["completed_s"]) < 60
]
warm_long = [request for request in warm if int(request["input_tokens"]) >= 8192]
t0_mono_ns = int(result["t0_mono_ns"])
stream = next((run_dir / "opprof").glob("*.jsonl"))
warm_descriptors: set[tuple[str, int]] = set()
clean_descriptors: set[tuple[str, int]] = set()
for line in stream.read_text().splitlines():
item = json.loads(line)
if "step_index" not in item or not item.get("model_executed"):
continue
graph = item["cudagraph"]
if not graph.get("hit"):
continue
relative_s = (int(item["submit_mono_ns"]) - t0_mono_ns) / 1e9
descriptor = (str(graph["runtime_mode"]), int(graph["bucket_tokens"]))
if 0 <= relative_s < 60:
warm_descriptors.add(descriptor)
elif 60 <= relative_s < 300:
clean_descriptors.add(descriptor)
log_lines = (run_dir / "server.log").read_text(errors="replace").splitlines()
ready_indices = [
index for index, line in enumerate(log_lines)
if "Application startup complete" in line
]
if len(ready_indices) != 1:
raise RuntimeError(f"expected one server ready marker: {run_dir}: {ready_indices}")
ready_index = ready_indices[0]
event_pattern = re.compile(
r"torch\.compile took|Directly load AOT compilation|\bCompiling\b|Capturing CUDA graphs",
re.IGNORECASE,
)
events = []
clean_boundary_wall_s = int(result["t0_wall_ns"]) / 1e9 + 60
clean_end_wall_s = int(result["t0_wall_ns"]) / 1e9 + 300
event_gate = True
clean_events = 0
for index, line in enumerate(log_lines):
if not event_pattern.search(line):
continue
timestamp = _log_event_time(line, int(result["t0_wall_ns"]))
phase = "pre-ready" if index < ready_index else "post-ready"
if index >= ready_index:
if timestamp is None:
event_gate = False
phase = "post-ready-unparseable"
elif timestamp >= clean_boundary_wall_s:
event_gate = False
phase = "clean" if timestamp < clean_end_wall_s else "post-clean"
if timestamp < clean_end_wall_s:
clean_events += 1
else:
phase = "warmup"
events.append(
{
"line": index + 1,
"phase": phase,
"timestamp_s": timestamp,
"message_prefix": line[:200],
}
)
config_match = re.search(
r"cudagraph_capture_sizes['\"]?:\s*\[([^\]]+)\]",
"\n".join(log_lines[:ready_index]),
)
startup_sizes = (
{int(value.strip()) for value in config_match.group(1).split(",")}
if config_match else set()
)
startup_modes = set()
for line in log_lines[:ready_index]:
if "Capturing CUDA graphs" not in line:
continue
if "PIECEWISE" in line:
startup_modes.add("PIECEWISE")
if "FULL" in line:
startup_modes.add("FULL")
uncovered = sorted(
descriptor for descriptor in clean_descriptors
if descriptor[0] not in startup_modes or descriptor[1] not in startup_sizes
)
passed = (
event_gate
and len(warm) >= 16
and len(warm_long) >= 1
and startup_modes == {"FULL", "PIECEWISE"}
and bool(startup_sizes)
and not uncovered
and clean_events == 0
)
return {
"amendment": "A-P5-1",
"passed": passed,
"warmup_completions": len(warm),
"warmup_long_completions": len(warm_long),
"warmup_long_min_tokens": min(
(int(request["input_tokens"]) for request in warm_long), default=None
),
"server_ready_line": ready_index + 1,
"compile_capture_events": events,
"event_gate_passed": event_gate,
"clean_capture_events": clean_events,
"startup_capture_sizes": sorted(startup_sizes),
"startup_capture_modes": sorted(startup_modes),
"warmup_descriptors": [list(item) for item in sorted(warm_descriptors)],
"clean_descriptors": [list(item) for item in sorted(clean_descriptors)],
"uncovered_clean_descriptors": [list(item) for item in uncovered],
"invariants": {
"events_preclean": event_gate,
"warmup_completions_ge_16": len(warm) >= 16,
"warmup_long_completion": len(warm_long) >= 1,
"startup_modes_complete": startup_modes == {"FULL", "PIECEWISE"},
"startup_sizes_present": bool(startup_sizes),
"clean_descriptors_covered": not uncovered,
"zero_clean_capture_events": clean_events == 0,
},
}
def validate_rate_client(run_dir: Path) -> dict[str, Any]:
result = json.loads((run_dir / "client/result.json").read_text())
sanity = json.loads((run_dir / "client/sanity.json").read_text())
requests = [
json.loads(line)
for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
]
failed_sanity = [
key for key, value in sanity["invariants"].items()
if not value and key != "drain_within_timeout"
]
failure_summary = m.summarize_request_failures(
requests, float(result["clean"]["start_s"]), float(result["clean"]["end_s"])
)
cold = cold_start_gate(run_dir)
quarantined = float(result["drain_seconds"]) > 600
invariants = {
"client_sanity": not failed_sanity,
"clean_duration": math.isclose(float(result["clean"]["duration_s"]), 240.0),
"clean_failures_zero": result["clean"]["failed"] == 0
and failure_summary["clean_failed"] == 0,
"failed_records_accounted": result["failed_records"] == failure_summary["failed"],
"manifest_no_wrap": not result["manifest_wrapped"]
and not result["manifest_exhausted"],
"warmup_cold_start_gate": cold["passed"],
"profile_count": len(result["profiles"]) == 0,
"drain_re_adjudicated": not quarantined,
}
non_drain = {key: value for key, value in invariants.items() if key != "drain_re_adjudicated"}
if not all(non_drain.values()):
raise RuntimeError(
f"A-P5-1 rate-client invariant failure: {run_dir}: "
f"invariants={invariants}; failed_sanity={failed_sanity}; cold={cold}"
)
return {
"result": result,
"sanity": sanity,
"request_count": len(requests),
"warmup_completions": cold["warmup_completions"],
"warmup_required": 16,
"warmup_gate_branch": "A-P5-1-cold-start",
"warmup_stability": None,
"cold_start_gate": cold,
"drain_budget_seconds": 600,
"drain_quarantined": quarantined,
"excluded_window_failures": failure_summary["excluded"],
"excluded_window_failure_kinds": failure_summary["excluded_kinds"],
"invariants": invariants,
}
def validate_run(
entry: dict[str, Any], profile: bool, burnin: bool, allow_missing_traces: bool = False
) -> dict[str, Any]:
del profile, allow_missing_traces
pattern = entry["assignment"].cell.pattern
if burnin or pattern.startswith("background"):
validation_pattern = "P06"
elif pattern.startswith("control-P03"):
validation_pattern = "P03"
elif pattern.startswith("control-P04"):
validation_pattern = "P04"
else:
validation_pattern = "P10"
if burnin or pattern.startswith("background"):
client = _ORIGINAL_VALIDATE_CLIENT(
entry["run_dir"], validation_pattern, False, burnin
)
else:
client = validate_rate_client(entry["run_dir"])
layer1 = m.validate_layer1(entry["run_dir"])
log = (entry["run_dir"] / "server.log").read_text(errors="replace")
invariants = {
"triton_moe": "Using TRITON Unquantized MoE backend" in log,
"chunked_mbt": "Chunked prefill is enabled with max_num_batched_tokens=8192" in log,
"tp1": "tensor_parallel_size=1" in log,
"drain_shutdown": "mode=drain timeout=600s" in log,
"a2_sizes": entry["assignment"].cell.config != "A2"
or all(str(size) in log for size in (3, 5, 6, 7)),
}
if not all(invariants.values()):
raise RuntimeError(f"server invariant failure: {entry['run_id']}: {invariants}")
forbidden = re.compile(r'"(?:prompt|messages|content|text)"\s*:')
for path in (
entry["run_dir"] / "client/requests.jsonl",
entry["run_dir"] / "client/result.json",
Path(layer1["stream"]),
):
if forbidden.search(path.read_text(errors="replace")):
raise RuntimeError(f"private text leaked: {path}")
summary = {
"schema": 1,
"run_id": entry["run_id"],
"pattern": pattern,
"config": entry["assignment"].cell.config,
"gpus": entry["assignment"].gpus,
"client": client,
"layer1": layer1,
"traces": [],
"missing_trace_files": 0,
"layer2_missing_after_controller_cleanup": False,
"drain_quarantined": client["drain_quarantined"],
"server_invariants": invariants,
}
m.atomic_json(entry["run_dir"] / "run-complete.json", summary)
return summary
m.validate_run = validate_run
def manifests() -> dict[str, Any]:
result = {}
for name in ("base", "A1", "A3"):
path = PRIVATE / f"P10-{name}.jsonl"
summary = json.loads(path.with_suffix(path.suffix + ".summary.json").read_text())
if summary["rows"] != 142 or summary["sha256"] != sha256_file(path):
raise RuntimeError(f"manifest verification failed: {name}")
result[name] = {"path": str(path), "sha256": summary["sha256"]}
return result
def fingerprint() -> dict[str, Any]:
return {
"source_commit": subprocess.check_output(
["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True
).strip(),
"source_tree": subprocess.check_output(
["git", "-C", str(SOURCE), "rev-parse", "HEAD^{tree}"], text=True
).strip(),
"client_sha256": sha256_file(CLIENT),
"controller_sha256": sha256_file(Path(__file__)),
"p3_client_sha256": sha256_file(P3_CLIENT),
"p3_matrix_sha256": sha256_file(Path(m.__file__).resolve()),
"manifests": manifests(),
"capture_sizes": list(CAPTURE_SIZES),
"rate": RATE,
}
def load_state(resume: bool) -> dict[str, Any]:
if STATE.exists():
if not resume:
raise RuntimeError("controller state exists; use --resume")
return json.loads(STATE.read_text())
return {
"schema": 1,
"status": "created",
"created_at": time.time(),
"controller_pid": os.getpid(),
"gpu_hours_total": 0.0,
"gpu_hours_this_stage": 0.0,
"completed_measured_runs": 0,
"completed_burnins": 0,
"drain_quarantined_runs": 0,
"clean_window_failures": 0,
"missing_trace_files": 0,
"stages": {},
"fingerprint": {},
}
def save_state(state: dict[str, Any]) -> None:
state["controller_pid"] = os.getpid()
state["updated_at"] = time.time()
m.atomic_json(STATE, state)
m.save_state = save_state
def ensure_provenance() -> None:
destination = RUN_ROOT / "provenance"
destination.mkdir(parents=True, exist_ok=True)
sources = [CLIENT, Path(__file__).resolve(), Path(m.__file__).resolve(), Path(m.common.__file__).resolve()]
hashes = {}
for source in sources:
target = destination / source.name
digest = sha256_file(source)
if target.exists() and sha256_file(target) != digest:
target = destination / f"{source.stem}.{digest[:12]}{source.suffix}"
if target.exists() and sha256_file(target) != digest:
raise RuntimeError(f"content-addressed provenance mismatch: {target}")
if not target.exists():
shutil.copy2(source, target)
hashes[target.name] = digest
m.atomic_json(destination / "sha256.json", hashes)
def primary_cells() -> list[m.Cell]:
config = {"base": "C00", "A1": "C00", "A2": "A2", "A3": "C00", "A4": "A4"}
items = [m.Cell(f"{arm}-r{replicate}", config[arm]) for arm in config for replicate in range(1, 4)]
return sorted(
items,
key=lambda cell: hashlib.sha256(
f"20260715:{cell.pattern.rsplit('-r',1)[1]}:{arm_name(cell.pattern)}".encode()
).hexdigest(),
)
def pack_unique(items: list[m.Cell], prefix: str) -> list[list[m.Assignment]]:
remaining = list(items)
waves: list[list[m.Assignment]] = []
wave_index = 0
while remaining:
selected: list[m.Cell] = []
used: set[str] = set()
for cell in list(remaining):
key = arm_name(cell.pattern)
if key in used:
continue
selected.append(cell)
used.add(key)
remaining.remove(cell)
if len(selected) == 4:
break
while len(selected) < 4:
selected.append(m.Cell(f"background-{prefix}-{wave_index}-{len(selected)}", "C00"))
assignments = []
for slot, cell in enumerate(selected):
gpu = (slot + wave_index) % 4
assignments.append(m.Assignment(cell, (gpu,)))
waves.append(assignments)
wave_index += 1
return waves
def pack_primary(items: list[m.Cell]) -> list[list[m.Assignment]]:
"""Pack SHA-ordered cells as 4/4/4/3 without duplicate arms per wave."""
capacities = (4, 4, 4, 3)
waves: list[list[m.Cell]] = [[] for _ in capacities]
def place(index: int) -> bool:
if index == len(items):
return all(len(wave) == capacity for wave, capacity in zip(waves, capacities, strict=True))
cell = items[index]
arm = arm_name(cell.pattern)
for wave_index, capacity in enumerate(capacities):
if len(waves[wave_index]) >= capacity:
continue
if any(arm_name(existing.pattern) == arm for existing in waves[wave_index]):
continue
waves[wave_index].append(cell)
if place(index + 1):
return True
waves[wave_index].pop()
return False
if not place(0):
raise RuntimeError("cannot pack frozen primary assignments into 4/4/4/3")
result: list[list[m.Assignment]] = []
for wave_index, cells in enumerate(waves):
if len(cells) == 3:
cells.append(m.Cell("background-primary-final", "C00"))
result.append(
[
m.Assignment(cell, ((slot + wave_index) % 4,))
for slot, cell in enumerate(cells)
]
)
return result
def execute_primary(resume: bool, amendment_a_p5_1: bool = False) -> None:
RUN_ROOT.mkdir(parents=True, exist_ok=True)
state = load_state(resume)
if resume:
m.cleanup_recorded(state)
current = fingerprint()
if state["fingerprint"] and state["fingerprint"] != current:
failure = str(state.get("stages", {}).get("primary-01", {}).get("failure", ""))
if not (
amendment_a_p5_1
and state.get("status") == "failed"
and "warmup" in failure.lower()
):
raise RuntimeError("resume fingerprint differs from frozen Phase-5 plan")
state.setdefault("amendments", {})["A-P5-1"] = {
"approved": True,
"applied_at": time.time(),
"reason": "replace rate-following drift gate with cold-start gates",
"prior_fingerprint": state["fingerprint"],
"replacement_fingerprint": current,
"retained_gpu_hours": state["gpu_hours_total"],
"burnins_reused": state["completed_burnins"],
}
state["fingerprint"] = current
state["status"] = "amended_resume_A-P5-1"
save_state(state)
state["fingerprint"] = current
state["status"] = "running_primary"
save_state(state)
ensure_provenance()
burnins = [
m.Assignment(m.Cell("burnin-C00", "C00"), (0,)),
m.Assignment(m.Cell("burnin-A2", "A2"), (1,)),
m.Assignment(m.Cell("burnin-A4", "A4"), (2,)),
]
m.run_stage(state, "burnins", burnins, "saturation", profile=False, burnin=True)
waves = pack_primary(primary_cells())
for index, wave in enumerate(waves, 1):
m.run_stage(state, f"primary-{index:02d}", wave, "moderate", profile=False)
primary = [
path for path in (RUN_ROOT / "primary").glob("*-r*-*/moderate/run-complete.json")
if "background" not in str(path)
]
if len(primary) != 15:
raise RuntimeError(f"primary completion mismatch: {len(primary)} != 15")
state["primary_runs"] = 15
state["background_runs"] = state["completed_measured_runs"] - 15
state["status"] = "primary_complete"
state["completed_at"] = time.time()
save_state(state)
def execute_controls(resume: bool) -> None:
state = load_state(resume)
m.cleanup_recorded(state)
if state.get("fingerprint") != fingerprint():
raise RuntimeError("control resume fingerprint mismatch")
cells = [
m.Cell(f"control-{pattern}-r{replicate}", "C00")
for pattern in ("P03", "P04") for replicate in range(1, 4)
]
for index, wave in enumerate(pack_unique(cells, "controls"), 1):
m.run_stage(state, f"controls-{index:02d}", wave, "moderate", profile=False)
state["status"] = "controls_complete"
state["controls_completed_at"] = time.time()
save_state(state)
def plan() -> dict[str, Any]:
return {
"schema": 1,
"primary_runs": 15,
"burnins": 3,
"waves": [[{"cell": a.cell.cell_id, "gpus": a.gpus} for a in wave] for wave in pack_primary(primary_cells())],
"rate": RATE,
"clean_seconds": 240,
"drain_seconds": 600,
"gpu_hour_limit": 6.0,
}
def main() -> None:
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
for name in ("primary", "controls"):
item = sub.add_parser(name)
item.add_argument("--resume", action="store_true")
if name == "primary":
item.add_argument("--amendment-a-p5-1", action="store_true")
sub.add_parser("plan")
sub.add_parser("status")
args = parser.parse_args()
if args.command == "primary":
execute_primary(args.resume, args.amendment_a_p5_1)
elif args.command == "controls":
execute_controls(args.resume)
elif args.command == "plan":
print(json.dumps(plan(), sort_keys=True, indent=2))
else:
print(STATE.read_text() if STATE.exists() else '{"status":"absent"}')
if __name__ == "__main__":
main()

212
runs/opprof-phase5/phase5/controller-state.json Normal file
View File

@@ -0,0 +1,212 @@
{
"clean_window_failures": 0,
"completed_burnins": 3,
"completed_measured_runs": 0,
"controller_pid": 2505724,
"created_at": 1783858074.2830184,
"drain_quarantined_runs": 0,
"fingerprint": {
"capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"client_sha256": "f935486ab2588c1fca514be29b59361f53118eb000ea037863de48ce4fc76b16",
"controller_sha256": "ea2e2066a8b6d9427c59fe80db44cf87a86776570ca9066bfdff0c0b0e7f4a46",
"manifests": {
"A1": {
"path": "/home/admin/cpfs/wjh/opprof-phase5-private/manifests/P10-A1.jsonl",
"sha256": "cab8468983fb7397ec88eb5e88a44c8b1b53d8021b7867e7bec6f58d3d903806"
},
"A3": {
"path": "/home/admin/cpfs/wjh/opprof-phase5-private/manifests/P10-A3.jsonl",
"sha256": "ec5eaa29fcd3ec421e4f68a902c7e7fea9c0bc6b16f1013cef30ccc1f97bae24"
},
"base": {
"path": "/home/admin/cpfs/wjh/opprof-phase5-private/manifests/P10-base.jsonl",
"sha256": "d3b4f540ddd629dd0e34fff55c3b3837f359bdd6e5947309a1ea2a358f811ffc"
}
},
"p3_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
"p3_matrix_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
"rate": 0.4725,
"source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
"source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
},
"gpu_hours_this_stage": 0.4224752351972792,
"gpu_hours_total": 0.6476566231913037,
"missing_trace_files": 0,
"schema": 1,
"stages": {
"burnins": {
"assignments": [
{
"cell": "burnin-C00-C00",
"gpus": [
0
]
},
{
"cell": "burnin-A2-A2",
"gpus": [
1
]
},
{
"cell": "burnin-A4-A4",
"gpus": [
2
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783858361.2828288,
"confirmation": false,
"gpu_hours": 0.22518138799402448,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783858074.4911764,
"status": "complete"
},
"primary-01": {
"assignments": [
{
"cell": "base-r2-C00",
"gpus": [
0
]
},
{
"cell": "A3-r1-C00",
"gpus": [
1
]
},
{
"cell": "A1-r2-C00",
"gpus": [
2
]
},
{
"cell": "A4-r1-A4",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {
"A1-r2-C00-moderate": {
"pgid": 2512122,
"pid": 2512122
},
"A3-r1-C00-moderate": {
"pgid": 2512121,
"pid": 2512121
},
"A4-r1-A4-moderate": {
"pgid": 2512123,
"pid": 2512123
},
"base-r2-C00-moderate": {
"pgid": 2512119,
"pid": 2512119
}
},
"confirmation": false,
"failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase5/primary/base-r2-C00/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=25; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [380, 201, 187], 'scheduled_tokens': [26927, 27616, 463], 'scheduled_token_throughput': [5385.4, 5523.2, 92.6], 'mean_scheduled_token_throughput': 3667.066666666666, 'slope_tokens_per_second_squared': -529.28, 'normalized_drift': 2.1650001817983497, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")",
"gpu_hours": 0.4224752351972792,
"load_point": "moderate",
"profile": false,
"servers": {
"A1-r2-C00-moderate": {
"gpus": [
2
],
"pgid": 2510424,
"pid": 2510424
},
"A3-r1-C00-moderate": {
"gpus": [
1
],
"pgid": 2510423,
"pid": 2510423
},
"A4-r1-A4-moderate": {
"gpus": [
3
],
"pgid": 2510425,
"pid": 2510425
},
"base-r2-C00-moderate": {
"gpus": [
0
],
"pgid": 2510422,
"pid": 2510422
}
},
"started_at": 1783858361.3184204,
"status": "failed"
}
},
"status": "failed",
"updated_at": 1783858758.067235
}

1
runs/opprof-phase5/phase5/launch-echo.log Normal file
View File

@@ -0,0 +1 @@
LAUNCH_ECHO utc=2026-07-12T12:07:54Z host=dash0 gpus=0-3 cpus=0-79 source=/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0@4b253fd manifests=/home/admin/cpfs/wjh/opprof-phase5-private/manifests outputs=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase5 runs=3burnin+15primary+1background rate=0.4725 warmup=60s clean=240s drain=600s est_wall=35-60min est_gpu=1.7-2.1_H20h hard_cap=6.0_H20h conditional_controls=6_if_bridge_fails

14178
runs/opprof-phase5/phase5/metrics.json Normal file
View File

File diff suppressed because it is too large Load Diff

115
runs/opprof-phase5/phase5/plan.json Normal file
View File

@@ -0,0 +1,115 @@
{
"burnins": 3,
"clean_seconds": 240,
"drain_seconds": 600,
"gpu_hour_limit": 6.0,
"primary_runs": 15,
"rate": 0.4725,
"schema": 1,
"waves": [
[
{
"cell": "base-r2-C00",
"gpus": [
0
]
},
{
"cell": "A3-r1-C00",
"gpus": [
1
]
},
{
"cell": "A1-r2-C00",
"gpus": [
2
]
},
{
"cell": "A4-r1-A4",
"gpus": [
3
]
}
],
[
{
"cell": "A1-r3-C00",
"gpus": [
1
]
},
{
"cell": "base-r1-C00",
"gpus": [
2
]
},
{
"cell": "A3-r3-C00",
"gpus": [
3
]
},
{
"cell": "A2-r1-A2",
"gpus": [
0
]
}
],
[
{
"cell": "base-r3-C00",
"gpus": [
2
]
},
{
"cell": "A3-r2-C00",
"gpus": [
3
]
},
{
"cell": "A4-r2-A4",
"gpus": [
0
]
},
{
"cell": "A2-r3-A2",
"gpus": [
1
]
}
],
[
{
"cell": "A4-r3-A4",
"gpus": [
3
]
},
{
"cell": "A1-r1-C00",
"gpus": [
0
]
},
{
"cell": "A2-r2-A2",
"gpus": [
1
]
},
{
"cell": "background-primary-final-C00",
"gpus": [
2
]
}
]
]
}

34
runs/opprof-phase5/test_phase5_analysis.py Normal file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env python3
from __future__ import annotations
import numpy as np
import analyze_phase5 as a
def main() -> None:
adjusted = a.holm({"A1": 0.001, "A2": 0.02, "A3": 0.04, "A4": 0.5})
assert adjusted == {"A1": 0.004, "A2": 0.06, "A3": 0.08, "A4": 0.5}
assert a.ci(np.arange(100, dtype=np.float64)) == [2.475, 96.52499999999999]
runs = [
{"blocks": np.asarray([[10.0, 2.0]] * 48)},
{"blocks": np.asarray([[20.0, 4.0]] * 48)},
{"blocks": np.asarray([[30.0, 6.0]] * 48)},
]
draws = a.hierarchical_draws(runs, np.random.default_rng(a.SEED))
assert draws.shape == (a.RESAMPLES,)
assert np.allclose(draws, 5.0)
assert a.point_efficiency(runs) == 5.0
idle_blocks = np.asarray([[0.0, 0.0]] + [[10.0, 2.0]] * 47)
idle_draws = a.hierarchical_draws(
[{"blocks": idle_blocks}], np.random.default_rng(a.SEED)
)
assert np.all(np.isfinite(idle_draws))
assert np.allclose(idle_draws, 5.0)
assert a.point_efficiency([{"blocks": idle_blocks}]) == 5.0
assert a.two_sided_p(np.asarray([-1.0, 1.0])) == 1.0
print("phase5 analysis: PASS")
if __name__ == "__main__":
main()

68
runs/opprof-phase5/test_phase5_tools.py Normal file
View File

@@ -0,0 +1,68 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import subprocess
import sys
import tempfile
from pathlib import Path
HERE = Path(__file__).resolve().parent
CLIENT = HERE / "opprof_phase5_client.py"
def write_jsonl(path: Path, rows: list[dict]) -> None:
path.write_text("".join(json.dumps(row) + "\n" for row in rows))
def run(command: list[str]) -> None:
completed = subprocess.run(command, text=True, capture_output=True)
if completed.returncode:
raise RuntimeError(f"command failed: {completed.stderr}")
def main() -> None:
with tempfile.TemporaryDirectory() as tmp_text:
tmp = Path(tmp_text)
manifest = tmp / "p3.jsonl"
source = tmp / "source.jsonl"
base = tmp / "base.jsonl"
a1 = tmp / "a1.jsonl"
rows = []
source_rows = []
lengths = [128, 8192, 256, 4096] * 8
for index, length in enumerate(lengths):
rows.append({
"request_id": f"P10-{index}", "pattern_id": "P10",
"input_tokens": length, "output_tokens": 8,
"arrival": "steady", "kind": "private-trace",
"source_index": index, "prompt": f"private-{index}",
})
source_rows.append({"timestamp": index * 0.1, "prompt": f"private-{index}"})
write_jsonl(manifest, rows)
write_jsonl(source, source_rows)
common = [
sys.executable, str(CLIENT), "transform", "--in", str(manifest),
"--take-first", "32", "--timestamp-source", str(source),
"--join-key", "source_index", "--timestamp-field", "timestamp",
"--arrival", "recorded-scaled", "--target-rate", "0.5",
]
run(common + ["--service-order", "original", "--out", str(base)])
run(common + [
"--service-order", "length-binned", "--reorder-block-size", "32",
"--analysis-cohort-size", "16", "--max-added-delay-seconds", "64",
"--out", str(a1),
])
base_summary = json.loads((base.with_suffix(".jsonl.summary.json")).read_text())
a1_summary = json.loads((a1.with_suffix(".jsonl.summary.json")).read_text())
assert base_summary["rows"] == a1_summary["rows"] == 32
assert base_summary["request_id_set_sha256"] == a1_summary["request_id_set_sha256"]
assert a1_summary["r16"] < base_summary["r16"]
assert a1_summary["max_added_delay_seconds"] <= 64
assert "private-" not in json.dumps(a1_summary)
print("phase5 tools: PASS")
if __name__ == "__main__":
main()