gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

250
runs/opprof-phase6/opprof_phase6_client.py Normal file
View File

@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Exact C1 anchor replay using the pinned AITuner trace/worker/SLO paths."""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import json
import math
import os
import sys
import time
from pathlib import Path
from typing import Any
AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2]))
sys.path.insert(0, str(AITUNER_ROOT / "src"))
os.environ.setdefault("AITUNER_CODEX_BASE_URL", "http://127.0.0.1:1")
from aituner.slo import evaluate_request, summarize_evaluations # noqa: E402
from aituner.spec import load_study_spec # noqa: E402
from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402
from aituner.worker import _probe_drain_deadline, _replay_requests # noqa: E402
def atomic_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n")
os.replace(tmp, path)
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def numeric(values: list[float | int | None]) -> dict[str, Any]:
finite = [float(x) for x in values if x is not None and math.isfinite(float(x))]
return {
"n": len(values), "finite_n": len(finite), "missing_n": len(values) - len(finite),
"min": min(finite) if finite else None, "max": max(finite) if finite else None,
"distinct_n": len(set(finite)),
}
def load_selected(study_path: Path, anchor: float):
study = load_study_spec(study_path)
window, requests = load_trace_requests(study, study_spec_path=study_path)
selected = select_requests_for_threshold(requests, threshold=anchor)
return study, window, requests, selected
def selected_summary(selected, duration_s: float, tp: int) -> dict[str, Any]:
ids = "\n".join(item.row_id for item in selected)
arrival = "\n".join(f"{item.arrival_s:.12f}" for item in selected)
lengths = "\n".join(str(item.prompt_tokens_hint) for item in selected)
return {
"count": len(selected),
"offered_req_s": len(selected) / duration_s,
"offered_req_s_per_gpu": len(selected) / duration_s / tp,
"request_id_order_sha256": hashlib.sha256(ids.encode()).hexdigest(),
"arrival_order_sha256": hashlib.sha256(arrival.encode()).hexdigest(),
"raw_length_order_sha256": hashlib.sha256(lengths.encode()).hexdigest(),
"arrival_s": numeric([item.arrival_s for item in selected]),
"raw_input_tokens": numeric([item.prompt_tokens_hint for item in selected]),
"long_gt4096": sum(int(item.prompt_tokens_hint or 0) > 4096 for item in selected),
}
def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]:
study_path = Path(args.study)
study, window, _requests, selected = load_selected(study_path, args.anchor)
if warmup:
first = selected[:16]
if not any(int(item.prompt_tokens_hint or 0) > 4096 for item in first):
long_item = next(item for item in selected if int(item.prompt_tokens_hint or 0) > 4096)
first = [*selected[:15], long_item]
first = sorted({item.row_id: item for item in first}.values(), key=lambda item: item.arrival_s)
if len(first) < 16:
raise RuntimeError("warmup set has fewer than 16 unique requests")
start = first[0].arrival_s
selected = [dataclasses.replace(item, arrival_s=item.arrival_s - start) for item in first]
duration_s = float(window.window_end - window.window_start)
interval_start_mono_ns = time.monotonic_ns()
interval_start_wall_ns = time.time_ns()
outcomes, early_stopped, early_stop_reason = _replay_requests(
selected,
base_url=args.base_url,
timeout_s=study.engine.request_timeout_s,
max_concurrency=study.trace.max_concurrency,
target_pass_rate=(0.0 if warmup else study.slo.target_pass_rate),
max_lag_s=study.trace.early_stop_max_lag_s,
max_elapsed_s=(
120.0 if warmup else _probe_drain_deadline(
selected, study.slo, ceiling=study.trace.early_stop_max_elapsed_s
)
),
evaluate_outcome=lambda outcome: evaluate_request(outcome, study.slo),
drain_inflight_on_early_stop=True,
)
interval_end_mono_ns = time.monotonic_ns()
interval_end_wall_ns = time.time_ns()
evaluations, slo_summary = summarize_evaluations(outcomes, study.slo)
by_id = {item.row_id: item for item in selected}
details = []
for outcome, evaluation in zip(outcomes, evaluations):
request = by_id[outcome.request_id]
details.append({
"request_id": outcome.request_id,
"sampling_u": request.sampling_u,
"arrival_s": request.arrival_s,
"raw_input_tokens": request.prompt_tokens_hint,
"success": outcome.success,
"ttft_ms": outcome.ttft_ms,
"tpot_ms": outcome.tpot_ms,
"completion_tokens": outcome.completion_tokens,
"completion_tokens_source": outcome.completion_tokens_source,
"slo_pass": evaluation.passed,
"reasons": evaluation.reasons,
"error": outcome.error,
})
out = Path(args.result_dir)
out.mkdir(parents=True, exist_ok=True)
with (out / "requests.jsonl").open("w") as f:
for item in details:
f.write(json.dumps(item, sort_keys=True) + "\n")
summary = selected_summary(selected, duration_s, args.tp)
exact = sum(
item.success and item.completion_tokens_source == "usage" and item.completion_tokens == 128
for item in outcomes
)
result = {
"schema": 1,
"kind": "warmup" if warmup else "anchor",
"cell": args.cell,
"anchor": args.anchor,
"tp": args.tp,
"mns": args.mns,
"study_sha256": sha256_file(study_path),
"interval": {
"start_mono_ns": interval_start_mono_ns, "end_mono_ns": interval_end_mono_ns,
"start_wall_ns": interval_start_wall_ns, "end_wall_ns": interval_end_wall_ns,
"elapsed_s": (interval_end_mono_ns - interval_start_mono_ns) / 1e9,
},
"selection": summary,
"observed_count": len(outcomes),
"exact_output_count": exact,
"slo_pass_count": slo_summary["slo_pass_count"],
"pass_rate": slo_summary["slo_pass_rate"],
"feasible": bool(slo_summary["feasible"]),
"early_stopped": early_stopped,
"early_stop_reason": early_stop_reason,
"ttft_ms": numeric([item.ttft_ms for item in outcomes]),
"tpot_ms": numeric([item.tpot_ms for item in outcomes]),
"invariants": {
"selected_nonempty": bool(selected),
"outcomes_cover_selected": len(outcomes) == len(selected),
"exact_output_or_failed": all(
(not item.success) or (
item.completion_tokens_source == "usage" and item.completion_tokens == 128
) for item in outcomes
),
"raw_lengths_present": all(item.prompt_tokens_hint is not None for item in selected),
"arrival_nondecreasing": all(
b.arrival_s >= a.arrival_s for a, b in zip(selected, selected[1:])
),
"warmup_16": (len(outcomes) >= 16 if warmup else True),
"warmup_exact_16": (exact >= 16 if warmup else True),
"warmup_long": (
any(int(item.prompt_tokens_hint or 0) > 4096 for item in selected)
if warmup else True
),
},
}
atomic_json(out / "result.json", result)
print(json.dumps({k: result[k] for k in ("cell", "anchor", "kind", "pass_rate", "feasible")}))
if not all(result["invariants"].values()):
raise RuntimeError(f"client invariants failed: {result['invariants']}")
return result
def preflight(args: argparse.Namespace) -> None:
ground = json.loads(Path(args.ground_truth).read_text())
studies = {1: Path(args.primary_study), 2: Path(args.primary_study), 4: Path(args.tp4_study)}
loaded = {}
mismatches = []
values = []
for cell in ground["cells"]:
tp = int(cell["tensor_parallel_size"])
if tp not in loaded:
_study, _window, requests, _selected = load_selected(studies[tp], 0.0)
loaded[tp] = requests
for historical in cell["probe_history"]:
selected = select_requests_for_threshold(
loaded[tp], threshold=float(historical["sampling_u"])
)
values.append(len(selected))
if len(selected) != int(historical["request_count"]):
mismatches.append({
"cell": cell["cell_id"], "anchor": historical["sampling_u"],
"expected": historical["request_count"], "actual": len(selected),
})
result = {
"schema": 1, "observations": len(values), "mismatches": mismatches,
"request_counts": numeric(values),
"invariants": {"observations_92": len(values) == 92, "counts_match": not mismatches},
}
atomic_json(Path(args.out), result)
print(json.dumps(result, sort_keys=True))
if not all(result["invariants"].values()):
raise RuntimeError("preflight count reconstruction failed")
def parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser()
sub = p.add_subparsers(dest="command", required=True)
pf = sub.add_parser("preflight")
pf.add_argument("--ground-truth", required=True)
pf.add_argument("--primary-study", required=True)
pf.add_argument("--tp4-study", required=True)
pf.add_argument("--out", required=True)
for name in ("warmup", "run-anchor"):
q = sub.add_parser(name)
q.add_argument("--study", required=True)
q.add_argument("--cell", required=True)
q.add_argument("--anchor", type=float, required=True)
q.add_argument("--tp", type=int, required=True)
q.add_argument("--mns", type=int, required=True)
q.add_argument("--base-url", required=True)
q.add_argument("--result-dir", required=True)
return p
def main() -> None:
args = parser().parse_args()
if args.command == "preflight":
preflight(args)
else:
run_replay(args, warmup=args.command == "warmup")
if __name__ == "__main__":
main()