Gahow Wang
d5b276180d
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1612 lines
63 KiB
Python
1612 lines
63 KiB
Python
#!/usr/bin/env python3
|
|
"""Frozen Phase-3 OpProf analysis; emits aggregate, prompt-free JSON only."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import bisect
|
|
import gzip
|
|
import json
|
|
import math
|
|
import re
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
import opprof_phase3_controller as common
|
|
|
|
|
|
SEED = 20260714
|
|
BOOTSTRAPS = 100_000
|
|
FAMILIES = (
|
|
"attention",
|
|
"moe_gemm",
|
|
"moe_router",
|
|
"collective",
|
|
"sampler",
|
|
"dense_gemm",
|
|
"norm_elementwise",
|
|
"kv_memory",
|
|
)
|
|
IRREGULAR_CONTROLS = (
|
|
("P05", "P01"),
|
|
("P05", "P03"),
|
|
("P06", "P02"),
|
|
("P06", "P04"),
|
|
("P09", "P01"),
|
|
("P09", "P03"),
|
|
("P10", "P03"),
|
|
("P10", "P04"),
|
|
)
|
|
AP37_MISSING_CELLS = (
|
|
"P03/C11",
|
|
"P05/C00",
|
|
"P10/C00-TP2",
|
|
"P11/C00",
|
|
)
|
|
AP37_MISSING_CONTRASTS = (
|
|
("P05", "P01"),
|
|
("P05", "P03"),
|
|
)
|
|
CAPTURE_BUCKETS = (
|
|
1,
|
|
2,
|
|
4,
|
|
8,
|
|
16,
|
|
24,
|
|
32,
|
|
40,
|
|
48,
|
|
56,
|
|
64,
|
|
72,
|
|
80,
|
|
88,
|
|
96,
|
|
104,
|
|
112,
|
|
120,
|
|
128,
|
|
136,
|
|
144,
|
|
152,
|
|
160,
|
|
168,
|
|
176,
|
|
184,
|
|
192,
|
|
200,
|
|
208,
|
|
216,
|
|
224,
|
|
232,
|
|
240,
|
|
248,
|
|
256,
|
|
272,
|
|
288,
|
|
304,
|
|
320,
|
|
336,
|
|
352,
|
|
368,
|
|
384,
|
|
400,
|
|
416,
|
|
432,
|
|
448,
|
|
464,
|
|
480,
|
|
496,
|
|
512,
|
|
)
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
return json.loads(path.read_text())
|
|
|
|
|
|
def numeric_sanity(values: list[float | int | None]) -> dict[str, Any]:
|
|
finite = [float(value) for value in values if value is not None and math.isfinite(value)]
|
|
return {
|
|
"n": len(values),
|
|
"finite_n": len(finite),
|
|
"missing_n": len(values) - len(finite),
|
|
"min": min(finite) if finite else None,
|
|
"max": max(finite) if finite else None,
|
|
"distinct_n": len(set(finite)),
|
|
}
|
|
|
|
|
|
def percentile_summary(values: list[float]) -> dict[str, Any]:
|
|
if not values:
|
|
return {"n": 0, "mean": None, "p50": None, "p95": None, "p99": None}
|
|
array = np.asarray(values, dtype=float)
|
|
return {
|
|
"n": len(values),
|
|
"mean": float(array.mean()),
|
|
"p50": float(np.quantile(array, 0.50)),
|
|
"p95": float(np.quantile(array, 0.95)),
|
|
"p99": float(np.quantile(array, 0.99)),
|
|
}
|
|
|
|
|
|
def jsonl(path: Path) -> list[dict[str, Any]]:
|
|
with path.open() as source:
|
|
return [json.loads(line) for line in source]
|
|
|
|
|
|
def expected_cells() -> set[str]:
|
|
cells = {f"P{index:02d}/C00" for index in range(1, 12)}
|
|
for pattern in ("P01", "P03", "P06", "P10"):
|
|
cells.update(f"{pattern}/{config}" for config in ("C10", "C01", "C11"))
|
|
cells.add("P10/C00-TP2")
|
|
return cells
|
|
|
|
|
|
def accepted_marker_paths(
|
|
root: Path,
|
|
) -> tuple[list[Path], list[Path], list[Path], list[str]]:
|
|
complete_stages = sorted(
|
|
path.parent for path in root.glob("stages/*/stage-complete.json")
|
|
)
|
|
accepted_ids = []
|
|
for stage in complete_stages:
|
|
stage_name = stage.name
|
|
if not (stage_name.startswith("primary-") or stage_name == "confirmations"):
|
|
continue
|
|
marker = load_json(stage / "stage-complete.json")
|
|
accepted_ids.extend(str(run_id) for run_id in marker["runs"])
|
|
if len(accepted_ids) != len(set(accepted_ids)):
|
|
raise RuntimeError("accepted run ID appears in more than one complete stage")
|
|
|
|
canonical = {}
|
|
for path in root.glob("primary/*/*/run-complete.json"):
|
|
if path.parent.name not in {"saturation", "moderate"}:
|
|
continue
|
|
run_id = str(load_json(path)["run_id"])
|
|
if run_id in canonical:
|
|
raise RuntimeError(f"duplicate canonical run marker: {run_id}")
|
|
canonical[run_id] = path
|
|
for path in root.glob("confirmations/*/run-complete.json"):
|
|
run_id = str(load_json(path)["run_id"])
|
|
if run_id in canonical:
|
|
raise RuntimeError(f"duplicate canonical run marker: {run_id}")
|
|
canonical[run_id] = path
|
|
missing = sorted(set(accepted_ids) - set(canonical))
|
|
if missing:
|
|
raise RuntimeError(f"complete-stage run markers missing: {missing}")
|
|
selected = [canonical[run_id] for run_id in accepted_ids]
|
|
primary = sorted(path for path in selected if "primary" in path.parts)
|
|
confirmations = sorted(path for path in selected if "confirmations" in path.parts)
|
|
unaccepted = sorted(set(canonical) - set(accepted_ids))
|
|
return primary, confirmations, complete_stages, unaccepted
|
|
|
|
|
|
def partial_verdict(has_hit: bool) -> str:
|
|
return "PASS" if has_hit else "INCONCLUSIVE"
|
|
|
|
|
|
def run_records(run_dir: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
result = load_json(run_dir / "client/result.json")
|
|
t0 = int(result["t0_mono_ns"])
|
|
start = t0 + int(60e9)
|
|
end = t0 + int(300e9)
|
|
stream = next((run_dir / "opprof").glob("*.jsonl"))
|
|
all_records = []
|
|
clean = []
|
|
for record in jsonl(stream):
|
|
if "step_index" not in record:
|
|
continue
|
|
all_records.append(record)
|
|
if start <= int(record["submit_mono_ns"]) < end:
|
|
clean.append(record)
|
|
return all_records, clean
|
|
|
|
|
|
def ap36_warmup_stability(run_dir: Path) -> dict[str, Any]:
|
|
result = load_json(run_dir / "client/result.json")
|
|
t0 = int(result["t0_mono_ns"])
|
|
requests = jsonl(run_dir / "client/requests.jsonl")
|
|
completions = sum(
|
|
bool(item["success"])
|
|
and 0 <= float(item["completed_s"]) < float(result["warmup_seconds"])
|
|
for item in requests
|
|
)
|
|
all_records, _ = run_records(run_dir)
|
|
counts = [0, 0, 0]
|
|
tokens = [0, 0, 0]
|
|
for item in all_records:
|
|
if not item["model_executed"]:
|
|
continue
|
|
relative_s = (int(item["submit_mono_ns"]) - t0) / 1e9
|
|
if not 45 <= relative_s < 60:
|
|
continue
|
|
index = min(2, int((relative_s - 45) // 5))
|
|
counts[index] += 1
|
|
tokens[index] += int(item["prefill_tokens"]) + int(item["decode_tokens"])
|
|
rates = [value / 5.0 for value in tokens]
|
|
mean = sum(rates) / 3
|
|
slope = (rates[2] - rates[0]) / 10.0
|
|
drift = abs(slope) * 15 / mean if mean > 0 else math.inf
|
|
return {
|
|
"warmup_completions": completions,
|
|
"step_counts": counts,
|
|
"scheduled_tokens": tokens,
|
|
"scheduled_token_throughput": rates,
|
|
"mean_scheduled_token_throughput": mean,
|
|
"slope_tokens_per_second_squared": slope,
|
|
"normalized_drift": drift,
|
|
"normalized_drift_limit": 0.10,
|
|
"passes": completions >= 16
|
|
and all(value >= 16 for value in counts)
|
|
and drift <= 0.10,
|
|
}
|
|
|
|
|
|
def mode(record: dict[str, Any]) -> str:
|
|
return str(record["cudagraph"]["runtime_mode"])
|
|
|
|
|
|
def clean_block(record: dict[str, Any], t0_ns: int) -> int:
|
|
return min(47, max(0, int((int(record["submit_mono_ns"]) - t0_ns - 60e9) // 5e9)))
|
|
|
|
|
|
def summarize_run(run_dir: Path, marker: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
client = load_json(run_dir / "client/result.json")
|
|
requests = jsonl(run_dir / "client/requests.jsonl")
|
|
all_records, records = run_records(run_dir)
|
|
completed = [
|
|
item
|
|
for item in requests
|
|
if item["success"] and 60 <= float(item["completed_s"]) < 300
|
|
]
|
|
e2e = [float(item["completed_s"] - item["admitted_s"]) for item in completed]
|
|
ttft = [float(item["first_token_s"] - item["admitted_s"]) for item in completed]
|
|
tpot = [
|
|
float(item["completed_s"] - item["first_token_s"])
|
|
/ max(1, int(item["actual_output_tokens"]) - 1)
|
|
for item in completed
|
|
]
|
|
duration_ms = [
|
|
(int(item["complete_mono_ns"]) - int(item["submit_mono_ns"])) / 1e6
|
|
for item in records
|
|
]
|
|
useful = [int(item["prefill_tokens"]) + int(item["decode_tokens"]) for item in records]
|
|
hit_records = [
|
|
item
|
|
for item in records
|
|
if item["model_executed"]
|
|
and item["cudagraph"]["hit"]
|
|
and int(item["cudagraph"]["bucket_tokens"]) > 0
|
|
]
|
|
model_records = [item for item in records if item["model_executed"]]
|
|
misses = [item for item in model_records if not item["cudagraph"]["hit"]]
|
|
eligible_misses = [
|
|
item for item in misses if int(item["cudagraph"]["unpadded_tokens"]) <= CAPTURE_BUCKETS[-1]
|
|
]
|
|
overflow = [
|
|
item for item in misses if int(item["cudagraph"]["unpadded_tokens"]) > CAPTURE_BUCKETS[-1]
|
|
]
|
|
theoretical_buckets = [
|
|
CAPTURE_BUCKETS[bisect.bisect_left(CAPTURE_BUCKETS, int(item["cudagraph"]["unpadded_tokens"]))]
|
|
for item in model_records
|
|
if 0 < int(item["cudagraph"]["unpadded_tokens"]) <= CAPTURE_BUCKETS[-1]
|
|
]
|
|
theoretical_unpadded = [
|
|
int(item["cudagraph"]["unpadded_tokens"])
|
|
for item in model_records
|
|
if 0 < int(item["cudagraph"]["unpadded_tokens"]) <= CAPTURE_BUCKETS[-1]
|
|
]
|
|
prefixes = []
|
|
for item in records:
|
|
for source in (item["prefix"].get("local"), item["prefix"].get("external")):
|
|
if source:
|
|
prefixes.append(source)
|
|
blocks = [defaultdict(float) for _ in range(48)]
|
|
t0 = int(client["t0_mono_ns"])
|
|
for item, step_ms in zip(records, duration_ms, strict=True):
|
|
block = blocks[clean_block(item, t0)]
|
|
token = int(item["prefill_tokens"]) + int(item["decode_tokens"])
|
|
graph = item["cudagraph"]
|
|
block["tokens"] += token
|
|
block["duration_ms"] += step_ms
|
|
block["model"] += bool(item["model_executed"])
|
|
block["miss"] += bool(item["model_executed"] and not graph["hit"])
|
|
block["eligible_miss"] += bool(
|
|
item["model_executed"]
|
|
and not graph["hit"]
|
|
and int(graph["unpadded_tokens"]) <= CAPTURE_BUCKETS[-1]
|
|
)
|
|
block["overflow"] += bool(
|
|
item["model_executed"]
|
|
and not graph["hit"]
|
|
and int(graph["unpadded_tokens"]) > CAPTURE_BUCKETS[-1]
|
|
)
|
|
if item["model_executed"] and graph["hit"] and int(graph["bucket_tokens"]) > 0:
|
|
block["padding"] += int(graph["padding_tokens"])
|
|
block["bucket"] += int(graph["bucket_tokens"])
|
|
denominator = sum(duration_ms)
|
|
bucket_sum = sum(int(item["cudagraph"]["bucket_tokens"]) for item in hit_records)
|
|
pad_sum = sum(int(item["cudagraph"]["padding_tokens"]) for item in hit_records)
|
|
model_n = len(model_records)
|
|
modes = Counter(mode(item) for item in records)
|
|
completion_times = [
|
|
float(item["completed_s"])
|
|
for item in requests
|
|
if item["success"]
|
|
]
|
|
clean_completion_rates = [
|
|
sum(start <= value < start + 10 for value in completion_times) / 10
|
|
for start in np.arange(220, 300, 10)
|
|
]
|
|
clean_waiting = [
|
|
item["queues"]["waiting"]
|
|
for item in records
|
|
if t0 + int(220e9) <= int(item["submit_mono_ns"]) < t0 + int(300e9)
|
|
]
|
|
clean_completion_median = float(np.median(clean_completion_rates))
|
|
clean_waiting_median = float(np.median(clean_waiting)) if clean_waiting else None
|
|
recoveries = []
|
|
profiles = client.get("profiles", [])
|
|
for index, profile in enumerate(profiles):
|
|
recovery_end = (
|
|
float(profiles[index + 1]["start_call_s"])
|
|
if index + 1 < len(profiles)
|
|
else float(client["admission_stop_s"])
|
|
)
|
|
tail_start = recovery_end - 10
|
|
completion_rate = (
|
|
sum(tail_start <= value < recovery_end for value in completion_times) / 10
|
|
)
|
|
tail_waiting = [
|
|
item["queues"]["waiting"]
|
|
for item in all_records
|
|
if t0 + int(tail_start * 1e9)
|
|
<= int(item["submit_mono_ns"])
|
|
< t0 + int(recovery_end * 1e9)
|
|
]
|
|
waiting_median = float(np.median(tail_waiting)) if tail_waiting else None
|
|
rate_ok = (
|
|
completion_rate == clean_completion_median
|
|
if clean_completion_median == 0
|
|
else abs(completion_rate / clean_completion_median - 1) <= 0.10
|
|
)
|
|
waiting_ok = (
|
|
waiting_median == clean_waiting_median
|
|
if clean_waiting_median == 0
|
|
else (
|
|
waiting_median is not None
|
|
and clean_waiting_median is not None
|
|
and abs(waiting_median / clean_waiting_median - 1) <= 0.10
|
|
)
|
|
)
|
|
recoveries.append(
|
|
{
|
|
"window": index + 1,
|
|
"tail_start_s": tail_start,
|
|
"tail_end_s": recovery_end,
|
|
"completion_rate_rps": completion_rate,
|
|
"clean_c_completion_rate_median_rps": clean_completion_median,
|
|
"waiting_median": waiting_median,
|
|
"clean_c_waiting_median": clean_waiting_median,
|
|
"completion_rate_within_10pct": rate_ok,
|
|
"waiting_within_10pct": waiting_ok,
|
|
"valid": rate_ok and waiting_ok,
|
|
}
|
|
)
|
|
summary = {
|
|
"run_id": marker["run_id"],
|
|
"pattern": marker["pattern"],
|
|
"config": marker["config"],
|
|
"load": client["load_point"],
|
|
"drain_seconds": float(client["drain_seconds"]),
|
|
"drain_quarantined": bool(marker["drain_quarantined"]),
|
|
"clean": client["clean"],
|
|
"requests_completed": len(completed),
|
|
"latency_s": {
|
|
"e2e": percentile_summary(e2e),
|
|
"ttft": percentile_summary(ttft),
|
|
"tpot": percentile_summary(tpot),
|
|
},
|
|
"layer1": {
|
|
"records_all": len(all_records),
|
|
"records_clean": len(records),
|
|
"step_duration_ms": percentile_summary(duration_ms),
|
|
"scheduled_tokens_per_step": percentile_summary([float(value) for value in useful]),
|
|
"requests_per_step": percentile_summary(
|
|
[float(item["scheduled_requests"]) for item in records]
|
|
),
|
|
"prefill_tokens": sum(int(item["prefill_tokens"]) for item in records),
|
|
"decode_tokens": sum(int(item["decode_tokens"]) for item in records),
|
|
"token_efficiency_per_ms": sum(useful) / denominator if denominator else None,
|
|
"queue_waiting_mean": float(np.mean([item["queues"]["waiting"] for item in records])),
|
|
"queue_waiting_max": max((item["queues"]["waiting"] for item in records), default=0),
|
|
"kv_usage_mean": float(np.mean([item["kv"]["usage"] for item in records])),
|
|
"kv_usage_max": max((item["kv"]["usage"] for item in records), default=0),
|
|
"preemptions": sum(int(item["preemptions"]) for item in records),
|
|
"prefix_query_hit_ratio": (
|
|
sum(int(item["hits"]) for item in prefixes)
|
|
/ sum(int(item["queries"]) for item in prefixes)
|
|
if sum(int(item["queries"]) for item in prefixes)
|
|
else 0.0
|
|
),
|
|
"mode_counts": dict(sorted(modes.items())),
|
|
"mode_shares": {key: value / len(records) for key, value in sorted(modes.items())},
|
|
},
|
|
"waste": {
|
|
"padding_fraction": pad_sum / bucket_sum if bucket_sum else 0.0,
|
|
"padded_tokens_per_useful_token": pad_sum / sum(useful) if sum(useful) else 0.0,
|
|
"graph_miss_rate": len(misses) / model_n if model_n else 0.0,
|
|
"eligible_miss_rate": len(eligible_misses) / model_n if model_n else 0.0,
|
|
"overflow_rate": len(overflow) / model_n if model_n else 0.0,
|
|
"bucket_slack": (
|
|
(sum(theoretical_buckets) - sum(theoretical_unpadded))
|
|
/ sum(theoretical_buckets)
|
|
if theoretical_buckets
|
|
else 0.0
|
|
),
|
|
"mixed_interference": None,
|
|
"moe_layer_cv": None,
|
|
},
|
|
"trace_files": len(marker.get("traces", [])),
|
|
"profile_recovery": recoveries,
|
|
"profile_recovery_valid": all(item["valid"] for item in recoveries),
|
|
"layer2_missing_after_controller_cleanup": bool(
|
|
marker.get("layer2_missing_after_controller_cleanup")
|
|
),
|
|
"blocks": [dict(block) for block in blocks],
|
|
}
|
|
return summary, records
|
|
|
|
|
|
def manifest_raggedness(path: Path, cohort: int) -> tuple[float, list[tuple[float, float]]]:
|
|
lengths = []
|
|
with path.open() as source:
|
|
for line in source:
|
|
lengths.append(int(json.loads(line)["input_tokens"]))
|
|
pieces = []
|
|
for start in range(0, len(lengths) - cohort + 1, cohort):
|
|
group = lengths[start : start + cohort]
|
|
denominator = float(cohort * max(group))
|
|
pieces.append((denominator - sum(group), denominator))
|
|
numerator = sum(item[0] for item in pieces)
|
|
denominator = sum(item[1] for item in pieces)
|
|
return numerator / denominator, pieces
|
|
|
|
|
|
EXECUTE = re.compile(r"execute_context_\d+\((\d+)\)_generation_\d+\((\d+)\)")
|
|
|
|
|
|
def trace_steps(path: Path, layer1: list[dict[str, Any]]) -> dict[str, Any]:
|
|
opener = gzip.open if path.suffix == ".gz" else open
|
|
with opener(path, "rt", encoding="utf-8") as source:
|
|
payload = json.load(source)
|
|
base = int(payload["baseTimeNanoseconds"])
|
|
events = payload["traceEvents"]
|
|
cpu = []
|
|
gpu_by_external: dict[int, list[dict[str, Any]]] = defaultdict(list)
|
|
kernels = []
|
|
for event in events:
|
|
name = str(event.get("name", ""))
|
|
if event.get("cat") == "user_annotation" and EXECUTE.fullmatch(name):
|
|
cpu.append(event)
|
|
elif event.get("cat") == "gpu_user_annotation" and EXECUTE.fullmatch(name):
|
|
gpu_by_external[int(event.get("args", {}).get("External id", -1))].append(event)
|
|
elif event.get("cat") == "kernel":
|
|
kernels.append(event)
|
|
cpu.sort(key=lambda item: float(item["ts"]))
|
|
if len(cpu) != 8:
|
|
raise RuntimeError(f"active execute count {len(cpu)} != 8: {path}")
|
|
walls = [int(item["submit_wall_ns"]) for item in layer1]
|
|
joined = []
|
|
used_steps = set()
|
|
unmatched: dict[str, float] = defaultdict(float)
|
|
for event in cpu:
|
|
external = int(event["args"]["External id"])
|
|
candidates = gpu_by_external.get(external, [])
|
|
if not candidates:
|
|
raise RuntimeError(f"GPU execute annotation absent: {path}: {external}")
|
|
gpu = max(candidates, key=lambda item: float(item.get("dur", 0)))
|
|
start = float(gpu["ts"])
|
|
end = start + float(gpu["dur"])
|
|
durations: dict[str, float] = defaultdict(float)
|
|
for kernel in kernels:
|
|
midpoint = float(kernel["ts"]) + float(kernel.get("dur", 0)) / 2
|
|
if start <= midpoint <= end:
|
|
kernel_name = str(kernel.get("name", ""))
|
|
duration = float(kernel.get("dur", 0))
|
|
family = common.classify_kernel(kernel_name)
|
|
durations[family] += duration
|
|
if family == "other":
|
|
unmatched[kernel_name] += duration
|
|
total = sum(durations.values())
|
|
if total <= 0:
|
|
raise RuntimeError(f"active execute has no kernels: {path}: {external}")
|
|
wall = base + int(float(event["ts"]) * 1000)
|
|
index = bisect.bisect_left(walls, wall)
|
|
choices = layer1[max(0, index - 3) : index + 3]
|
|
record = min(choices, key=lambda item: abs(int(item["submit_wall_ns"]) - wall))
|
|
match = EXECUTE.fullmatch(str(event["name"]))
|
|
assert match
|
|
expected = (int(match.group(1)), int(match.group(2)))
|
|
actual = (int(record["prefill_tokens"]), int(record["decode_tokens"]))
|
|
delta_ms = (int(record["submit_wall_ns"]) - wall) / 1e6
|
|
if actual != expected or abs(delta_ms) > 100 or record["step_index"] in used_steps:
|
|
raise RuntimeError(
|
|
f"ambiguous Layer-1 join: {path}: expected={expected} actual={actual} "
|
|
f"delta_ms={delta_ms}"
|
|
)
|
|
used_steps.add(record["step_index"])
|
|
shares = {family: durations.get(family, 0.0) / total for family in FAMILIES}
|
|
shares["other"] = durations.get("other", 0.0) / total
|
|
joined.append(
|
|
{
|
|
"step_index": int(record["step_index"]),
|
|
"join_delta_ms": delta_ms,
|
|
"prefill_tokens": actual[0],
|
|
"decode_tokens": actual[1],
|
|
"scheduled_requests": int(record["scheduled_requests"]),
|
|
"decode_batch_size": int(record["decode_batch_size"]),
|
|
"runtime_mode": mode(record),
|
|
"duration_us": dict(durations),
|
|
"shares": shares,
|
|
"classifiable_fraction": 1.0 - shares["other"],
|
|
}
|
|
)
|
|
aggregate = defaultdict(float)
|
|
for step in joined:
|
|
for family, duration in step["duration_us"].items():
|
|
aggregate[family] += duration
|
|
total = sum(aggregate.values())
|
|
attention_subduration = defaultdict(float)
|
|
mode_duration: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float))
|
|
mode_steps = Counter()
|
|
for step in joined:
|
|
if step["prefill_tokens"] and step["decode_tokens"]:
|
|
attention_key = "attention_mixed"
|
|
elif step["prefill_tokens"]:
|
|
attention_key = "attention_prefill"
|
|
else:
|
|
attention_key = "attention_decode"
|
|
attention_subduration[attention_key] += step["duration_us"].get("attention", 0.0)
|
|
mode_steps[step["runtime_mode"]] += 1
|
|
for family, duration in step["duration_us"].items():
|
|
mode_duration[step["runtime_mode"]][family] += duration
|
|
return {
|
|
"path": path.name,
|
|
"steps": joined,
|
|
"shares": {family: aggregate.get(family, 0.0) / total for family in FAMILIES},
|
|
"other_share": aggregate.get("other", 0.0) / total,
|
|
"classifiable_fraction": 1.0 - aggregate.get("other", 0.0) / total,
|
|
"attention_subshares": {
|
|
key: attention_subduration.get(key, 0.0) / total
|
|
for key in ("attention_prefill", "attention_decode", "attention_mixed")
|
|
},
|
|
"mode_steps": dict(sorted(mode_steps.items())),
|
|
"mode_shares": {
|
|
key: {
|
|
family: durations.get(family, 0.0) / sum(durations.values())
|
|
for family in FAMILIES
|
|
}
|
|
for key, durations in sorted(mode_duration.items())
|
|
},
|
|
"top_unmatched": [
|
|
{"name": name, "duration_us": duration}
|
|
for name, duration in sorted(unmatched.items(), key=lambda item: item[1], reverse=True)[
|
|
:20
|
|
]
|
|
],
|
|
}
|
|
|
|
|
|
def smd(profile: list[float], clean: list[float]) -> float:
|
|
a = np.asarray(profile, dtype=float)
|
|
b = np.asarray(clean, dtype=float)
|
|
if np.all(a == a[0]) and np.all(b == b[0]):
|
|
return 0.0 if a[0] == b[0] else math.inf
|
|
denominator = math.sqrt((float(a.var(ddof=1)) + float(b.var(ddof=1))) / 2)
|
|
if denominator == 0:
|
|
return math.inf
|
|
return float((a.mean() - b.mean()) / denominator)
|
|
|
|
|
|
def representativeness(window: dict[str, Any], clean_c: list[dict[str, Any]]) -> dict[str, Any]:
|
|
steps = window["steps"]
|
|
features: dict[str, tuple[list[float], list[float]]] = {
|
|
"scheduled_tokens": (
|
|
[step["prefill_tokens"] + step["decode_tokens"] for step in steps],
|
|
[item["prefill_tokens"] + item["decode_tokens"] for item in clean_c],
|
|
),
|
|
"prefill_fraction": (
|
|
[
|
|
step["prefill_tokens"]
|
|
/ max(1, step["prefill_tokens"] + step["decode_tokens"])
|
|
for step in steps
|
|
],
|
|
[
|
|
item["prefill_tokens"]
|
|
/ max(1, item["prefill_tokens"] + item["decode_tokens"])
|
|
for item in clean_c
|
|
],
|
|
),
|
|
"decode_batch_size": (
|
|
[step["decode_batch_size"] for step in steps],
|
|
[item["decode_batch_size"] for item in clean_c],
|
|
),
|
|
}
|
|
modes = ("FULL", "PIECEWISE", "NONE")
|
|
for key in modes:
|
|
features[f"mode_{key}"] = (
|
|
[float(step["runtime_mode"] == key) for step in steps],
|
|
[float(mode(item) == key) for item in clean_c],
|
|
)
|
|
values = {key: smd(*pair) for key, pair in features.items()}
|
|
return {
|
|
"smd": values,
|
|
"valid": all(math.isfinite(value) and abs(value) <= 0.25 for value in values.values()),
|
|
}
|
|
|
|
|
|
def convex_hull(points: list[tuple[float, float]]) -> list[tuple[float, float]]:
|
|
unique = sorted(set(points))
|
|
if len(unique) <= 1:
|
|
return unique
|
|
|
|
def cross(origin, left, right):
|
|
return (left[0] - origin[0]) * (right[1] - origin[1]) - (
|
|
left[1] - origin[1]
|
|
) * (right[0] - origin[0])
|
|
|
|
lower = []
|
|
for point in unique:
|
|
while len(lower) >= 2 and cross(lower[-2], lower[-1], point) <= 0:
|
|
lower.pop()
|
|
lower.append(point)
|
|
upper = []
|
|
for point in reversed(unique):
|
|
while len(upper) >= 2 and cross(upper[-2], upper[-1], point) <= 0:
|
|
upper.pop()
|
|
upper.append(point)
|
|
return lower[:-1] + upper[:-1]
|
|
|
|
|
|
def inside_convex(hull: list[tuple[float, float]], point: tuple[float, float]) -> bool:
|
|
if not hull:
|
|
return False
|
|
if len(hull) == 1:
|
|
return point == hull[0]
|
|
if len(hull) == 2:
|
|
left, right = hull
|
|
cross = (right[0] - left[0]) * (point[1] - left[1]) - (
|
|
right[1] - left[1]
|
|
) * (point[0] - left[0])
|
|
return (
|
|
abs(cross) <= 1e-9
|
|
and min(left[0], right[0]) <= point[0] <= max(left[0], right[0])
|
|
and min(left[1], right[1]) <= point[1] <= max(left[1], right[1])
|
|
)
|
|
signs = []
|
|
for index, left in enumerate(hull):
|
|
right = hull[(index + 1) % len(hull)]
|
|
cross = (right[0] - left[0]) * (point[1] - left[1]) - (
|
|
right[1] - left[1]
|
|
) * (point[0] - left[0])
|
|
if abs(cross) > 1e-9:
|
|
signs.append(math.copysign(1, cross))
|
|
return not signs or min(signs) == max(signs)
|
|
|
|
|
|
def fit_nonnegative_robust(rows: list[tuple[float, float, float]]):
|
|
values = np.asarray(rows, dtype=float)
|
|
x = np.log1p(values[:, 0])
|
|
n = np.log1p(values[:, 1])
|
|
y = values[:, 2]
|
|
x_knots = np.quantile(x, [0.25, 0.5, 0.75])
|
|
n_knots = np.quantile(n, [0.25, 0.5, 0.75])
|
|
|
|
def design(raw_x, raw_n):
|
|
a = np.log1p(np.asarray(raw_x, dtype=float))
|
|
b = np.log1p(np.asarray(raw_n, dtype=float))
|
|
columns = [np.ones_like(a), a, b, a * b]
|
|
columns.extend(np.maximum(0, a - knot) for knot in x_knots)
|
|
columns.extend(np.maximum(0, b - knot) for knot in n_knots)
|
|
return np.column_stack(columns)
|
|
|
|
matrix = design(values[:, 0], values[:, 1])
|
|
weights = np.ones(len(y))
|
|
coef = np.zeros(matrix.shape[1])
|
|
for _ in range(20):
|
|
weighted = np.sqrt(weights)
|
|
coef = np.linalg.lstsq(matrix * weighted[:, None], y * weighted, rcond=None)[0]
|
|
coef = np.maximum(0, coef)
|
|
residual = y - matrix @ coef
|
|
scale = 1.4826 * np.median(np.abs(residual - np.median(residual))) + 1e-9
|
|
weights = np.minimum(1.0, 1.345 * scale / np.maximum(np.abs(residual), 1e-9))
|
|
|
|
def predict(raw_x: float, raw_n: float) -> float:
|
|
return float(max(0.0, design([raw_x], [raw_n])[0] @ coef))
|
|
|
|
return predict, convex_hull(
|
|
[(float(row[0]), float(row[1])) for row in values]
|
|
)
|
|
|
|
|
|
def add_mixed_interference(
|
|
summaries: dict[str, dict[str, Any]],
|
|
records: dict[str, list[dict[str, Any]]],
|
|
) -> None:
|
|
groups: dict[tuple[str, str], list[str]] = defaultdict(list)
|
|
for run_id, summary in summaries.items():
|
|
if run_id.endswith("-confirmation"):
|
|
continue
|
|
groups[(summary["config"], summary["load"])].append(run_id)
|
|
for run_ids in groups.values():
|
|
for target in run_ids:
|
|
pattern = summaries[target]["pattern"]
|
|
training = [item for item in run_ids if summaries[item]["pattern"] != pattern]
|
|
prefill = []
|
|
decode = []
|
|
zero = []
|
|
for run_id in training:
|
|
for item in records[run_id]:
|
|
p = int(item["prefill_tokens"])
|
|
d = int(item["decode_tokens"])
|
|
n = int(item["scheduled_requests"])
|
|
duration = (item["complete_mono_ns"] - item["submit_mono_ns"]) / 1e6
|
|
if p > 0 and d == 0:
|
|
prefill.append((p, n, duration))
|
|
elif d > 0 and p == 0:
|
|
decode.append((d, n, duration))
|
|
elif p == 0 and d == 0:
|
|
zero.append(duration)
|
|
if len(prefill) < 30 or len(decode) < 30:
|
|
continue
|
|
fp, p_support = fit_nonnegative_robust(prefill)
|
|
fd, d_support = fit_nonnegative_robust(decode)
|
|
alpha = float(np.median(zero)) if zero else 0.0
|
|
supported = []
|
|
for item in records[target]:
|
|
p = int(item["prefill_tokens"])
|
|
d = int(item["decode_tokens"])
|
|
n = int(item["scheduled_requests"])
|
|
if not (p > 0 and d > 0):
|
|
continue
|
|
if not inside_convex(p_support, (p, n)) or not inside_convex(
|
|
d_support, (d, n)
|
|
):
|
|
continue
|
|
predicted = fp(p, n) + fd(d, n) - alpha
|
|
if predicted <= 0:
|
|
continue
|
|
observed = (item["complete_mono_ns"] - item["submit_mono_ns"]) / 1e6
|
|
supported.append((item, observed - predicted, predicted))
|
|
summaries[target]["waste"]["mixed_supported_steps"] = len(supported)
|
|
summaries[target]["waste"]["mixed_total_steps"] = sum(
|
|
item["prefill_tokens"] > 0 and item["decode_tokens"] > 0
|
|
for item in records[target]
|
|
)
|
|
if len(supported) < 30:
|
|
continue
|
|
numerator = sum(item[1] for item in supported)
|
|
denominator = sum(item[2] for item in supported)
|
|
summaries[target]["waste"]["mixed_interference"] = numerator / denominator
|
|
blocks = summaries[target]["blocks"]
|
|
t0 = int(load_json(Path(summaries[target]["run_dir"]) / "client/result.json")["t0_mono_ns"])
|
|
for record, residual, predicted in supported:
|
|
block = blocks[clean_block(record, t0)]
|
|
block["mix_residual"] = block.get("mix_residual", 0.0) + residual
|
|
block["mix_predicted"] = block.get("mix_predicted", 0.0) + predicted
|
|
|
|
|
|
def ratio_blocks(summary: dict[str, Any], metric: str) -> np.ndarray | None:
|
|
keys = {
|
|
"padding_fraction": ("padding", "bucket"),
|
|
"graph_miss_rate": ("miss", "model"),
|
|
"overflow_rate": ("overflow", "model"),
|
|
"mixed_interference": ("mix_residual", "mix_predicted"),
|
|
"efficiency": ("tokens", "duration_ms"),
|
|
}
|
|
numerator, denominator = keys[metric]
|
|
values = np.asarray(
|
|
[[block.get(numerator, 0.0), block.get(denominator, 0.0)] for block in summary["blocks"]],
|
|
dtype=float,
|
|
)
|
|
if values[:, 1].sum() <= 0:
|
|
return None
|
|
return values
|
|
|
|
|
|
def bootstrap_difference(
|
|
left: np.ndarray, right: np.ndarray, rng: np.random.Generator
|
|
) -> dict[str, float]:
|
|
point = left[:, 0].sum() / left[:, 1].sum() - right[:, 0].sum() / right[:, 1].sum()
|
|
draws = np.empty(BOOTSTRAPS)
|
|
chunk = 5000
|
|
for start in range(0, BOOTSTRAPS, chunk):
|
|
count = min(chunk, BOOTSTRAPS - start)
|
|
li = rng.integers(0, len(left), size=(count, len(left)))
|
|
ri = rng.integers(0, len(right), size=(count, len(right)))
|
|
ln = left[li, 0].sum(axis=1)
|
|
ld = left[li, 1].sum(axis=1)
|
|
rn = right[ri, 0].sum(axis=1)
|
|
rd = right[ri, 1].sum(axis=1)
|
|
draws[start : start + count] = ln / np.maximum(ld, 1e-12) - rn / np.maximum(rd, 1e-12)
|
|
p = min(1.0, 2 * min(float(np.mean(draws <= 0)), float(np.mean(draws >= 0))))
|
|
return {
|
|
"point": float(point),
|
|
"ci95_low": float(np.quantile(draws, 0.025)),
|
|
"ci95_high": float(np.quantile(draws, 0.975)),
|
|
"simultaneous_low": float(np.quantile(draws, 0.05 / 16)),
|
|
"simultaneous_high": float(np.quantile(draws, 1 - 0.05 / 16)),
|
|
"p": p,
|
|
}
|
|
|
|
|
|
def holm(results: list[dict[str, Any]], total_tests: int | None = None) -> None:
|
|
ordered = sorted(results, key=lambda item: item["p"])
|
|
count = total_tests or len(ordered)
|
|
running = 0.0
|
|
for index, item in enumerate(ordered):
|
|
running = max(running, min(1.0, (count - index) * item["p"]))
|
|
item["p_holm"] = running
|
|
|
|
|
|
def permutation_p(
|
|
left: list[dict[str, Any]],
|
|
right: list[dict[str, Any]],
|
|
family_a: str,
|
|
family_b: str,
|
|
orientation: float,
|
|
rng: np.random.Generator,
|
|
) -> float:
|
|
observed_windows = []
|
|
null_windows = []
|
|
for window in range(2):
|
|
a = np.asarray(
|
|
[
|
|
orientation * (step["shares"][family_a] - step["shares"][family_b])
|
|
for step in left[window]["steps"]
|
|
]
|
|
)
|
|
b = np.asarray(
|
|
[
|
|
orientation * (step["shares"][family_a] - step["shares"][family_b])
|
|
for step in right[window]["steps"]
|
|
]
|
|
)
|
|
observed_windows.append(float(a.mean() - b.mean()))
|
|
pooled = np.concatenate([a, b])
|
|
values = np.empty(BOOTSTRAPS)
|
|
for start in range(0, BOOTSTRAPS, 5000):
|
|
count = min(5000, BOOTSTRAPS - start)
|
|
random = rng.random((count, 16))
|
|
indices = np.argpartition(random, 8, axis=1)[:, :8]
|
|
selected = np.take_along_axis(np.broadcast_to(pooled, (count, 16)), indices, axis=1)
|
|
values[start : start + count] = 2 * selected.mean(axis=1) - pooled.mean() * 2
|
|
null_windows.append(values)
|
|
observed = min(observed_windows)
|
|
null = np.minimum(null_windows[0], null_windows[1])
|
|
return float((np.count_nonzero(null >= observed) + 1) / (BOOTSTRAPS + 1))
|
|
|
|
|
|
def kendall_tau_b(left: list[float], right: list[float]) -> float:
|
|
concordant = discordant = tie_left = tie_right = 0
|
|
for i in range(len(left)):
|
|
for j in range(i + 1, len(left)):
|
|
a = np.sign(left[i] - left[j])
|
|
b = np.sign(right[i] - right[j])
|
|
if a == 0 and b != 0:
|
|
tie_left += 1
|
|
elif b == 0 and a != 0:
|
|
tie_right += 1
|
|
elif a * b > 0:
|
|
concordant += 1
|
|
elif a * b < 0:
|
|
discordant += 1
|
|
denominator = math.sqrt(
|
|
(concordant + discordant + tie_left) * (concordant + discordant + tie_right)
|
|
)
|
|
return (concordant - discordant) / denominator if denominator else 0.0
|
|
|
|
|
|
def ranked_families(shares: dict[str, float]) -> list[dict[str, Any]]:
|
|
ordered = sorted(FAMILIES, key=lambda family: shares[family], reverse=True)
|
|
result = []
|
|
rank = 1
|
|
group_top = None
|
|
for index, family in enumerate(ordered):
|
|
if group_top is None or group_top - shares[family] > 0.01:
|
|
rank = index + 1
|
|
group_top = shares[family]
|
|
result.append({"family": family, "share": shares[family], "rank": rank})
|
|
return result
|
|
|
|
|
|
def operator_window_valid(
|
|
window: dict[str, Any], summary: dict[str, Any]
|
|
) -> bool:
|
|
representative = window.get("representativeness")
|
|
return bool(
|
|
window["classifiable_fraction"] >= 0.70
|
|
and representative
|
|
and representative["valid"]
|
|
and summary["profile_recovery_valid"]
|
|
)
|
|
|
|
|
|
def analyze(root: Path, private: Path) -> dict[str, Any]:
|
|
markers, confirmation_markers, complete_stages, unaccepted_markers = (
|
|
accepted_marker_paths(root)
|
|
)
|
|
if len(markers) != 40 or confirmation_markers:
|
|
raise RuntimeError(
|
|
f"A-P3-7 run count before analysis: primary={len(markers)} "
|
|
f"confirm={len(confirmation_markers)}"
|
|
)
|
|
summaries: dict[str, dict[str, Any]] = {}
|
|
layer_records: dict[str, list[dict[str, Any]]] = {}
|
|
for path in markers + confirmation_markers:
|
|
marker = load_json(path)
|
|
run_id = marker["run_id"]
|
|
summary, records = summarize_run(path.parent, marker)
|
|
summary["run_dir"] = str(path.parent)
|
|
summaries[run_id] = summary
|
|
layer_records[run_id] = records
|
|
cell_loads: dict[str, set[str]] = defaultdict(set)
|
|
for summary in summaries.values():
|
|
cell_loads[f"{summary['pattern']}/{summary['config']}"].add(summary["load"])
|
|
complete_cells = sorted(
|
|
cell for cell, loads in cell_loads.items() if loads == {"saturation", "moderate"}
|
|
)
|
|
missing_cells = sorted(expected_cells() - set(complete_cells))
|
|
if missing_cells != sorted(AP37_MISSING_CELLS):
|
|
raise RuntimeError(f"A-P3-7 missing-cell mismatch: {missing_cells}")
|
|
completed_c00_patterns = sorted(
|
|
cell.split("/", 1)[0] for cell in complete_cells if cell.endswith("/C00")
|
|
)
|
|
ragged = {}
|
|
ragged_pieces = {}
|
|
for pattern in (f"P{index:02d}" for index in range(1, 12)):
|
|
values = {}
|
|
for cohort in (32, 64, 128):
|
|
value, pieces = manifest_raggedness(private / f"{pattern}.jsonl", cohort)
|
|
values[f"R{cohort}"] = value
|
|
if cohort == 64:
|
|
ragged_pieces[pattern] = np.asarray(pieces)
|
|
ragged[pattern] = values
|
|
for summary in summaries.values():
|
|
summary["waste"].update(ragged[summary["pattern"]])
|
|
add_mixed_interference(summaries, layer_records)
|
|
|
|
layer2: dict[str, list[dict[str, Any]]] = {}
|
|
layer2_issues = []
|
|
for run_id, summary in summaries.items():
|
|
run_dir = Path(summary["run_dir"])
|
|
traces = sorted((run_dir / "traces").glob("*.pt.trace.json*"))
|
|
if not traces:
|
|
if not summary["layer2_missing_after_controller_cleanup"] and not run_id.endswith("confirmation"):
|
|
layer2_issues.append(f"unexpected missing traces: {run_id}")
|
|
continue
|
|
all_layer, _ = run_records(run_dir)
|
|
windows = [trace_steps(path, all_layer) for path in traces]
|
|
if summary["config"] != "C00-TP2":
|
|
clean_result = load_json(run_dir / "client/result.json")
|
|
t0 = int(clean_result["t0_mono_ns"])
|
|
clean_c = [
|
|
item
|
|
for item in all_layer
|
|
if t0 + int(220e9) <= int(item["submit_mono_ns"]) < t0 + int(300e9)
|
|
]
|
|
for window in windows:
|
|
window["representativeness"] = representativeness(window, clean_c)
|
|
layer2[run_id] = windows
|
|
|
|
all_operator_windows = {
|
|
run_id: [
|
|
{
|
|
"trace": window["path"],
|
|
"shares": window["shares"],
|
|
"other_share": window["other_share"],
|
|
"classifiable_fraction": window["classifiable_fraction"],
|
|
"attention_subshares": window["attention_subshares"],
|
|
"mode_steps": window["mode_steps"],
|
|
"mode_shares": window["mode_shares"],
|
|
"top_unmatched": window["top_unmatched"],
|
|
"representativeness": window.get("representativeness"),
|
|
}
|
|
for window in windows
|
|
]
|
|
for run_id, windows in sorted(layer2.items())
|
|
}
|
|
|
|
patterns = [f"P{i:02d}" for i in range(1, 12)]
|
|
primary_ids = [f"{pattern}-C00-moderate" for pattern in completed_c00_patterns]
|
|
operator_table = {}
|
|
operator_share_table = {}
|
|
operator_mode_segments = {}
|
|
invalid_primary_windows = []
|
|
for pattern in patterns:
|
|
operator_share_table[pattern] = {}
|
|
for load in ("saturation", "moderate"):
|
|
run_id = f"{pattern}-C00-{load}"
|
|
windows = layer2.get(run_id, [])
|
|
rows = []
|
|
for index, window in enumerate(windows, 1):
|
|
valid = operator_window_valid(window, summaries[run_id])
|
|
rows.append(
|
|
{
|
|
"window": index,
|
|
"shares": window["shares"],
|
|
"ranking": ranked_families(window["shares"]),
|
|
"classifiable_fraction": window["classifiable_fraction"],
|
|
"representativeness": window["representativeness"],
|
|
"attention_subshares": window["attention_subshares"],
|
|
"mode_steps": window["mode_steps"],
|
|
"mode_shares": window["mode_shares"],
|
|
"top_unmatched": window["top_unmatched"],
|
|
"valid": valid,
|
|
}
|
|
)
|
|
mean_shares = (
|
|
{
|
|
family: float(
|
|
np.mean([window["shares"][family] for window in windows])
|
|
)
|
|
for family in FAMILIES
|
|
}
|
|
if windows
|
|
else None
|
|
)
|
|
status = (
|
|
"EVALUABLE"
|
|
if len(rows) == 2 and all(row["valid"] for row in rows)
|
|
else "NOT_EVALUABLE"
|
|
)
|
|
reason = None
|
|
if run_id not in summaries:
|
|
reason = "pattern/config cell missing under A-P3-7"
|
|
elif len(rows) != 2:
|
|
reason = f"accepted trace windows {len(rows)}/2"
|
|
elif status != "EVALUABLE":
|
|
reason = "classifiability, representativeness, or recovery gate failed"
|
|
operator_share_table[pattern][load] = {
|
|
"run_id": run_id,
|
|
"status": status,
|
|
"reason": reason,
|
|
"window_count": len(rows),
|
|
"valid_window_count": sum(row["valid"] for row in rows),
|
|
"mean_shares": mean_shares,
|
|
"mean_ranking": ranked_families(mean_shares) if mean_shares else None,
|
|
"windows": rows,
|
|
}
|
|
if run_id in primary_ids:
|
|
if len(rows) != 2:
|
|
invalid_primary_windows.append(
|
|
f"{run_id}: trace count {len(rows)}"
|
|
)
|
|
else:
|
|
operator_table[run_id] = rows
|
|
invalid_primary_windows.extend(
|
|
f"{run_id}: window {row['window']}"
|
|
for row in rows
|
|
if not row["valid"]
|
|
)
|
|
if not windows:
|
|
continue
|
|
by_mode: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for window in windows:
|
|
for step in window["steps"]:
|
|
by_mode[step["runtime_mode"]].append(step)
|
|
operator_mode_segments[run_id] = {}
|
|
for runtime_mode, steps in sorted(by_mode.items()):
|
|
if len(steps) < 8:
|
|
continue
|
|
durations = defaultdict(float)
|
|
for step in steps:
|
|
for family, duration in step["duration_us"].items():
|
|
durations[family] += duration
|
|
total = sum(durations.values())
|
|
operator_mode_segments[run_id][runtime_mode] = {
|
|
"steps": len(steps),
|
|
"shares": {
|
|
family: durations.get(family, 0.0) / total
|
|
for family in FAMILIES
|
|
},
|
|
"other_share": durations.get("other", 0.0) / total,
|
|
}
|
|
|
|
rng = np.random.default_rng(SEED)
|
|
inversions = []
|
|
total_ranking_tests = math.comb(11, 2) * math.comb(len(FAMILIES), 2)
|
|
for pi, left_pattern in enumerate(patterns):
|
|
left_id = f"{left_pattern}-C00-moderate"
|
|
if len(operator_table.get(left_id, [])) != 2 or any(
|
|
not item["valid"] for item in operator_table.get(left_id, [])
|
|
):
|
|
continue
|
|
for right_pattern in patterns[pi + 1 :]:
|
|
right_id = f"{right_pattern}-C00-moderate"
|
|
if len(operator_table.get(right_id, [])) != 2 or any(
|
|
not item["valid"] for item in operator_table.get(right_id, [])
|
|
):
|
|
continue
|
|
for ai, family_a in enumerate(FAMILIES):
|
|
for family_b in FAMILIES[ai + 1 :]:
|
|
left_gaps = [
|
|
window["shares"][family_a] - window["shares"][family_b]
|
|
for window in layer2[left_id]
|
|
]
|
|
right_gaps = [
|
|
window["shares"][family_a] - window["shares"][family_b]
|
|
for window in layer2[right_id]
|
|
]
|
|
orientation = None
|
|
if min(left_gaps) >= 0.05 and max(right_gaps) <= -0.05:
|
|
orientation = 1.0
|
|
elif max(left_gaps) <= -0.05 and min(right_gaps) >= 0.05:
|
|
orientation = -1.0
|
|
if orientation is None:
|
|
continue
|
|
p = permutation_p(
|
|
layer2[left_id],
|
|
layer2[right_id],
|
|
family_a,
|
|
family_b,
|
|
orientation,
|
|
rng,
|
|
)
|
|
inversions.append(
|
|
{
|
|
"left": left_pattern,
|
|
"right": right_pattern,
|
|
"family_a": family_a,
|
|
"family_b": family_b,
|
|
"left_gaps": left_gaps,
|
|
"right_gaps": right_gaps,
|
|
"orientation": orientation,
|
|
"p": p,
|
|
}
|
|
)
|
|
holm(inversions, total_tests=total_ranking_tests)
|
|
accepted_inversions = [item for item in inversions if item["p_holm"] < 0.05]
|
|
|
|
tau = []
|
|
for index, left in enumerate(patterns):
|
|
left_id = f"{left}-C00-moderate"
|
|
if len(operator_table.get(left_id, [])) != 2 or any(
|
|
not item["valid"] for item in operator_table.get(left_id, [])
|
|
):
|
|
continue
|
|
left_mean = {
|
|
family: float(np.mean([window["shares"][family] for window in layer2[left_id]]))
|
|
for family in FAMILIES
|
|
}
|
|
left_rank = {item["family"]: item["rank"] for item in ranked_families(left_mean)}
|
|
for right in patterns[index + 1 :]:
|
|
right_id = f"{right}-C00-moderate"
|
|
if len(operator_table.get(right_id, [])) != 2 or any(
|
|
not item["valid"] for item in operator_table.get(right_id, [])
|
|
):
|
|
continue
|
|
right_mean = {
|
|
family: float(
|
|
np.mean([window["shares"][family] for window in layer2[right_id]])
|
|
)
|
|
for family in FAMILIES
|
|
}
|
|
right_rank = {
|
|
item["family"]: item["rank"] for item in ranked_families(right_mean)
|
|
}
|
|
tau.append(
|
|
{
|
|
"left": left,
|
|
"right": right,
|
|
"tau_b": kendall_tau_b(
|
|
[left_rank[family] for family in FAMILIES],
|
|
[right_rank[family] for family in FAMILIES],
|
|
),
|
|
}
|
|
)
|
|
|
|
waste_contrasts = []
|
|
contrast_status = []
|
|
by_metric: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for irregular, control in IRREGULAR_CONTROLS:
|
|
left_id = f"{irregular}-C00-moderate"
|
|
right_id = f"{control}-C00-moderate"
|
|
missing = [
|
|
run_id.removesuffix("-moderate")
|
|
for run_id in (left_id, right_id)
|
|
if run_id not in summaries
|
|
]
|
|
if missing:
|
|
contrast_status.append(
|
|
{
|
|
"irregular": irregular,
|
|
"control": control,
|
|
"evaluable": False,
|
|
"verdict": "NOT_EVALUABLE",
|
|
"missing_cells": missing,
|
|
"efficiency_loss": None,
|
|
"metrics": [],
|
|
"passing_metrics": [],
|
|
"passes_h1b": False,
|
|
}
|
|
)
|
|
continue
|
|
left = summaries[left_id]
|
|
right = summaries[right_id]
|
|
efficiency_loss = 1 - (
|
|
left["layer1"]["token_efficiency_per_ms"]
|
|
/ right["layer1"]["token_efficiency_per_ms"]
|
|
)
|
|
metric_results = []
|
|
for metric in ("padding_fraction", "graph_miss_rate", "overflow_rate", "mixed_interference"):
|
|
a = ratio_blocks(left, metric)
|
|
b = ratio_blocks(right, metric)
|
|
if a is None or b is None:
|
|
result = {
|
|
"point": None,
|
|
"ci95_low": None,
|
|
"ci95_high": None,
|
|
"simultaneous_low": None,
|
|
"simultaneous_high": None,
|
|
"p": 1.0,
|
|
"available": False,
|
|
}
|
|
else:
|
|
result = bootstrap_difference(a, b, rng)
|
|
result["available"] = True
|
|
result.update(
|
|
{
|
|
"irregular": irregular,
|
|
"control": control,
|
|
"metric": metric,
|
|
"efficiency_loss": efficiency_loss,
|
|
"evaluable": True,
|
|
}
|
|
)
|
|
by_metric[metric].append(result)
|
|
waste_contrasts.append(result)
|
|
metric_results.append(result)
|
|
result = bootstrap_difference(ragged_pieces[irregular], ragged_pieces[control], rng)
|
|
result["available"] = True
|
|
result.update(
|
|
{
|
|
"irregular": irregular,
|
|
"control": control,
|
|
"metric": "R64",
|
|
"efficiency_loss": efficiency_loss,
|
|
"evaluable": True,
|
|
}
|
|
)
|
|
by_metric["R64"].append(result)
|
|
waste_contrasts.append(result)
|
|
metric_results.append(result)
|
|
result = {
|
|
"irregular": irregular,
|
|
"control": control,
|
|
"metric": "moe_layer_cv",
|
|
"efficiency_loss": efficiency_loss,
|
|
"evaluable": True,
|
|
"point": None,
|
|
"ci95_low": None,
|
|
"ci95_high": None,
|
|
"simultaneous_low": None,
|
|
"simultaneous_high": None,
|
|
"p": 1.0,
|
|
"available": False,
|
|
}
|
|
by_metric["moe_layer_cv"].append(result)
|
|
waste_contrasts.append(result)
|
|
metric_results.append(result)
|
|
contrast_status.append(
|
|
{
|
|
"irregular": irregular,
|
|
"control": control,
|
|
"evaluable": True,
|
|
"verdict": None,
|
|
"missing_cells": [],
|
|
"efficiency_loss": efficiency_loss,
|
|
"metrics": metric_results,
|
|
"passing_metrics": [],
|
|
"passes_h1b": False,
|
|
}
|
|
)
|
|
for values in by_metric.values():
|
|
holm(values, total_tests=len(IRREGULAR_CONTROLS))
|
|
thresholds = {
|
|
"padding_fraction": 0.05,
|
|
"graph_miss_rate": 0.10,
|
|
"overflow_rate": 0.10,
|
|
"R64": 0.15,
|
|
"mixed_interference": 0.10,
|
|
"moe_layer_cv": 0.15,
|
|
}
|
|
for item in waste_contrasts:
|
|
item["material"] = (
|
|
item["point"] is not None
|
|
and item["point"] >= thresholds[item["metric"]]
|
|
and item["simultaneous_low"] is not None
|
|
and item["simultaneous_low"] > 0
|
|
and item["p_holm"] < 0.05
|
|
)
|
|
item["coincident_efficiency_or_residual"] = (
|
|
item["efficiency_loss"] >= 0.05
|
|
or (
|
|
summaries[f"{item['irregular']}-C00-moderate"]["waste"].get(
|
|
"mixed_interference"
|
|
)
|
|
or -math.inf
|
|
)
|
|
> 0
|
|
)
|
|
item["passes_h1b"] = item["material"] and item["coincident_efficiency_or_residual"]
|
|
for contrast in contrast_status:
|
|
if not contrast["evaluable"]:
|
|
continue
|
|
hits = [item for item in contrast["metrics"] if item["passes_h1b"]]
|
|
contrast["passing_metrics"] = [item["metric"] for item in hits]
|
|
contrast["passes_h1b"] = bool(hits)
|
|
contrast["verdict"] = "PASS" if hits else "NO_QUALIFYING_METRIC"
|
|
observed_missing_contrasts = sorted(
|
|
(item["irregular"], item["control"])
|
|
for item in contrast_status
|
|
if not item["evaluable"]
|
|
)
|
|
if observed_missing_contrasts != sorted(AP37_MISSING_CONTRASTS):
|
|
raise RuntimeError(
|
|
f"A-P3-7 missing-contrast mismatch: {observed_missing_contrasts}"
|
|
)
|
|
h1b_hits = [item for item in waste_contrasts if item["passes_h1b"]]
|
|
|
|
controller = load_json(root / "controller-state.json")
|
|
trace_count = sum(
|
|
len(list((Path(summary["run_dir"]) / "traces").glob("*.pt.trace.json*")))
|
|
for summary in summaries.values()
|
|
)
|
|
other_gpu_processes = []
|
|
for stage in complete_stages:
|
|
path = stage / "other-gpu-processes.json"
|
|
if path.exists():
|
|
other_gpu_processes.extend(load_json(path))
|
|
clean_failures = sum(int(item["clean"]["failed"]) for item in summaries.values())
|
|
moderate_rate_ok = []
|
|
for run_id, summary in summaries.items():
|
|
if summary["load"] == "moderate":
|
|
offered = float(summary["clean"]["offered_rps"])
|
|
requested = float(load_json(Path(summary["run_dir"]) / "client/result.json")["request_rate"])
|
|
moderate_rate_ok.append(abs(offered / requested - 1) <= 0.05)
|
|
ratios = []
|
|
for summary in summaries.values():
|
|
ratios.extend(
|
|
[
|
|
summary["waste"]["padding_fraction"],
|
|
summary["waste"]["graph_miss_rate"],
|
|
summary["waste"]["overflow_rate"],
|
|
summary["waste"]["R64"],
|
|
summary["layer1"]["kv_usage_mean"],
|
|
summary["layer1"]["kv_usage_max"],
|
|
]
|
|
)
|
|
p10_ap36 = ap36_warmup_stability(
|
|
root / "primary/P10-C00-TP2/saturation"
|
|
)
|
|
same_wave_warmup = {}
|
|
for cell in ("P11-C00", "P03-C11"):
|
|
result = load_json(root / f"primary/{cell}/saturation/client/result.json")
|
|
requests = jsonl(root / f"primary/{cell}/saturation/client/requests.jsonl")
|
|
same_wave_warmup[cell] = sum(
|
|
bool(item["success"])
|
|
and 0
|
|
<= float(item["completed_s"])
|
|
< float(result["warmup_seconds"])
|
|
for item in requests
|
|
)
|
|
boundary_marker = load_json(
|
|
Path(summaries["P01-C01-moderate"]["run_dir"]) / "run-complete.json"
|
|
)
|
|
operational_findings = {
|
|
"p10_tp2_non_stabilization": {
|
|
**p10_ap36,
|
|
"status": "PATTERN_CONDITIONED_OPERATIONAL_FINDING",
|
|
"accepted_measurement": False,
|
|
"comparison": (
|
|
"all synthetic pattern runs passed their applicable registered "
|
|
"warm-up gates; orchestrator adjudication"
|
|
),
|
|
"same_wave_synthetic_warmup_completions": same_wave_warmup,
|
|
},
|
|
"long_context_drain": {
|
|
"run_id": "P10-C01-saturation",
|
|
"drain_seconds": summaries["P10-C01-saturation"]["drain_seconds"],
|
|
"amended_budget_seconds": 600,
|
|
"quarantined": False,
|
|
},
|
|
"failure_boundary": {
|
|
"run_id": "P01-C01-moderate",
|
|
"clean_failures": summaries["P01-C01-moderate"]["clean"]["failed"],
|
|
"excluded_window_failures": boundary_marker["client"][
|
|
"excluded_window_failures"
|
|
],
|
|
"excluded_failure_kinds": boundary_marker["client"][
|
|
"excluded_window_failure_kinds"
|
|
],
|
|
},
|
|
"layer2_sampling": {
|
|
"completed_c00_moderate_patterns": len(completed_c00_patterns),
|
|
"evaluable_c00_moderate_patterns": sum(
|
|
operator_share_table[pattern]["moderate"]["status"]
|
|
== "EVALUABLE"
|
|
for pattern in completed_c00_patterns
|
|
),
|
|
"invalid_windows": len(invalid_primary_windows),
|
|
"classifiable_fraction_min": min(
|
|
window["classifiable_fraction"]
|
|
for windows in layer2.values()
|
|
for window in windows
|
|
),
|
|
},
|
|
}
|
|
sanity = {
|
|
"numeric": {
|
|
"completed_throughput_rps": numeric_sanity(
|
|
[item["clean"]["completed_throughput_rps"] for item in summaries.values()]
|
|
),
|
|
"token_efficiency_per_ms": numeric_sanity(
|
|
[item["layer1"]["token_efficiency_per_ms"] for item in summaries.values()]
|
|
),
|
|
"drain_seconds": numeric_sanity([item["drain_seconds"] for item in summaries.values()]),
|
|
"layer1_clean_steps": numeric_sanity(
|
|
[item["layer1"]["records_clean"] for item in summaries.values()]
|
|
),
|
|
"operator_classifiable_fraction": numeric_sanity(
|
|
[window["classifiable_fraction"] for windows in layer2.values() for window in windows]
|
|
),
|
|
"waste_ratios": numeric_sanity(ratios),
|
|
"kendall_tau_b": numeric_sanity([item["tau_b"] for item in tau]),
|
|
"operator_share": numeric_sanity(
|
|
[
|
|
share
|
|
for windows in layer2.values()
|
|
for window in windows
|
|
for share in window["shares"].values()
|
|
]
|
|
),
|
|
"waste_contrast_effect": numeric_sanity(
|
|
[item["point"] for item in waste_contrasts]
|
|
),
|
|
},
|
|
"invariants": {
|
|
"ap37_primary_runs_40": len(markers) == 40,
|
|
"ap37_confirmation_runs_0": len(confirmation_markers) == 0,
|
|
"accepted_run_ids_unique": len(summaries) == len(markers),
|
|
"complete_cells_20": len(complete_cells) == 20,
|
|
"missing_cells_exact": missing_cells == sorted(AP37_MISSING_CELLS),
|
|
"completed_c00_patterns_9": len(completed_c00_patterns) == 9,
|
|
"clean_duration_240": all(item["clean"]["duration_s"] == 240 for item in summaries.values()),
|
|
"clean_failures_zero": clean_failures == 0,
|
|
"moderate_rate_within_5pct": all(moderate_rate_ok),
|
|
"layer1_footer_invariants": all(
|
|
all(load_json(Path(item["run_dir"]) / "run-complete.json")["layer1"]["invariants"].values())
|
|
for item in summaries.values()
|
|
),
|
|
"ratios_in_unit_interval": all(0 <= value <= 1 for value in ratios),
|
|
"trace_count_accepted_72": trace_count == 72,
|
|
"missing_trace_count_accepted_8": controller.get("missing_trace_files") == 8,
|
|
"controller_frozen_at_ap36_failure": controller.get("status") == "failed"
|
|
and controller.get("completed_measured_runs") == 40,
|
|
"complete_stage_count_12": len(complete_stages) == 12,
|
|
"clock_and_load_snapshots_complete": all(
|
|
(stage / "clocks-before.txt").exists()
|
|
and (stage / "clocks-after.txt").exists()
|
|
and (stage / "loadavg-before.txt").exists()
|
|
and (stage / "loadavg-after.txt").exists()
|
|
for stage in complete_stages
|
|
),
|
|
"other_gpu_processes_absent": not other_gpu_processes,
|
|
"drain_quarantine_under_20pct": controller.get("drain_quarantined_runs", 0) / 40 <= 0.20,
|
|
"no_layer2_parser_issues": not layer2_issues,
|
|
"h1b_evaluable_contrasts_6": sum(
|
|
item["evaluable"] for item in contrast_status
|
|
)
|
|
== 6,
|
|
"h1b_missing_contrasts_exact": observed_missing_contrasts
|
|
== sorted(AP37_MISSING_CONTRASTS),
|
|
"ap36_operational_finding_reproduced": math.isclose(
|
|
p10_ap36["normalized_drift"], 0.367639109533929
|
|
)
|
|
and p10_ap36["warmup_completions"] == 17,
|
|
"patterns_not_all_identical_throughput": len(
|
|
{round(item["clean"]["completed_throughput_rps"], 9) for item in summaries.values()}
|
|
)
|
|
> 1,
|
|
},
|
|
"declared_deviations": {
|
|
"missing_saturation_traces": 8,
|
|
"missing_cells": missing_cells,
|
|
"missing_confirmations": ["P10", "P06", "P03", "P01"],
|
|
"unaccepted_canonical_run_markers_excluded": unaccepted_markers,
|
|
"invalid_primary_layer2_windows": invalid_primary_windows,
|
|
"moe_layer_cv": "N/A: layer scopes do not cover >=80% of MoE GEMM time",
|
|
},
|
|
}
|
|
if not all(sanity["invariants"].values()):
|
|
raise RuntimeError(f"data sanity red flag: {sanity['invariants']}")
|
|
return {
|
|
"schema": 1,
|
|
"analysis_seed": SEED,
|
|
"bootstrap_resamples": BOOTSTRAPS,
|
|
"matrix": {
|
|
"primary_runs": len(markers),
|
|
"confirmation_runs": len(confirmation_markers),
|
|
"complete_cells": complete_cells,
|
|
"missing_cells": missing_cells,
|
|
"completed_c00_patterns": completed_c00_patterns,
|
|
"trace_files": trace_count,
|
|
"drain_quarantined_runs": controller.get("drain_quarantined_runs", 0),
|
|
"clean_window_failures": clean_failures,
|
|
"gpu_hours_total": controller["gpu_hours_total"],
|
|
},
|
|
"runs": summaries,
|
|
"all_operator_windows": all_operator_windows,
|
|
"operator_windows": operator_table,
|
|
"operator_share_table": operator_share_table,
|
|
"operator_mode_segments": operator_mode_segments,
|
|
"ranking": {
|
|
"tests": total_ranking_tests,
|
|
"completed_patterns": completed_c00_patterns,
|
|
"evaluable_patterns": [
|
|
pattern
|
|
for pattern in completed_c00_patterns
|
|
if operator_share_table[pattern]["moderate"]["status"]
|
|
== "EVALUABLE"
|
|
],
|
|
"missing_patterns": ["P05", "P11"],
|
|
"candidates": inversions,
|
|
"accepted_inversions": accepted_inversions,
|
|
"kendall_tau_b": tau,
|
|
"invalid_primary_windows": invalid_primary_windows,
|
|
},
|
|
"waste_contrasts": waste_contrasts,
|
|
"waste_contrast_status": contrast_status,
|
|
"waste_thresholds": thresholds,
|
|
"robustness": {
|
|
"mixed_interference": "leave-one-pattern-out fits applied within config/load",
|
|
"operator_ranking": "two fixed wall-separated windows; no separate LOAO procedure was preregistered",
|
|
"confirmation_runs": "NOT_EVALUABLE: all four confirmations are missing",
|
|
},
|
|
"operational_findings": operational_findings,
|
|
"hypothesis": {
|
|
"H1a": partial_verdict(bool(accepted_inversions)),
|
|
"H1b": partial_verdict(bool(h1b_hits)),
|
|
"compound": "CONFIRMED" if accepted_inversions and h1b_hits else "PARTIAL" if accepted_inversions or h1b_hits else "INCONCLUSIVE",
|
|
"h1b_hits": h1b_hits,
|
|
"refutation_allowed": False,
|
|
"logical_asymmetry": "A-P3-7 permits existential confirmation but incomplete coverage forbids refutation",
|
|
},
|
|
"sanity": sanity,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--root", type=Path, required=True)
|
|
parser.add_argument("--private", type=Path, required=True)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
result = analyze(args.root, args.private)
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = args.out.with_suffix(args.out.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n")
|
|
temporary.replace(args.out)
|
|
print(json.dumps({"out": str(args.out), "hypothesis": result["hypothesis"], "sanity": result["sanity"]}, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|