Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
419
runs/opprof-phase6/opprof_phase6_solo_controller.py
Normal file
419
runs/opprof-phase6/opprof_phase6_solo_controller.py
Normal file
@@ -0,0 +1,419 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A-P6-2 serialized solo controller for authoritative Phase-6 frontiers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import opprof_phase6_controller as base
|
||||
|
||||
|
||||
SOLO_ROOT = base.RUN_ROOT / "solo-authoritative"
|
||||
STATE = SOLO_ROOT / "controller-state.json"
|
||||
CAMPAIGN_STATE = base.RUN_ROOT / "controller-state.json"
|
||||
GPU_LIMIT = 6.0
|
||||
SAFETY_HOURS = 0.20
|
||||
MARKER = "opprof-phase6-solo-A-P6-2"
|
||||
TRACE = base.AITUNER / "trace_windows/traces/chat_w20260311_1000.jsonl"
|
||||
|
||||
ORDER = [
|
||||
"tp4_mns32", "tp4_mns64", "tp2_mns32", "tp2_mns64",
|
||||
"tp4_mns16", "tp2_mns8", "tp2_mns16", "tp4_mns8",
|
||||
"tp1_mns8", "tp1_mns16", "tp1_mns32", "tp1_mns64",
|
||||
]
|
||||
|
||||
# Exact co-located primaries are remeasured before adaptive crawl. W4 had no
|
||||
# prior primary, so it starts at P and selects L/U from the solo result.
|
||||
CORE = {
|
||||
"tp1_mns8": [base.CELLS["tp1_mns8"]["peak"], base.CELLS["tp1_mns8"]["lower"]],
|
||||
"tp1_mns16": [base.CELLS["tp1_mns16"]["peak"], base.CELLS["tp1_mns16"]["upper"]],
|
||||
"tp1_mns32": [base.CELLS["tp1_mns32"]["peak"], base.CELLS["tp1_mns32"]["upper"]],
|
||||
"tp1_mns64": [base.CELLS["tp1_mns64"]["peak"], base.CELLS["tp1_mns64"]["upper"]],
|
||||
"tp2_mns8": [base.CELLS["tp2_mns8"]["peak"], base.CELLS["tp2_mns8"]["lower"]],
|
||||
"tp2_mns16": [base.CELLS["tp2_mns16"]["peak"], base.CELLS["tp2_mns16"]["lower"]],
|
||||
"tp2_mns32": [base.CELLS["tp2_mns32"]["peak"], base.CELLS["tp2_mns32"]["lower"]],
|
||||
"tp2_mns64": [base.CELLS["tp2_mns64"]["peak"], base.CELLS["tp2_mns64"]["lower"]],
|
||||
"tp4_mns8": [base.CELLS["tp4_mns8"]["peak"], base.CELLS["tp4_mns8"]["lower"]],
|
||||
"tp4_mns16": [
|
||||
base.CELLS["tp4_mns16"]["peak"], base.CELLS["tp4_mns16"]["lower"],
|
||||
base.CELLS["tp4_mns16"]["upper"],
|
||||
],
|
||||
"tp4_mns32": [base.CELLS["tp4_mns32"]["peak"]],
|
||||
"tp4_mns64": [base.CELLS["tp4_mns64"]["peak"]],
|
||||
}
|
||||
|
||||
CELL_ESTIMATE = {cell: {1: .11, 2: .22, 4: .48}[cfg["tp"]] for cell, cfg in base.CELLS.items()}
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: Any) -> None:
|
||||
base.atomic_json(path, value)
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
if STATE.exists():
|
||||
return json.loads(STATE.read_text())
|
||||
campaign = json.loads(CAMPAIGN_STATE.read_text())
|
||||
return {
|
||||
"schema": 1, "amendment": "A-P6-2", "status": "initialized",
|
||||
"hard_cap_h20_hours": GPU_LIMIT,
|
||||
"prior_h20_hours": float(campaign["gpu_hours_total"]),
|
||||
"gpu_hours_total": float(campaign["gpu_hours_total"]),
|
||||
"solo_gpu_hours": 0.0, "completed_cells": 0,
|
||||
"primary_anchors": 0, "confirmations": 0,
|
||||
"cells": {}, "failures": [], "started_at": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
atomic_json(STATE, state)
|
||||
|
||||
|
||||
def historical() -> tuple[dict[tuple[str, float], dict[str, Any]], dict[str, list[float]]]:
|
||||
ground = json.loads(base.GROUND.read_text())
|
||||
expected = {}
|
||||
histories = {}
|
||||
for cell in ground["cells"]:
|
||||
anchors = []
|
||||
for probe in cell["probe_history"]:
|
||||
anchor = float(probe["sampling_u"])
|
||||
expected[(cell["cell_id"], anchor)] = probe
|
||||
anchors.append(anchor)
|
||||
histories[cell["cell_id"]] = sorted(anchors)
|
||||
return expected, histories
|
||||
|
||||
|
||||
def same_anchor(left: float, right: float) -> bool:
|
||||
return math.isclose(left, right, rel_tol=0, abs_tol=1e-15)
|
||||
|
||||
|
||||
def colocated_primary(cell: str, anchor: float) -> dict[str, Any] | None:
|
||||
cell_dir = base.RUN_ROOT / "cells" / cell
|
||||
for path in cell_dir.glob("anchor-*/result.json"):
|
||||
item = json.loads(path.read_text())
|
||||
if same_anchor(float(item["anchor"]), anchor):
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def append_echo(line: str) -> None:
|
||||
SOLO_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
with (SOLO_ROOT / "launch-echo.log").open("a") as handle:
|
||||
handle.write(line + "\n")
|
||||
print(line, flush=True)
|
||||
|
||||
|
||||
def wait_all_idle(timeout: float = 30.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
base.assert_all_idle()
|
||||
return
|
||||
except RuntimeError as current:
|
||||
error = current
|
||||
time.sleep(1)
|
||||
raise error or RuntimeError("GPU cleanup did not reach idle")
|
||||
|
||||
|
||||
def remaining_projection(index: int) -> float:
|
||||
return sum(CELL_ESTIMATE[cell] for cell in ORDER[index:]) + SAFETY_HOURS
|
||||
|
||||
|
||||
def start_entry(cell: str, index: int) -> dict[str, Any]:
|
||||
cfg = base.CELLS[cell]
|
||||
gpus = tuple(range(int(cfg["tp"])))
|
||||
cell_dir = SOLO_ROOT / "cells" / cell
|
||||
cell_dir.mkdir(parents=True, exist_ok=True)
|
||||
port = 8700 + index
|
||||
command = base.server_command(cell, gpus, port)
|
||||
with (cell_dir / "commands.log").open("a") as handle:
|
||||
handle.write(f"SERVER {shlex.join(command)}\n")
|
||||
server_handle = (cell_dir / "server.log").open("ab", buffering=0)
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"CUDA_VISIBLE_DEVICES": ",".join(map(str, gpus)),
|
||||
"VLLM_OPPROF_DIR": str(cell_dir / "opprof"),
|
||||
"OPPROF_PHASE6_MARKER": MARKER, "AITUNER_ROOT": str(base.AITUNER),
|
||||
"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
|
||||
})
|
||||
server = subprocess.Popen(
|
||||
command, cwd=base.SOURCE, env=env, stdout=server_handle,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
)
|
||||
base.OWNED_PGIDS.add(server.pid)
|
||||
return {
|
||||
"cell": cell, "gpus": gpus, "port": port, "dir": cell_dir,
|
||||
"server": server, "server_handle": server_handle,
|
||||
"spawned_at": time.time(), "results": [],
|
||||
}
|
||||
|
||||
|
||||
def run_one(
|
||||
entry: dict[str, Any], anchor: float, out: Path, state: dict[str, Any],
|
||||
cell_state: dict[str, Any], role: str,
|
||||
) -> dict[str, Any]:
|
||||
result = base.run_clients(
|
||||
[entry], [(entry, anchor, out)], state, f"solo-{entry['cell']}"
|
||||
)[0]
|
||||
expected_count = cell_state["expected_counts"][str(anchor)]
|
||||
if int(result["selection"]["count"]) != int(expected_count):
|
||||
raise RuntimeError(
|
||||
f"selection mismatch {entry['cell']} {anchor}: "
|
||||
f"{result['selection']['count']} != {expected_count}"
|
||||
)
|
||||
cell_state.setdefault("runs", []).append({
|
||||
"anchor": anchor, "role": role, "dir": str(out),
|
||||
"pass_rate": result["pass_rate"], "feasible": result["feasible"],
|
||||
})
|
||||
save_state(state)
|
||||
return result
|
||||
|
||||
|
||||
def anchor_trials(cell_state: dict[str, Any], anchor: float) -> list[dict[str, Any]]:
|
||||
return [
|
||||
item for item in cell_state.get("runs", [])
|
||||
if same_anchor(float(item["anchor"]), anchor)
|
||||
]
|
||||
|
||||
|
||||
def accepted_feasible(cell_state: dict[str, Any], anchor: float) -> bool | None:
|
||||
trials = anchor_trials(cell_state, anchor)
|
||||
votes = [bool(item["feasible"]) for item in trials]
|
||||
if not votes:
|
||||
return None
|
||||
if len(votes) == 1 or len(set(votes)) == 1:
|
||||
return votes[0]
|
||||
if len(votes) >= 3:
|
||||
return sum(votes) >= 2
|
||||
return None
|
||||
|
||||
|
||||
def optional_fits(
|
||||
state: dict[str, Any], entry: dict[str, Any], future_after: float,
|
||||
) -> bool:
|
||||
replay = len(entry["gpus"]) * 80 / 3600
|
||||
projected = (
|
||||
float(state["gpu_hours_total"]) + base.live_gpu_hours([entry])
|
||||
+ future_after + replay + SAFETY_HOURS
|
||||
)
|
||||
return projected < GPU_LIMIT
|
||||
|
||||
|
||||
def maybe_confirm(
|
||||
entry: dict[str, Any], anchor: float, primary: dict[str, Any],
|
||||
state: dict[str, Any], cell_state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]],
|
||||
future_after: float,
|
||||
) -> None:
|
||||
old = expected[(entry["cell"], anchor)]
|
||||
coloc = colocated_primary(entry["cell"], anchor)
|
||||
disagreement = (
|
||||
bool(primary["feasible"]) != bool(old["feasible"])
|
||||
or (coloc is not None and bool(primary["feasible"]) != bool(coloc["feasible"]))
|
||||
)
|
||||
boundary = .93 <= float(primary["pass_rate"]) <= .97
|
||||
if not (disagreement or boundary):
|
||||
return
|
||||
while len(anchor_trials(cell_state, anchor)) < 3:
|
||||
trials = anchor_trials(cell_state, anchor)
|
||||
if len(trials) >= 2 and len({bool(item["feasible"]) for item in trials}) == 1:
|
||||
return
|
||||
if not optional_fits(state, entry, future_after):
|
||||
cell_state.setdefault("deferred_confirmations", []).append(anchor)
|
||||
return
|
||||
trial = len(trials) + 1
|
||||
out = entry["dir"] / f"confirm-{trial - 1}-anchor-{anchor}"
|
||||
run_one(entry, anchor, out, state, cell_state, f"confirmation-{trial}")
|
||||
state["confirmations"] += 1
|
||||
|
||||
|
||||
def run_primary(
|
||||
entry: dict[str, Any], anchor: float, state: dict[str, Any],
|
||||
cell_state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]],
|
||||
future_after: float, role: str,
|
||||
) -> dict[str, Any]:
|
||||
existing = [item for item in cell_state.get("runs", []) if item["role"].startswith("primary")]
|
||||
if any(same_anchor(float(item["anchor"]), anchor) for item in existing):
|
||||
path = next(
|
||||
Path(item["dir"]) for item in existing
|
||||
if same_anchor(float(item["anchor"]), anchor)
|
||||
)
|
||||
return json.loads((path / "result.json").read_text())
|
||||
out = entry["dir"] / f"anchor-{anchor}"
|
||||
result = run_one(entry, anchor, out, state, cell_state, role)
|
||||
state["primary_anchors"] += 1
|
||||
maybe_confirm(entry, anchor, result, state, cell_state, expected, future_after)
|
||||
return result
|
||||
|
||||
|
||||
def next_below(history: list[float], tested: set[float]) -> float | None:
|
||||
if not tested:
|
||||
return None
|
||||
candidates = [x for x in history if x < min(tested) and x not in tested]
|
||||
return max(candidates) if candidates else None
|
||||
|
||||
|
||||
def next_above(history: list[float], tested: set[float]) -> float | None:
|
||||
if not tested:
|
||||
return None
|
||||
candidates = [x for x in history if x > max(tested) and x not in tested]
|
||||
return min(candidates) if candidates else None
|
||||
|
||||
|
||||
def execute_cell(
|
||||
index: int, cell: str, state: dict[str, Any],
|
||||
expected: dict[tuple[str, float], dict[str, Any]], histories: dict[str, list[float]],
|
||||
) -> None:
|
||||
if state["cells"].get(cell, {}).get("status") == "complete":
|
||||
return
|
||||
future = remaining_projection(index)
|
||||
if float(state["gpu_hours_total"]) + future >= GPU_LIMIT:
|
||||
state["status"] = "budget_projection_stop"
|
||||
state["budget_stop"] = {
|
||||
"before_cell": cell, "spent_h20_hours": state["gpu_hours_total"],
|
||||
"remaining_projection_h20_hours": future,
|
||||
"projected_total_h20_hours": state["gpu_hours_total"] + future,
|
||||
"hard_cap_h20_hours": GPU_LIMIT,
|
||||
}
|
||||
save_state(state)
|
||||
raise RuntimeError(f"projected budget exceeds cap before {cell}")
|
||||
cfg = base.CELLS[cell]
|
||||
echo = (
|
||||
f"SOLO_WAVE_ECHO cell={cell} tp={cfg['tp']} mns={cfg['mns']} "
|
||||
f"gpus=0-{int(cfg['tp'])-1} mandatory={','.join(map(str, CORE[cell]))} "
|
||||
f"spent_h20h={state['gpu_hours_total']:.6f} cell_est_h20h={CELL_ESTIMATE[cell]:.3f} "
|
||||
f"remaining_projection_h20h={future:.3f} cap_h20h={GPU_LIMIT:.1f} "
|
||||
f"ground_truth={base.GROUND} trace={TRACE}"
|
||||
)
|
||||
append_echo(echo)
|
||||
wait_all_idle()
|
||||
cell_state = {
|
||||
"status": "starting", "started_at": time.time(), "tp": cfg["tp"], "mns": cfg["mns"],
|
||||
"mandatory": CORE[cell],
|
||||
"expected_counts": {
|
||||
str(anchor): expected[(cell, anchor)]["request_count"] for anchor in histories[cell]
|
||||
},
|
||||
"runs": [],
|
||||
}
|
||||
state["status"] = "running"
|
||||
state["cells"][cell] = cell_state
|
||||
save_state(state)
|
||||
entry = start_entry(cell, index)
|
||||
failure = None
|
||||
future_after = sum(CELL_ESTIMATE[item] for item in ORDER[index + 1:])
|
||||
try:
|
||||
base.wait_ready(entry)
|
||||
cell_state["status"] = "warmup"
|
||||
save_state(state)
|
||||
warm = base.run_clients(
|
||||
[entry], [(entry, cfg["peak"], entry["dir"] / "warmup")],
|
||||
state, f"solo-{cell}", warmup=True,
|
||||
)[0]
|
||||
cell_state["warmup"] = {
|
||||
"exact_output_count": warm["exact_output_count"],
|
||||
"long_gt4096": warm["selection"]["long_gt4096"],
|
||||
}
|
||||
cell_state["status"] = "mandatory"
|
||||
save_state(state)
|
||||
for anchor in CORE[cell]:
|
||||
run_primary(entry, anchor, state, cell_state, expected, future_after, "primary-mandatory")
|
||||
|
||||
peak = float(cfg["peak"])
|
||||
peak_vote = accepted_feasible(cell_state, peak)
|
||||
if cell in {"tp4_mns32", "tp4_mns64"} and peak_vote is not None:
|
||||
direction = float(cfg["upper"] if peak_vote else cfg["lower"])
|
||||
run_primary(entry, direction, state, cell_state, expected, future_after, "primary-direction")
|
||||
|
||||
cell_state["status"] = "crawl"
|
||||
save_state(state)
|
||||
while True:
|
||||
primary_anchors = {
|
||||
float(item["anchor"]) for item in cell_state["runs"]
|
||||
if item["role"].startswith("primary")
|
||||
}
|
||||
votes = {anchor: accepted_feasible(cell_state, anchor) for anchor in primary_anchors}
|
||||
pass_anchors = [anchor for anchor, vote in votes.items() if vote is True]
|
||||
fail_anchors = [anchor for anchor, vote in votes.items() if vote is False]
|
||||
if pass_anchors and fail_anchors and max(pass_anchors) < min(fail_anchors):
|
||||
break
|
||||
if any(vote is None for vote in votes.values()):
|
||||
cell_state["censor"] = "UNRESOLVED_SOLO_ANCHOR"
|
||||
break
|
||||
if pass_anchors and not fail_anchors:
|
||||
candidate = next_above(histories[cell], primary_anchors)
|
||||
elif fail_anchors and not pass_anchors:
|
||||
candidate = next_below(histories[cell], primary_anchors)
|
||||
else:
|
||||
# Non-monotonic anchors already contain both states but no valid bracket.
|
||||
cell_state["censor"] = "NONMONOTONIC_SOLO_ANCHORS"
|
||||
break
|
||||
if candidate is None:
|
||||
cell_state["censor"] = "HISTORY_EDGE"
|
||||
break
|
||||
if not optional_fits(state, entry, future_after):
|
||||
cell_state["censor"] = "BUDGET_CENSORED"
|
||||
break
|
||||
run_primary(entry, candidate, state, cell_state, expected, future_after, "primary-crawl")
|
||||
cell_state["status"] = "stopping"
|
||||
save_state(state)
|
||||
except Exception as error:
|
||||
failure = error
|
||||
finally:
|
||||
try:
|
||||
base.stop_entry(entry)
|
||||
except Exception as error:
|
||||
failure = failure or error
|
||||
time.sleep(2)
|
||||
try:
|
||||
wait_all_idle()
|
||||
except Exception as error:
|
||||
failure = failure or error
|
||||
hours = base.live_gpu_hours([entry])
|
||||
state["gpu_hours_total"] += hours
|
||||
state["solo_gpu_hours"] += hours
|
||||
cell_state["gpu_hours"] = hours
|
||||
if failure is not None:
|
||||
cell_state["status"] = "failed"
|
||||
cell_state["failure"] = repr(failure)
|
||||
state["status"] = "failed"
|
||||
state["failures"].append({"cell": cell, "failure": repr(failure)})
|
||||
save_state(state)
|
||||
raise failure
|
||||
validation = base.validate_cell(entry)
|
||||
cell_state["validation"] = validation
|
||||
cell_state["status"] = "complete"
|
||||
cell_state["completed_at"] = time.time()
|
||||
state["completed_cells"] += 1
|
||||
save_state(state)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
SOLO_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
base.GPU_LIMIT = GPU_LIMIT
|
||||
base.MARKER = MARKER
|
||||
expected, histories = historical()
|
||||
state = load_state()
|
||||
state["status"] = "running"
|
||||
save_state(state)
|
||||
for index, cell in enumerate(ORDER):
|
||||
execute_cell(index, cell, state, expected, histories)
|
||||
state["status"] = "complete"
|
||||
state["completed_at"] = time.time()
|
||||
save_state(state)
|
||||
print(json.dumps({
|
||||
"status": state["status"], "cells": state["completed_cells"],
|
||||
"primary_anchors": state["primary_anchors"],
|
||||
"confirmations": state["confirmations"],
|
||||
"solo_gpu_hours": state["solo_gpu_hours"],
|
||||
"campaign_gpu_hours": state["gpu_hours_total"],
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user