Diagnose Qwen30 Fixed-PD admission state
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the schematic for Qwen30 Fixed-PD TTFT admission diagnosis."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
|
||||
OUTPUT = Path(__file__).with_name("qwen30-fixed-pd-ttft-admission-mock.png")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
figure, axes = plt.subplots(1, 2, figsize=(10.8, 4.2))
|
||||
|
||||
labels = ["TP2/MNS64", "TP4/MNS64"]
|
||||
x = np.arange(len(labels))
|
||||
axes[0].bar(x, [190, 135], label="Prefill execution (mock)", color="#4c78a8")
|
||||
axes[0].bar(
|
||||
x,
|
||||
[30, 6000],
|
||||
bottom=[190, 135],
|
||||
label="Admission wait (mock)",
|
||||
color="#f58518",
|
||||
)
|
||||
axes[0].set_xticks(x, labels)
|
||||
axes[0].set_yscale("log")
|
||||
axes[0].set_ylabel("TTFT decomposition (ms, log scale)")
|
||||
axes[0].set_title("(a) Execution error or queue amplification?")
|
||||
axes[0].legend(frameon=False)
|
||||
|
||||
width = 0.34
|
||||
axes[1].bar(x - width / 2, [8, 15], width, label="Real required slots (mock)")
|
||||
axes[1].bar(x + width / 2, [58, 80], width, label="Frontier required slots (mock)")
|
||||
axes[1].axhline(64, color="#d62728", linestyle="--", label="MNS=64")
|
||||
axes[1].set_xticks(x, labels)
|
||||
axes[1].set_ylabel("arrival rate × residence time")
|
||||
axes[1].set_title("(b) Does modeled state cross admission cap?")
|
||||
axes[1].legend(frameon=False)
|
||||
axes[1].text(
|
||||
0.02,
|
||||
0.97,
|
||||
"SCHEMATIC / MOCK DATA",
|
||||
transform=axes[1].transAxes,
|
||||
va="top",
|
||||
fontsize=9,
|
||||
color="#8c2d04",
|
||||
)
|
||||
|
||||
figure.suptitle("Qwen30 Fixed-PD TTFT diagnosis", fontsize=13)
|
||||
figure.tight_layout()
|
||||
figure.savefig(OUTPUT, dpi=180, bbox_inches="tight")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 100 KiB |
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay the decisive Qwen30 Fixed-PD cells with Frontier state outputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
CONFIGS = {
|
||||
"tp2_mns64": "tp2",
|
||||
"tp4_mns32": "tp4",
|
||||
"tp4_mns64": "tp4",
|
||||
}
|
||||
|
||||
|
||||
def load_module(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
BASE = load_module(
|
||||
"qwen30_fixed_pd_state_base", HERE / "run_qwen235_fixed_pd_state_replay.py"
|
||||
)
|
||||
Q30 = load_module(
|
||||
"qwen30_fixed_pd_surface", HERE / "run_frontier_qwen30_exact_trace_surface.py"
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-sim-root", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
parser.add_argument("--frontier-source", type=Path, required=True)
|
||||
parser.add_argument("--python-deps", type=Path, required=True)
|
||||
parser.add_argument("--config", action="append", choices=CONFIGS)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=1800)
|
||||
parser.add_argument("--resume", action="store_true")
|
||||
parser.add_argument("--op-trace", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run_cell(
|
||||
*,
|
||||
config: str,
|
||||
base_sim_root: Path,
|
||||
output_root: Path,
|
||||
frontier_source: Path,
|
||||
python_deps: Path,
|
||||
timeout_seconds: float,
|
||||
resume: bool,
|
||||
op_trace: bool,
|
||||
) -> dict[str, Any]:
|
||||
base_run = base_sim_root / "runs" / config / CONFIGS[config]
|
||||
base_command_path = base_run / "command.json"
|
||||
base_result_path = base_run / "result.json"
|
||||
if not base_command_path.is_file() or not base_result_path.is_file():
|
||||
raise FileNotFoundError(f"frozen inputs missing under {base_run}")
|
||||
base_command = json.loads(base_command_path.read_text())
|
||||
base_result = json.loads(base_result_path.read_text())
|
||||
if base_result.get("status") != "completed":
|
||||
raise ValueError(f"frozen result is not completed: {base_result_path}")
|
||||
|
||||
run_root = output_root / config
|
||||
result_path = run_root / "result.json"
|
||||
if resume and result_path.is_file():
|
||||
previous = json.loads(result_path.read_text())
|
||||
if (
|
||||
previous.get("status") == "PASS"
|
||||
and previous.get("inputs", {}).get("base_command_sha256")
|
||||
== BASE.sha256_file(base_command_path)
|
||||
and previous.get("inputs", {}).get("base_result_sha256")
|
||||
== BASE.sha256_file(base_result_path)
|
||||
and bool(previous.get("op_trace_enabled")) == op_trace
|
||||
):
|
||||
return previous
|
||||
if run_root.exists() and any(run_root.iterdir()):
|
||||
raise FileExistsError(f"refusing non-empty output: {run_root}")
|
||||
run_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
command = BASE.transform_command(
|
||||
base_command,
|
||||
metrics_root=run_root / "frontier_metrics",
|
||||
run_id=f"qwen30_fixed_pd_state_{config}",
|
||||
op_trace=op_trace,
|
||||
)
|
||||
BASE.atomic_json(run_root / "command.json", command)
|
||||
inputs = {
|
||||
"base_run": str(base_run.resolve()),
|
||||
"base_command_sha256": BASE.sha256_file(base_command_path),
|
||||
"base_result_sha256": BASE.sha256_file(base_result_path),
|
||||
"base_request_metrics_sha256": base_result["request_metrics_sha256"],
|
||||
}
|
||||
manifest = {
|
||||
"schema": "qwen30-fixed-pd-state-replay-v1",
|
||||
"config": config,
|
||||
"inputs": inputs,
|
||||
"command_sha256": BASE.sha256_file(run_root / "command.json"),
|
||||
"frontier_git_head": subprocess.check_output(
|
||||
["git", "-C", str(frontier_source), "rev-parse", "HEAD"], text=True
|
||||
).strip(),
|
||||
"op_trace_enabled": op_trace,
|
||||
"controlled_changes": [
|
||||
"metrics output directory",
|
||||
"metrics run id",
|
||||
"full Frontier stage/batch ledger enabled",
|
||||
"individual batch metrics enabled",
|
||||
]
|
||||
+ (["op-level tracing enabled"] if op_trace else []),
|
||||
}
|
||||
BASE.atomic_json(run_root / "run_manifest.json", manifest)
|
||||
|
||||
started = time.monotonic()
|
||||
with (run_root / "stdout.log").open("w") as stdout, (
|
||||
run_root / "stderr.log"
|
||||
).open("w") as stderr:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=frontier_source,
|
||||
env=BASE.environment(frontier_source, python_deps),
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
timeout=timeout_seconds,
|
||||
check=False,
|
||||
)
|
||||
returncode = int(completed.returncode)
|
||||
except subprocess.TimeoutExpired:
|
||||
returncode = 124
|
||||
elapsed_seconds = time.monotonic() - started
|
||||
if returncode != 0:
|
||||
failure = {
|
||||
"status": "STOP",
|
||||
"config": config,
|
||||
"returncode": returncode,
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
}
|
||||
BASE.atomic_json(run_root / "failure.json", failure)
|
||||
raise RuntimeError(f"state replay failed: {failure}")
|
||||
|
||||
fallback_evidence = Q30.collective_fallback_evidence(run_root)
|
||||
if fallback_evidence:
|
||||
raise RuntimeError(f"collective-profile fallback detected: {fallback_evidence}")
|
||||
paths = BASE.find_state_metrics(run_root)
|
||||
request_metrics_sha256 = BASE.sha256_file(paths["requests"])
|
||||
scorer_equivalent = request_metrics_sha256 == base_result["request_metrics_sha256"]
|
||||
if not scorer_equivalent:
|
||||
raise RuntimeError(
|
||||
f"observation changed scorer input for {config}: "
|
||||
f"{request_metrics_sha256} != {base_result['request_metrics_sha256']}"
|
||||
)
|
||||
state = BASE.summarize_frontier(
|
||||
system_metrics_path=paths["system"],
|
||||
request_metrics_path=paths["requests"],
|
||||
batch_metrics_path=paths["batches"],
|
||||
ledger_path=paths["ledger"],
|
||||
)
|
||||
BASE.atomic_json(run_root / "common-state.json", state)
|
||||
if op_trace:
|
||||
matches = sorted(run_root.glob("frontier_metrics/**/op_traces.jsonl"))
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"expected one op trace, found {len(matches)}")
|
||||
paths["op_trace"] = matches[0]
|
||||
result = {
|
||||
"schema": "qwen30-fixed-pd-state-replay-result-v1",
|
||||
"status": "PASS",
|
||||
"config": config,
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
"returncode": returncode,
|
||||
"inputs": inputs,
|
||||
"scorer_equivalence": {
|
||||
"request_metrics_byte_identical": scorer_equivalent,
|
||||
"request_metrics_sha256": request_metrics_sha256,
|
||||
"base_metrics": base_result["metrics"],
|
||||
},
|
||||
"state_artifacts": {
|
||||
name: {"path": str(path.resolve()), "sha256": BASE.sha256_file(path)}
|
||||
for name, path in paths.items()
|
||||
},
|
||||
"common_state": state,
|
||||
"collective_fallback_evidence": fallback_evidence,
|
||||
"op_trace_enabled": op_trace,
|
||||
}
|
||||
BASE.atomic_json(result_path, result)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
for name in ("base_sim_root", "output_root", "frontier_source", "python_deps"):
|
||||
setattr(args, name, getattr(args, name).resolve())
|
||||
results = []
|
||||
for config in tuple(args.config or CONFIGS):
|
||||
result = run_cell(
|
||||
config=config,
|
||||
base_sim_root=args.base_sim_root,
|
||||
output_root=args.output_root,
|
||||
frontier_source=args.frontier_source,
|
||||
python_deps=args.python_deps,
|
||||
timeout_seconds=args.timeout_seconds,
|
||||
resume=args.resume,
|
||||
op_trace=args.op_trace,
|
||||
)
|
||||
results.append(result)
|
||||
print(json.dumps({"config": config, "status": result["status"]}), flush=True)
|
||||
BASE.atomic_json(
|
||||
args.output_root / "state_replay.json",
|
||||
{
|
||||
"schema": "qwen30-fixed-pd-state-replay-aggregate-v1",
|
||||
"status": "PASS",
|
||||
"results": results,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user