Orchestrate Qwen235 four-case fidelity campaign
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare pooled real and Frontier Qwen235 latency-selection surfaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128")
|
||||
CASES = ("fixed-pd", "fixed-po", "trace-pd", "trace-po")
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float):
|
||||
if not values:
|
||||
return None
|
||||
return sorted(values)[math.ceil(len(values) * fraction) - 1]
|
||||
|
||||
|
||||
def summarize(values: list[float]):
|
||||
if not values:
|
||||
return {"mean": None, "p90": None}
|
||||
return {"mean": sum(values) / len(values), "p90": percentile(values, 0.9)}
|
||||
|
||||
|
||||
def real_surface(root: Path, case: str):
|
||||
surface = {}
|
||||
for config in CONFIGS:
|
||||
requests = []
|
||||
trials = []
|
||||
for trial in (1, 2, 3):
|
||||
path = root / "real" / case / "real" / config / f"trial{trial}" / "results" / "result.json"
|
||||
payload = json.loads(path.read_text())
|
||||
summary = payload["summary"]
|
||||
if summary["failed"] or summary["completed"] != len(payload["requests"]):
|
||||
raise ValueError(f"invalid real result: {path}")
|
||||
if any(not row["success"] for row in payload["requests"]):
|
||||
raise ValueError(f"failed request: {path}")
|
||||
requests.extend(payload["requests"])
|
||||
trials.append(str(path.resolve()))
|
||||
ttft = summarize([float(row["ttft_ms"]) for row in requests])
|
||||
tpot = summarize([float(row["tpot_ms"]) for row in requests if row["tpot_ms"] is not None])
|
||||
e2e = summarize([float(row["e2e_ms"]) for row in requests])
|
||||
surface[config] = {
|
||||
"ttft_mean_ms": ttft["mean"], "ttft_p90_ms": ttft["p90"],
|
||||
"tpot_mean_ms": tpot["mean"], "tpot_p90_ms": tpot["p90"],
|
||||
"e2e_mean_ms": e2e["mean"], "e2e_p90_ms": e2e["p90"],
|
||||
"request_samples": len(requests), "trials": trials,
|
||||
}
|
||||
return surface
|
||||
|
||||
|
||||
def sim_surface(root: Path, case: str):
|
||||
payload = json.loads((root / "sim" / case / "frontier_surface.json").read_text())
|
||||
surface = {}
|
||||
for result in payload["results"]:
|
||||
if result["status"] != "completed":
|
||||
continue
|
||||
config = result["config"]["name"]
|
||||
if config in surface:
|
||||
raise ValueError(f"duplicate simulator config: {config}")
|
||||
surface[config] = {key: value for key, value in result["metrics"].items() if key.endswith("_ms")}
|
||||
if set(surface) != set(CONFIGS):
|
||||
raise ValueError(f"simulator coverage failure for {case}: {sorted(surface)}")
|
||||
return surface
|
||||
|
||||
|
||||
def compare_metric(real: dict, sim: dict, metric: str):
|
||||
applicable = [config for config in CONFIGS if real[config][metric] is not None and sim[config][metric] is not None]
|
||||
real_winner = min(applicable, key=lambda config: (real[config][metric], config))
|
||||
sim_winner = min(applicable, key=lambda config: (sim[config][metric], config))
|
||||
regret = real[sim_winner][metric] / real[real_winner][metric] - 1
|
||||
informative = agreement = 0
|
||||
reversals = []
|
||||
for left, right in itertools.combinations(applicable, 2):
|
||||
real_direction = (real[left][metric] > real[right][metric]) - (real[left][metric] < real[right][metric])
|
||||
sim_direction = (sim[left][metric] > sim[right][metric]) - (sim[left][metric] < sim[right][metric])
|
||||
if real_direction and sim_direction:
|
||||
informative += 1
|
||||
agreement += int(real_direction == sim_direction)
|
||||
if real_direction != sim_direction:
|
||||
reversals.append([left, right])
|
||||
return {
|
||||
"real_winner": real_winner,
|
||||
"sim_winner": sim_winner,
|
||||
"winner_match": real_winner == sim_winner,
|
||||
"selected_real_regret": regret,
|
||||
"informative_pairs": informative,
|
||||
"pair_direction_agreement": agreement / informative if informative else None,
|
||||
"reversals": reversals,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--campaign-root", type=Path, required=True)
|
||||
parser.add_argument("--json-output", type=Path, required=True)
|
||||
parser.add_argument("--markdown-output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
cases = {}
|
||||
lines = ["# Qwen235 vLLM 0.20 Frontier vs real", "", "| case | metric | Frontier winner | real winner | match | regret | pair agreement |", "|---|---|---|---|---:|---:|---:|"]
|
||||
for case in CASES:
|
||||
real = real_surface(args.campaign_root, case)
|
||||
sim = sim_surface(args.campaign_root, case)
|
||||
metrics = ["ttft_mean_ms", "ttft_p90_ms", "e2e_mean_ms", "e2e_p90_ms"]
|
||||
if case.endswith("pd"):
|
||||
metrics[2:2] = ["tpot_mean_ms", "tpot_p90_ms"]
|
||||
comparisons = {}
|
||||
for metric in metrics:
|
||||
item = compare_metric(real, sim, metric)
|
||||
comparisons[metric] = item
|
||||
lines.append(
|
||||
f"| {case} | {metric} | {item['sim_winner']} | {item['real_winner']} | "
|
||||
f"{'yes' if item['winner_match'] else 'no'} | {item['selected_real_regret']:.1%} | "
|
||||
f"{item['pair_direction_agreement']:.1%} |"
|
||||
)
|
||||
cases[case] = {"real": real, "sim": sim, "comparison": comparisons}
|
||||
payload = {"schema": "qwen235-v020-simulator-real-comparison-v1", "cases": cases}
|
||||
args.json_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
args.markdown_output.write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract simulator-visible graph/KV metadata from Qwen235 server starts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--case-root", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
configs = {}
|
||||
for config in CONFIGS:
|
||||
log = args.case_root / "real" / config / "trial1" / "logs" / "server.log"
|
||||
text = log.read_text(errors="replace")
|
||||
token_matches = re.findall(r"GPU KV cache size:\s*([0-9,]+) tokens", text)
|
||||
capture_matches = re.findall(r"cudagraph_capture_sizes': \[([^]]+)\]", text)
|
||||
if not token_matches or not capture_matches:
|
||||
raise ValueError(f"missing runtime metadata in {log}")
|
||||
tokens = int(token_matches[-1].replace(",", ""))
|
||||
if tokens % 16:
|
||||
raise ValueError(f"KV token count is not block aligned: {tokens}")
|
||||
capture = [int(value.strip()) for value in capture_matches[-1].split(",")]
|
||||
configs[config] = {
|
||||
"num_gpu_blocks": tokens // 16,
|
||||
"capture_sizes": capture,
|
||||
"source_log": str(log.resolve()),
|
||||
}
|
||||
payload = {"schema": "qwen235-v020-runtime-contract-v1", "configs": configs}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
print(json.dumps(payload, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -60,7 +60,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--python-deps", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
parser.add_argument("--runtime-contract", type=Path, required=True)
|
||||
parser.add_argument("--trace", action="append", required=True)
|
||||
parser.add_argument("--trace-tp", action="append", required=True, help="TP=PATH")
|
||||
parser.add_argument("--config", action="append")
|
||||
parser.add_argument("--prefix-caching", action=argparse.BooleanOptionalAction, default=True)
|
||||
parser.add_argument("--allreduce-csv", type=Path, required=True)
|
||||
@@ -140,10 +140,17 @@ def main() -> None:
|
||||
setattr(args, name, getattr(args, name).resolve())
|
||||
paths = profile_paths(args.profile_root)
|
||||
contract = json.loads(args.runtime_contract.read_text())["configs"]
|
||||
traces = [
|
||||
Q30.parse_trace(spec, rate_contract="trace-window", prefix_caching=args.prefix_caching)
|
||||
for spec in args.trace
|
||||
]
|
||||
trace_by_tp = {}
|
||||
for specification in args.trace_tp:
|
||||
raw_tp, separator, path = specification.partition("=")
|
||||
if not separator:
|
||||
raise ValueError(f"trace-tp must be TP=PATH: {specification}")
|
||||
tp = int(raw_tp)
|
||||
trace_by_tp[tp] = Q30.parse_trace(
|
||||
f"eval={path}", rate_contract="trace-window", prefix_caching=args.prefix_caching
|
||||
)
|
||||
if set(trace_by_tp) != {4, 8}:
|
||||
raise ValueError("trace-tp must provide exactly TP4 and TP8")
|
||||
selected = list(GRID)
|
||||
if args.config:
|
||||
wanted = set(args.config)
|
||||
@@ -169,7 +176,7 @@ def main() -> None:
|
||||
results = []
|
||||
for config in selected:
|
||||
config_knobs = knobs(config, paths, contract, args.output_root / "cache", args.prefix_caching)
|
||||
for trace in traces:
|
||||
for trace in (trace_by_tp[config.tp],):
|
||||
run_dir = args.output_root / "runs" / config.name / trace["label"]
|
||||
result_path = run_dir / "result.json"
|
||||
if args.resume and result_path.is_file():
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}"
|
||||
PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}"
|
||||
RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
|
||||
FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
|
||||
REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}"
|
||||
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
||||
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
|
||||
PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}"
|
||||
ALLREDUCE_CSV="${ALLREDUCE_CSV:-${RUNNER_DIR}/profiles/measured-allreduce.csv}"
|
||||
BASE_PUBLIC=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv
|
||||
BASE_PRIVATE=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl
|
||||
FIXED_PER_GPU_RATE="${FIXED_PER_GPU_RATE:-0.2}"
|
||||
REQUESTS=129
|
||||
|
||||
mkdir -p "${CAMPAIGN_ROOT}"/{provenance,traces,real,sim,analysis}
|
||||
exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1
|
||||
echo "Q235_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 simulator=Frontier-5b953f5-piecewise cases={Fixed-PD:4096x256@0.2rps/gpu,Fixed-PO:4096x1@0.2rps/gpu,Trace-PD:u0p01-original-OSL,Trace-PO:u0p01-OSL1} requests=129 transform=t_prime=t/TP configs={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored expected_wall=10-30h expected_cost=90-220_H20-GPUh profile=${PROFILE_ROOT} output=${CAMPAIGN_ROOT}"
|
||||
date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
|
||||
sha256sum "${BASH_SOURCE[0]}" "${BASE_PUBLIC}" "${BASE_PRIVATE}" "${MODEL_ROOT}/config.json" \
|
||||
"${PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" > "${CAMPAIGN_ROOT}/provenance/input.sha256"
|
||||
git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit"
|
||||
git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit"
|
||||
|
||||
NORMALIZER="${RUNNER_DIR}/../simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py"
|
||||
MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py"
|
||||
for tp in 4 8; do
|
||||
"${VENV_ROOT}/bin/python" "${NORMALIZER}" --base-public-csv "${BASE_PUBLIC}" \
|
||||
--base-private-jsonl "${BASE_PRIVATE}" --tp "${tp}" \
|
||||
--output-root "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}"
|
||||
"${VENV_ROOT}/bin/python" "${MATERIALIZER}" trace \
|
||||
--base-public "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/public/frontier.csv" \
|
||||
--base-private "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/private/real_requests.jsonl" \
|
||||
--output-tokens 1 --tp "${tp}" --output-root "${CAMPAIGN_ROOT}/traces/trace-po/tp${tp}"
|
||||
for spec in fixed-pd:256 fixed-po:1; do
|
||||
case_name="${spec%%:*}"; osl="${spec##*:}"
|
||||
"${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed --model "${MODEL_ROOT}" \
|
||||
--input-tokens 4096 --output-tokens "${osl}" --requests "${REQUESTS}" \
|
||||
--per-gpu-rate "${FIXED_PER_GPU_RATE}" --tp "${tp}" \
|
||||
--output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}"
|
||||
done
|
||||
done
|
||||
|
||||
run_real() {
|
||||
local case_name="$1" prefix="$2" port="$3"
|
||||
CASE_NAME="${case_name}" PREFIX_CACHING="${prefix}" \
|
||||
TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \
|
||||
RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \
|
||||
BASE_PORT="${port}" bash "${RUNNER_DIR}/run_qwen235_v020_real_surface.sh"
|
||||
}
|
||||
run_real fixed-pd false 9300
|
||||
run_real fixed-po false 9400
|
||||
run_real trace-pd true 9500
|
||||
run_real trace-po true 9600
|
||||
|
||||
"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \
|
||||
--case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \
|
||||
--output "${CAMPAIGN_ROOT}/provenance/runtime-contract.json"
|
||||
|
||||
run_sim() {
|
||||
local case_name="$1" prefix_flag="$2"
|
||||
"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/run_frontier_qwen235_v020_surface.py" \
|
||||
--frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \
|
||||
--profile-root "${PROFILE_ROOT}" --python-deps "${PYTHON_DEPS}" \
|
||||
--output-root "${CAMPAIGN_ROOT}/sim/${case_name}" \
|
||||
--runtime-contract "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" \
|
||||
--trace-tp "4=${CAMPAIGN_ROOT}/traces/${case_name}/tp4/public/frontier.csv" \
|
||||
--trace-tp "8=${CAMPAIGN_ROOT}/traces/${case_name}/tp8/public/frontier.csv" \
|
||||
"${prefix_flag}" --allreduce-csv "${ALLREDUCE_CSV}" --resume
|
||||
}
|
||||
run_sim fixed-pd --no-prefix-caching
|
||||
run_sim fixed-po --no-prefix-caching
|
||||
run_sim trace-pd --prefix-caching
|
||||
run_sim trace-po --prefix-caching
|
||||
|
||||
"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/analyze_qwen235_v020_campaign.py" \
|
||||
--campaign-root "${CAMPAIGN_ROOT}" \
|
||||
--json-output "${CAMPAIGN_ROOT}/analysis/comparison.json" \
|
||||
--markdown-output "${CAMPAIGN_ROOT}/analysis/comparison.md"
|
||||
find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
|
||||
| sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256"
|
||||
date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
|
||||
echo Q235_V020_CAMPAIGN_COMPLETE
|
||||
@@ -58,9 +58,9 @@ wait_for_wave() {
|
||||
}
|
||||
|
||||
launch_config() {
|
||||
local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false
|
||||
[[ "${tp}" == 8 ]] && ep=true
|
||||
local config="tp${tp}_ep${ep}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}"
|
||||
local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1
|
||||
if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi
|
||||
local config="tp${tp}_ep${ep_size}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}"
|
||||
local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}"
|
||||
PORT=$((PORT + 1))
|
||||
if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then
|
||||
|
||||
Reference in New Issue
Block a user