diff --git a/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py b/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py new file mode 100644 index 0000000..25853e6 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Compare pooled real and Frontier Qwen235 latency-selection surfaces.""" + +from __future__ import annotations + +import argparse +import itertools +import json +import math +from pathlib import Path + + +CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128") +CASES = ("fixed-pd", "fixed-po", "trace-pd", "trace-po") + + +def percentile(values: list[float], fraction: float): + if not values: + return None + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def summarize(values: list[float]): + if not values: + return {"mean": None, "p90": None} + return {"mean": sum(values) / len(values), "p90": percentile(values, 0.9)} + + +def real_surface(root: Path, case: str): + surface = {} + for config in CONFIGS: + requests = [] + trials = [] + for trial in (1, 2, 3): + path = root / "real" / case / "real" / config / f"trial{trial}" / "results" / "result.json" + payload = json.loads(path.read_text()) + summary = payload["summary"] + if summary["failed"] or summary["completed"] != len(payload["requests"]): + raise ValueError(f"invalid real result: {path}") + if any(not row["success"] for row in payload["requests"]): + raise ValueError(f"failed request: {path}") + requests.extend(payload["requests"]) + trials.append(str(path.resolve())) + ttft = summarize([float(row["ttft_ms"]) for row in requests]) + tpot = summarize([float(row["tpot_ms"]) for row in requests if row["tpot_ms"] is not None]) + e2e = summarize([float(row["e2e_ms"]) for row in requests]) + surface[config] = { + "ttft_mean_ms": ttft["mean"], "ttft_p90_ms": ttft["p90"], + "tpot_mean_ms": tpot["mean"], "tpot_p90_ms": tpot["p90"], + "e2e_mean_ms": e2e["mean"], "e2e_p90_ms": e2e["p90"], + "request_samples": len(requests), "trials": trials, + } + return surface + + +def sim_surface(root: Path, case: str): + payload = json.loads((root / "sim" / case / "frontier_surface.json").read_text()) + surface = {} + for result in payload["results"]: + if result["status"] != "completed": + continue + config = result["config"]["name"] + if config in surface: + raise ValueError(f"duplicate simulator config: {config}") + surface[config] = {key: value for key, value in result["metrics"].items() if key.endswith("_ms")} + if set(surface) != set(CONFIGS): + raise ValueError(f"simulator coverage failure for {case}: {sorted(surface)}") + return surface + + +def compare_metric(real: dict, sim: dict, metric: str): + applicable = [config for config in CONFIGS if real[config][metric] is not None and sim[config][metric] is not None] + real_winner = min(applicable, key=lambda config: (real[config][metric], config)) + sim_winner = min(applicable, key=lambda config: (sim[config][metric], config)) + regret = real[sim_winner][metric] / real[real_winner][metric] - 1 + informative = agreement = 0 + reversals = [] + for left, right in itertools.combinations(applicable, 2): + real_direction = (real[left][metric] > real[right][metric]) - (real[left][metric] < real[right][metric]) + sim_direction = (sim[left][metric] > sim[right][metric]) - (sim[left][metric] < sim[right][metric]) + if real_direction and sim_direction: + informative += 1 + agreement += int(real_direction == sim_direction) + if real_direction != sim_direction: + reversals.append([left, right]) + return { + "real_winner": real_winner, + "sim_winner": sim_winner, + "winner_match": real_winner == sim_winner, + "selected_real_regret": regret, + "informative_pairs": informative, + "pair_direction_agreement": agreement / informative if informative else None, + "reversals": reversals, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--campaign-root", type=Path, required=True) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--markdown-output", type=Path, required=True) + args = parser.parse_args() + cases = {} + lines = ["# Qwen235 vLLM 0.20 Frontier vs real", "", "| case | metric | Frontier winner | real winner | match | regret | pair agreement |", "|---|---|---|---|---:|---:|---:|"] + for case in CASES: + real = real_surface(args.campaign_root, case) + sim = sim_surface(args.campaign_root, case) + metrics = ["ttft_mean_ms", "ttft_p90_ms", "e2e_mean_ms", "e2e_p90_ms"] + if case.endswith("pd"): + metrics[2:2] = ["tpot_mean_ms", "tpot_p90_ms"] + comparisons = {} + for metric in metrics: + item = compare_metric(real, sim, metric) + comparisons[metric] = item + lines.append( + f"| {case} | {metric} | {item['sim_winner']} | {item['real_winner']} | " + f"{'yes' if item['winner_match'] else 'no'} | {item['selected_real_regret']:.1%} | " + f"{item['pair_direction_agreement']:.1%} |" + ) + cases[case] = {"real": real, "sim": sim, "comparison": comparisons} + payload = {"schema": "qwen235-v020-simulator-real-comparison-v1", "cases": cases} + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + args.markdown_output.write_text("\n".join(lines) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py b/runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py new file mode 100644 index 0000000..1b7b4ee --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Extract simulator-visible graph/KV metadata from Qwen235 server starts.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--case-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + configs = {} + for config in CONFIGS: + log = args.case_root / "real" / config / "trial1" / "logs" / "server.log" + text = log.read_text(errors="replace") + token_matches = re.findall(r"GPU KV cache size:\s*([0-9,]+) tokens", text) + capture_matches = re.findall(r"cudagraph_capture_sizes': \[([^]]+)\]", text) + if not token_matches or not capture_matches: + raise ValueError(f"missing runtime metadata in {log}") + tokens = int(token_matches[-1].replace(",", "")) + if tokens % 16: + raise ValueError(f"KV token count is not block aligned: {tokens}") + capture = [int(value.strip()) for value in capture_matches[-1].split(",")] + configs[config] = { + "num_gpu_blocks": tokens // 16, + "capture_sizes": capture, + "source_log": str(log.resolve()), + } + payload = {"schema": "qwen235-v020-runtime-contract-v1", "configs": configs} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py index 6f7be30..2f25871 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py @@ -60,7 +60,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--python-deps", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--runtime-contract", type=Path, required=True) - parser.add_argument("--trace", action="append", required=True) + parser.add_argument("--trace-tp", action="append", required=True, help="TP=PATH") parser.add_argument("--config", action="append") parser.add_argument("--prefix-caching", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--allreduce-csv", type=Path, required=True) @@ -140,10 +140,17 @@ def main() -> None: setattr(args, name, getattr(args, name).resolve()) paths = profile_paths(args.profile_root) contract = json.loads(args.runtime_contract.read_text())["configs"] - traces = [ - Q30.parse_trace(spec, rate_contract="trace-window", prefix_caching=args.prefix_caching) - for spec in args.trace - ] + trace_by_tp = {} + for specification in args.trace_tp: + raw_tp, separator, path = specification.partition("=") + if not separator: + raise ValueError(f"trace-tp must be TP=PATH: {specification}") + tp = int(raw_tp) + trace_by_tp[tp] = Q30.parse_trace( + f"eval={path}", rate_contract="trace-window", prefix_caching=args.prefix_caching + ) + if set(trace_by_tp) != {4, 8}: + raise ValueError("trace-tp must provide exactly TP4 and TP8") selected = list(GRID) if args.config: wanted = set(args.config) @@ -169,7 +176,7 @@ def main() -> None: results = [] for config in selected: config_knobs = knobs(config, paths, contract, args.output_root / "cache", args.prefix_caching) - for trace in traces: + for trace in (trace_by_tp[config.tp],): run_dir = args.output_root / "runs" / config.name / trace["label"] result_path = run_dir / "result.json" if args.resume and result_path.is_file(): diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh new file mode 100644 index 0000000..0663035 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}" +PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" +PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}" +ALLREDUCE_CSV="${ALLREDUCE_CSV:-${RUNNER_DIR}/profiles/measured-allreduce.csv}" +BASE_PUBLIC=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv +BASE_PRIVATE=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl +FIXED_PER_GPU_RATE="${FIXED_PER_GPU_RATE:-0.2}" +REQUESTS=129 + +mkdir -p "${CAMPAIGN_ROOT}"/{provenance,traces,real,sim,analysis} +exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1 +echo "Q235_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 simulator=Frontier-5b953f5-piecewise cases={Fixed-PD:4096x256@0.2rps/gpu,Fixed-PO:4096x1@0.2rps/gpu,Trace-PD:u0p01-original-OSL,Trace-PO:u0p01-OSL1} requests=129 transform=t_prime=t/TP configs={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored expected_wall=10-30h expected_cost=90-220_H20-GPUh profile=${PROFILE_ROOT} output=${CAMPAIGN_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +sha256sum "${BASH_SOURCE[0]}" "${BASE_PUBLIC}" "${BASE_PRIVATE}" "${MODEL_ROOT}/config.json" \ + "${PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" > "${CAMPAIGN_ROOT}/provenance/input.sha256" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit" +git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit" + +NORMALIZER="${RUNNER_DIR}/../simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py" +MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py" +for tp in 4 8; do + "${VENV_ROOT}/bin/python" "${NORMALIZER}" --base-public-csv "${BASE_PUBLIC}" \ + --base-private-jsonl "${BASE_PRIVATE}" --tp "${tp}" \ + --output-root "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}" + "${VENV_ROOT}/bin/python" "${MATERIALIZER}" trace \ + --base-public "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/public/frontier.csv" \ + --base-private "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/private/real_requests.jsonl" \ + --output-tokens 1 --tp "${tp}" --output-root "${CAMPAIGN_ROOT}/traces/trace-po/tp${tp}" + for spec in fixed-pd:256 fixed-po:1; do + case_name="${spec%%:*}"; osl="${spec##*:}" + "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed --model "${MODEL_ROOT}" \ + --input-tokens 4096 --output-tokens "${osl}" --requests "${REQUESTS}" \ + --per-gpu-rate "${FIXED_PER_GPU_RATE}" --tp "${tp}" \ + --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}" + done +done + +run_real() { + local case_name="$1" prefix="$2" port="$3" + CASE_NAME="${case_name}" PREFIX_CACHING="${prefix}" \ + TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \ + RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \ + BASE_PORT="${port}" bash "${RUNNER_DIR}/run_qwen235_v020_real_surface.sh" +} +run_real fixed-pd false 9300 +run_real fixed-po false 9400 +run_real trace-pd true 9500 +run_real trace-po true 9600 + +"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \ + --case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \ + --output "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" + +run_sim() { + local case_name="$1" prefix_flag="$2" + "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/run_frontier_qwen235_v020_surface.py" \ + --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" --python-deps "${PYTHON_DEPS}" \ + --output-root "${CAMPAIGN_ROOT}/sim/${case_name}" \ + --runtime-contract "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" \ + --trace-tp "4=${CAMPAIGN_ROOT}/traces/${case_name}/tp4/public/frontier.csv" \ + --trace-tp "8=${CAMPAIGN_ROOT}/traces/${case_name}/tp8/public/frontier.csv" \ + "${prefix_flag}" --allreduce-csv "${ALLREDUCE_CSV}" --resume +} +run_sim fixed-pd --no-prefix-caching +run_sim fixed-po --no-prefix-caching +run_sim trace-pd --prefix-caching +run_sim trace-po --prefix-caching + +"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/analyze_qwen235_v020_campaign.py" \ + --campaign-root "${CAMPAIGN_ROOT}" \ + --json-output "${CAMPAIGN_ROOT}/analysis/comparison.json" \ + --markdown-output "${CAMPAIGN_ROOT}/analysis/comparison.md" +find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_V020_CAMPAIGN_COMPLETE diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh index 5d04bb2..b7fd213 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh @@ -58,9 +58,9 @@ wait_for_wave() { } launch_config() { - local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false - [[ "${tp}" == 8 ]] && ep=true - local config="tp${tp}_ep${ep}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}" + local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1 + if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi + local config="tp${tp}_ep${ep_size}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}" local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}" PORT=$((PORT + 1)) if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then