Orchestrate Qwen235 four-case fidelity campaign

runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Compare pooled real and Frontier Qwen235 latency-selection surfaces."""
+
+from __future__ import annotations
+
+import argparse
+import itertools
+import json
+import math
+from pathlib import Path
+
+
+CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128")
+CASES = ("fixed-pd", "fixed-po", "trace-pd", "trace-po")
+
+
+def percentile(values: list[float], fraction: float):
+    if not values:
+        return None
+    return sorted(values)[math.ceil(len(values) * fraction) - 1]
+
+
+def summarize(values: list[float]):
+    if not values:
+        return {"mean": None, "p90": None}
+    return {"mean": sum(values) / len(values), "p90": percentile(values, 0.9)}
+
+
+def real_surface(root: Path, case: str):
+    surface = {}
+    for config in CONFIGS:
+        requests = []
+        trials = []
+        for trial in (1, 2, 3):
+            path = root / "real" / case / "real" / config / f"trial{trial}" / "results" / "result.json"
+            payload = json.loads(path.read_text())
+            summary = payload["summary"]
+            if summary["failed"] or summary["completed"] != len(payload["requests"]):
+                raise ValueError(f"invalid real result: {path}")
+            if any(not row["success"] for row in payload["requests"]):
+                raise ValueError(f"failed request: {path}")
+            requests.extend(payload["requests"])
+            trials.append(str(path.resolve()))
+        ttft = summarize([float(row["ttft_ms"]) for row in requests])
+        tpot = summarize([float(row["tpot_ms"]) for row in requests if row["tpot_ms"] is not None])
+        e2e = summarize([float(row["e2e_ms"]) for row in requests])
+        surface[config] = {
+            "ttft_mean_ms": ttft["mean"], "ttft_p90_ms": ttft["p90"],
+            "tpot_mean_ms": tpot["mean"], "tpot_p90_ms": tpot["p90"],
+            "e2e_mean_ms": e2e["mean"], "e2e_p90_ms": e2e["p90"],
+            "request_samples": len(requests), "trials": trials,
+        }
+    return surface
+
+
+def sim_surface(root: Path, case: str):
+    payload = json.loads((root / "sim" / case / "frontier_surface.json").read_text())
+    surface = {}
+    for result in payload["results"]:
+        if result["status"] != "completed":
+            continue
+        config = result["config"]["name"]
+        if config in surface:
+            raise ValueError(f"duplicate simulator config: {config}")
+        surface[config] = {key: value for key, value in result["metrics"].items() if key.endswith("_ms")}
+    if set(surface) != set(CONFIGS):
+        raise ValueError(f"simulator coverage failure for {case}: {sorted(surface)}")
+    return surface
+
+
+def compare_metric(real: dict, sim: dict, metric: str):
+    applicable = [config for config in CONFIGS if real[config][metric] is not None and sim[config][metric] is not None]
+    real_winner = min(applicable, key=lambda config: (real[config][metric], config))
+    sim_winner = min(applicable, key=lambda config: (sim[config][metric], config))
+    regret = real[sim_winner][metric] / real[real_winner][metric] - 1
+    informative = agreement = 0
+    reversals = []
+    for left, right in itertools.combinations(applicable, 2):
+        real_direction = (real[left][metric] > real[right][metric]) - (real[left][metric] < real[right][metric])
+        sim_direction = (sim[left][metric] > sim[right][metric]) - (sim[left][metric] < sim[right][metric])
+        if real_direction and sim_direction:
+            informative += 1
+            agreement += int(real_direction == sim_direction)
+            if real_direction != sim_direction:
+                reversals.append([left, right])
+    return {
+        "real_winner": real_winner,
+        "sim_winner": sim_winner,
+        "winner_match": real_winner == sim_winner,
+        "selected_real_regret": regret,
+        "informative_pairs": informative,
+        "pair_direction_agreement": agreement / informative if informative else None,
+        "reversals": reversals,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--campaign-root", type=Path, required=True)
+    parser.add_argument("--json-output", type=Path, required=True)
+    parser.add_argument("--markdown-output", type=Path, required=True)
+    args = parser.parse_args()
+    cases = {}
+    lines = ["# Qwen235 vLLM 0.20 Frontier vs real", "", "| case | metric | Frontier winner | real winner | match | regret | pair agreement |", "|---|---|---|---|---:|---:|---:|"]
+    for case in CASES:
+        real = real_surface(args.campaign_root, case)
+        sim = sim_surface(args.campaign_root, case)
+        metrics = ["ttft_mean_ms", "ttft_p90_ms", "e2e_mean_ms", "e2e_p90_ms"]
+        if case.endswith("pd"):
+            metrics[2:2] = ["tpot_mean_ms", "tpot_p90_ms"]
+        comparisons = {}
+        for metric in metrics:
+            item = compare_metric(real, sim, metric)
+            comparisons[metric] = item
+            lines.append(
+                f"| {case} | {metric} | {item['sim_winner']} | {item['real_winner']} | "
+                f"{'yes' if item['winner_match'] else 'no'} | {item['selected_real_regret']:.1%} | "
+                f"{item['pair_direction_agreement']:.1%} |"
+            )
+        cases[case] = {"real": real, "sim": sim, "comparison": comparisons}
+    payload = {"schema": "qwen235-v020-simulator-real-comparison-v1", "cases": cases}
+    args.json_output.parent.mkdir(parents=True, exist_ok=True)
+    args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    args.markdown_output.write_text("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Extract simulator-visible graph/KV metadata from Qwen235 server starts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--case-root", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    configs = {}
+    for config in CONFIGS:
+        log = args.case_root / "real" / config / "trial1" / "logs" / "server.log"
+        text = log.read_text(errors="replace")
+        token_matches = re.findall(r"GPU KV cache size:\s*([0-9,]+) tokens", text)
+        capture_matches = re.findall(r"cudagraph_capture_sizes': \[([^]]+)\]", text)
+        if not token_matches or not capture_matches:
+            raise ValueError(f"missing runtime metadata in {log}")
+        tokens = int(token_matches[-1].replace(",", ""))
+        if tokens % 16:
+            raise ValueError(f"KV token count is not block aligned: {tokens}")
+        capture = [int(value.strip()) for value in capture_matches[-1].split(",")]
+        configs[config] = {
+            "num_gpu_blocks": tokens // 16,
+            "capture_sizes": capture,
+            "source_log": str(log.resolve()),
+        }
+    payload = {"schema": "qwen235-v020-runtime-contract-v1", "configs": configs}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(payload, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py

@@ -60,7 +60,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--python-deps", type=Path, required=True)
     parser.add_argument("--output-root", type=Path, required=True)
     parser.add_argument("--runtime-contract", type=Path, required=True)
-    parser.add_argument("--trace", action="append", required=True)
+    parser.add_argument("--trace-tp", action="append", required=True, help="TP=PATH")
     parser.add_argument("--config", action="append")
     parser.add_argument("--prefix-caching", action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--allreduce-csv", type=Path, required=True)
@@ -140,10 +140,17 @@ def main() -> None:
         setattr(args, name, getattr(args, name).resolve())
     paths = profile_paths(args.profile_root)
     contract = json.loads(args.runtime_contract.read_text())["configs"]
-    traces = [
+    trace_by_tp = {}
-        Q30.parse_trace(spec, rate_contract="trace-window", prefix_caching=args.prefix_caching)
+    for specification in args.trace_tp:
-        for spec in args.trace
+        raw_tp, separator, path = specification.partition("=")
-    ]
+        if not separator:
+            raise ValueError(f"trace-tp must be TP=PATH: {specification}")
+        tp = int(raw_tp)
+        trace_by_tp[tp] = Q30.parse_trace(
+            f"eval={path}", rate_contract="trace-window", prefix_caching=args.prefix_caching
+        )
+    if set(trace_by_tp) != {4, 8}:
+        raise ValueError("trace-tp must provide exactly TP4 and TP8")
     selected = list(GRID)
     if args.config:
         wanted = set(args.config)
@@ -169,7 +176,7 @@ def main() -> None:
     results = []
     for config in selected:
         config_knobs = knobs(config, paths, contract, args.output_root / "cache", args.prefix_caching)
-        for trace in traces:
+        for trace in (trace_by_tp[config.tp],):
             run_dir = args.output_root / "runs" / config.name / trace["label"]
             result_path = run_dir / "result.json"
             if args.resume and result_path.is_file():

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}"
+PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
+PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}"
+ALLREDUCE_CSV="${ALLREDUCE_CSV:-${RUNNER_DIR}/profiles/measured-allreduce.csv}"
+BASE_PUBLIC=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv
+BASE_PRIVATE=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl
+FIXED_PER_GPU_RATE="${FIXED_PER_GPU_RATE:-0.2}"
+REQUESTS=129
+
+mkdir -p "${CAMPAIGN_ROOT}"/{provenance,traces,real,sim,analysis}
+exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1
+echo "Q235_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 simulator=Frontier-5b953f5-piecewise cases={Fixed-PD:4096x256@0.2rps/gpu,Fixed-PO:4096x1@0.2rps/gpu,Trace-PD:u0p01-original-OSL,Trace-PO:u0p01-OSL1} requests=129 transform=t_prime=t/TP configs={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored expected_wall=10-30h expected_cost=90-220_H20-GPUh profile=${PROFILE_ROOT} output=${CAMPAIGN_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+sha256sum "${BASH_SOURCE[0]}" "${BASE_PUBLIC}" "${BASE_PRIVATE}" "${MODEL_ROOT}/config.json" \
+  "${PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" > "${CAMPAIGN_ROOT}/provenance/input.sha256"
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit"
+git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit"
+
+NORMALIZER="${RUNNER_DIR}/../simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py"
+MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py"
+for tp in 4 8; do
+  "${VENV_ROOT}/bin/python" "${NORMALIZER}" --base-public-csv "${BASE_PUBLIC}" \
+    --base-private-jsonl "${BASE_PRIVATE}" --tp "${tp}" \
+    --output-root "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}"
+  "${VENV_ROOT}/bin/python" "${MATERIALIZER}" trace \
+    --base-public "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/public/frontier.csv" \
+    --base-private "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/private/real_requests.jsonl" \
+    --output-tokens 1 --tp "${tp}" --output-root "${CAMPAIGN_ROOT}/traces/trace-po/tp${tp}"
+  for spec in fixed-pd:256 fixed-po:1; do
+    case_name="${spec%%:*}"; osl="${spec##*:}"
+    "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed --model "${MODEL_ROOT}" \
+      --input-tokens 4096 --output-tokens "${osl}" --requests "${REQUESTS}" \
+      --per-gpu-rate "${FIXED_PER_GPU_RATE}" --tp "${tp}" \
+      --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}"
+  done
+done
+
+run_real() {
+  local case_name="$1" prefix="$2" port="$3"
+  CASE_NAME="${case_name}" PREFIX_CACHING="${prefix}" \
+    TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \
+    RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \
+    BASE_PORT="${port}" bash "${RUNNER_DIR}/run_qwen235_v020_real_surface.sh"
+}
+run_real fixed-pd false 9300
+run_real fixed-po false 9400
+run_real trace-pd true 9500
+run_real trace-po true 9600
+
+"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \
+  --case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \
+  --output "${CAMPAIGN_ROOT}/provenance/runtime-contract.json"
+
+run_sim() {
+  local case_name="$1" prefix_flag="$2"
+  "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/run_frontier_qwen235_v020_surface.py" \
+    --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \
+    --profile-root "${PROFILE_ROOT}" --python-deps "${PYTHON_DEPS}" \
+    --output-root "${CAMPAIGN_ROOT}/sim/${case_name}" \
+    --runtime-contract "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" \
+    --trace-tp "4=${CAMPAIGN_ROOT}/traces/${case_name}/tp4/public/frontier.csv" \
+    --trace-tp "8=${CAMPAIGN_ROOT}/traces/${case_name}/tp8/public/frontier.csv" \
+    "${prefix_flag}" --allreduce-csv "${ALLREDUCE_CSV}" --resume
+}
+run_sim fixed-pd --no-prefix-caching
+run_sim fixed-po --no-prefix-caching
+run_sim trace-pd --prefix-caching
+run_sim trace-po --prefix-caching
+
+"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/analyze_qwen235_v020_campaign.py" \
+  --campaign-root "${CAMPAIGN_ROOT}" \
+  --json-output "${CAMPAIGN_ROOT}/analysis/comparison.json" \
+  --markdown-output "${CAMPAIGN_ROOT}/analysis/comparison.md"
+find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
+  | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256"
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_V020_CAMPAIGN_COMPLETE

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh

@@ -58,9 +58,9 @@ wait_for_wave() {
 }
 
 launch_config() {
-  local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false
+  local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1
-  [[ "${tp}" == 8 ]] && ep=true
+  if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi
-  local config="tp${tp}_ep${ep}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}"
+  local config="tp${tp}_ep${ep_size}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}"
   local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}"
   PORT=$((PORT + 1))
   if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then