diff --git a/runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md b/runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md new file mode 100644 index 0000000..365943c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md @@ -0,0 +1,49 @@ +# EXP-SIMFID-Q30-FIXED-PRESSURE-SURFACE: high-pressure Fixed-PD / Fixed-PO + +> Status: approved and launching (2026-07-19). + +## Question + +When uniform Fixed-PD is moved from the near-singleton workload to +`4096 -> 256` at `1.125` request/s/GPU, does Frontier still choose a different +latency-optimal configuration from real community vLLM 0.20? Does removing +decode (`4096 -> 1`) change that selection boundary? + +## Setup and decision rule + +- Qwen3-30B-A3B BF16 on dash0 H20; community vLLM 0.20.0. +- Fixed-PD `4096 -> 256` and Fixed-PO `4096 -> 1`; 257 uniform, + prefix-disjoint requests; prefix caching off. +- Rate is frozen at 1.125 request/s/GPU, so global rates are 1.125, 2.25, and + 4.5 request/s for TP1, TP2, and TP4. +- Surface: `TP in {1,2,4}` x `MNS in {8,16,32,64}`, MBT=8192. Real + measurements use three fresh-server trials with rotated order. +- Frontier is `deadc4a`, `piecewise`, the frozen CUDA-event profile plus the + graph-compatible KERNEL_ONLY profile and measured collectives used by the + prior Trace-PD comparison. No latency calibration is fitted to this case. +- Compare exact winners, selected-config real regret, and pairwise order for + mean/p90 TTFT, E2E, and TPOT where decode exists. SLO is not scored. + +The pressure-probe measurements selected the workload but are excluded from +the real evaluation pool. A failed or incomplete Frontier cell is a coverage +failure, not a high-latency observation. + +## Expected output and decision + +The final table has one row per case/objective with Frontier winner, real +winner, regret, and pairwise agreement. If the old Fixed-PD reversal persists, +the failure is not an artifact of singleton load. If it disappears, simulator +fidelity has a load-regime boundary that must be localized. Fixed-PO isolates +whether decode-state composition is necessary for either outcome. + +The previously reviewed matrix schematic remains the figure prototype: +`../simulator-tuning-latency-matrix-v0/latency-selection-matrix-schematic.svg`. + +## Provenance + +Remote output root: +`dash0:/home/admin/cpfs/wjh/aituner/qwen30-fixed-pressure-surface-20260719-r1`. +The campaign records source/profile/model hashes, runtime versions, GPU +inventory, per-cell commands, raw request records, and an artifact checksum +manifest. + diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh new file mode 100644 index 0000000..2f0ade6 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# Frozen Qwen30 Fixed-PD/Fixed-PO Frontier-versus-real campaign. The pressure +# calibration artifacts are deliberately not reused as evaluation results. + +set -euo pipefail + +CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" +REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}" +PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}" +PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}" +KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}" +ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:?FLASHINFER_WORKSPACE_BASE is required}" +REQUESTS="${REQUESTS:-257}" +PER_GPU_RATE="${PER_GPU_RATE:-1.125}" + +[[ "${REQUESTS}" == "257" ]] || { echo 'ERROR: frozen campaign requires REQUESTS=257' >&2; exit 1; } +[[ "${PER_GPU_RATE}" == "1.125" ]] || { echo 'ERROR: frozen campaign requires PER_GPU_RATE=1.125' >&2; exit 1; } + +MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py" +REAL_RUNNER="${RUNNER_DIR}/run_qwen30_latency_case_real_surface.sh" +SIM_RUNNER="${RUNNER_DIR}/run_frontier_qwen30_exact_trace_surface.py" +REAL_AUDITOR="${RUNNER_DIR}/audit_qwen30_latency_case.py" +COMPARATOR="${RUNNER_DIR}/analyze_qwen30_latency_case.py" + +mkdir -p "${CAMPAIGN_ROOT}/provenance" "${CAMPAIGN_ROOT}/traces" \ + "${CAMPAIGN_ROOT}/real" "${CAMPAIGN_ROOT}/sim" "${CAMPAIGN_ROOT}/analysis" +exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1 + +printf '%s\n' "Q30_FIXED_PRESSURE_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 cases={fixed-pd:4096_to_256,fixed-po:4096_to_1} prefix=false requests=${REQUESTS} per_gpu_rate=${PER_GPU_RATE} global_rates={TP1:1.125,TP2:2.25,TP4:4.5} surface_per_case=TP{1,2,4}xMNS{8,16,32,64} real_trials=3 fresh_server=true simulator=Frontier-deadc4a_piecewise_graph-kernel-only metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} expected_wall=2-5h expected_cost=12-24_H20-GPUh output=${CAMPAIGN_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ + +sha256sum "${BASH_SOURCE[0]}" "${MATERIALIZER}" "${REAL_RUNNER}" \ + "${SIM_RUNNER}" "${REAL_AUDITOR}" "${COMPARATOR}" \ + "${MODEL_ROOT}/config.json" "${PROFILE_ROOT}/manifest.json" \ + "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" \ + > "${CAMPAIGN_ROOT}/provenance/input.sha256" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit" +git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit" +"${VENV_ROOT}/bin/vllm" --version > "${CAMPAIGN_ROOT}/provenance/vllm.version" +nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \ + > "${CAMPAIGN_ROOT}/provenance/gpus.before.csv" + +prepare_case() { + local case_name="$1" output_tokens="$2" + local tp + for tp in 1 2 4; do + "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed \ + --model "${MODEL_ROOT}" --input-tokens 4096 --output-tokens "${output_tokens}" \ + --requests "${REQUESTS}" --per-gpu-rate "${PER_GPU_RATE}" --tp "${tp}" \ + --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}" + done +} + +run_real_case() { + local case_name="$1" base_port="$2" + CASE_NAME="${case_name}" PREFIX_CACHING=false \ + TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" \ + OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \ + RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \ + FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_WORKSPACE_BASE}" \ + BASE_PORT="${base_port}" RESUME_VALID_CELLS=true \ + bash "${REAL_RUNNER}" +} + +run_sim_tp() { + local case_name="$1" tp="$2" + local sim_root="${CAMPAIGN_ROOT}/sim/${case_name}" + local -a configs=() + local mns + for mns in 8 16 32 64; do + configs+=(--config "tp${tp}_mns${mns}") + done + /usr/bin/python3 "${SIM_RUNNER}" \ + --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \ + --python-deps "${PYTHON_DEPS}" --output-root "${sim_root}" \ + --trace "tp${tp}=${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}/public/frontier.csv" \ + "${configs[@]}" --rate-contract uniform-spacing --no-prefix-caching \ + --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \ + --timeout-seconds 3600 --predictor-training-job-threads 4 \ + --decode-cuda-graph-mode piecewise --align-real-graph-runtime \ + --fresh-predictor-cache --resume --continue-on-failure +} + +run_sim_case() { + local case_name="$1" failed=0 pid + local -a pids=() + mkdir -p "${CAMPAIGN_ROOT}/sim/${case_name}" + for tp in 1 2 4; do + run_sim_tp "${case_name}" "${tp}" \ + > "${CAMPAIGN_ROOT}/sim/${case_name}/launcher-tp${tp}.log" 2>&1 & + pids+=("$!") + done + for pid in "${pids[@]}"; do + wait "${pid}" || failed=1 + done + [[ "${failed}" -eq 0 ]] || return 1 +} + +analyze_case() { + local case_name="$1" + local analysis_root="${CAMPAIGN_ROOT}/analysis/${case_name}" + mkdir -p "${analysis_root}" + "${VENV_ROOT}/bin/python" "${REAL_AUDITOR}" \ + --case-root "${CAMPAIGN_ROOT}/real/${case_name}" \ + --traces-root "${CAMPAIGN_ROOT}/traces/${case_name}" \ + --json-output "${analysis_root}/real-audit.json" \ + --markdown-output "${analysis_root}/real-audit.md" + "${VENV_ROOT}/bin/python" "${COMPARATOR}" \ + --sim-root "${CAMPAIGN_ROOT}/sim/${case_name}" \ + --real-audit "${analysis_root}/real-audit.json" \ + --json-output "${analysis_root}/comparison.json" \ + --markdown-output "${analysis_root}/comparison.md" +} + +prepare_case fixed-pd 256 +prepare_case fixed-po 1 + +run_real_case fixed-pd 9000 +run_real_case fixed-po 9100 + +run_sim_case fixed-pd +analyze_case fixed-pd +run_sim_case fixed-po +analyze_case fixed-po + +nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \ + > "${CAMPAIGN_ROOT}/provenance/gpus.after.csv" +find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +printf '%s\n' 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index 7de86fc..e6a68ff 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -21,6 +21,14 @@ SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}" IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-false}" PORT="${BASE_PORT:-8300}" +REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp1/private/real_requests.jsonl")" + +for tp in 2 4; do + [[ "$(wc -l < "${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || { + echo "ERROR: TP-specific request counts differ" >&2 + exit 1 + } +done case "${PREFIX_CACHING}" in true|false) ;; @@ -186,7 +194,7 @@ run_tp_wave() { } { - printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" + printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=${REQUEST_COUNT} transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}" sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \