diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh new file mode 100644 index 0000000..a64da12 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# Capture vLLM's resolved runtime state for Fixed-P/PD before their Frontier +# surfaces are frozen. No benchmark request is issued in this preflight. + +set -euo pipefail + +OUT="${1:?output root is required}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-shared-workspace}" +PORT=8600 + +mkdir -p "${OUT}/runs" "${OUT}/provenance" "${FLASHINFER_WORKSPACE_BASE}" +exec > >(tee -a "${OUT}/controller.log") 2>&1 + +echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 configs=TP{1,2,4}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +sha256sum "${BASH_SOURCE[0]}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" +"${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" + +if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then + echo 'ERROR: a GPU is not idle before runtime preflight' >&2 + exit 1 +fi + +declare -a PIDS=() + +cleanup_pid() { + local pid="$1" + if kill -0 "${pid}" 2>/dev/null; then + kill -TERM -- "-${pid}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${pid}" 2>/dev/null || true + fi +} + +cleanup_all() { + local pid + for pid in "${PIDS[@]:-}"; do + cleanup_pid "${pid}" + done +} +trap cleanup_all EXIT INT TERM + +launch_one() { + local tp="$1" mns="$2" gpus="$3" port="$4" + local name="tp${tp}_mns${mns}" + local root="${OUT}/runs/${name}" + mkdir -p "${root}" + ( + export HOME=/tmp/wjh + export XDG_CACHE_HOME=/tmp/wjh/.cache + export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm + export TOKENIZERS_PARALLELISM=false + export VLLM_USE_V1=1 + export TORCH_CUDA_ARCH_LIST=9.0 + export HF_HUB_OFFLINE=1 + export TRANSFORMERS_OFFLINE=1 + export CUDA_VISIBLE_DEVICES="${gpus}" + setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${port}" --served-model-name qwen3-30b-fixed-preflight \ + --tensor-parallel-size "${tp}" --gpu-memory-utilization 0.92 \ + --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${mns}" \ + --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + > "${root}/server.log" 2>&1 & + local server_pid=$! + local ready=0 + for _ in $(seq 1 150); do + if curl -fsS --max-time 2 "http://127.0.0.1:${port}/v1/models" > "${root}/models.json" 2>/dev/null; then + ready=1 + break + fi + if ! kill -0 "${server_pid}" 2>/dev/null; then + break + fi + sleep 2 + done + grep -E -i 'cuda graph|cudagraph|capture|gpu blocks|num_gpu_blocks|kv cache' "${root}/server.log" > "${root}/resolved-state.log" || true + if [[ "${ready}" -ne 1 ]]; then + echo "status=failed" > "${root}/status" + cleanup_pid "${server_pid}" + exit 1 + fi + echo "status=ready" > "${root}/status" + cleanup_pid "${server_pid}" + ) & + PIDS+=("$!") +} + +wait_wave() { + local failed=0 pid + for pid in "${PIDS[@]}"; do + wait "${pid}" || failed=1 + done + PIDS=() + if [[ "${failed}" -ne 0 ]]; then + echo 'ERROR: one or more runtime preflight servers failed' >&2 + exit 1 + fi + if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then + echo 'ERROR: a GPU remains allocated after runtime preflight wave' >&2 + exit 1 + fi +} + +for tp in 4 2 1; do + echo "PREFLIGHT_WAVE_START tp=${tp}" + case "${tp}" in + 4) + launch_one 4 8 '0,1,2,3' "${PORT}"; PORT=$((PORT + 1)) + launch_one 4 16 '4,5,6,7' "${PORT}"; PORT=$((PORT + 1)) + wait_wave + launch_one 4 32 '0,1,2,3' "${PORT}"; PORT=$((PORT + 1)) + launch_one 4 64 '4,5,6,7' "${PORT}"; PORT=$((PORT + 1)) + ;; + 2) + launch_one 2 8 '0,1' "${PORT}"; PORT=$((PORT + 1)) + launch_one 2 16 '2,3' "${PORT}"; PORT=$((PORT + 1)) + launch_one 2 32 '4,5' "${PORT}"; PORT=$((PORT + 1)) + launch_one 2 64 '6,7' "${PORT}"; PORT=$((PORT + 1)) + ;; + 1) + launch_one 1 8 '0' "${PORT}"; PORT=$((PORT + 1)) + launch_one 1 16 '1' "${PORT}"; PORT=$((PORT + 1)) + launch_one 1 32 '2' "${PORT}"; PORT=$((PORT + 1)) + launch_one 1 64 '3' "${PORT}"; PORT=$((PORT + 1)) + ;; + esac + wait_wave + echo "PREFLIGHT_WAVE_COMPLETE tp=${tp}" +done + +find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo 'FIXED_RUNTIME_PREFLIGHT_COMPLETE'