Record fixed-case vLLM runtime state

2026-07-18 01:05:46 +08:00
parent 44104bd96e
commit a7e3c0fdf3
1 changed files with 139 additions and 0 deletions
--- a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh
+++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+# Capture vLLM's resolved runtime state for Fixed-P/PD before their Frontier
+# surfaces are frozen.  No benchmark request is issued in this preflight.
+
+set -euo pipefail
+
+OUT="${1:?output root is required}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
+FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-shared-workspace}"
+PORT=8600
+
+mkdir -p "${OUT}/runs" "${OUT}/provenance" "${FLASHINFER_WORKSPACE_BASE}"
+exec > >(tee -a "${OUT}/controller.log") 2>&1
+
+echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 configs=TP{1,2,4}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+sha256sum "${BASH_SOURCE[0]}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256"
+"${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version"
+
+if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then
+  echo 'ERROR: a GPU is not idle before runtime preflight' >&2
+  exit 1
+fi
+
+declare -a PIDS=()
+
+cleanup_pid() {
+  local pid="$1"
+  if kill -0 "${pid}" 2>/dev/null; then
+    kill -TERM -- "-${pid}" 2>/dev/null || true
+    for _ in $(seq 1 30); do
+      kill -0 "${pid}" 2>/dev/null || break
+      sleep 1
+    done
+    kill -KILL -- "-${pid}" 2>/dev/null || true
+  fi
+}
+
+cleanup_all() {
+  local pid
+  for pid in "${PIDS[@]:-}"; do
+    cleanup_pid "${pid}"
+  done
+}
+trap cleanup_all EXIT INT TERM
+
+launch_one() {
+  local tp="$1" mns="$2" gpus="$3" port="$4"
+  local name="tp${tp}_mns${mns}"
+  local root="${OUT}/runs/${name}"
+  mkdir -p "${root}"
+  (
+    export HOME=/tmp/wjh
+    export XDG_CACHE_HOME=/tmp/wjh/.cache
+    export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm
+    export TOKENIZERS_PARALLELISM=false
+    export VLLM_USE_V1=1
+    export TORCH_CUDA_ARCH_LIST=9.0
+    export HF_HUB_OFFLINE=1
+    export TRANSFORMERS_OFFLINE=1
+    export CUDA_VISIBLE_DEVICES="${gpus}"
+    setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
+      --host 127.0.0.1 --port "${port}" --served-model-name qwen3-30b-fixed-preflight \
+      --tensor-parallel-size "${tp}" --gpu-memory-utilization 0.92 \
+      --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${mns}" \
+      --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \
+      > "${root}/server.log" 2>&1 &
+    local server_pid=$!
+    local ready=0
+    for _ in $(seq 1 150); do
+      if curl -fsS --max-time 2 "http://127.0.0.1:${port}/v1/models" > "${root}/models.json" 2>/dev/null; then
+        ready=1
+        break
+      fi
+      if ! kill -0 "${server_pid}" 2>/dev/null; then
+        break
+      fi
+      sleep 2
+    done
+    grep -E -i 'cuda graph|cudagraph|capture|gpu blocks|num_gpu_blocks|kv cache' "${root}/server.log" > "${root}/resolved-state.log" || true
+    if [[ "${ready}" -ne 1 ]]; then
+      echo "status=failed" > "${root}/status"
+      cleanup_pid "${server_pid}"
+      exit 1
+    fi
+    echo "status=ready" > "${root}/status"
+    cleanup_pid "${server_pid}"
+  ) &
+  PIDS+=("$!")
+}
+
+wait_wave() {
+  local failed=0 pid
+  for pid in "${PIDS[@]}"; do
+    wait "${pid}" || failed=1
+  done
+  PIDS=()
+  if [[ "${failed}" -ne 0 ]]; then
+    echo 'ERROR: one or more runtime preflight servers failed' >&2
+    exit 1
+  fi
+  if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then
+    echo 'ERROR: a GPU remains allocated after runtime preflight wave' >&2
+    exit 1
+  fi
+}
+
+for tp in 4 2 1; do
+  echo "PREFLIGHT_WAVE_START tp=${tp}"
+  case "${tp}" in
+    4)
+      launch_one 4 8  '0,1,2,3' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 4 16 '4,5,6,7' "${PORT}"; PORT=$((PORT + 1))
+      wait_wave
+      launch_one 4 32 '0,1,2,3' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 4 64 '4,5,6,7' "${PORT}"; PORT=$((PORT + 1))
+      ;;
+    2)
+      launch_one 2 8  '0,1' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 2 16 '2,3' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 2 32 '4,5' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 2 64 '6,7' "${PORT}"; PORT=$((PORT + 1))
+      ;;
+    1)
+      launch_one 1 8  '0' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 1 16 '1' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 1 32 '2' "${PORT}"; PORT=$((PORT + 1))
+      launch_one 1 64 '3' "${PORT}"; PORT=$((PORT + 1))
+      ;;
+  esac
+  wait_wave
+  echo "PREFLIGHT_WAVE_COMPLETE tp=${tp}"
+done
+
+find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256"
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo 'FIXED_RUNTIME_PREFLIGHT_COMPLETE'