gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow fixed runtime preflight resume

Browse Source
This commit is contained in:
Gahow Wang
2026-07-18 01:11:08 +08:00
parent 12705411f9
commit 7a437b4d91
1 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh
View File

@@ -9,17 +9,28 @@ OUT="${1:?output root is required}"
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-shared-workspace}" FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-shared-workspace}"
TPS="${TPS:-4 2 1}"
PORT=8600 PORT=8600
mkdir -p "${OUT}/runs" "${OUT}/provenance" "${FLASHINFER_WORKSPACE_BASE}" mkdir -p "${OUT}/runs" "${OUT}/provenance" "${FLASHINFER_WORKSPACE_BASE}"
exec > >(tee -a "${OUT}/controller.log") 2>&1 exec > >(tee -a "${OUT}/controller.log") 2>&1
echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 configs=TP{1,2,4}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h" echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 tps=${TPS}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h"
date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
sha256sum "${BASH_SOURCE[0]}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" sha256sum "${BASH_SOURCE[0]}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256"
"${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version"
if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then wait_for_idle() {
for _ in $(seq 1 30); do
if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then
return 0
fi
sleep 2
done
return 1
}
if ! wait_for_idle; then
echo 'ERROR: a GPU is not idle before runtime preflight' >&2 echo 'ERROR: a GPU is not idle before runtime preflight' >&2
exit 1 exit 1
fi fi
@@ -101,13 +112,13 @@ wait_wave() {
echo 'ERROR: one or more runtime preflight servers failed' >&2 echo 'ERROR: one or more runtime preflight servers failed' >&2
exit 1 exit 1
fi fi
if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then if ! wait_for_idle; then
echo 'ERROR: a GPU remains allocated after runtime preflight wave' >&2 echo 'ERROR: a GPU remains allocated after runtime preflight wave' >&2
exit 1 exit 1
fi fi
} }
for tp in 4 2 1; do for tp in ${TPS}; do
echo "PREFLIGHT_WAVE_START tp=${tp}" echo "PREFLIGHT_WAVE_START tp=${tp}"
case "${tp}" in case "${tp}" in
4) 4)