diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh index 29a98f7..235cb91 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh @@ -13,6 +13,7 @@ ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:?ENABLE_EXPERT_PARALLEL is requ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" CLIENT="${CLIENT:?CLIENT is required}" +READY_TIMEOUT_SECONDS="${READY_TIMEOUT_SECONDS:-1800}" SERVED_MODEL="qwen3-235b-v020-smoke" SERVER_PID="" @@ -42,8 +43,12 @@ case "${ENABLE_EXPERT_PARALLEL}" in false) EP_FLAG=() ;; *) echo "ERROR: ENABLE_EXPERT_PARALLEL must be true or false" >&2; exit 1 ;; esac +if ! [[ "${READY_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: READY_TIMEOUT_SECONDS must be a positive integer" >&2 + exit 1 +fi -echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=8-20m expected_gpu_cap=$((${TP} * 1))_H20h" +echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" @@ -71,7 +76,8 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ "${EP_FLAG[@]}" > "${OUT}/logs/server.log" 2>&1 & SERVER_PID=$! READY=0 -for _ in $(seq 1 180); do +READY_DEADLINE=$((SECONDS + READY_TIMEOUT_SECONDS)) +while (( SECONDS < READY_DEADLINE )); do if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" > "${OUT}/results/models.json" 2>/dev/null; then READY=1 break