diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh index 4b33f35..f2affde 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh @@ -15,6 +15,7 @@ MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" CLIENT="${CLIENT:?CLIENT is required}" READY_TIMEOUT_SECONDS="${READY_TIMEOUT_SECONDS:-1800}" TRTLLM_DG_CACHE_DIR="${TRTLLM_DG_CACHE_DIR:-${OUT}/deep_gemm}" +TRTLLM_DG_NVCC_COMPILER="${TRTLLM_DG_NVCC_COMPILER:-/usr/local/cuda/bin/nvcc}" DEEP_GEMM_JIT_DEBUG="${DEEP_GEMM_JIT_DEBUG:-false}" SERVED_MODEL="qwen3-235b-v020-smoke" SERVER_PID="" @@ -49,12 +50,16 @@ if ! [[ "${READY_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: READY_TIMEOUT_SECONDS must be a positive integer" >&2 exit 1 fi +if [[ ! -x "${TRTLLM_DG_NVCC_COMPILER}" ]]; then + echo "ERROR: TRTLLM_DG_NVCC_COMPILER is not executable: ${TRTLLM_DG_NVCC_COMPILER}" >&2 + exit 1 +fi case "${DEEP_GEMM_JIT_DEBUG}" in true|false) ;; *) echo "ERROR: DEEP_GEMM_JIT_DEBUG must be true or false" >&2; exit 1 ;; esac -echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s deep_gemm_cache=${TRTLLM_DG_CACHE_DIR} deep_gemm_jit_debug=${DEEP_GEMM_JIT_DEBUG} role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" +echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s deep_gemm_cache=${TRTLLM_DG_CACHE_DIR} deep_gemm_nvcc=${TRTLLM_DG_NVCC_COMPILER} deep_gemm_jit_debug=${DEEP_GEMM_JIT_DEBUG} role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" @@ -74,6 +79,7 @@ export HOME=/tmp/wjh export XDG_CACHE_HOME=/tmp/wjh/.cache export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm export TRTLLM_DG_CACHE_DIR +export TRTLLM_DG_NVCC_COMPILER if [[ "${DEEP_GEMM_JIT_DEBUG}" == true ]]; then export TRTLLM_DG_JIT_DEBUG=1 fi