diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh index 41b63c0..91733c5 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh @@ -15,6 +15,7 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}" IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" +IDLE_GPU_SETTLE_ATTEMPTS="${IDLE_GPU_SETTLE_ATTEMPTS:-120}" RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}" PORT="${BASE_PORT:-9300}" REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")" @@ -43,10 +44,18 @@ PY } preflight_gpus() { + local attempt nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \ | tee -a "${OUT}/controller.log" - nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ - | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}' + for ((attempt = 1; attempt <= IDLE_GPU_SETTLE_ATTEMPTS; attempt++)); do + if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ + | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then + return 0 + fi + sleep 1 + done + echo "ERROR: GPU memory did not settle below ${IDLE_GPU_MEMORY_TOLERANCE_MIB} MiB after ${IDLE_GPU_SETTLE_ATTEMPTS}s" >&2 + return 1 } assert_no_server() {