Wait for Qwen235 GPU memory cleanup

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh

@@ -15,6 +15,7 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
 SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}"
 IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
+IDLE_GPU_SETTLE_ATTEMPTS="${IDLE_GPU_SETTLE_ATTEMPTS:-120}"
 RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}"
 PORT="${BASE_PORT:-9300}"
 REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")"
@@ -43,10 +44,18 @@ PY
 }
 
 preflight_gpus() {
+  local attempt
   nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \
     | tee -a "${OUT}/controller.log"
-  nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
-    | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'
+  for ((attempt = 1; attempt <= IDLE_GPU_SETTLE_ATTEMPTS; attempt++)); do
+    if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
+      | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then
+      return 0
+    fi
+    sleep 1
+  done
+  echo "ERROR: GPU memory did not settle below ${IDLE_GPU_MEMORY_TOLERANCE_MIB} MiB after ${IDLE_GPU_SETTLE_ATTEMPTS}s" >&2
+  return 1
 }
 
 assert_no_server() {