gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Wait for Qwen235 GPU memory cleanup

Browse Source
This commit is contained in:
Gahow Wang
2026-07-19 07:07:08 +08:00
parent b2f97927de
commit b72de0fd15
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh
View File

@@ -15,6 +15,7 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}"
IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
IDLE_GPU_SETTLE_ATTEMPTS="${IDLE_GPU_SETTLE_ATTEMPTS:-120}"
RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}" RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}"
PORT="${BASE_PORT:-9300}" PORT="${BASE_PORT:-9300}"
REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")" REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")"
@@ -43,10 +44,18 @@ PY
} }
preflight_gpus() { preflight_gpus() {
local attempt
nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \ nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \
| tee -a "${OUT}/controller.log" | tee -a "${OUT}/controller.log"
nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ for ((attempt = 1; attempt <= IDLE_GPU_SETTLE_ATTEMPTS; attempt++)); do
| awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}' if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
| awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then
return 0
fi
sleep 1
done
echo "ERROR: GPU memory did not settle below ${IDLE_GPU_MEMORY_TOLERANCE_MIB} MiB after ${IDLE_GPU_SETTLE_ATTEMPTS}s" >&2
return 1
} }
assert_no_server() { assert_no_server() {