diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index d9b1d65..7de86fc 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -18,6 +18,8 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-${OUT}/flashinfer-shared-workspace}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}" +IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" +RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-false}" PORT="${BASE_PORT:-8300}" case "${PREFIX_CACHING}" in @@ -30,6 +32,16 @@ case "${CASE_NAME}" in *) echo "ERROR: unsupported CASE_NAME=${CASE_NAME}" >&2; exit 1 ;; esac +case "${RESUME_VALID_CELLS}" in + true|false) ;; + *) echo "ERROR: RESUME_VALID_CELLS must be true or false" >&2; exit 1 ;; +esac + +[[ "${IDLE_GPU_MEMORY_TOLERANCE_MIB}" =~ ^[0-9]+$ ]] || { + echo "ERROR: IDLE_GPU_MEMORY_TOLERANCE_MIB must be a non-negative integer" >&2 + exit 1 +} + for path in "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" "${MODEL_ROOT}/config.json"; do [[ -f "${path}" ]] || { echo "ERROR: required path is missing: ${path}" >&2; exit 1; } done @@ -37,6 +49,23 @@ mkdir -p "${OUT}" declare -a WAVE_PIDS=() +has_valid_result() { + local result="$1" expected_requests="$2" + [[ -s "${result}" ]] || return 1 + "${VENV_ROOT}/bin/python" - "${result}" "${expected_requests}" <<'PY' +import json +import sys + +with open(sys.argv[1]) as source: + summary = json.load(source).get("summary", {}) +raise SystemExit( + 0 + if summary.get("completed") == int(sys.argv[2]) and summary.get("failed") == 0 + else 1 +) +PY +} + wait_for_wave() { local failed=0 pid for pid in "${WAVE_PIDS[@]}"; do @@ -56,10 +85,11 @@ preflight_gpus() { nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \ | tee -a "${OUT}/controller.log" if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ - | awk '$1 != 0 {exit 1}'; then + | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then return 0 fi - printf '%s\n' 'ERROR non-idle GPU before a real-serving wave; refusing to start.' \ + printf 'ERROR GPU memory exceeds idle tolerance (%s MiB) before a real-serving wave; refusing to start.\n' \ + "${IDLE_GPU_MEMORY_TOLERANCE_MIB}" \ | tee -a "${OUT}/controller.log" >&2 return 1 } @@ -77,8 +107,16 @@ launch_config() { local config="tp${tp}_mns${mns}" local run_out="${OUT}/real/${config}/trial${trial}" local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" + local request_count local port="${PORT}" PORT=$((PORT + 1)) + request_count="$(wc -l < "${requests}")" + if [[ "${RESUME_VALID_CELLS}" == true ]] \ + && has_valid_result "${run_out}/results/result.json" "${request_count}"; then + printf 'CONFIG_REUSED_VALID case=%s trial=%s tp=%s mns=%s\n' \ + "${CASE_NAME}" "${trial}" "${tp}" "${mns}" + return 0 + fi mkdir -p "${run_out}" ( cd "${RUNNER_DIR}" @@ -148,7 +186,7 @@ run_tp_wave() { } { - printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" + printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}" sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \