Resume valid Qwen30 latency cells safely

runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh

@@ -18,6 +18,8 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
 FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-${OUT}/flashinfer-shared-workspace}"
 SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}"
+IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
+RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-false}"
 PORT="${BASE_PORT:-8300}"
 
 case "${PREFIX_CACHING}" in
@@ -30,6 +32,16 @@ case "${CASE_NAME}" in
   *) echo "ERROR: unsupported CASE_NAME=${CASE_NAME}" >&2; exit 1 ;;
 esac
 
+case "${RESUME_VALID_CELLS}" in
+  true|false) ;;
+  *) echo "ERROR: RESUME_VALID_CELLS must be true or false" >&2; exit 1 ;;
+esac
+
+[[ "${IDLE_GPU_MEMORY_TOLERANCE_MIB}" =~ ^[0-9]+$ ]] || {
+  echo "ERROR: IDLE_GPU_MEMORY_TOLERANCE_MIB must be a non-negative integer" >&2
+  exit 1
+}
+
 for path in "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" "${MODEL_ROOT}/config.json"; do
   [[ -f "${path}" ]] || { echo "ERROR: required path is missing: ${path}" >&2; exit 1; }
 done
@@ -37,6 +49,23 @@ mkdir -p "${OUT}"
 
 declare -a WAVE_PIDS=()
 
+has_valid_result() {
+  local result="$1" expected_requests="$2"
+  [[ -s "${result}" ]] || return 1
+  "${VENV_ROOT}/bin/python" - "${result}" "${expected_requests}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1]) as source:
+    summary = json.load(source).get("summary", {})
+raise SystemExit(
+    0
+    if summary.get("completed") == int(sys.argv[2]) and summary.get("failed") == 0
+    else 1
+)
+PY
+}
+
 wait_for_wave() {
   local failed=0 pid
   for pid in "${WAVE_PIDS[@]}"; do
@@ -56,10 +85,11 @@ preflight_gpus() {
   nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \
     | tee -a "${OUT}/controller.log"
   if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
-    | awk '$1 != 0 {exit 1}'; then
+    | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then
     return 0
   fi
-  printf '%s\n' 'ERROR non-idle GPU before a real-serving wave; refusing to start.' \
+  printf 'ERROR GPU memory exceeds idle tolerance (%s MiB) before a real-serving wave; refusing to start.\n' \
+    "${IDLE_GPU_MEMORY_TOLERANCE_MIB}" \
     | tee -a "${OUT}/controller.log" >&2
   return 1
 }
@@ -77,8 +107,16 @@ launch_config() {
   local config="tp${tp}_mns${mns}"
   local run_out="${OUT}/real/${config}/trial${trial}"
   local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl"
+  local request_count
   local port="${PORT}"
   PORT=$((PORT + 1))
+  request_count="$(wc -l < "${requests}")"
+  if [[ "${RESUME_VALID_CELLS}" == true ]] \
+    && has_valid_result "${run_out}/results/result.json" "${request_count}"; then
+    printf 'CONFIG_REUSED_VALID case=%s trial=%s tp=%s mns=%s\n' \
+      "${CASE_NAME}" "${trial}" "${tp}" "${mns}"
+    return 0
+  fi
   mkdir -p "${run_out}"
   (
     cd "${RUNNER_DIR}"
@@ -148,7 +186,7 @@ run_tp_wave() {
 }
 
 {
-  printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real"
+  printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real"
   date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
   mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}"
   sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \