Avoid occupied Qwen235 trace ports

runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh

@@ -10,6 +10,13 @@ RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
 FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
 VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 
+# FlashInfer's TensorRT-LLM MoE runtime invokes `nvcc` by name when it
+# autotunes a previously unseen FP8 grouped-GEMM shape.
+export PATH="/usr/local/cuda/bin:${PATH}"
+export FRONTIER_VLLM_MODEL_TYPE=qwen3_moe
+export VLLM_KV_CACHE_LAYOUT=NHD
+command -v nvcc >/dev/null
+
 mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}"
 exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1
 echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}"

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh

@@ -53,8 +53,8 @@ run_real() {
 }
 run_real fixed-pd false 9300
 run_real fixed-po false 9400
-run_real trace-pd true 9500
-run_real trace-po true 9600
+run_real trace-pd true 9800
+run_real trace-po true 9900
 
 "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \
   --case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh

@@ -30,12 +30,14 @@ common_profile() {
     "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \
       --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
       --num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \
-      --num_tokens_list "${TOKENS[@]}" --profile_method "${method}" --yes
+      --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" \
+      --profile_method "${method}" --yes
   env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
     "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
       --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
-      --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 8192 \
-      --batch_size_list "${BATCHES[@]}" --decode_kv_cache_size_list "${KV[@]}" \
+      --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 40960 \
+      --max_batch_size 256 --batch_size_list "${BATCHES[@]}" \
+      --decode_kv_cache_size_list "${KV[@]}" \
       --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
       --profile_method "${method}" --yes
 }
@@ -46,7 +48,7 @@ moe_profile() {
     "${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \
       --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
       --num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \
-      --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
+      --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
       --num_samples_per_distribution 1 --profile_method "${method}" --yes
 }
 
@@ -59,7 +61,7 @@ moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_RO
 moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!")
 failed=0
 for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
-[[ "${failed}" -eq 0 ]] || { tail -80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
+[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
 
 "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \
   --cuda-common "${OUTPUT_ROOT}/cuda-common" \

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh

@@ -15,10 +15,14 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
 SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}"
 IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
+IDLE_GPU_SETTLE_ATTEMPTS="${IDLE_GPU_SETTLE_ATTEMPTS:-120}"
 RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}"
 PORT="${BASE_PORT:-9300}"
 REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")"
 
+export PATH="/usr/local/cuda/bin:${PATH}"
+command -v nvcc >/dev/null
+
 [[ "$(wc -l < "${TRACE_ROOT}/tp8/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || {
   echo 'ERROR: TP-specific request counts differ' >&2
   exit 1
@@ -40,10 +44,18 @@ PY
 }
 
 preflight_gpus() {
+  local attempt
   nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \
     | tee -a "${OUT}/controller.log"
-  nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
-    | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'
+  for ((attempt = 1; attempt <= IDLE_GPU_SETTLE_ATTEMPTS; attempt++)); do
+    if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
+      | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then
+      return 0
+    fi
+    sleep 1
+  done
+  echo "ERROR: GPU memory did not settle below ${IDLE_GPU_MEMORY_TOLERANCE_MIB} MiB after ${IDLE_GPU_SETTLE_ATTEMPTS}s" >&2
+  return 1
 }
 
 assert_no_server() {
@@ -60,7 +72,8 @@ wait_for_wave() {
 launch_config() {
   local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1
   if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi
-  local config="tp${tp}_ep${ep_size}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}"
+  local config="tp${tp}_ep${ep_size}_mns${mns}"
+  local run_out="${OUT}/real/${config}/trial${trial}"
   local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}"
   PORT=$((PORT + 1))
   if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then
@@ -97,12 +110,16 @@ run_trial() {
     "${CASE_NAME}" "${trial}" "${mnss[0]}" "${mnss[1]}" | tee -a "${OUT}/controller.log"
   launch_config "${trial}" 4 "${mnss[0]}" '0,1,2,3'
   launch_config "${trial}" 4 "${mnss[1]}" '4,5,6,7'
-  wait_for_wave && preflight_gpus && assert_no_server
+  wait_for_wave
+  preflight_gpus
+  assert_no_server
   for mns in "${mnss[@]}"; do
     printf 'WAVE_START case=%s trial=%s tp=8 mns=%s\n' \
       "${CASE_NAME}" "${trial}" "${mns}" | tee -a "${OUT}/controller.log"
     launch_config "${trial}" 8 "${mns}" '0,1,2,3,4,5,6,7'
-    wait_for_wave && preflight_gpus && assert_no_server
+    wait_for_wave
+    preflight_gpus
+    assert_no_server
   done
 }