Profile Qwen235 true mixed attention

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_true_mixed_profiles.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL=Qwen3-235B-A22B
+PREFILL_BATCHES=(1 2 4 8)
+PREFILL_CHUNKS=(128 512 2048 8192)
+DECODE_BATCHES=(1 8 32 64 120)
+DECODE_KV=(128 1024 4096 16384 32768)
+
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
+exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1
+
+echo "Q235_TRUE_MIXED_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} tp={4,8} grid=4x4x5x5 parallel_gpus=4 expected_wall=10-30m expected_cost=0.7-2_H20-GPUh output=${OUTPUT_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \
+  | awk '$2 > 16 {exit 1}'
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
+"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version"
+cd "${FRONTIER_SOURCE}"
+
+profile() {
+  local gpu="$1" method="$2" tp="$3" root="$4"
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers "${tp}" --max_model_len 40960 --max_seq_len 40960 \
+      --max_batch_size 128 --batch_size_list 1 --decode_kv_cache_size_list 128 \
+      --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
+      --enable_true_mixed \
+      --true_mixed_prefill_batch_sizes "${PREFILL_BATCHES[@]}" \
+      --true_mixed_prefill_chunk_sizes "${PREFILL_CHUNKS[@]}" \
+      --true_mixed_decode_batch_sizes "${DECODE_BATCHES[@]}" \
+      --true_mixed_decode_kv_cache_sizes "${DECODE_KV[@]}" \
+      --true_mixed_prefill_kv_cache_size 0 \
+      --profile_method "${method}" --yes
+}
+
+declare -a pids=()
+profile 0 cuda_event 4 "${OUTPUT_ROOT}/cuda-tp4" > "${OUTPUT_ROOT}/logs/cuda-tp4.log" 2>&1 & pids+=("$!")
+profile 1 record_function 4 "${OUTPUT_ROOT}/kernel-tp4" > "${OUTPUT_ROOT}/logs/kernel-tp4.log" 2>&1 & pids+=("$!")
+profile 2 cuda_event 8 "${OUTPUT_ROOT}/cuda-tp8" > "${OUTPUT_ROOT}/logs/cuda-tp8.log" 2>&1 & pids+=("$!")
+profile 3 record_function 8 "${OUTPUT_ROOT}/kernel-tp8" > "${OUTPUT_ROOT}/logs/kernel-tp8.log" 2>&1 & pids+=("$!")
+failed=0
+for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
+[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_V020_TRUE_MIXED_PROFILES_COMPLETE