73 lines
4.5 KiB
Bash
73 lines
4.5 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
|
|
FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
|
|
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
|
RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
|
|
MODEL=Qwen3-235B-A22B
|
|
TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 512 1024 2048 4096 8192)
|
|
BATCHES=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256)
|
|
KV=(128 1024 2048 4096 8192 16384 32768 40960)
|
|
|
|
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
|
|
exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1
|
|
|
|
echo "Q235_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} attention_tp={4,8} moe_paths={TP4/EP1:Triton,TP1/EP8:FlashInfer-CUTLASS} token_points=${#TOKENS[@]} batch_points=${#BATCHES[@]} kv_points=${#KV[@]} parallel_gpus=6 expected_wall=20-75m expected_cost=2-7_H20-GPUh output=${OUTPUT_ROOT}"
|
|
date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
|
|
nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \
|
|
| awk '$2 > 16 {exit 1}'
|
|
git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
|
|
"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version"
|
|
# Frontier's model-config loader resolves data/config/models from the current
|
|
# source checkout, so all profiling entrypoints must run from this directory.
|
|
cd "${FRONTIER_SOURCE}"
|
|
|
|
common_profile() {
|
|
local gpu="$1" method="$2" root="$3"
|
|
env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
|
|
"${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \
|
|
--disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
|
|
--num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \
|
|
--max_tokens 8192 --num_tokens_list "${TOKENS[@]}" \
|
|
--profile_method "${method}" --yes
|
|
env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
|
|
"${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
|
|
--disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
|
|
--num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 40960 \
|
|
--batch_size_list "${BATCHES[@]}" --decode_kv_cache_size_list "${KV[@]}" \
|
|
--fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
|
|
--profile_method "${method}" --yes
|
|
}
|
|
|
|
moe_profile() {
|
|
local gpu="$1" method="$2" tp="$3" ep="$4" root="$5"
|
|
env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
|
|
"${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \
|
|
--disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
|
|
--num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \
|
|
--max_tokens 8192 --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
|
|
--num_samples_per_distribution 1 --profile_method "${method}" --yes
|
|
}
|
|
|
|
declare -a pids=()
|
|
common_profile 0 cuda_event "${OUTPUT_ROOT}/cuda-common" > "${OUTPUT_ROOT}/logs/cuda-common.log" 2>&1 & pids+=("$!")
|
|
common_profile 1 record_function "${OUTPUT_ROOT}/kernel-common" > "${OUTPUT_ROOT}/logs/kernel-common.log" 2>&1 & pids+=("$!")
|
|
moe_profile 2 cuda_event 4 1 "${OUTPUT_ROOT}/cuda-moe-tp4" > "${OUTPUT_ROOT}/logs/cuda-moe-tp4.log" 2>&1 & pids+=("$!")
|
|
moe_profile 3 cuda_event 1 8 "${OUTPUT_ROOT}/cuda-moe-ep8" > "${OUTPUT_ROOT}/logs/cuda-moe-ep8.log" 2>&1 & pids+=("$!")
|
|
moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_ROOT}/logs/kernel-moe-tp4.log" 2>&1 & pids+=("$!")
|
|
moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!")
|
|
failed=0
|
|
for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
|
|
[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
|
|
|
|
"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \
|
|
--cuda-common "${OUTPUT_ROOT}/cuda-common" \
|
|
--cuda-moe-tp4 "${OUTPUT_ROOT}/cuda-moe-tp4" --cuda-moe-ep8 "${OUTPUT_ROOT}/cuda-moe-ep8" \
|
|
--kernel-common "${OUTPUT_ROOT}/kernel-common" \
|
|
--kernel-moe-tp4 "${OUTPUT_ROOT}/kernel-moe-tp4" --kernel-moe-ep8 "${OUTPUT_ROOT}/kernel-moe-ep8" \
|
|
--output-root "${OUTPUT_ROOT}/frozen"
|
|
date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
|
|
echo Q235_V020_PROFILES_COMPLETE
|