76 lines
3.4 KiB
Bash
76 lines
3.4 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
TP="${TP:?TP must be 1, 2, or 4}"
|
|
case "${TP}" in
|
|
1|2|4) ;;
|
|
*) echo "invalid TP=${TP}" >&2; exit 1 ;;
|
|
esac
|
|
|
|
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}"
|
|
RUN_DIR="$(pwd -P)"
|
|
PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}"
|
|
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
|
PROFILE_HOME="${PROFILE_HOME:-/tmp/wjh}"
|
|
VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}"
|
|
FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}"
|
|
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
|
|
CAPTURE_BUCKETS="${CAPTURE_BUCKETS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128}"
|
|
KV_CONTEXTS="${KV_CONTEXTS:-128 1024 2048 4096 8192 16384 32768 40960}"
|
|
WARMUP_ITERS="${WARMUP_ITERS:-3}"
|
|
REPEATS="${REPEATS:-5}"
|
|
export HOME="${PROFILE_HOME}"
|
|
export XDG_CACHE_HOME="${PROFILE_HOME}/.cache"
|
|
export VLLM_CACHE_ROOT="${XDG_CACHE_HOME}/vllm"
|
|
|
|
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw"
|
|
exec > >(tee -a "${OUTPUT_ROOT}/logs/attention-tp${TP}.log") 2>&1
|
|
|
|
IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?a fleet-allocated GPU is required}"
|
|
if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
|
|
echo "expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
BATCH_SPECS=()
|
|
for bucket in ${CAPTURE_BUCKETS}; do
|
|
prefix=""
|
|
if [[ "${bucket}" -ne 1 ]]; then
|
|
prefix="${bucket}"
|
|
fi
|
|
for context in ${KV_CONTEXTS}; do
|
|
BATCH_SPECS+=("${prefix}q1s${context}")
|
|
done
|
|
done
|
|
|
|
echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} role=FA3-decode-kernel-only tp=${TP} buckets='${CAPTURE_BUCKETS}' kv_contexts='${KV_CONTEXTS}' method=Frontier-RecordFunctionTracer output=${OUTPUT_ROOT} expected_wall=10-35m expected_gpu_cap=1.0_H20h"
|
|
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
|
|
nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader
|
|
test "$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD)" = "deadc4a321f0baaa534c6ebd17f974123733cdc2"
|
|
test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1"
|
|
test -f "${MODEL_ROOT}/config.json"
|
|
git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit"
|
|
git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
|
|
git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm.commit"
|
|
printf '%s\n' "${BATCH_SPECS[@]}" > "${OUTPUT_ROOT}/provenance/batch-specs.txt"
|
|
sha256sum "${PROFILE_DIR}/profile_vllm020_flashattn.py" "${RUN_DIR}/run_graph_kernel_only_attention.sh" > "${OUTPUT_ROOT}/provenance/source.sha256"
|
|
|
|
timeout --signal=TERM --kill-after=30s 2400 \
|
|
"${VENV_ROOT}/bin/python" "${PROFILE_DIR}/profile_vllm020_flashattn.py" \
|
|
--vllm-source "${VLLM_SOURCE}" \
|
|
--frontier-source "${FRONTIER_SOURCE}" \
|
|
--model "${MODEL_ROOT}" \
|
|
--output "${OUTPUT_ROOT}/raw/attention-tp${TP}.json" \
|
|
--tp "${TP}" \
|
|
--batch-specs "${BATCH_SPECS[@]}" \
|
|
--warmup-iters "${WARMUP_ITERS}" \
|
|
--repeats "${REPEATS}" \
|
|
--profile-kv-update \
|
|
--profile-method record_function
|
|
|
|
test -s "${OUTPUT_ROOT}/raw/attention-tp${TP}.json"
|
|
sha256sum "${OUTPUT_ROOT}/raw/attention-tp${TP}.json" "${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256"
|
|
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
|
|
echo "GRAPH_KERNEL_ONLY_ATTENTION_COMPLETE tp=${TP} rows=${#BATCH_SPECS[@]}"
|