57 lines
2.6 KiB
Bash
57 lines
2.6 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}"
|
|
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
|
FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}"
|
|
VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}"
|
|
RUN_DIR="$(pwd -P)"
|
|
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles"
|
|
exec > >(tee -a "${OUTPUT_ROOT}/logs/smoke.log") 2>&1
|
|
|
|
IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}"
|
|
if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
|
|
echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=qwen3-a3b-30b-moe runtime=vLLM-0.20.0+cu129 profiler=Frontier@d9cfeb6 operator=linear,norm,rope,embedding tp=1,2,4 tokens=8 precision=BF16 method=cuda_event output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=900s hard_gpu_cap=0.25_H20h"
|
|
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
|
|
nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader
|
|
|
|
test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e"
|
|
test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1"
|
|
git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit"
|
|
git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
|
|
git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit"
|
|
sha256sum run_frontier_linear_smoke.sh frontier_vllm020_compat.py \
|
|
> "${OUTPUT_ROOT}/provenance/source.sha256"
|
|
uv pip freeze --python "${VENV_ROOT}/bin/python" \
|
|
> "${OUTPUT_ROOT}/provenance/pip-freeze.txt"
|
|
nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \
|
|
--format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv"
|
|
|
|
cd "${FRONTIER_ROOT}"
|
|
timeout --signal=TERM --kill-after=30s 780 \
|
|
"${VENV_ROOT}/bin/python" "${RUN_DIR}/frontier_vllm020_compat.py" \
|
|
--disable_ray \
|
|
--num_gpus 1 \
|
|
--output_dir "${OUTPUT_ROOT}/profiles" \
|
|
--device h20 \
|
|
--models qwen3-a3b-30b-moe \
|
|
--num_tensor_parallel_workers 1 2 4 \
|
|
--num_tokens_list 8 \
|
|
--profile_method cuda_event \
|
|
--precision BF16 \
|
|
--is_moe \
|
|
--yes
|
|
|
|
find "${OUTPUT_ROOT}/profiles" -type f -size +0c -print \
|
|
> "${OUTPUT_ROOT}/provenance/profile-files.txt"
|
|
test -s "${OUTPUT_ROOT}/provenance/profile-files.txt"
|
|
find "${OUTPUT_ROOT}/profiles" -type f -print0 | sort -z | xargs -0 sha256sum \
|
|
> "${OUTPUT_ROOT}/profiles.sha256"
|
|
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
|
|
echo "FRONTIER_LINEAR_SMOKE_COMPLETE"
|