51 lines
2.7 KiB
Bash
51 lines
2.7 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}"
|
|
FIXTURE="${FIXTURE:?FIXTURE must be set}"
|
|
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
|
VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}"
|
|
MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
|
|
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw"
|
|
exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1
|
|
|
|
IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}"
|
|
if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
|
|
echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "ROUTING_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 trace_fixture=${FIXTURE} fixture_sha256=e9e7f5b4e0d3a788dcd99d432f939d9e36bff2a64e412649b407b0609f0e39bb requests=8 input_tokens_trace=23673 output_tokens_trace=6502 TP=1 MBT=8192 MNS=64 prefix_cache=true chunked_prefill=true dtype=BF16 output=${OUTPUT_ROOT} expected_wall=15-40m hard_wall=3600s hard_gpu_cap=1.0_H20h"
|
|
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
|
|
nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader
|
|
|
|
test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1"
|
|
test -s "${MODEL}/config.json"
|
|
echo "e9e7f5b4e0d3a788dcd99d432f939d9e36bff2a64e412649b407b0609f0e39bb ${FIXTURE}" | sha256sum -c -
|
|
git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit"
|
|
git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit"
|
|
sha256sum capture_trace_routing.py run_trace_routing.sh \
|
|
> "${OUTPUT_ROOT}/provenance/source.sha256"
|
|
sha256sum "${MODEL}/config.json" > "${OUTPUT_ROOT}/provenance/model-config.sha256"
|
|
sha256sum "${FIXTURE}" > "${OUTPUT_ROOT}/provenance/fixture.sha256"
|
|
uv pip freeze --python "${VENV_ROOT}/bin/python" \
|
|
> "${OUTPUT_ROOT}/provenance/pip-freeze.txt"
|
|
nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \
|
|
--format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv"
|
|
|
|
timeout --signal=TERM --kill-after=30s 3300 \
|
|
"${VENV_ROOT}/bin/python" capture_trace_routing.py \
|
|
--vllm-source "${VLLM_SOURCE}" \
|
|
--model "${MODEL}" \
|
|
--fixture "${FIXTURE}" \
|
|
--output "${OUTPUT_ROOT}/raw/routing.json" \
|
|
--routes "${OUTPUT_ROOT}/raw/routes.npz"
|
|
|
|
test -s "${OUTPUT_ROOT}/raw/routing.json"
|
|
test -s "${OUTPUT_ROOT}/raw/routes.npz"
|
|
sha256sum "${OUTPUT_ROOT}/raw/routing.json" "${OUTPUT_ROOT}/raw/routes.npz" \
|
|
"${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256"
|
|
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
|
|
echo "TRACE_ROUTING_COMPLETE requests=8"
|