Add Qwen30 prefill fidelity experiment
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
|
||||
TP="${TP:?TP is required}"
|
||||
MNS="${MNS:?MNS is required}"
|
||||
RATES="${RATES:-4 8 16 32 64}"
|
||||
SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
|
||||
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
||||
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
|
||||
SERVED_MODEL="qwen3-30b-prefill-only"
|
||||
SERVER_PID=""
|
||||
|
||||
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
|
||||
exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
|
||||
for _ in $(seq 1 30); do
|
||||
kill -0 "${SERVER_PID}" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
|
||||
fi
|
||||
SERVER_PID=""
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:-}"
|
||||
if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then
|
||||
echo "ERROR: expected ${TP} allocated GPUs, got ${CUDA_VISIBLE_DEVICES:-unset}" >&2
|
||||
exit 1
|
||||
fi
|
||||
read -r -a RATE_ARRAY <<< "${RATES}"
|
||||
|
||||
echo "QWEN30_PREFILL_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL2048_OSL1 arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}"
|
||||
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
|
||||
sha256sum run_qwen30_prefill_real_config.sh qwen30_prefill_client.py \
|
||||
> "${OUTPUT_ROOT}/provenance/source.sha256"
|
||||
"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" \
|
||||
> "${OUTPUT_ROOT}/provenance/contract.json" <<'PY'
|
||||
import importlib.metadata as metadata
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
|
||||
tp, mns, rates = sys.argv[1:]
|
||||
print(json.dumps({
|
||||
"python": platform.python_version(),
|
||||
"torch": metadata.version("torch"),
|
||||
"transformers": metadata.version("transformers"),
|
||||
"vllm": metadata.version("vllm"),
|
||||
"config": {"tp": int(tp), "mns": int(mns), "mbt": 8192},
|
||||
"rates": [float(value) for value in rates.split()],
|
||||
"rounds": 2,
|
||||
"requests_per_anchor": 64,
|
||||
"anchor_isolation": "fresh_server_per_rate_per_round",
|
||||
"target_rate_warmup_requests": "min(32, max(4, ceil(rate * 2)))",
|
||||
"input_tokens": 2048,
|
||||
"output_tokens": 1,
|
||||
"ttft_slo_ms": 1256.0,
|
||||
"target_pass_rate": 0.95,
|
||||
}, indent=2, sort_keys=True))
|
||||
PY
|
||||
nvidia-smi --query-gpu=index,name,uuid,driver_version --format=csv,noheader \
|
||||
> "${OUTPUT_ROOT}/provenance/gpus.csv"
|
||||
sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256"
|
||||
|
||||
export TOKENIZERS_PARALLELISM=false
|
||||
export VLLM_USE_V1=1
|
||||
export TORCH_CUDA_ARCH_LIST=9.0
|
||||
export HF_HUB_OFFLINE=1
|
||||
export TRANSFORMERS_OFFLINE=1
|
||||
|
||||
for ROUND in 1 2; do
|
||||
ROUND_ROOT="${OUTPUT_ROOT}/round${ROUND}"
|
||||
mkdir -p "${ROUND_ROOT}/logs" "${ROUND_ROOT}/results"
|
||||
ORDERED_RATES=("${RATE_ARRAY[@]}")
|
||||
if [[ "${ROUND}" -eq 2 ]]; then
|
||||
ORDERED_RATES=()
|
||||
for ((index=${#RATE_ARRAY[@]}-1; index>=0; index--)); do
|
||||
ORDERED_RATES+=("${RATE_ARRAY[index]}")
|
||||
done
|
||||
fi
|
||||
for RATE in "${ORDERED_RATES[@]}"; do
|
||||
KEY="$(printf 'r%.2f' "${RATE}" | tr '.' 'p')"
|
||||
SERVER_LOG="${ROUND_ROOT}/logs/server_${KEY}.log"
|
||||
setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
|
||||
--host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \
|
||||
--tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \
|
||||
--max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
|
||||
--no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \
|
||||
> "${SERVER_LOG}" 2>&1 &
|
||||
SERVER_PID=$!
|
||||
READY=0
|
||||
for _ in $(seq 1 120); do
|
||||
if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \
|
||||
> "${ROUND_ROOT}/results/models_${KEY}.json" 2>/dev/null; then
|
||||
READY=1
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
tail -200 "${SERVER_LOG}"
|
||||
exit 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
if [[ "${READY}" -ne 1 ]]; then
|
||||
tail -200 "${SERVER_LOG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY'
|
||||
import math
|
||||
import sys
|
||||
print(min(32, max(4, math.ceil(float(sys.argv[1]) * 2.0))))
|
||||
PY
|
||||
)"
|
||||
"${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \
|
||||
--served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
|
||||
--requests "${WARMUP_REQUESTS}" \
|
||||
--output "${ROUND_ROOT}/results/warmup_${KEY}.json"
|
||||
"${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \
|
||||
--served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
|
||||
--requests 64 --output "${ROUND_ROOT}/results/${KEY}.json"
|
||||
cleanup
|
||||
done
|
||||
done
|
||||
|
||||
find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
|
||||
| sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256"
|
||||
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
|
||||
echo "QWEN30_PREFILL_REAL_CONFIG_COMPLETE"
|
||||
Reference in New Issue
Block a user