aituner/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh

#!/usr/bin/env bash

set -euo pipefail

OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
REQUESTS_FILE="${REQUESTS_FILE:?REQUESTS_FILE is required}"
TP="${TP:?TP is required}"
MNS="${MNS:?MNS is required}"
TRACE_LABEL="${TRACE_LABEL:?TRACE_LABEL is required}"
SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
PREFIX_CACHING="${PREFIX_CACHING:-true}"
ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:-false}"
MODEL_QUANTIZATION="${MODEL_QUANTIZATION:-}"
DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-false}"
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}"
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}"
SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-120}"
CLIENT_TIMEOUT_SECONDS="${CLIENT_TIMEOUT_SECONDS:-1800}"
SERVED_MODEL="${SERVED_MODEL:-qwen3-30b-exact-trace}"
EXACT_TRACE_CLIENT="${EXACT_TRACE_CLIENT:-qwen30_exact_trace_client.py}"
SERVER_PID=""

mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/results" \
  "${FLASHINFER_WORKSPACE_BASE}"
exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1

cleanup() {
  if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
    kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
    for _ in $(seq 1 30); do
      kill -0 "${SERVER_PID}" 2>/dev/null || break
      sleep 1
    done
    kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
  fi
  SERVER_PID=""
}
trap cleanup EXIT INT TERM

IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU allocation is required}"
if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then
  echo "ERROR: expected ${TP} GPUs, got ${CUDA_VISIBLE_DEVICES}" >&2
  exit 1
fi

case "${PREFIX_CACHING}" in
  true)
    PREFIX_CACHING_FLAG="--enable-prefix-caching"
    ;;
  false)
    PREFIX_CACHING_FLAG="--no-enable-prefix-caching"
    ;;
  *)
    echo "ERROR: PREFIX_CACHING must be true or false, got ${PREFIX_CACHING}" >&2
    exit 1
    ;;
esac

case "${ENABLE_EXPERT_PARALLEL}" in
  true) EP_FLAG=(--enable-expert-parallel) ;;
  false) EP_FLAG=() ;;
  *) echo "ERROR: ENABLE_EXPERT_PARALLEL must be true or false" >&2; exit 1 ;;
esac
case "${DISABLE_CUSTOM_ALL_REDUCE}" in
  true) CUSTOM_AR_FLAG=(--disable-custom-all-reduce) ;;
  false) CUSTOM_AR_FLAG=() ;;
  *) echo "ERROR: DISABLE_CUSTOM_ALL_REDUCE must be true or false" >&2; exit 1 ;;
esac
QUANT_FLAG=()
if [[ -n "${MODEL_QUANTIZATION}" ]]; then
  QUANT_FLAG=(--quantization "${MODEL_QUANTIZATION}")
fi

REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")"
echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}"
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \
  ../frontier-phase-factorial-v0/qwen30_prefill_client.py \
  > "${OUTPUT_ROOT}/provenance/source.sha256"
sha256sum "${REQUESTS_FILE}" > "${OUTPUT_ROOT}/provenance/requests.sha256"
sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256"
nvidia-smi --query-gpu=index,name,uuid,driver_version --format=csv,noheader \
  > "${OUTPUT_ROOT}/provenance/gpus.csv"

export TOKENIZERS_PARALLELISM=false
export VLLM_USE_V1=1
export TORCH_CUDA_ARCH_LIST=9.0
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
export FLASHINFER_WORKSPACE_BASE
ulimit -n 65536

setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
  --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \
  --tensor-parallel-size "${TP}" --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
  --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
  "${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \
  "${CUSTOM_AR_FLAG[@]}" "${QUANT_FLAG[@]}" "${EP_FLAG[@]}" \
  > "${OUTPUT_ROOT}/logs/server.log" 2>&1 &
SERVER_PID=$!
READY=0
for _ in $(seq 1 "${SERVER_READY_ATTEMPTS}"); do
  if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \
    > "${OUTPUT_ROOT}/results/models.json" 2>/dev/null; then
    READY=1
    break
  fi
  if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
    tail -200 "${OUTPUT_ROOT}/logs/server.log"
    exit 1
  fi
  sleep 3
done
if [[ "${READY}" -ne 1 ]]; then
  echo "ERROR server did not become ready within $((SERVER_READY_ATTEMPTS * 3)) seconds" >&2
  tail -200 "${OUTPUT_ROOT}/logs/server.log"
  exit 1
fi

# Warm execution kernels with four prefix-disjoint requests, then start the
# measured replay from an empty scheduler queue.  The unrelated 2K-token KV
# footprint remains explicit and negligible relative to the configured cache.
"${VENV_ROOT}/bin/python" ../frontier-phase-factorial-v0/qwen30_prefill_client.py \
  --port "${SERVER_PORT}" --served-model "${SERVED_MODEL}" \
  --model-path "${MODEL_ROOT}" --rate 1 --requests 4 --input-tokens 512 \
  --output-tokens 1 --output "${OUTPUT_ROOT}/results/warmup.json"

"${VENV_ROOT}/bin/python" "${EXACT_TRACE_CLIENT}" \
  --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \
  --served-model "${SERVED_MODEL}" \
  --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \
  --timeout-seconds "${CLIENT_TIMEOUT_SECONDS}"

cleanup
find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
  | sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256"
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
echo "QWEN30_EXACT_TRACE_REAL_ANCHOR_COMPLETE"