Track simulator fidelity experiment artifacts

2026-07-19 15:31:09 +08:00
parent e0ea7e9961
commit 4c8d581a5b
115 changed files with 42355 additions and 0 deletions
--- a/runs/frontier-multicase-sufficiency-v1/run_t0_real_config.sh
+++ b/runs/frontier-multicase-sufficiency-v1/run_t0_real_config.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+TP="${TP:?TP is required}"
+MNS="${MNS:?MNS is required}"
+MBT="${MBT:?MBT is required}"
+RATES="${RATES:?RATES is required}"
+SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh-frontier-vllm0102-smoke/.venv}"
+MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
+SERVED_MODEL="qwen3-235b-t0-surface"
+SERVER_PID=""
+
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
+exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1
+
+cleanup() {
+  if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+    kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
+    for _ in $(seq 1 30); do
+      kill -0 "${SERVER_PID}" 2>/dev/null || break
+      sleep 1
+    done
+    kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
+  fi
+  SERVER_PID=""
+}
+trap cleanup EXIT INT TERM
+
+IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:-}"
+if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then
+  echo "ERROR: expected ${TP} allocated GPUs, got ${CUDA_VISIBLE_DEVICES:-unset}" >&2
+  exit 1
+fi
+read -r -a RATE_ARRAY <<< "${RATES}"
+if [[ "${#RATE_ARRAY[@]}" -lt 1 ]]; then
+  echo "ERROR: at least one frozen rate is required" >&2
+  exit 1
+fi
+
+echo "REAL_CONFIG_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} config=TP${TP}_MNS${MNS}_MBT${MBT} rates=${RATES// /,} repeats=2 requests_per_anchor=64 isolation=fresh_server_per_anchor target_warmup=min32_max4_ceil_rate_x20 trace=fixed_ISL2048_OSL128 prefix=off runtime=community_vllm_0.10.2 execution=eager kv=BF16 output=${OUTPUT_ROOT}"
+date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
+sha256sum run_t0_real_config.sh t0_rate_client.py > "${OUTPUT_ROOT}/provenance/source.sha256"
+"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${MBT}" "${RATES}" \
+  > "${OUTPUT_ROOT}/provenance/contract.json" <<'PY'
+import importlib.metadata as metadata
+import json
+import platform
+import sys
+
+tp, mns, mbt, rates = sys.argv[1:]
+print(json.dumps({
+    "python": platform.python_version(),
+    "torch": metadata.version("torch"),
+    "transformers": metadata.version("transformers"),
+    "vllm": metadata.version("vllm"),
+    "config": {"tp": int(tp), "mns": int(mns), "mbt": int(mbt)},
+    "rates": [float(value) for value in rates.split()],
+    "rounds": 2,
+    "requests_per_anchor": 64,
+    "anchor_isolation": "fresh_server_per_rate_per_round",
+    "target_rate_warmup_requests": "min(32, max(4, ceil(rate * 20)))",
+    "ttft_slo_ms": 1256.0,
+    "tpot_slos_ms": [40.0, 120.0, 150.0, 180.0],
+}, indent=2, sort_keys=True))
+PY
+nvidia-smi --query-gpu=index,name,uuid,driver_version --format=csv,noheader \
+  > "${OUTPUT_ROOT}/provenance/gpus.csv"
+sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256"
+
+export TOKENIZERS_PARALLELISM=false
+export VLLM_USE_V1=1
+export VLLM_ATTENTION_BACKEND=FLASHINFER
+export TORCH_CUDA_ARCH_LIST=9.0
+
+EXTRA_FLAGS=()
+NUM_BLOCKS=26101
+if [[ "${TP}" -eq 8 ]]; then
+  EXTRA_FLAGS+=(--enable-expert-parallel)
+  NUM_BLOCKS=62351
+fi
+
+for ROUND in 1 2; do
+  ROUND_ROOT="${OUTPUT_ROOT}/round${ROUND}"
+  mkdir -p "${ROUND_ROOT}/logs" "${ROUND_ROOT}/results"
+  ORDERED_RATES=("${RATE_ARRAY[@]}")
+  if [[ "${ROUND}" -eq 2 ]]; then
+    ORDERED_RATES=()
+    for ((index=${#RATE_ARRAY[@]}-1; index>=0; index--)); do ORDERED_RATES+=("${RATE_ARRAY[index]}"); done
+  fi
+  for RATE in "${ORDERED_RATES[@]}"; do
+    KEY="$(printf 'r%.2f' "${RATE}" | tr '.' 'p')"
+    SERVER_LOG="${ROUND_ROOT}/logs/server_${KEY}.log"
+    setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
+      --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \
+      --tensor-parallel-size "${TP}" --disable-custom-all-reduce --quantization fp8 \
+      --gpu-memory-utilization 0.80 --num-gpu-blocks-override "${NUM_BLOCKS}" \
+      --kv-cache-dtype auto --max-model-len 40960 --max-num-batched-tokens "${MBT}" \
+      --max-num-seqs "${MNS}" --no-enable-prefix-caching --enable-chunked-prefill \
+      --enforce-eager --disable-log-requests "${EXTRA_FLAGS[@]}" \
+      > "${SERVER_LOG}" 2>&1 &
+    SERVER_PID=$!
+    READY=0
+    for _ in $(seq 1 180); do
+      if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \
+        > "${ROUND_ROOT}/results/models_${KEY}.json" 2>/dev/null; then
+        READY=1
+        break
+      fi
+      if ! kill -0 "${SERVER_PID}" 2>/dev/null; then tail -200 "${SERVER_LOG}"; exit 1; fi
+      sleep 5
+    done
+    if [[ "${READY}" -ne 1 ]]; then tail -200 "${SERVER_LOG}"; exit 1; fi
+
+    WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY'
+import math
+import sys
+print(min(32, max(4, math.ceil(float(sys.argv[1]) * 20.0))))
+PY
+)"
+    "${VENV_ROOT}/bin/python" t0_rate_client.py --port "${SERVER_PORT}" \
+      --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
+      --requests "${WARMUP_REQUESTS}" \
+      --output "${ROUND_ROOT}/results/warmup_${KEY}.json"
+    "${VENV_ROOT}/bin/python" t0_rate_client.py --port "${SERVER_PORT}" \
+      --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
+      --requests 64 --output "${ROUND_ROOT}/results/${KEY}.json"
+    cleanup
+  done
+done
+
+find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
+  | sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256"
+date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
+echo "T0_REAL_CONFIG_COMPLETE"