Invalidate prior A/B results + add proper experiment harness
Prior cross-machine comparison (commit 1e86285) was invalid: dash0
baseline used warm instances with residual KV cache, inflating TTFT
by 2x. Evidence: inst_7 APC=68.3% impossible from 25 cold-start
requests; WARM TTFT p90=3.3s vs fresh=0.26s.
Fair same-machine comparison (both fresh restart on dash0):
Baseline: TTFT50=1.075 TPOT90=0.076 E2E50=5.075 OK=198/200
Elastic P2P: TTFT50=1.018 TPOT90=0.085 E2E50=6.977 OK=195/200
Elastic is WORSE due to Mooncake kv_both memory overhead.
Changes:
- REPORT.md: rewrite §3-4 with corrected results, add §3.5 errata
- pd_separation_analysis.md: update elastic TL;DR with correct numbers
- cache_aware_proxy.py: fix double-decrement bugs in offload path,
add 120s prefill timeout with co-located fallback (HEAVY_COLO_FALLBACK)
- bench.sh: standardized experiment harness with guaranteed GPU cleanup
and fresh-state verification (nvidia-smi check before start)
- run_elastic_stability_test.sh: two-phase elastic vs baseline test
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
300
scripts/bench.sh
Executable file
300
scripts/bench.sh
Executable file
@@ -0,0 +1,300 @@
|
||||
#!/bin/bash
|
||||
# Standardized single-experiment harness with guaranteed fresh state.
|
||||
#
|
||||
# GUARANTEES:
|
||||
# 1. All GPU processes killed before start (verified via nvidia-smi)
|
||||
# 2. All GPU processes killed after finish (clean for next experiment)
|
||||
# 3. Fresh vLLM instances + proxy for every run
|
||||
# 4. All outputs saved to outputs/<tag>/ with metrics, breakdown, APC, GPU snapshot
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/bench.sh --tag my_experiment --mode baseline
|
||||
# bash scripts/bench.sh --tag my_experiment --mode elastic
|
||||
# bash scripts/bench.sh --tag my_experiment --mode baseline --policy lmetric
|
||||
# bash scripts/bench.sh --tag my_experiment --mode elastic --requests 1000
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}"
|
||||
PYTHON="$VENV/python"
|
||||
VLLM="$VENV/vllm"
|
||||
MODEL="${MODEL_PATH:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
||||
TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
|
||||
|
||||
# Defaults
|
||||
TAG=""
|
||||
MODE="baseline" # baseline | elastic
|
||||
POLICY="linear" # linear | lmetric
|
||||
N_INSTANCES=8
|
||||
BASE_PORT=8000
|
||||
PROXY_PORT=9090
|
||||
REQUESTS=200
|
||||
TIME_SCALE=20
|
||||
MAX_SESSIONS=8
|
||||
HEAVY_THRESHOLD=20000
|
||||
|
||||
# Parse args
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--tag) TAG="$2"; shift 2 ;;
|
||||
--mode) MODE="$2"; shift 2 ;;
|
||||
--policy) POLICY="$2"; shift 2 ;;
|
||||
--requests) REQUESTS="$2"; shift 2 ;;
|
||||
--time-scale) TIME_SCALE="$2"; shift 2 ;;
|
||||
--sessions) MAX_SESSIONS="$2"; shift 2 ;;
|
||||
--heavy-threshold) HEAVY_THRESHOLD="$2"; shift 2 ;;
|
||||
*) echo "Unknown: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "Usage: bench.sh --tag NAME --mode {baseline|elastic} [--policy {linear|lmetric}] [--requests N]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OUTDIR="$PROJECT_DIR/outputs/$TAG"
|
||||
if [ -d "$OUTDIR" ] && [ -f "$OUTDIR/metrics.jsonl" ]; then
|
||||
echo "[ERROR] Output directory $OUTDIR already exists with data. Use a different --tag."
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$OUTDIR"
|
||||
|
||||
# Save experiment config
|
||||
cat > "$OUTDIR/config.json" << CONF
|
||||
{
|
||||
"tag": "$TAG",
|
||||
"mode": "$MODE",
|
||||
"policy": "$POLICY",
|
||||
"model": "$MODEL",
|
||||
"n_instances": $N_INSTANCES,
|
||||
"requests": $REQUESTS,
|
||||
"time_scale": $TIME_SCALE,
|
||||
"max_sessions": $MAX_SESSIONS,
|
||||
"heavy_threshold": $HEAVY_THRESHOLD,
|
||||
"timestamp": "$(date -Iseconds)",
|
||||
"hostname": "$(hostname)"
|
||||
}
|
||||
CONF
|
||||
|
||||
# ─── GPU Cleanup (verified) ────────────────────────────────────────────────
|
||||
|
||||
cleanup_gpu() {
|
||||
echo "[cleanup] Killing all vLLM/proxy processes..."
|
||||
for p in $(ps aux | grep -E 'vllm serve|cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 3
|
||||
# Kill any remaining GPU holders
|
||||
local gpu_pids
|
||||
gpu_pids=$(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true)
|
||||
if [ -n "$gpu_pids" ]; then
|
||||
echo "[cleanup] Killing GPU-holding PIDs: $gpu_pids"
|
||||
echo "$gpu_pids" | xargs -r kill -9 2>/dev/null || true
|
||||
sleep 5
|
||||
fi
|
||||
# Verify GPUs are free
|
||||
local used
|
||||
used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | awk '{s+=$1} END{print s}')
|
||||
if [ "${used:-0}" -gt 100 ]; then
|
||||
echo "[ERROR] GPUs still have ${used}MB allocated after cleanup. Aborting."
|
||||
nvidia-smi --query-gpu=index,memory.used --format=csv,noheader
|
||||
exit 1
|
||||
fi
|
||||
echo "[cleanup] All GPUs verified free."
|
||||
}
|
||||
|
||||
# ─── Launch vLLM instances ─────────────────────────────────────────────────
|
||||
|
||||
launch_instances() {
|
||||
echo "[launch] Starting $N_INSTANCES vLLM instances (mode=$MODE)..."
|
||||
|
||||
local kv_config=""
|
||||
if [ "$MODE" = "elastic" ]; then
|
||||
kv_config='--kv-transfer-config {"kv_connector":"MooncakeConnector","kv_role":"kv_both"}'
|
||||
fi
|
||||
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local master=$((29500 + i))
|
||||
local logfile="$OUTDIR/vllm_inst_${i}.log"
|
||||
|
||||
local env_prefix="MASTER_PORT=$master CUDA_VISIBLE_DEVICES=$i"
|
||||
if [ "$MODE" = "elastic" ]; then
|
||||
env_prefix="VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998 + i)) $env_prefix"
|
||||
fi
|
||||
|
||||
eval "$env_prefix $VLLM serve '$MODEL' \
|
||||
--host 0.0.0.0 --port $port \
|
||||
--tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
$kv_config \
|
||||
> '$logfile' 2>&1 &"
|
||||
|
||||
echo " inst_$i: GPU=$i port=$port"
|
||||
sleep 2 # stagger to avoid port collision
|
||||
done
|
||||
|
||||
# Wait for health
|
||||
echo "[launch] Waiting for instances to become healthy..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$port/health" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 120 ]; then
|
||||
echo "[FAIL] Instance $i (port $port) failed to start. Log:"
|
||||
tail -10 "$OUTDIR/vllm_inst_${i}.log"
|
||||
cleanup_gpu
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo " inst_$i healthy"
|
||||
done
|
||||
|
||||
# Wait for bootstrap (elastic only)
|
||||
if [ "$MODE" = "elastic" ]; then
|
||||
echo "[launch] Waiting for Mooncake bootstrap servers..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local bp=$((8998 + i))
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$bp/query" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 60 ]; then
|
||||
echo "[FAIL] Bootstrap $bp failed"
|
||||
cleanup_gpu
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " bootstrap $bp ready"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Launch proxy ──────────────────────────────────────────────────────────
|
||||
|
||||
launch_proxy() {
|
||||
echo "[proxy] Starting (mode=$MODE, policy=$POLICY)..."
|
||||
local combined_args=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))"
|
||||
done
|
||||
|
||||
local extra_args="--policy $POLICY"
|
||||
if [ "$MODE" = "elastic" ]; then
|
||||
local bp_list=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
bp_list="${bp_list:+$bp_list,}$((8998 + i))"
|
||||
done
|
||||
extra_args="$extra_args --offload --heavy-threshold $HEAVY_THRESHOLD --bootstrap-ports $bp_list"
|
||||
fi
|
||||
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined $combined_args \
|
||||
--port $PROXY_PORT \
|
||||
$extra_args \
|
||||
> "$OUTDIR/proxy.log" 2>&1 &
|
||||
|
||||
# Wait for proxy
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$PROXY_PORT/stats" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 30 ]; then
|
||||
echo "[FAIL] Proxy failed to start"
|
||||
cleanup_gpu
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "[proxy] Ready on port $PROXY_PORT"
|
||||
}
|
||||
|
||||
# ─── Run benchmark ─────────────────────────────────────────────────────────
|
||||
|
||||
run_benchmark() {
|
||||
echo "[bench] Running $REQUESTS requests (time_scale=$TIME_SCALE, sessions=$MAX_SESSIONS)..."
|
||||
$PYTHON -m replayer \
|
||||
--trace "$TRACE" \
|
||||
--output "$OUTDIR/metrics.jsonl" \
|
||||
--endpoint "http://localhost:$PROXY_PORT" \
|
||||
--model "$MODEL" \
|
||||
--time-scale "$TIME_SCALE" \
|
||||
--max-inflight-sessions "$MAX_SESSIONS" \
|
||||
--request-limit "$REQUESTS" \
|
||||
-v 2>&1 | tee "$OUTDIR/replayer.log"
|
||||
}
|
||||
|
||||
# ─── Collect artifacts ─────────────────────────────────────────────────────
|
||||
|
||||
collect_artifacts() {
|
||||
echo "[collect] Saving artifacts..."
|
||||
curl -sf "http://localhost:$PROXY_PORT/breakdown" > "$OUTDIR/breakdown.json" 2>/dev/null || true
|
||||
curl -sf "http://localhost:$PROXY_PORT/stats" > "$OUTDIR/stats.json" 2>/dev/null || true
|
||||
nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total \
|
||||
--format=csv > "$OUTDIR/gpu_snapshot.csv" 2>/dev/null || true
|
||||
|
||||
# APC from vLLM logs
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
pch=$(grep "Prefix cache hit rate" "$OUTDIR/vllm_inst_${i}.log" 2>/dev/null | tail -1 | grep -oP "Prefix cache hit rate: \K[0-9.]+" || echo "0")
|
||||
ech=$(grep "External prefix cache hit rate" "$OUTDIR/vllm_inst_${i}.log" 2>/dev/null | tail -1 | grep -oP "External prefix cache hit rate: \K[0-9.]+" || echo "")
|
||||
ext_str=""
|
||||
[ -n "$ech" ] && ext_str=" ext=$ech%"
|
||||
echo "inst_$i: prefix=$pch%$ext_str"
|
||||
done | tee "$OUTDIR/apc.txt"
|
||||
}
|
||||
|
||||
# ─── Summary ───────────────────────────────────────────────────────────────
|
||||
|
||||
print_summary() {
|
||||
$PYTHON -c "
|
||||
import json
|
||||
rows = [json.loads(l) for l in open('$OUTDIR/metrics.jsonl')]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
err = [r for r in rows if r.get('error')]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
|
||||
tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e2es = sorted([r['latency_s'] for r in ok])
|
||||
print()
|
||||
print('=' * 70)
|
||||
print(' RESULT: $TAG ($MODE, $POLICY)')
|
||||
print('=' * 70)
|
||||
print(' OK=%d/%d (%.1f%%) TTFT50=%.3f TTFT90=%.3f TPOT90=%.4f E2E50=%.3f' % (
|
||||
len(ok), len(rows), len(ok)*100/len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)))
|
||||
for lo,hi,cl in [(0,5000,'WARM'),(5000,20000,'MEDIUM'),(20000,200000,'HEAVY')]:
|
||||
sub = [r for r in ok if lo <= r['input_length'] < hi and r.get('ttft_s')]
|
||||
if sub:
|
||||
t = sorted([r['ttft_s'] for r in sub])
|
||||
tp = sorted([r['tpot_s'] for r in sub if r.get('tpot_s') and r['tpot_s']>0])
|
||||
print(' %-8s n=%3d TTFT50=%.3f TTFT90=%.3f TPOT90=%.4f' % (
|
||||
cl, len(sub), p(t,.5), p(t,.9), p(tp,.9) if tp else 0))
|
||||
if err:
|
||||
print(' Errors (%d):' % len(err))
|
||||
for e in err[:5]:
|
||||
print(' input=%d %s' % (e['input_length'], str(e.get('error',''))[:60]))
|
||||
print(' Output: $OUTDIR/')
|
||||
print('=' * 70)
|
||||
"
|
||||
}
|
||||
|
||||
# ─── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "================================================================"
|
||||
echo " bench.sh: $TAG"
|
||||
echo " mode=$MODE policy=$POLICY requests=$REQUESTS"
|
||||
echo " $(date)"
|
||||
echo "================================================================"
|
||||
|
||||
cleanup_gpu
|
||||
launch_instances
|
||||
launch_proxy
|
||||
run_benchmark
|
||||
collect_artifacts
|
||||
print_summary
|
||||
cleanup_gpu
|
||||
|
||||
echo "[done] $(date)"
|
||||
@@ -361,13 +361,22 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h
|
||||
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||
|
||||
|
||||
PREFILL_TIMEOUT_S = 120 # max seconds to wait for P-instance prefill
|
||||
|
||||
|
||||
async def _handle_heavy_offload(api, req_data, headers, token_ids, input_length,
|
||||
p_inst, d_inst, breakdown):
|
||||
"""HEAVY request: prefill on p_inst, KV via Mooncake, decode on d_inst."""
|
||||
"""HEAVY request: prefill on p_inst, KV via Mooncake, decode on d_inst.
|
||||
|
||||
On prefill timeout/failure, falls back to co-located decode on d_inst.
|
||||
"""
|
||||
global _offload_inflight
|
||||
request_id = headers.get("X-Request-Id", "")
|
||||
estimated_new = breakdown.get("estimated_new_tokens", 0)
|
||||
|
||||
# Step 1: Await prefill on p_inst (ongoing_tokens already reserved by caller)
|
||||
breakdown["t_prefill_sent"] = _time.monotonic()
|
||||
prefill_ok = False
|
||||
try:
|
||||
prefill_data = req_data.copy()
|
||||
prefill_data["kv_transfer_params"] = {
|
||||
@@ -381,25 +390,56 @@ async def _handle_heavy_offload(api, req_data, headers, token_ids, input_length,
|
||||
prefill_data.pop("stream_options", None)
|
||||
|
||||
p_headers = {**headers, "X-data-parallel-rank": "0"}
|
||||
resp = await p_inst.client.post(api, json=prefill_data, headers=p_headers)
|
||||
resp = await asyncio.wait_for(
|
||||
p_inst.client.post(api, json=prefill_data, headers=p_headers),
|
||||
timeout=PREFILL_TIMEOUT_S,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
await resp.aclose()
|
||||
p_inst.record_prefix(token_ids)
|
||||
breakdown["t_prefill_done"] = _time.monotonic()
|
||||
prefill_ok = True
|
||||
except Exception as e:
|
||||
breakdown["t_prefill_done"] = _time.monotonic()
|
||||
breakdown["error"] = str(e)
|
||||
_breakdown_log.append(breakdown)
|
||||
global _offload_inflight
|
||||
_offload_inflight = max(0, _offload_inflight - 1)
|
||||
p_inst.num_requests -= 1
|
||||
raise HTTPException(status_code=502, detail="Prefill failed: %s" % e)
|
||||
breakdown["prefill_error"] = str(e)
|
||||
finally:
|
||||
# Always release P-instance resources exactly once
|
||||
p_inst.ongoing_tokens -= input_length
|
||||
p_inst.pending_prefill_tokens -= breakdown.get("estimated_new_tokens", 0)
|
||||
p_inst.pending_prefill_tokens -= estimated_new
|
||||
p_inst.num_requests -= 1
|
||||
_offload_inflight = max(0, _offload_inflight - 1)
|
||||
|
||||
p_inst.num_requests -= 1
|
||||
if not prefill_ok:
|
||||
# Fallback: co-located prefill+decode on d_inst (no KV transfer)
|
||||
breakdown["route_class"] = "HEAVY_COLO_FALLBACK"
|
||||
d_inst.ongoing_tokens += input_length
|
||||
d_inst.pending_prefill_tokens += estimated_new
|
||||
d_inst.num_requests += 1
|
||||
|
||||
async def generate_fallback():
|
||||
prefill_done = False
|
||||
try:
|
||||
async with d_inst.client.stream("POST", api, json=req_data, headers=headers) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if not prefill_done:
|
||||
d_inst.pending_prefill_tokens -= estimated_new
|
||||
d_inst.ongoing_decode_tokens += input_length
|
||||
breakdown["t_first_token"] = _time.monotonic()
|
||||
prefill_done = True
|
||||
yield chunk
|
||||
d_inst.record_prefix(token_ids)
|
||||
finally:
|
||||
if not prefill_done:
|
||||
d_inst.pending_prefill_tokens -= estimated_new
|
||||
else:
|
||||
d_inst.ongoing_decode_tokens -= input_length
|
||||
d_inst.ongoing_tokens -= input_length
|
||||
d_inst.num_requests -= 1
|
||||
breakdown["t_done"] = _time.monotonic()
|
||||
_breakdown_log.append(breakdown)
|
||||
|
||||
return StreamingResponse(generate_fallback(), media_type="text/event-stream")
|
||||
|
||||
# Step 2: Stream decode on d_inst (pulls KV from Mooncake)
|
||||
d_inst.ongoing_tokens += input_length
|
||||
|
||||
329
scripts/run_elastic_stability_test.sh
Executable file
329
scripts/run_elastic_stability_test.sh
Executable file
@@ -0,0 +1,329 @@
|
||||
#!/bin/bash
|
||||
# Elastic P2P stability test: runs 200-request benchmark with offload mode
|
||||
# and baseline mode, then compares success rates.
|
||||
#
|
||||
# Must be run on dash0 (8 GPUs with Mooncake support).
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_elastic_stability_test.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}"
|
||||
PYTHON="$VENV/python"
|
||||
VLLM="$VENV/vllm"
|
||||
|
||||
MODEL="${MODEL_PATH:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
||||
TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
|
||||
N_INSTANCES=8
|
||||
BASE_PORT=8000
|
||||
PROXY_PORT=9090
|
||||
REQUEST_LIMIT=200
|
||||
TIME_SCALE=20
|
||||
MAX_INFLIGHT=8
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
OUT_ELASTIC="$PROJECT_DIR/outputs/elastic_stability_${TIMESTAMP}"
|
||||
OUT_BASELINE="$PROJECT_DIR/outputs/baseline_stability_${TIMESTAMP}"
|
||||
|
||||
# ─── Helper functions ────────────────────────────────────────────────────────
|
||||
|
||||
kill_all() {
|
||||
echo "[cleanup] Killing vLLM and proxy processes..."
|
||||
for p in $(ps aux | grep 'vllm serve' | grep -v grep | awk '{print $2}' 2>/dev/null); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
for p in $(ps aux | grep 'cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 5
|
||||
echo "[cleanup] Releasing GPUs..."
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 10
|
||||
echo "[cleanup] Done."
|
||||
}
|
||||
|
||||
wait_for_instances() {
|
||||
local n=$1
|
||||
echo "[wait] Waiting for $n vLLM instances to become healthy..."
|
||||
for i in $(seq 0 $((n - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$port/health" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 120 ]; then
|
||||
echo "[FAIL] Instance $i (port $port) did not start in 600s"
|
||||
return 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo " Instance $i (port $port) healthy"
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_bootstrap() {
|
||||
echo "[wait] Waiting for Mooncake bootstrap servers..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local bp=$((8998 + i))
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$bp/query" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 60 ]; then
|
||||
echo "[FAIL] Bootstrap $bp did not start in 120s"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " Bootstrap $bp ready"
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_proxy() {
|
||||
echo "[wait] Waiting for proxy on port $PROXY_PORT..."
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$PROXY_PORT/stats" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 30 ]; then
|
||||
echo "[FAIL] Proxy did not start in 60s"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " Proxy ready"
|
||||
}
|
||||
|
||||
launch_vllm_kv_both() {
|
||||
echo ""
|
||||
echo "=== Launching $N_INSTANCES vLLM instances (kv_both) ==="
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local bp=$((8998 + i))
|
||||
local master=$((29500 + i))
|
||||
local log="/tmp/elastic_test_${i}.log"
|
||||
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$bp \
|
||||
MASTER_PORT=$master \
|
||||
CUDA_VISIBLE_DEVICES=$i \
|
||||
$VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port "$port" --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$log" 2>&1 &
|
||||
|
||||
echo " Instance $i: GPU=$i port=$port bootstrap=$bp log=$log"
|
||||
sleep 2
|
||||
done
|
||||
wait_for_instances $N_INSTANCES
|
||||
wait_for_bootstrap
|
||||
}
|
||||
|
||||
launch_vllm_baseline() {
|
||||
echo ""
|
||||
echo "=== Launching $N_INSTANCES vLLM instances (baseline, no Mooncake) ==="
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local master=$((29500 + i))
|
||||
local log="/tmp/baseline_test_${i}.log"
|
||||
|
||||
MASTER_PORT=$master \
|
||||
CUDA_VISIBLE_DEVICES=$i \
|
||||
$VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port "$port" --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
> "$log" 2>&1 &
|
||||
|
||||
echo " Instance $i: GPU=$i port=$port log=$log"
|
||||
sleep 2
|
||||
done
|
||||
wait_for_instances $N_INSTANCES
|
||||
}
|
||||
|
||||
launch_proxy_elastic() {
|
||||
echo ""
|
||||
echo "=== Starting proxy (elastic offload mode) ==="
|
||||
local combined_args=""
|
||||
local bp_list=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))"
|
||||
bp_list="${bp_list:+$bp_list,}$((8998 + i))"
|
||||
done
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined $combined_args \
|
||||
--bootstrap-ports "$bp_list" \
|
||||
--offload --heavy-threshold 20000 \
|
||||
--port $PROXY_PORT \
|
||||
> /tmp/proxy_elastic.log 2>&1 &
|
||||
wait_for_proxy
|
||||
}
|
||||
|
||||
launch_proxy_baseline() {
|
||||
echo ""
|
||||
echo "=== Starting proxy (baseline, no offload) ==="
|
||||
local combined_args=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))"
|
||||
done
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined $combined_args \
|
||||
--port $PROXY_PORT \
|
||||
> /tmp/proxy_baseline.log 2>&1 &
|
||||
wait_for_proxy
|
||||
}
|
||||
|
||||
run_benchmark() {
|
||||
local tag=$1
|
||||
local output_dir=$2
|
||||
mkdir -p "$output_dir"
|
||||
|
||||
echo ""
|
||||
echo "=== Running benchmark: $tag ($REQUEST_LIMIT requests) ==="
|
||||
$PYTHON -m replayer \
|
||||
--trace "$TRACE" \
|
||||
--output "$output_dir/metrics.jsonl" \
|
||||
--endpoint "http://localhost:$PROXY_PORT" \
|
||||
--model "$MODEL" \
|
||||
--time-scale "$TIME_SCALE" \
|
||||
--max-inflight-sessions "$MAX_INFLIGHT" \
|
||||
--request-limit "$REQUEST_LIMIT" \
|
||||
-v 2>&1 | tee "$output_dir/replayer.log"
|
||||
|
||||
# Save proxy breakdown and stats
|
||||
curl -sf "http://localhost:$PROXY_PORT/breakdown" > "$output_dir/breakdown.json" 2>/dev/null || true
|
||||
curl -sf "http://localhost:$PROXY_PORT/stats" > "$output_dir/stats.json" 2>/dev/null || true
|
||||
}
|
||||
|
||||
collect_gpu_util() {
|
||||
local output_dir=$1
|
||||
nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total \
|
||||
--format=csv > "$output_dir/gpu_snapshot.csv" 2>/dev/null || true
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
local label=$1
|
||||
local output_dir=$2
|
||||
local metrics="$output_dir/metrics.jsonl"
|
||||
|
||||
if [ ! -f "$metrics" ]; then
|
||||
echo " [$label] No metrics file found!"
|
||||
return
|
||||
fi
|
||||
|
||||
# Count total, success, error from metrics JSONL
|
||||
local total=$(wc -l < "$metrics")
|
||||
local success=$(grep -c '"error":null\|"error": null' "$metrics" 2>/dev/null || grep -c '"ttft":[0-9]' "$metrics" 2>/dev/null || echo 0)
|
||||
local errors=$((total - success))
|
||||
local rate="N/A"
|
||||
if [ "$total" -gt 0 ]; then
|
||||
rate=$(awk "BEGIN{printf \"%.1f\", ($success/$total)*100}")
|
||||
fi
|
||||
|
||||
echo " [$label]"
|
||||
echo " Total requests: $total"
|
||||
echo " Successful: $success"
|
||||
echo " Errors: $errors"
|
||||
echo " Success rate: ${rate}%"
|
||||
|
||||
# Print summary.json if it exists
|
||||
local summary="$output_dir/metrics.summary.json"
|
||||
if [ -f "$summary" ]; then
|
||||
echo " Summary: $(cat "$summary")"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "================================================================"
|
||||
echo " Elastic P2P Stability Test"
|
||||
echo " $(date)"
|
||||
echo " Model: $MODEL"
|
||||
echo " Requests: $REQUEST_LIMIT"
|
||||
echo " Output: elastic → $OUT_ELASTIC"
|
||||
echo " baseline → $OUT_BASELINE"
|
||||
echo "================================================================"
|
||||
|
||||
# Sanity checks
|
||||
if [ ! -f "$TRACE" ]; then
|
||||
echo "[ERROR] Trace file not found: $TRACE"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$PYTHON" ]; then
|
||||
echo "[ERROR] Python not found: $PYTHON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ─── Phase 1: Elastic P2P offload ────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "############################################################"
|
||||
echo " Phase 1: Elastic P2P Offload"
|
||||
echo "############################################################"
|
||||
|
||||
kill_all
|
||||
launch_vllm_kv_both
|
||||
launch_proxy_elastic
|
||||
collect_gpu_util "$OUT_ELASTIC"
|
||||
run_benchmark "elastic_p2p" "$OUT_ELASTIC"
|
||||
|
||||
echo ""
|
||||
echo "[phase1] Saving APC stats..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
port=$((BASE_PORT + i))
|
||||
curl -sf "http://127.0.0.1:$port/metrics" 2>/dev/null \
|
||||
| grep -E 'vllm:cache_hit|prefix_cache' \
|
||||
>> "$OUT_ELASTIC/apc_metrics.txt" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# ─── Phase 2: Baseline (no offload) ─────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "############################################################"
|
||||
echo " Phase 2: Baseline (no offload, no Mooncake)"
|
||||
echo "############################################################"
|
||||
|
||||
kill_all
|
||||
launch_vllm_baseline
|
||||
launch_proxy_baseline
|
||||
collect_gpu_util "$OUT_BASELINE"
|
||||
run_benchmark "baseline" "$OUT_BASELINE"
|
||||
|
||||
echo ""
|
||||
echo "[phase2] Saving APC stats..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
port=$((BASE_PORT + i))
|
||||
curl -sf "http://127.0.0.1:$port/metrics" 2>/dev/null \
|
||||
| grep -E 'vllm:cache_hit|prefix_cache' \
|
||||
>> "$OUT_BASELINE/apc_metrics.txt" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# ─── Cleanup ─────────────────────────────────────────────────────────────────
|
||||
|
||||
kill_all
|
||||
|
||||
# ─── Comparison ──────────────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Results Comparison"
|
||||
echo "================================================================"
|
||||
|
||||
print_summary "Elastic P2P" "$OUT_ELASTIC"
|
||||
echo ""
|
||||
print_summary "Baseline" "$OUT_BASELINE"
|
||||
|
||||
echo ""
|
||||
echo "Detailed outputs:"
|
||||
echo " Elastic: $OUT_ELASTIC/"
|
||||
echo " Baseline: $OUT_BASELINE/"
|
||||
echo ""
|
||||
echo "Breakdown analysis:"
|
||||
echo " python scripts/analyze_breakdown.py $OUT_ELASTIC/breakdown.json"
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Done. $(date)"
|
||||
echo "================================================================"
|
||||
Reference in New Issue
Block a user