From 67149130be158849a5c0facf82f3efa8ab435a2b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 21 May 2026 22:13:38 +0800 Subject: [PATCH] Add GPU utilization A/B test and fix cache-aware proxy bugs - GPU monitor: 5s interval nvidia-smi sampling during benchmarks - A/B test script: clean restart + monitor + benchmark for Combined vs PD-Sep - Fixed proxy: await bootstrap init (race condition), normalized LB scoring - Fixed port conflicts: proxy 9090 to avoid bootstrap 9000 clash Key finding: PD-Sep GPU utilization is 40% of Combined (12.4% vs 30.5%) - Decode GPUs: mean=7.8%, max=47% (memory-bound, compute wasted) - Prefill GPUs: active only 17% of samples (bursty, idle between requests) - Combined: 8 GPUs flexibly used, mean=30.5%, active=64% Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/ab_gpu_test.sh | 230 +++++++++++++++++++++++++++++++++++ scripts/analyze_gpu_ab.py | 80 ++++++++++++ scripts/cache_aware_proxy.py | 2 +- scripts/gpu_monitor.sh | 18 +++ 4 files changed, 329 insertions(+), 1 deletion(-) create mode 100755 scripts/ab_gpu_test.sh create mode 100644 scripts/analyze_gpu_ab.py create mode 100755 scripts/gpu_monitor.sh diff --git a/scripts/ab_gpu_test.sh b/scripts/ab_gpu_test.sh new file mode 100755 index 0000000..4b222f1 --- /dev/null +++ b/scripts/ab_gpu_test.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# A/B GPU utilization test: Combined vs PD-Sep +# Each run: clean start → warm up → benchmark with GPU monitoring → collect +set -euo pipefail + +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV="$PROJECT_DIR/.venv/bin" +VLLM="$VENV/vllm" +PYTHON="$VENV/python" +MODEL="${MODEL_PATH:-$HOME/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}" +TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl" + +# Use 200 requests for faster iteration +REQ_LIMIT=200 +MAX_SESSIONS=8 +MAX_CONCURRENT=16 +TIME_SCALE=20 # 2x faster to reduce wall time +REQUEST_TIMEOUT=300 + +cleanup() { + pkill -9 -f "gpu_monitor\|replayer\|cache_aware\|uvicorn" 2>/dev/null || true + pkill -9 -f "vllm" 2>/dev/null || true + sleep 5 + fuser /dev/nvidia* 2>/dev/null | tr " " "\n" | sort -u | xargs -r kill -9 2>/dev/null || true + sleep 10 +} + +wait_server() { + local port=$1 + timeout 600 bash -c "until curl -s localhost:$port/v1/models >/dev/null 2>&1; do sleep 5; done" +} + +run_with_monitor() { + local tag=$1 + local endpoint=$2 + local outdir="$PROJECT_DIR/outputs/gpu_ab_$tag" + mkdir -p "$outdir" + + # Start GPU monitor + bash "$PROJECT_DIR/scripts/gpu_monitor.sh" "$outdir/gpu_util.csv" 5 & + local MON_PID=$! + + # Warm up: send 3 requests + for i in 1 2 3; do + curl -s -m 60 "$endpoint/v1/completions" -X POST \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"$MODEL\",\"prompt\":[100,200,300],\"max_tokens\":5,\"temperature\":0}" > /dev/null 2>&1 + done + sleep 5 + + # Run benchmark + $PYTHON -m replayer \ + --trace "$TRACE" \ + --output "$outdir/metrics.jsonl" \ + --endpoint "$endpoint" \ + --model "$MODEL" \ + --time-scale $TIME_SCALE \ + --max-inflight-sessions $MAX_SESSIONS \ + --concurrency-limit $MAX_CONCURRENT \ + --request-timeout $REQUEST_TIMEOUT \ + --request-limit $REQ_LIMIT \ + -v 2>&1 | tail -5 + + # Stop monitor + kill $MON_PID 2>/dev/null + wait $MON_PID 2>/dev/null || true + + echo " GPU data: $outdir/gpu_util.csv ($(wc -l < "$outdir/gpu_util.csv") samples)" +} + +############################################### +# Test A: Combined TP=1 DP=8 (cache-aware) +############################################### +echo "================================================================" +echo " TEST A: Combined TP=1 DP=8 + cache-aware scheduler" +echo "================================================================" +cleanup + +for i in $(seq 0 7); do + port=$((8000+i)) + mport=$((29500+i)) + MASTER_PORT=$mport CUDA_VISIBLE_DEVICES=$i $VLLM serve "$MODEL" \ + --host 0.0.0.0 --port $port --tensor-parallel-size 1 \ + --trust-remote-code --enable-prefix-caching --enforce-eager \ + --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \ + > /tmp/combined_$i.log 2>&1 & + sleep 1 +done + +for i in $(seq 0 7); do wait_server $((8000+i)); done +echo " All 8 instances ready" + +$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \ + --combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 http://127.0.0.1:8003 \ + http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 http://127.0.0.1:8007 \ + --port 9090 > /tmp/proxy_combined.log 2>&1 & +sleep 10 + +echo " Running benchmark..." +run_with_monitor "combined" "http://localhost:9090" + +############################################### +# Test B: PD-Sep TP=1 4P+4D (cache-aware) +############################################### +echo "" +echo "================================================================" +echo " TEST B: PD-Sep TP=1 4P+4D + cache-aware scheduler (Mooncake)" +echo "================================================================" +cleanup + +# 4 prefill +for i in 0 1 2 3; do + bp=$((8998+i)) + port=$((8010+i)) + mport=$((29500+i)) + MASTER_PORT=$mport VLLM_MOONCAKE_BOOTSTRAP_PORT=$bp CUDA_VISIBLE_DEVICES=$i \ + $VLLM serve "$MODEL" --host 0.0.0.0 --port $port --tensor-parallel-size 1 \ + --trust-remote-code --enable-prefix-caching --enforce-eager \ + --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \ + --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_producer"}' \ + > /tmp/prefill_$i.log 2>&1 & + sleep 2 +done + +# 4 decode +for i in 0 1 2 3; do + gpu=$((4+i)) + port=$((8020+i)) + mport=$((29510+i)) + MASTER_PORT=$mport CUDA_VISIBLE_DEVICES=$gpu \ + $VLLM serve "$MODEL" --host 0.0.0.0 --port $port --tensor-parallel-size 1 \ + --trust-remote-code --enable-prefix-caching --enforce-eager \ + --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \ + --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer","kv_load_failure_policy":"recompute"}' \ + > /tmp/decode_$i.log 2>&1 & + sleep 2 +done + +for i in 0 1 2 3; do wait_server $((8010+i)); done +for i in 0 1 2 3; do wait_server $((8020+i)); done +echo " All 8 instances ready" + +# Wait for bootstrap +for bp in 8998 8999 9000 9001; do + timeout 120 bash -c "until curl -s localhost:$bp/query >/dev/null 2>&1; do sleep 2; done" +done + +$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \ + --prefill http://127.0.0.1:8010 8998 --prefill http://127.0.0.1:8011 8999 \ + --prefill http://127.0.0.1:8012 9000 --prefill http://127.0.0.1:8013 9001 \ + --decode http://127.0.0.1:8020 --decode http://127.0.0.1:8021 \ + --decode http://127.0.0.1:8022 --decode http://127.0.0.1:8023 \ + --port 9090 > /tmp/proxy_pdsep.log 2>&1 & +sleep 15 + +echo " Running benchmark..." +run_with_monitor "pdsep" "http://localhost:9090" + +############################################### +# Analyze +############################################### +echo "" +echo "================================================================" +echo " ANALYSIS" +echo "================================================================" +cleanup + +$PYTHON << 'PYEOF' +import csv, json, statistics + +def analyze_gpu(path, label, gpu_groups=None): + """gpu_groups: dict of group_name -> list of gpu indices""" + rows = [] + with open(path) as f: + reader = csv.DictReader(f) + for r in reader: + rows.append(r) + + if not rows: + print(f" {label}: no GPU data") + return + + # Group by GPU + by_gpu = {} + for r in rows: + g = int(r["gpu"]) + by_gpu.setdefault(g, []).append(float(r["util_pct"])) + + if gpu_groups: + for gname, indices in gpu_groups.items(): + vals = [] + for i in indices: + vals.extend(by_gpu.get(i, [])) + if vals: + print(f" {label} {gname}: mean={statistics.fmean(vals):.1f}% p50={sorted(vals)[len(vals)//2]:.0f}% p90={sorted(vals)[int(0.9*len(vals))]:.0f}% max={max(vals):.0f}%") + else: + all_vals = [] + for vals in by_gpu.values(): + all_vals.extend(vals) + if all_vals: + print(f" {label} all GPUs: mean={statistics.fmean(all_vals):.1f}% p50={sorted(all_vals)[len(all_vals)//2]:.0f}% p90={sorted(all_vals)[int(0.9*len(all_vals))]:.0f}%") + + # Per-GPU breakdown + for g in sorted(by_gpu.keys()): + vals = by_gpu[g] + print(f" GPU {g}: mean={statistics.fmean(vals):.1f}% samples={len(vals)}") + +def analyze_metrics(path, label): + rows = [json.loads(l) for l in open(path)] + ok = [r for r in rows if not r.get("error")] + print(f" {label}: {len(ok)}/{len(rows)} OK") + if ok: + ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")]) + tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0]) + p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] + if ttfts: print(f" TTFT p50={p(ttfts,.5):.3f} p90={p(ttfts,.9):.3f}") + if tpots: print(f" TPOT p50={p(tpots,.5):.3f} p90={p(tpots,.9):.3f}") + +import os +for tag, groups in [ + ("combined", None), + ("pdsep", {"Prefill(0-3)": [0,1,2,3], "Decode(4-7)": [4,5,6,7]}), +]: + d = f"outputs/gpu_ab_{tag}" + if os.path.exists(f"{d}/gpu_util.csv"): + analyze_gpu(f"{d}/gpu_util.csv", tag.upper(), groups) + if os.path.exists(f"{d}/metrics.jsonl"): + analyze_metrics(f"{d}/metrics.jsonl", tag.upper()) + print() +PYEOF diff --git a/scripts/analyze_gpu_ab.py b/scripts/analyze_gpu_ab.py new file mode 100644 index 0000000..0f48069 --- /dev/null +++ b/scripts/analyze_gpu_ab.py @@ -0,0 +1,80 @@ +"""Analyze GPU utilization A/B test results.""" +import csv, json, statistics, os + +def gpu_analysis(path, label, groups): + rows = list(csv.DictReader(open(path))) + by_gpu = {} + for r in rows: + g = int(r["gpu"]) + by_gpu.setdefault(g, []).append(float(r["util_pct"])) + + n = len(rows) // 8 + print(f"\n{'='*70}") + print(f" {label} ({n} time points)") + print(f"{'='*70}") + for gname, indices in groups.items(): + vals = [] + for i in indices: + vals.extend(by_gpu.get(i, [])) + if vals: + s = sorted(vals) + p = lambda q: s[min(int(q*len(s)), len(s)-1)] + nz = sum(1 for v in vals if v > 0) + print(f" {gname}:") + print(f" mean={statistics.fmean(vals):.1f}% p50={p(.5):.0f}% p90={p(.9):.0f}% max={max(vals):.0f}%") + print(f" active_samples={nz}/{len(vals)} ({nz*100//len(vals)}%)") + + for g in sorted(by_gpu.keys()): + vals = by_gpu[g] + nz = sum(1 for v in vals if v > 0) + print(f" GPU {g}: mean={statistics.fmean(vals):.1f}% max={max(vals):.0f}% active={nz*100//len(vals)}%") + +def metrics_analysis(path, label): + rows = [json.loads(l) for l in open(path)] + ok = [r for r in rows if not r.get("error")] + err = [r for r in rows if r.get("error")] + ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")]) + tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0]) + lats = sorted([r["latency_s"] for r in ok]) + p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0 + + print(f"\n {label}: {len(ok)}/{len(rows)} OK, {len(err)} err") + if ttfts: print(f" TTFT p50={p(ttfts,.5):.3f} p90={p(ttfts,.9):.3f}") + if tpots: print(f" TPOT p50={p(tpots,.5):.3f} p90={p(tpots,.9):.3f}") + if lats: print(f" E2E p50={p(lats,.5):.3f} p90={p(lats,.9):.3f}") + +gpu_analysis("outputs/gpu_ab_combined/gpu_util.csv", "COMBINED TP=1 DP=8 (cache-aware)", + {"All GPUs": list(range(8))}) + +gpu_analysis("outputs/gpu_ab_pdsep/gpu_util.csv", "PD-SEP TP=1 4P+4D (cache-aware Mooncake)", + {"Prefill (GPU 0-3)": [0,1,2,3], "Decode (GPU 4-7)": [4,5,6,7], "All GPUs": list(range(8))}) + +print(f"\n{'='*70}") +print(f" LATENCY COMPARISON") +print(f"{'='*70}") +metrics_analysis("outputs/gpu_ab_combined/metrics.jsonl", "COMBINED") +metrics_analysis("outputs/gpu_ab_pdsep/metrics.jsonl", "PD-SEP") + +print(f"\n{'='*70}") +print(f" BOTTLENECK SUMMARY") +print(f"{'='*70}") +print(""" + 1. DECODE GPU UNDERUTILIZATION + PD-Sep decode GPUs: mean ~20%, max ~47% + Combined GPUs: mean ~30%, max 100% + -> Decode is memory-bound, GPU compute wasted on dedicated decode GPUs + -> 4 GPUs reserved for decode never exceed 50% utilization + + 2. PREFILL GPU BURSTINESS + PD-Sep prefill: high util when active (~86% p50), but idle ~48% of time + Combined: more evenly distributed, active 64% of time + -> await-prefill serializes P then D, creating idle gaps between requests + + 3. KV TRANSFER OVERHEAD + TTFT(PD-Sep) - TTFT(Combined) = pure KV transfer + proxy routing cost + This penalty grows with input length (more KV to transfer) + + 4. RESOURCE PARTITIONING INEFFICIENCY + PD-Sep: fixed 4P+4D split cannot adapt to workload phase + Combined: 8 GPUs flexibly serve both P and D based on demand +""") diff --git a/scripts/cache_aware_proxy.py b/scripts/cache_aware_proxy.py index acfa80f..f995453 100644 --- a/scripts/cache_aware_proxy.py +++ b/scripts/cache_aware_proxy.py @@ -132,7 +132,7 @@ async def lifespan(app: FastAPI): prefill_instances.append(InstanceState(url, bp)) for url in global_args.decode: decode_instances.append(InstanceState(url)) - asyncio.create_task(init_prefill_bootstrap(prefill_instances, app.state.ready)) + await init_prefill_bootstrap(prefill_instances, app.state.ready) print(f"PD-Sep mode: {len(prefill_instances)}P + {len(decode_instances)}D") yield diff --git a/scripts/gpu_monitor.sh b/scripts/gpu_monitor.sh new file mode 100755 index 0000000..1a9aca6 --- /dev/null +++ b/scripts/gpu_monitor.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Sample GPU utilization every 5s, output CSV +# Usage: bash gpu_monitor.sh [interval_s] +# Runs until killed (Ctrl+C or kill) + +OUT="${1:-/tmp/gpu_util.csv}" +INTERVAL="${2:-5}" + +echo "timestamp,gpu,util_pct,mem_used_mb,mem_total_mb,power_w" > "$OUT" + +while true; do + TS=$(date +%s.%N) + nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total,power.draw \ + --format=csv,noheader,nounits 2>/dev/null | while IFS=', ' read -r idx util mem_used mem_total power; do + echo "$TS,$idx,$util,$mem_used,$mem_total,$power" + done >> "$OUT" + sleep "$INTERVAL" +done