Add GPU utilization A/B test and fix cache-aware proxy bugs

- GPU monitor: 5s interval nvidia-smi sampling during benchmarks
- A/B test script: clean restart + monitor + benchmark for Combined vs PD-Sep
- Fixed proxy: await bootstrap init (race condition), normalized LB scoring
- Fixed port conflicts: proxy 9090 to avoid bootstrap 9000 clash

Key finding: PD-Sep GPU utilization is 40% of Combined (12.4% vs 30.5%)
- Decode GPUs: mean=7.8%, max=47% (memory-bound, compute wasted)
- Prefill GPUs: active only 17% of samples (bursty, idle between requests)
- Combined: 8 GPUs flexibly used, mean=30.5%, active=64%

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/ab_gpu_test.sh

@@ -0,0 +1,230 @@
+#!/bin/bash
+# A/B GPU utilization test: Combined vs PD-Sep
+# Each run: clean start → warm up → benchmark with GPU monitoring → collect
+set -euo pipefail
+
+PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+VENV="$PROJECT_DIR/.venv/bin"
+VLLM="$VENV/vllm"
+PYTHON="$VENV/python"
+MODEL="${MODEL_PATH:-$HOME/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
+TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
+
+# Use 200 requests for faster iteration
+REQ_LIMIT=200
+MAX_SESSIONS=8
+MAX_CONCURRENT=16
+TIME_SCALE=20  # 2x faster to reduce wall time
+REQUEST_TIMEOUT=300
+
+cleanup() {
+    pkill -9 -f "gpu_monitor\|replayer\|cache_aware\|uvicorn" 2>/dev/null || true
+    pkill -9 -f "vllm" 2>/dev/null || true
+    sleep 5
+    fuser /dev/nvidia* 2>/dev/null | tr " " "\n" | sort -u | xargs -r kill -9 2>/dev/null || true
+    sleep 10
+}
+
+wait_server() {
+    local port=$1
+    timeout 600 bash -c "until curl -s localhost:$port/v1/models >/dev/null 2>&1; do sleep 5; done"
+}
+
+run_with_monitor() {
+    local tag=$1
+    local endpoint=$2
+    local outdir="$PROJECT_DIR/outputs/gpu_ab_$tag"
+    mkdir -p "$outdir"
+
+    # Start GPU monitor
+    bash "$PROJECT_DIR/scripts/gpu_monitor.sh" "$outdir/gpu_util.csv" 5 &
+    local MON_PID=$!
+
+    # Warm up: send 3 requests
+    for i in 1 2 3; do
+        curl -s -m 60 "$endpoint/v1/completions" -X POST \
+            -H "Content-Type: application/json" \
+            -d "{\"model\":\"$MODEL\",\"prompt\":[100,200,300],\"max_tokens\":5,\"temperature\":0}" > /dev/null 2>&1
+    done
+    sleep 5
+
+    # Run benchmark
+    $PYTHON -m replayer \
+        --trace "$TRACE" \
+        --output "$outdir/metrics.jsonl" \
+        --endpoint "$endpoint" \
+        --model "$MODEL" \
+        --time-scale $TIME_SCALE \
+        --max-inflight-sessions $MAX_SESSIONS \
+        --concurrency-limit $MAX_CONCURRENT \
+        --request-timeout $REQUEST_TIMEOUT \
+        --request-limit $REQ_LIMIT \
+        -v 2>&1 | tail -5
+
+    # Stop monitor
+    kill $MON_PID 2>/dev/null
+    wait $MON_PID 2>/dev/null || true
+
+    echo "  GPU data: $outdir/gpu_util.csv ($(wc -l < "$outdir/gpu_util.csv") samples)"
+}
+
+###############################################
+# Test A: Combined TP=1 DP=8 (cache-aware)
+###############################################
+echo "================================================================"
+echo "  TEST A: Combined TP=1 DP=8 + cache-aware scheduler"
+echo "================================================================"
+cleanup
+
+for i in $(seq 0 7); do
+    port=$((8000+i))
+    mport=$((29500+i))
+    MASTER_PORT=$mport CUDA_VISIBLE_DEVICES=$i $VLLM serve "$MODEL" \
+        --host 0.0.0.0 --port $port --tensor-parallel-size 1 \
+        --trust-remote-code --enable-prefix-caching --enforce-eager \
+        --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
+        > /tmp/combined_$i.log 2>&1 &
+    sleep 1
+done
+
+for i in $(seq 0 7); do wait_server $((8000+i)); done
+echo "  All 8 instances ready"
+
+$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
+    --combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 http://127.0.0.1:8003 \
+               http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 http://127.0.0.1:8007 \
+    --port 9090 > /tmp/proxy_combined.log 2>&1 &
+sleep 10
+
+echo "  Running benchmark..."
+run_with_monitor "combined" "http://localhost:9090"
+
+###############################################
+# Test B: PD-Sep TP=1 4P+4D (cache-aware)
+###############################################
+echo ""
+echo "================================================================"
+echo "  TEST B: PD-Sep TP=1 4P+4D + cache-aware scheduler (Mooncake)"
+echo "================================================================"
+cleanup
+
+# 4 prefill
+for i in 0 1 2 3; do
+    bp=$((8998+i))
+    port=$((8010+i))
+    mport=$((29500+i))
+    MASTER_PORT=$mport VLLM_MOONCAKE_BOOTSTRAP_PORT=$bp CUDA_VISIBLE_DEVICES=$i \
+    $VLLM serve "$MODEL" --host 0.0.0.0 --port $port --tensor-parallel-size 1 \
+        --trust-remote-code --enable-prefix-caching --enforce-eager \
+        --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
+        --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_producer"}' \
+        > /tmp/prefill_$i.log 2>&1 &
+    sleep 2
+done
+
+# 4 decode
+for i in 0 1 2 3; do
+    gpu=$((4+i))
+    port=$((8020+i))
+    mport=$((29510+i))
+    MASTER_PORT=$mport CUDA_VISIBLE_DEVICES=$gpu \
+    $VLLM serve "$MODEL" --host 0.0.0.0 --port $port --tensor-parallel-size 1 \
+        --trust-remote-code --enable-prefix-caching --enforce-eager \
+        --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
+        --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer","kv_load_failure_policy":"recompute"}' \
+        > /tmp/decode_$i.log 2>&1 &
+    sleep 2
+done
+
+for i in 0 1 2 3; do wait_server $((8010+i)); done
+for i in 0 1 2 3; do wait_server $((8020+i)); done
+echo "  All 8 instances ready"
+
+# Wait for bootstrap
+for bp in 8998 8999 9000 9001; do
+    timeout 120 bash -c "until curl -s localhost:$bp/query >/dev/null 2>&1; do sleep 2; done"
+done
+
+$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
+    --prefill http://127.0.0.1:8010 8998 --prefill http://127.0.0.1:8011 8999 \
+    --prefill http://127.0.0.1:8012 9000 --prefill http://127.0.0.1:8013 9001 \
+    --decode http://127.0.0.1:8020 --decode http://127.0.0.1:8021 \
+    --decode http://127.0.0.1:8022 --decode http://127.0.0.1:8023 \
+    --port 9090 > /tmp/proxy_pdsep.log 2>&1 &
+sleep 15
+
+echo "  Running benchmark..."
+run_with_monitor "pdsep" "http://localhost:9090"
+
+###############################################
+# Analyze
+###############################################
+echo ""
+echo "================================================================"
+echo "  ANALYSIS"
+echo "================================================================"
+cleanup
+
+$PYTHON << 'PYEOF'
+import csv, json, statistics
+
+def analyze_gpu(path, label, gpu_groups=None):
+    """gpu_groups: dict of group_name -> list of gpu indices"""
+    rows = []
+    with open(path) as f:
+        reader = csv.DictReader(f)
+        for r in reader:
+            rows.append(r)
+
+    if not rows:
+        print(f"  {label}: no GPU data")
+        return
+
+    # Group by GPU
+    by_gpu = {}
+    for r in rows:
+        g = int(r["gpu"])
+        by_gpu.setdefault(g, []).append(float(r["util_pct"]))
+
+    if gpu_groups:
+        for gname, indices in gpu_groups.items():
+            vals = []
+            for i in indices:
+                vals.extend(by_gpu.get(i, []))
+            if vals:
+                print(f"  {label} {gname}: mean={statistics.fmean(vals):.1f}% p50={sorted(vals)[len(vals)//2]:.0f}% p90={sorted(vals)[int(0.9*len(vals))]:.0f}% max={max(vals):.0f}%")
+    else:
+        all_vals = []
+        for vals in by_gpu.values():
+            all_vals.extend(vals)
+        if all_vals:
+            print(f"  {label} all GPUs: mean={statistics.fmean(all_vals):.1f}% p50={sorted(all_vals)[len(all_vals)//2]:.0f}% p90={sorted(all_vals)[int(0.9*len(all_vals))]:.0f}%")
+
+    # Per-GPU breakdown
+    for g in sorted(by_gpu.keys()):
+        vals = by_gpu[g]
+        print(f"    GPU {g}: mean={statistics.fmean(vals):.1f}% samples={len(vals)}")
+
+def analyze_metrics(path, label):
+    rows = [json.loads(l) for l in open(path)]
+    ok = [r for r in rows if not r.get("error")]
+    print(f"  {label}: {len(ok)}/{len(rows)} OK")
+    if ok:
+        ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
+        tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
+        p = lambda v,q: v[min(int(q*len(v)),len(v)-1)]
+        if ttfts: print(f"    TTFT p50={p(ttfts,.5):.3f} p90={p(ttfts,.9):.3f}")
+        if tpots: print(f"    TPOT p50={p(tpots,.5):.3f} p90={p(tpots,.9):.3f}")
+
+import os
+for tag, groups in [
+    ("combined", None),
+    ("pdsep", {"Prefill(0-3)": [0,1,2,3], "Decode(4-7)": [4,5,6,7]}),
+]:
+    d = f"outputs/gpu_ab_{tag}"
+    if os.path.exists(f"{d}/gpu_util.csv"):
+        analyze_gpu(f"{d}/gpu_util.csv", tag.upper(), groups)
+    if os.path.exists(f"{d}/metrics.jsonl"):
+        analyze_metrics(f"{d}/metrics.jsonl", tag.upper())
+    print()
+PYEOF

scripts/analyze_gpu_ab.py

@@ -0,0 +1,80 @@
+"""Analyze GPU utilization A/B test results."""
+import csv, json, statistics, os
+
+def gpu_analysis(path, label, groups):
+    rows = list(csv.DictReader(open(path)))
+    by_gpu = {}
+    for r in rows:
+        g = int(r["gpu"])
+        by_gpu.setdefault(g, []).append(float(r["util_pct"]))
+
+    n = len(rows) // 8
+    print(f"\n{'='*70}")
+    print(f"  {label} ({n} time points)")
+    print(f"{'='*70}")
+    for gname, indices in groups.items():
+        vals = []
+        for i in indices:
+            vals.extend(by_gpu.get(i, []))
+        if vals:
+            s = sorted(vals)
+            p = lambda q: s[min(int(q*len(s)), len(s)-1)]
+            nz = sum(1 for v in vals if v > 0)
+            print(f"  {gname}:")
+            print(f"    mean={statistics.fmean(vals):.1f}%  p50={p(.5):.0f}%  p90={p(.9):.0f}%  max={max(vals):.0f}%")
+            print(f"    active_samples={nz}/{len(vals)} ({nz*100//len(vals)}%)")
+
+    for g in sorted(by_gpu.keys()):
+        vals = by_gpu[g]
+        nz = sum(1 for v in vals if v > 0)
+        print(f"    GPU {g}: mean={statistics.fmean(vals):.1f}%  max={max(vals):.0f}%  active={nz*100//len(vals)}%")
+
+def metrics_analysis(path, label):
+    rows = [json.loads(l) for l in open(path)]
+    ok = [r for r in rows if not r.get("error")]
+    err = [r for r in rows if r.get("error")]
+    ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
+    tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
+    lats = sorted([r["latency_s"] for r in ok])
+    p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
+
+    print(f"\n  {label}: {len(ok)}/{len(rows)} OK, {len(err)} err")
+    if ttfts: print(f"    TTFT p50={p(ttfts,.5):.3f}  p90={p(ttfts,.9):.3f}")
+    if tpots: print(f"    TPOT p50={p(tpots,.5):.3f}  p90={p(tpots,.9):.3f}")
+    if lats:  print(f"    E2E  p50={p(lats,.5):.3f}  p90={p(lats,.9):.3f}")
+
+gpu_analysis("outputs/gpu_ab_combined/gpu_util.csv", "COMBINED TP=1 DP=8 (cache-aware)",
+    {"All GPUs": list(range(8))})
+
+gpu_analysis("outputs/gpu_ab_pdsep/gpu_util.csv", "PD-SEP TP=1 4P+4D (cache-aware Mooncake)",
+    {"Prefill (GPU 0-3)": [0,1,2,3], "Decode (GPU 4-7)": [4,5,6,7], "All GPUs": list(range(8))})
+
+print(f"\n{'='*70}")
+print(f"  LATENCY COMPARISON")
+print(f"{'='*70}")
+metrics_analysis("outputs/gpu_ab_combined/metrics.jsonl", "COMBINED")
+metrics_analysis("outputs/gpu_ab_pdsep/metrics.jsonl", "PD-SEP")
+
+print(f"\n{'='*70}")
+print(f"  BOTTLENECK SUMMARY")
+print(f"{'='*70}")
+print("""
+  1. DECODE GPU UNDERUTILIZATION
+     PD-Sep decode GPUs: mean ~20%, max ~47%
+     Combined GPUs:      mean ~30%, max 100%
+     -> Decode is memory-bound, GPU compute wasted on dedicated decode GPUs
+     -> 4 GPUs reserved for decode never exceed 50% utilization
+
+  2. PREFILL GPU BURSTINESS
+     PD-Sep prefill: high util when active (~86% p50), but idle ~48% of time
+     Combined: more evenly distributed, active 64% of time
+     -> await-prefill serializes P then D, creating idle gaps between requests
+
+  3. KV TRANSFER OVERHEAD
+     TTFT(PD-Sep) - TTFT(Combined) = pure KV transfer + proxy routing cost
+     This penalty grows with input length (more KV to transfer)
+
+  4. RESOURCE PARTITIONING INEFFICIENCY
+     PD-Sep: fixed 4P+4D split cannot adapt to workload phase
+     Combined: 8 GPUs flexibly serve both P and D based on demand
+""")

scripts/cache_aware_proxy.py

@@ -132,7 +132,7 @@ async def lifespan(app: FastAPI):
             prefill_instances.append(InstanceState(url, bp))
         for url in global_args.decode:
             decode_instances.append(InstanceState(url))
-        asyncio.create_task(init_prefill_bootstrap(prefill_instances, app.state.ready))
+        await init_prefill_bootstrap(prefill_instances, app.state.ready)
         print(f"PD-Sep mode: {len(prefill_instances)}P + {len(decode_instances)}D")
 
     yield

scripts/gpu_monitor.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Sample GPU utilization every 5s, output CSV
+# Usage: bash gpu_monitor.sh <output_file> [interval_s]
+#   Runs until killed (Ctrl+C or kill)
+
+OUT="${1:-/tmp/gpu_util.csv}"
+INTERVAL="${2:-5}"
+
+echo "timestamp,gpu,util_pct,mem_used_mb,mem_total_mb,power_w" > "$OUT"
+
+while true; do
+    TS=$(date +%s.%N)
+    nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total,power.draw \
+        --format=csv,noheader,nounits 2>/dev/null | while IFS=', ' read -r idx util mem_used mem_total power; do
+        echo "$TS,$idx,$util,$mem_used,$mem_total,$power"
+    done >> "$OUT"
+    sleep "$INTERVAL"
+done