PS experiments + H4 cache-gate + GPU profiling + Mooncake elif→if fix

Experiments run:
- Phase 0: kv_both has zero idle overhead (TPOT +1.3%, noise)
- PS V1 (cold prefill): REJECTED — PS always slower than cached C
- PS V1+flexD: 92.5% OK, HEAVY TTFT 7.8s (baseline 5.0s) — PS bottleneck
- V2 (C_s prefill + flexible D): E2E -9% but 6 errors, RDMA bimodal
- H4 (cache-gate): 198/200 OK, GPU imbalance 4.0x→2.0x, but HEAVY_OFFLOAD
  TTFT=11.5s due to RDMA. HEAVY_COLO improved 10.5% from better balance.
- H5: Mooncake RDMA transfer R²=0.095, bimodal (0.6s or 18-30s)

Key findings:
- Mooncake lacks layerwise KV transfer → RDMA is pure sequential overhead
- 92% of HEAVY are turn-1 cold → offloading cold requests always loses
- GPU balance improvement from routing IS real (-10.5% HEAVY_COLO TTFT)
- RDMA transfer negates the routing benefit for offloaded requests

Code changes:
- bench.sh: add GPU timeline monitoring (gpu_monitor.sh during benchmark)
- cache_aware_proxy.py: H4 cache-gate, flexible D, PS routing
- mooncake_connector.py: elif→if fix (allow dual prefill+decode flags)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/bench.sh

@@ -34,6 +34,7 @@ REQUESTS=200
 TIME_SCALE=20
 MAX_SESSIONS=8
 HEAVY_THRESHOLD=20000
+NO_OFFLOAD=false
 
 # Parse args
 while [[ $# -gt 0 ]]; do
@@ -41,16 +42,18 @@ while [[ $# -gt 0 ]]; do
         --tag) TAG="$2"; shift 2 ;;
         --mode) MODE="$2"; shift 2 ;;
         --policy) POLICY="$2"; shift 2 ;;
+        --instances) N_INSTANCES="$2"; shift 2 ;;
         --requests) REQUESTS="$2"; shift 2 ;;
         --time-scale) TIME_SCALE="$2"; shift 2 ;;
         --sessions) MAX_SESSIONS="$2"; shift 2 ;;
         --heavy-threshold) HEAVY_THRESHOLD="$2"; shift 2 ;;
+        --no-offload) NO_OFFLOAD=true; shift ;;
         *) echo "Unknown: $1"; exit 1 ;;
     esac
 done
 
 if [ -z "$TAG" ]; then
-    echo "Usage: bench.sh --tag NAME --mode {baseline|elastic} [--policy {linear|lmetric}] [--requests N]"
+    echo "Usage: bench.sh --tag NAME --mode {baseline|elastic} [--instances N] [--policy {linear|lmetric}] [--requests N]"
     exit 1
 fi
 
@@ -73,6 +76,7 @@ cat > "$OUTDIR/config.json" << CONF
     "time_scale": $TIME_SCALE,
     "max_sessions": $MAX_SESSIONS,
     "heavy_threshold": $HEAVY_THRESHOLD,
+    "no_offload": "$NO_OFFLOAD",
     "timestamp": "$(date -Iseconds)",
     "hostname": "$(hostname)"
 }
@@ -110,29 +114,33 @@ cleanup_gpu() {
 launch_instances() {
     echo "[launch] Starting $N_INSTANCES vLLM instances (mode=$MODE)..."
 
-    local kv_config=""
-    if [ "$MODE" = "elastic" ]; then
-        kv_config='--kv-transfer-config {"kv_connector":"MooncakeConnector","kv_role":"kv_both"}'
-    fi
-
     for i in $(seq 0 $((N_INSTANCES - 1))); do
         local port=$((BASE_PORT + i))
         local master=$((29500 + i))
         local logfile="$OUTDIR/vllm_inst_${i}.log"
 
-        local env_prefix="MASTER_PORT=$master CUDA_VISIBLE_DEVICES=$i"
         if [ "$MODE" = "elastic" ]; then
-            env_prefix="VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998 + i)) $env_prefix"
+            VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998 + i)) \
+            MASTER_PORT=$master \
+            CUDA_VISIBLE_DEVICES=$i \
+            $VLLM serve "$MODEL" \
+                --host 0.0.0.0 --port $port \
+                --tensor-parallel-size 1 \
+                --trust-remote-code --enable-prefix-caching --enforce-eager \
+                --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
+                --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
+                > "$logfile" 2>&1 &
+        else
+            MASTER_PORT=$master \
+            CUDA_VISIBLE_DEVICES=$i \
+            $VLLM serve "$MODEL" \
+                --host 0.0.0.0 --port $port \
+                --tensor-parallel-size 1 \
+                --trust-remote-code --enable-prefix-caching --enforce-eager \
+                --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
+                > "$logfile" 2>&1 &
         fi
 
-        eval "$env_prefix $VLLM serve '$MODEL' \
-            --host 0.0.0.0 --port $port \
-            --tensor-parallel-size 1 \
-            --trust-remote-code --enable-prefix-caching --enforce-eager \
-            --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
-            $kv_config \
-            > '$logfile' 2>&1 &"
-
         echo "  inst_$i: GPU=$i port=$port"
         sleep 2  # stagger to avoid port collision
     done
@@ -190,7 +198,11 @@ launch_proxy() {
         for i in $(seq 0 $((N_INSTANCES - 1))); do
             bp_list="${bp_list:+$bp_list,}$((8998 + i))"
         done
-        extra_args="$extra_args --offload --heavy-threshold $HEAVY_THRESHOLD --bootstrap-ports $bp_list"
+        if [ "$NO_OFFLOAD" = "true" ]; then
+            extra_args="$extra_args --bootstrap-ports $bp_list"
+        else
+            extra_args="$extra_args --offload --heavy-threshold $HEAVY_THRESHOLD --bootstrap-ports $bp_list"
+        fi
     fi
 
     $PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
@@ -217,6 +229,11 @@ launch_proxy() {
 
 run_benchmark() {
     echo "[bench] Running $REQUESTS requests (time_scale=$TIME_SCALE, sessions=$MAX_SESSIONS)..."
+
+    # Start GPU monitor in background
+    bash "$PROJECT_DIR/scripts/gpu_monitor.sh" "$OUTDIR/gpu_util.csv" 5 &
+    GPU_MON_PID=$!
+
     $PYTHON -m replayer \
         --trace "$TRACE" \
         --output "$OUTDIR/metrics.jsonl" \
@@ -226,6 +243,11 @@ run_benchmark() {
         --max-inflight-sessions "$MAX_SESSIONS" \
         --request-limit "$REQUESTS" \
         -v 2>&1 | tee "$OUTDIR/replayer.log"
+
+    # Stop GPU monitor
+    kill $GPU_MON_PID 2>/dev/null || true
+    wait $GPU_MON_PID 2>/dev/null || true
+    echo "[bench] GPU util saved: $(wc -l < "$OUTDIR/gpu_util.csv") samples"
 }
 
 # ─── Collect artifacts ─────────────────────────────────────────────────────