From 85b230455e9be85a7a89cd9c3f2c09d5b4f2bfa3 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 23 May 2026 03:04:02 +0800 Subject: [PATCH] H7 OVERLOAD_FACTOR sweep: negative result + H4 GPU profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H7: Sweeping OVERLOAD_FACTOR (2.0/1.5/1.3/1.0) has no effect on GPU imbalance (~3.5-4x across all settings). Root cause: imbalance is from workload skew at session placement (turn 1), not from routing at turn 2+. H4 GPU profiling confirms: GPU balance improvement IS real (4.0x→2.0x), and it directly improves HEAVY_COLO TTFT by 10.5%. But RDMA-offloaded requests have bimodal transfer times (0.6s or 18-31s) that negate the routing benefit. Updated elastic_hypotheses.md with H7 results and next directions: higher load experiments where contention amplifies routing differences. Co-Authored-By: Claude Opus 4.6 (1M context) --- analysis/elastic_hypotheses.md | 59 ++++++++++++++++++++++++++++++++++ scripts/bench.sh | 8 ++++- scripts/cache_aware_proxy.py | 5 ++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/analysis/elastic_hypotheses.md b/analysis/elastic_hypotheses.md index d883628..ddbd828 100644 --- a/analysis/elastic_hypotheses.md +++ b/analysis/elastic_hypotheses.md @@ -132,3 +132,62 @@ Offload (V2): ❌ Adds RDMA overhead ✅ C_s fast prefill + D load balan The optimal strategy is **hybrid**: co-locate cold turn-1, offload cached turn-2+. This is the key insight for the paper: **the offload decision should be cache-aware, not size-based**. A 80k-token request with 90% cache hit is effectively a 8k-token prefill — MEDIUM, not HEAVY. The "heaviness" that matters for PD disaggregation is `new_tokens_to_compute`, not `total_input_length`. + +--- + +## H7: OVERLOAD_FACTOR tuning improves GPU balance + +**Claim**: Lowering OVERLOAD_FACTOR (from 2.0 to 1.5/1.3/1.0) breaks session affinity earlier, improving GPU utilization balance. + +**Experiment**: 4 baseline runs (no Mooncake) with OF=2.0, 1.5, 1.3, 1.0. 200 req each, fresh restart. + +**Result**: +``` +OF=2.0: imbalance=3.71x TTFT50=1.077 E2E50=5.093 +OF=1.5: imbalance=3.45x TTFT50=1.068 E2E50=5.480 +OF=1.3: imbalance=3.96x TTFT50=1.073 E2E50=5.144 +OF=1.0: imbalance=3.47x TTFT50=1.085 E2E50=5.496 +``` + +All within noise. APC unchanged (~30%). + +**Verdict**: **REJECTED**. The imbalance is driven by workload skew (some sessions are inherently heavier), not by sticky routing. The OVERLOAD_FACTOR threshold rarely fires because per-instance load fluctuates too quickly. The hot GPU just rotates to different instances across runs. + +**Key learning**: The root cause of GPU imbalance is at **session placement time (turn 1)**, not at affinity-breaking time (turn 2+). Turn-1 placement uses `ongoing_tokens` scoring, which is a snapshot that doesn't account for cumulative or future load. + +--- + +## H4 Validated: Cache-gate improves GPU balance but RDMA kills TTFT + +**Experiment**: H4 cache-gate (8C kv_both, offload only when cache_ratio >= 0.3) with GPU profiling. + +**Result**: +``` + Baseline H4 cache-gate +GPU Imbalance: 3.97x 2.04x ← 2x better balance +GPU Std: 14.9% 6.7% ← less variance +GPU Max: 63.3% 35.3% ← no extreme hotspot +HEAVY_COLO TTFT: 7.02s 6.28s ← -10.5% from better balance! +HEAVY_OFFLOAD TTFT: N/A 11.45s ← RDMA penalty +OK/N: 198/200 198/200 ← same reliability +``` + +**Key finding**: The 10.5% HEAVY_COLO improvement proves GPU balance → better latency. But the 7 RDMA-offloaded requests (TTFT=11.45s) pull down the aggregate. RDMA transfer is bimodal: 3/7 fast (0.6-1.2s), 3/7 slow (18-31s). + +--- + +## Current Understanding (updated) + +1. **PD-Sep**: net negative (memory wall) ← proven +2. **LMetric**: ≈ baseline for agentic (session affinity limits routing freedom) ← proven +3. **Elastic P2P (RDMA)**: net negative on single machine (Mooncake lacks layerwise transfer → RDMA is pure overhead) ← proven +4. **OVERLOAD_FACTOR tuning**: no effect (imbalance from workload skew, not routing) ← proven +5. **GPU balance improvement → HEAVY TTFT -10.5%**: validated (H4 HEAVY_COLO data) +6. **The bottleneck is at time_scale=20 with 200 req**: system is only 30% loaded. Higher load may reveal more optimization opportunities. + +## Next directions + +- **Higher load (time_scale=10, 500+ req)**: increase contention to amplify routing differences +- **1000 req at time_scale=20**: reduce statistical noise (±7% → ±3%) +- **Session-count-aware placement**: balance number of active heavy sessions per instance, not just ongoing tokens +- **Layerwise KV transfer**: modify Mooncake to pipeline KV transfer with compute (requires deep vLLM change) diff --git a/scripts/bench.sh b/scripts/bench.sh index 117d008..2a64c44 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -35,6 +35,7 @@ TIME_SCALE=20 MAX_SESSIONS=8 HEAVY_THRESHOLD=20000 NO_OFFLOAD=false +OVERLOAD_FACTOR_ARG="" # Parse args while [[ $# -gt 0 ]]; do @@ -48,6 +49,7 @@ while [[ $# -gt 0 ]]; do --sessions) MAX_SESSIONS="$2"; shift 2 ;; --heavy-threshold) HEAVY_THRESHOLD="$2"; shift 2 ;; --no-offload) NO_OFFLOAD=true; shift ;; + --overload-factor) OVERLOAD_FACTOR_ARG="$2"; shift 2 ;; *) echo "Unknown: $1"; exit 1 ;; esac done @@ -77,6 +79,7 @@ cat > "$OUTDIR/config.json" << CONF "max_sessions": $MAX_SESSIONS, "heavy_threshold": $HEAVY_THRESHOLD, "no_offload": "$NO_OFFLOAD", + "overload_factor": "${OVERLOAD_FACTOR_ARG:-2.0}", "timestamp": "$(date -Iseconds)", "hostname": "$(hostname)" } @@ -193,6 +196,9 @@ launch_proxy() { done local extra_args="--policy $POLICY" + if [ -n "$OVERLOAD_FACTOR_ARG" ]; then + extra_args="$extra_args --overload-factor $OVERLOAD_FACTOR_ARG" + fi if [ "$MODE" = "elastic" ]; then local bp_list="" for i in $(seq 0 $((N_INSTANCES - 1))); do @@ -307,7 +313,7 @@ print('=' * 70) echo "================================================================" echo " bench.sh: $TAG" -echo " mode=$MODE policy=$POLICY requests=$REQUESTS" +echo " mode=$MODE policy=$POLICY requests=$REQUESTS overload_factor=${OVERLOAD_FACTOR_ARG:-2.0}" echo " $(date)" echo "================================================================" diff --git a/scripts/cache_aware_proxy.py b/scripts/cache_aware_proxy.py index 746d3d9..30c4c0c 100644 --- a/scripts/cache_aware_proxy.py +++ b/scripts/cache_aware_proxy.py @@ -28,7 +28,7 @@ from fastapi.responses import StreamingResponse BLOCK_SIZE = 512 CACHE_HIT_ALPHA = 1.0 HEAVY_THRESHOLD = 20000 # default; overridden by --heavy-threshold -OVERLOAD_FACTOR = 2.0 +OVERLOAD_FACTOR = 2.0 # default; overridden by --overload-factor class InstanceState: @@ -627,6 +627,8 @@ def parse_args(): help="Comma-separated bootstrap ports for combined instances (for offload mode)") p.add_argument("--policy", type=str, default="linear", choices=["linear", "lmetric"], help="Routing policy: linear (default) or lmetric (P_tokens × BS, OSDI'26)") + p.add_argument("--overload-factor", type=float, default=2.0, + help="Break session affinity when instance load > factor * avg") args = p.parse_args() args.prefill = [] @@ -645,4 +647,5 @@ def parse_args(): if __name__ == "__main__": global_args = parse_args() HEAVY_THRESHOLD = global_args.heavy_threshold + OVERLOAD_FACTOR = global_args.overload_factor uvicorn.run(app, host=global_args.host, port=global_args.port)