From 3594f7dce0dc735e77ea2f66cf8ee7adc7fcb855 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 23 May 2026 11:56:58 +0800 Subject: [PATCH] Fix LMetric routing: remove session affinity, align with OSDI'26 spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LMetric was incorrectly sharing session-sticky logic with Linear policy. Fixed to pure per-request routing: score = P_tokens × BS where P = pending_prefill + (input - cache_hit), BS = num_requests. Experiment result (200 req, fresh restart): Linear vs corrected LMetric show <2% difference on all metrics — LMetric's cache-hit estimation provides implicit soft affinity that preserves locality without explicit session stickiness. Also fix bench.sh missing cd (replayer module not found from non-project cwd) and rewrite run_lmetric_ab.sh as thin wrapper around bench.sh to eliminate duplicated launch/cleanup logic that broke under set -euo. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/bench.sh | 1 + scripts/cache_aware_proxy.py | 20 ++--- scripts/run_lmetric_ab.sh | 157 +++++++---------------------------- 3 files changed, 38 insertions(+), 140 deletions(-) diff --git a/scripts/bench.sh b/scripts/bench.sh index 1c301c3..beb8e17 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -328,6 +328,7 @@ echo " mode=$MODE policy=$POLICY requests=$REQUESTS overload_factor=${OVERLO echo " $(date)" echo "================================================================" +cd "$PROJECT_DIR" cleanup_gpu launch_instances launch_proxy diff --git a/scripts/cache_aware_proxy.py b/scripts/cache_aware_proxy.py index 30c4c0c..8839c33 100644 --- a/scripts/cache_aware_proxy.py +++ b/scripts/cache_aware_proxy.py @@ -130,31 +130,21 @@ def pick_instance_lmetric(instances: list[InstanceState], token_ids: list[int] | affinity: dict[str, int]) -> tuple[InstanceState, int]: """LMetric routing: score = P_tokens × BS (OSDI'26). - Instances doing P-role offloads get a large penalty. + Pure per-request load-based routing, no session affinity. + P = pending_prefill_tokens + (input_length - cache_hit) + BS = num_requests (current batch size) """ - avg_load = max(sum(i.ongoing_tokens for i in instances) / len(instances), 1.0) - - if session_id and session_id in affinity: - idx = affinity[session_id] - if idx < len(instances): - inst = instances[idx] - if (inst.ongoing_tokens <= avg_load * OVERLOAD_FACTOR - and inst.active_p_offloads == 0): - return inst, idx - best_idx, best_score = 0, float("inf") for i, inst in enumerate(instances): cache_hit = inst.estimate_cache_hit(token_ids) new_prefill = max(0, input_length - cache_hit) - p_tokens = inst.pending_prefill_tokens + new_prefill + _p_offload_penalty(inst) - bs = inst.num_requests + 1 + p_tokens = inst.pending_prefill_tokens + new_prefill + bs = inst.num_requests score = p_tokens * bs if score < best_score: best_score = score best_idx = i - if session_id: - affinity[session_id] = best_idx return instances[best_idx], best_idx diff --git a/scripts/run_lmetric_ab.sh b/scripts/run_lmetric_ab.sh index 8fa4340..3eb6d51 100755 --- a/scripts/run_lmetric_ab.sh +++ b/scripts/run_lmetric_ab.sh @@ -1,148 +1,55 @@ #!/bin/bash -# A/B comparison: linear (current baseline) vs lmetric (OSDI'26) routing policy. -# Both use same 8× TP=1 combined instances, fresh restart between experiments. -set -euo pipefail +# A/B comparison: linear (session-sticky) vs lmetric (OSDI'26, no affinity). +# Wraps bench.sh for guaranteed fresh state between experiments. +# +# Usage: +# bash scripts/run_lmetric_ab.sh # defaults: 200 req +# bash scripts/run_lmetric_ab.sh --requests 1000 # override request count -PROJECT_DIR="/home/admin/cpfs/wjh/agentic-kv" -VENV="$PROJECT_DIR/.venv/bin" -VLLM="$VENV/vllm" -PYTHON="$VENV/python" -MODEL="/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct" -TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl" - -N_INSTANCES=8 -BASE_PORT=8000 -PROXY_PORT=9090 -REQUEST_LIMIT=200 -TIME_SCALE=20 -MAX_SESSIONS=8 - -cleanup() { - for p in $(ps aux | grep 'vllm serve' | grep -v grep | awk '{print $2}'); do kill -9 $p 2>/dev/null; done - for p in $(ps aux | grep 'cache_aware_proxy' | grep -v grep | awk '{print $2}'); do kill -9 $p 2>/dev/null; done - sleep 5 - for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u); do kill -9 $p 2>/dev/null; done - sleep 10 -} - -start_instances() { - echo " Starting $N_INSTANCES vLLM instances..." - for i in $(seq 0 $((N_INSTANCES - 1))); do - port=$((BASE_PORT + i)) - MASTER_PORT=$((29500 + i)) CUDA_VISIBLE_DEVICES=$i \ - $VLLM serve "$MODEL" \ - --host 0.0.0.0 --port $port \ - --tensor-parallel-size 1 \ - --trust-remote-code --enable-prefix-caching --enforce-eager \ - --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \ - > /tmp/lmetric_ab_inst_$i.log 2>&1 & - done - - echo " Waiting for instances..." - for i in $(seq 0 $((N_INSTANCES - 1))); do - port=$((BASE_PORT + i)) - timeout 600 bash -c "until curl -s localhost:$port/v1/models > /dev/null 2>&1; do sleep 5; done" - echo " Instance $i (port $port) ready" - done -} - -run_experiment() { - local policy=$1 - local tag=$2 - local outdir="$PROJECT_DIR/outputs/$tag" - mkdir -p "$outdir" - - echo " Starting proxy (policy=$policy)..." - $PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \ - --combined $(for i in $(seq 0 $((N_INSTANCES - 1))); do echo -n "http://127.0.0.1:$((BASE_PORT + i)) "; done) \ - --policy "$policy" \ - --port $PROXY_PORT > /tmp/lmetric_ab_proxy_${policy}.log 2>&1 & - PROXY_PID=$! - sleep 3 - - # Smoke test - result=$(curl -s -m 30 http://localhost:$PROXY_PORT/v1/completions \ - -X POST -H "Content-Type: application/json" \ - -d "{\"model\":\"$MODEL\",\"prompt\":[100,200,300],\"max_tokens\":3,\"temperature\":0}" 2>&1) - if ! echo "$result" | grep -q "choices"; then - echo " ERROR: Smoke test failed: $result" - kill $PROXY_PID 2>/dev/null - return 1 - fi - echo " Smoke test passed" - - # Start GPU monitor - bash "$PROJECT_DIR/scripts/gpu_monitor.sh" > "$outdir/gpu_util.csv" & - GPU_MON_PID=$! - - # Run benchmark - echo " Running benchmark (policy=$policy, $REQUEST_LIMIT requests)..." - $PYTHON -m replayer \ - --trace "$TRACE" \ - --output "$outdir/metrics.jsonl" \ - --endpoint "http://localhost:$PROXY_PORT" \ - --model "$MODEL" \ - --time-scale $TIME_SCALE \ - --max-inflight-sessions $MAX_SESSIONS \ - --request-limit $REQUEST_LIMIT \ - -v - - # Save breakdown - curl -s http://localhost:$PROXY_PORT/breakdown > "$outdir/breakdown.json" 2>/dev/null - curl -s http://localhost:$PROXY_PORT/stats > "$outdir/stats.json" 2>/dev/null - - # Collect APC from vLLM logs - echo " Collecting APC..." - for i in $(seq 0 $((N_INSTANCES - 1))); do - pch=$(grep "Prefix cache hit rate" /tmp/lmetric_ab_inst_$i.log 2>/dev/null | tail -1 | grep -oP "Prefix cache hit rate: \K[0-9.]+" || echo "0") - echo " inst_$i: prefix=$pch%" - done | tee "$outdir/apc.txt" - - kill $GPU_MON_PID 2>/dev/null - kill $PROXY_PID 2>/dev/null - wait $PROXY_PID 2>/dev/null - echo " Done: $(wc -l < "$outdir/metrics.jsonl") requests -> $outdir" -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}" +TS=$(date +%Y%m%d_%H%M%S) echo "================================================================" echo " A/B: Linear vs LMetric routing policy" echo " $(date)" echo "================================================================" -# Experiment 1: Linear (current baseline) echo "" -echo "=== Experiment 1: Linear policy ===" -cleanup -start_instances -run_experiment "linear" "ab_linear" +echo "=== Experiment 1/2: Linear (session-sticky) ===" +bash "$SCRIPT_DIR/bench.sh" --tag "ab_linear_${TS}" --mode baseline --policy linear "$@" -# Experiment 2: LMetric (OSDI'26) echo "" -echo "=== Experiment 2: LMetric policy ===" -cleanup -start_instances -run_experiment "lmetric" "ab_lmetric" +echo "=== Experiment 2/2: LMetric (no affinity) ===" +bash "$SCRIPT_DIR/bench.sh" --tag "ab_lmetric_${TS}" --mode baseline --policy lmetric "$@" -# Compare echo "" echo "================================================================" echo " Results comparison" echo "================================================================" -$PYTHON -c " -import json, statistics +"$VENV/python" -c " +import json -def summarize(path, label): +def summarize(path): rows = [json.loads(l) for l in open(path)] ok = [r for r in rows if not r.get('error')] - p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0 - ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')]) - tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0]) - e2es = sorted([r['latency_s'] for r in ok]) - print('%-20s OK=%3d/%3d TTFT50=%.3f TTFT90=%.3f TPOT90=%.3f E2E50=%.3f' % ( - label, len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5))) + p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0 + ttfts = [r['ttft_s'] for r in ok if r.get('ttft_s')] + tpots = [r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0] + e2es = [r['latency_s'] for r in ok] + return len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5) -summarize('$PROJECT_DIR/outputs/ab_linear/metrics.jsonl', 'Linear') -summarize('$PROJECT_DIR/outputs/ab_lmetric/metrics.jsonl', 'LMetric') +lin = summarize('$PROJECT_DIR/outputs/ab_linear_${TS}/metrics.jsonl') +lm = summarize('$PROJECT_DIR/outputs/ab_lmetric_${TS}/metrics.jsonl') + +fmt = ' %-20s OK=%3d/%3d TTFT50=%7.3f TTFT90=%7.3f TPOT90=%6.4f E2E50=%7.3f' +print(fmt % ('Linear', *lin)) +print(fmt % ('LMetric', *lm)) +print() +for name, i in [('TTFT50',2),('TTFT90',3),('TPOT90',4),('E2E50',5)]: + d = (lm[i] - lin[i]) / lin[i] * 100 if lin[i] else 0 + print(' %s delta: %+.1f%%' % (name, d)) " echo ""