Fix LMetric routing: remove session affinity, align with OSDI'26 spec

LMetric was incorrectly sharing session-sticky logic with Linear policy.
Fixed to pure per-request routing: score = P_tokens × BS where
P = pending_prefill + (input - cache_hit), BS = num_requests.

Experiment result (200 req, fresh restart): Linear vs corrected LMetric
show <2% difference on all metrics — LMetric's cache-hit estimation
provides implicit soft affinity that preserves locality without explicit
session stickiness.

Also fix bench.sh missing cd (replayer module not found from non-project
cwd) and rewrite run_lmetric_ab.sh as thin wrapper around bench.sh to
eliminate duplicated launch/cleanup logic that broke under set -euo.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/bench.sh

@@ -328,6 +328,7 @@ echo "  mode=$MODE  policy=$POLICY  requests=$REQUESTS  overload_factor=${OVERLO
 echo "  $(date)"
 echo "================================================================"
 
+cd "$PROJECT_DIR"
 cleanup_gpu
 launch_instances
 launch_proxy

scripts/cache_aware_proxy.py

@@ -130,31 +130,21 @@ def pick_instance_lmetric(instances: list[InstanceState], token_ids: list[int] |
                           affinity: dict[str, int]) -> tuple[InstanceState, int]:
     """LMetric routing: score = P_tokens × BS (OSDI'26).
 
-    Instances doing P-role offloads get a large penalty.
+    Pure per-request load-based routing, no session affinity.
+    P = pending_prefill_tokens + (input_length - cache_hit)
+    BS = num_requests (current batch size)
     """
-    avg_load = max(sum(i.ongoing_tokens for i in instances) / len(instances), 1.0)
-
-    if session_id and session_id in affinity:
-        idx = affinity[session_id]
-        if idx < len(instances):
-            inst = instances[idx]
-            if (inst.ongoing_tokens <= avg_load * OVERLOAD_FACTOR
-                    and inst.active_p_offloads == 0):
-                return inst, idx
-
     best_idx, best_score = 0, float("inf")
     for i, inst in enumerate(instances):
         cache_hit = inst.estimate_cache_hit(token_ids)
         new_prefill = max(0, input_length - cache_hit)
-        p_tokens = inst.pending_prefill_tokens + new_prefill + _p_offload_penalty(inst)
+        p_tokens = inst.pending_prefill_tokens + new_prefill
-        bs = inst.num_requests + 1
+        bs = inst.num_requests
         score = p_tokens * bs
         if score < best_score:
             best_score = score
             best_idx = i
 
-    if session_id:
-        affinity[session_id] = best_idx
     return instances[best_idx], best_idx
 
 

scripts/run_lmetric_ab.sh

@@ -1,148 +1,55 @@
 #!/bin/bash
-# A/B comparison: linear (current baseline) vs lmetric (OSDI'26) routing policy.
+# A/B comparison: linear (session-sticky) vs lmetric (OSDI'26, no affinity).
-# Both use same 8× TP=1 combined instances, fresh restart between experiments.
+# Wraps bench.sh for guaranteed fresh state between experiments.
-set -euo pipefail
+#
+# Usage:
+#   bash scripts/run_lmetric_ab.sh                    # defaults: 200 req
+#   bash scripts/run_lmetric_ab.sh --requests 1000    # override request count
 
-PROJECT_DIR="/home/admin/cpfs/wjh/agentic-kv"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-VENV="$PROJECT_DIR/.venv/bin"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
-VLLM="$VENV/vllm"
+VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}"
-PYTHON="$VENV/python"
+TS=$(date +%Y%m%d_%H%M%S)
-MODEL="/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct"
-TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
-
-N_INSTANCES=8
-BASE_PORT=8000
-PROXY_PORT=9090
-REQUEST_LIMIT=200
-TIME_SCALE=20
-MAX_SESSIONS=8
-
-cleanup() {
-    for p in $(ps aux | grep 'vllm serve' | grep -v grep | awk '{print $2}'); do kill -9 $p 2>/dev/null; done
-    for p in $(ps aux | grep 'cache_aware_proxy' | grep -v grep | awk '{print $2}'); do kill -9 $p 2>/dev/null; done
-    sleep 5
-    for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u); do kill -9 $p 2>/dev/null; done
-    sleep 10
-}
-
-start_instances() {
-    echo "  Starting $N_INSTANCES vLLM instances..."
-    for i in $(seq 0 $((N_INSTANCES - 1))); do
-        port=$((BASE_PORT + i))
-        MASTER_PORT=$((29500 + i)) CUDA_VISIBLE_DEVICES=$i \
-        $VLLM serve "$MODEL" \
-            --host 0.0.0.0 --port $port \
-            --tensor-parallel-size 1 \
-            --trust-remote-code --enable-prefix-caching --enforce-eager \
-            --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
-            > /tmp/lmetric_ab_inst_$i.log 2>&1 &
-    done
-
-    echo "  Waiting for instances..."
-    for i in $(seq 0 $((N_INSTANCES - 1))); do
-        port=$((BASE_PORT + i))
-        timeout 600 bash -c "until curl -s localhost:$port/v1/models > /dev/null 2>&1; do sleep 5; done"
-        echo "    Instance $i (port $port) ready"
-    done
-}
-
-run_experiment() {
-    local policy=$1
-    local tag=$2
-    local outdir="$PROJECT_DIR/outputs/$tag"
-    mkdir -p "$outdir"
-
-    echo "  Starting proxy (policy=$policy)..."
-    $PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
-        --combined $(for i in $(seq 0 $((N_INSTANCES - 1))); do echo -n "http://127.0.0.1:$((BASE_PORT + i)) "; done) \
-        --policy "$policy" \
-        --port $PROXY_PORT > /tmp/lmetric_ab_proxy_${policy}.log 2>&1 &
-    PROXY_PID=$!
-    sleep 3
-
-    # Smoke test
-    result=$(curl -s -m 30 http://localhost:$PROXY_PORT/v1/completions \
-        -X POST -H "Content-Type: application/json" \
-        -d "{\"model\":\"$MODEL\",\"prompt\":[100,200,300],\"max_tokens\":3,\"temperature\":0}" 2>&1)
-    if ! echo "$result" | grep -q "choices"; then
-        echo "  ERROR: Smoke test failed: $result"
-        kill $PROXY_PID 2>/dev/null
-        return 1
-    fi
-    echo "  Smoke test passed"
-
-    # Start GPU monitor
-    bash "$PROJECT_DIR/scripts/gpu_monitor.sh" > "$outdir/gpu_util.csv" &
-    GPU_MON_PID=$!
-
-    # Run benchmark
-    echo "  Running benchmark (policy=$policy, $REQUEST_LIMIT requests)..."
-    $PYTHON -m replayer \
-        --trace "$TRACE" \
-        --output "$outdir/metrics.jsonl" \
-        --endpoint "http://localhost:$PROXY_PORT" \
-        --model "$MODEL" \
-        --time-scale $TIME_SCALE \
-        --max-inflight-sessions $MAX_SESSIONS \
-        --request-limit $REQUEST_LIMIT \
-        -v
-
-    # Save breakdown
-    curl -s http://localhost:$PROXY_PORT/breakdown > "$outdir/breakdown.json" 2>/dev/null
-    curl -s http://localhost:$PROXY_PORT/stats > "$outdir/stats.json" 2>/dev/null
-
-    # Collect APC from vLLM logs
-    echo "  Collecting APC..."
-    for i in $(seq 0 $((N_INSTANCES - 1))); do
-        pch=$(grep "Prefix cache hit rate" /tmp/lmetric_ab_inst_$i.log 2>/dev/null | tail -1 | grep -oP "Prefix cache hit rate: \K[0-9.]+" || echo "0")
-        echo "    inst_$i: prefix=$pch%"
-    done | tee "$outdir/apc.txt"
-
-    kill $GPU_MON_PID 2>/dev/null
-    kill $PROXY_PID 2>/dev/null
-    wait $PROXY_PID 2>/dev/null
-    echo "  Done: $(wc -l < "$outdir/metrics.jsonl") requests -> $outdir"
-}
 
 echo "================================================================"
 echo "  A/B: Linear vs LMetric routing policy"
 echo "  $(date)"
 echo "================================================================"
 
-# Experiment 1: Linear (current baseline)
 echo ""
-echo "=== Experiment 1: Linear policy ==="
+echo "=== Experiment 1/2: Linear (session-sticky) ==="
-cleanup
+bash "$SCRIPT_DIR/bench.sh" --tag "ab_linear_${TS}" --mode baseline --policy linear "$@"
-start_instances
-run_experiment "linear" "ab_linear"
 
-# Experiment 2: LMetric (OSDI'26)
 echo ""
-echo "=== Experiment 2: LMetric policy ==="
+echo "=== Experiment 2/2: LMetric (no affinity) ==="
-cleanup
+bash "$SCRIPT_DIR/bench.sh" --tag "ab_lmetric_${TS}" --mode baseline --policy lmetric "$@"
-start_instances
-run_experiment "lmetric" "ab_lmetric"
 
-# Compare
 echo ""
 echo "================================================================"
 echo "  Results comparison"
 echo "================================================================"
-$PYTHON -c "
+"$VENV/python" -c "
-import json, statistics
+import json
 
-def summarize(path, label):
+def summarize(path):
     rows = [json.loads(l) for l in open(path)]
     ok = [r for r in rows if not r.get('error')]
-    p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
+    p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
-    ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
+    ttfts = [r['ttft_s'] for r in ok if r.get('ttft_s')]
-    tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
+    tpots = [r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0]
-    e2es = sorted([r['latency_s'] for r in ok])
+    e2es = [r['latency_s'] for r in ok]
-    print('%-20s OK=%3d/%3d  TTFT50=%.3f  TTFT90=%.3f  TPOT90=%.3f  E2E50=%.3f' % (
+    return len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)
-        label, len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)))
 
-summarize('$PROJECT_DIR/outputs/ab_linear/metrics.jsonl', 'Linear')
+lin = summarize('$PROJECT_DIR/outputs/ab_linear_${TS}/metrics.jsonl')
-summarize('$PROJECT_DIR/outputs/ab_lmetric/metrics.jsonl', 'LMetric')
+lm = summarize('$PROJECT_DIR/outputs/ab_lmetric_${TS}/metrics.jsonl')
+
+fmt = '  %-20s OK=%3d/%3d  TTFT50=%7.3f  TTFT90=%7.3f  TPOT90=%6.4f  E2E50=%7.3f'
+print(fmt % ('Linear', *lin))
+print(fmt % ('LMetric', *lm))
+print()
+for name, i in [('TTFT50',2),('TTFT90',3),('TPOT90',4),('E2E50',5)]:
+    d = (lm[i] - lin[i]) / lin[i] * 100 if lin[i] else 0
+    print('  %s delta: %+.1f%%' % (name, d))
 "
 
 echo ""