From 3ae99293fd076e8b86ba12c377938060808502b8 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Mon, 25 May 2026 00:03:28 +0800 Subject: [PATCH] Relax _push_allowed: gate on request size, not cache savings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old gate blocked offload when push_new (= input - cache_hit) < 20K, which prevented migration of high-cache sessions — exactly the ones that benefit most. After PD-sep, the target receives full KV via RDMA and has the same cache as the source, so cache_hit is irrelevant to the offload decision. New gate: only check input_length >= heavy_threshold (request must be HEAVY) and max_offload_inflight (concurrency cap). Let the cost model decide whether the contention difference justifies migration. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/cache_aware_proxy.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/cache_aware_proxy.py b/scripts/cache_aware_proxy.py index 89e0591..1dead20 100644 --- a/scripts/cache_aware_proxy.py +++ b/scripts/cache_aware_proxy.py @@ -521,13 +521,8 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h def _push_allowed(cache_hit: int) -> bool: if _current_offloads() >= SETTINGS.max_offload_inflight: return False - push_new = max(0, input_length - cache_hit) - if push_new < SETTINGS.heavy_threshold: + if input_length < SETTINGS.heavy_threshold: return False - if SETTINGS.cache_gate_ratio > 0: - cache_ratio = cache_hit / max(input_length, 1) - if cache_ratio < SETTINGS.cache_gate_ratio: - return False return True def _instance_cost(i: int) -> tuple[float, bool]: