gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P2: all routing policies read real state via eff_ accessors + ablation harness

Browse Source 
InstanceState.eff_{num_requests,pending_prefill,ongoing_decode,ongoing_tokens}
= max(shadow, real) when feed fresh (fixes 30s-stale under-count, keeps
in-flight RaceFix), plus real-only r_max_prefill_remaining / r_kv_used_frac.
Wired into load_only, lmetric, sticky, unified(_kv_both), unified_v3, and
snapshot logging. Feed off => identical to before. run_v3_trace.sh gains ES=1
toggle (always deploys enhanced proxy); run_ablation_es.sh runs each config
ES0-vs-ES1 to test whether real state changes policy performance/ranking.
All unit-tested without GPU.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-28 20:21:12 +08:00
parent be948d32b8
commit 5b26c345f4
4 changed files with 161 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
microbench/connector_tax/layerwise/cache_aware_proxy.WRITEMODE.py
View File

@@ -217,6 +217,50 @@ class InstanceState:
# when the feed is disabled/stale. Set by _engine_state_poll_loop.
self.real_state: dict | None = None
# ---- effective-load accessors (P2): prefer REAL engine state when the
# feed is fresh, else the proxy shadow counter. We take max(shadow, real)
# for load so we never under-count: REAL fixes the 30s-stale under-count,
# while the shadow's atomic pre-await reservation still covers the
# in-flight window (preserving the RaceFix against concurrent picks).
def eff_num_requests(self) -> float:
rs = self.real_state
if rs is not None:
return max(self.num_requests,
rs.get("num_running", 0) + rs.get("num_waiting", 0))
return self.num_requests
def eff_pending_prefill(self) -> float:
rs = self.real_state
if rs is not None:
return max(self.pending_prefill_tokens,
rs.get("pending_prefill_tokens", 0))
return self.pending_prefill_tokens
def eff_ongoing_decode(self) -> float:
rs = self.real_state
if rs is not None:
return max(self.ongoing_decode_tokens,
rs.get("ongoing_decode_tokens", 0))
return self.ongoing_decode_tokens
def eff_ongoing_tokens(self) -> float:
rs = self.real_state
if rs is not None:
return max(self.ongoing_tokens,
rs.get("pending_prefill_tokens", 0)
+ rs.get("ongoing_decode_tokens", 0))
return self.ongoing_tokens
def r_max_prefill_remaining(self) -> int:
rs = self.real_state
return int(rs.get("max_prefill_remaining", 0)) if rs is not None else 0
def r_kv_used_frac(self) -> float:
rs = self.real_state
if rs is not None:
return float(rs.get("gpu_kv_used_frac", 0.0) or 0.0)
return 0.0
def estimate_cache_hit(self, token_ids: list[int] | None) -> int:
if not token_ids or len(token_ids) < BLOCK_SIZE:
return 0
@@ -277,11 +321,19 @@ def snapshot_workers(
"cached_blocks": len(inst.cached_blocks),
"cache_hit": cache_hit,
"new_prefill": new_prefill,
"score_linear": (inst.ongoing_tokens
# scores reflect the ACTUAL decision basis (eff = real-or-shadow).
"score_linear": (inst.eff_ongoing_tokens()
+ _p_offload_penalty(inst)
- CACHE_HIT_ALPHA * cache_hit),
"score_lmetric": (inst.pending_prefill_tokens + new_prefill)
* inst.num_requests,
"score_lmetric": (inst.eff_pending_prefill() + new_prefill)
* inst.eff_num_requests(),
# P2: real-state fields when the feed is fresh (None otherwise).
"real_num_requests": (inst.eff_num_requests()
if inst.real_state is not None else None),
"real_max_prefill_remaining": (inst.r_max_prefill_remaining()
if inst.real_state is not None else None),
"real_kv_used_frac": (inst.r_kv_used_frac()
if inst.real_state is not None else None),
})
return snap
@@ -297,20 +349,20 @@ def pick_instance(instances: list[InstanceState], token_ids: list[int] | None,
Instances doing P-role offloads get a large penalty to steer
WARM/MEDIUM traffic away.
"""
avg_load = max(sum(i.ongoing_tokens for i in instances) / len(instances), 1.0)
avg_load = max(sum(i.eff_ongoing_tokens() for i in instances) / len(instances), 1.0)
if session_id and session_id in affinity:
idx = affinity[session_id]
if idx < len(instances):
inst = instances[idx]
if (inst.ongoing_tokens <= avg_load * SETTINGS.overload_factor
if (inst.eff_ongoing_tokens() <= avg_load * SETTINGS.overload_factor
and inst.active_p_offloads == 0):
return inst, idx
best_idx, best_score = 0, float("inf")
for i, inst in enumerate(instances):
cache_hit = inst.estimate_cache_hit(token_ids)
score = (inst.ongoing_tokens + _p_offload_penalty(inst)
score = (inst.eff_ongoing_tokens() + _p_offload_penalty(inst)
- CACHE_HIT_ALPHA * cache_hit)
if score < best_score:
best_score = score
@@ -334,7 +386,7 @@ def pick_instance_load_only(
isolate the locality contribution of cache-aware policies.
"""
best_idx = min(range(len(instances)),
key=lambda i: instances[i].num_requests)
key=lambda i: instances[i].eff_num_requests())
return instances[best_idx], best_idx
@@ -357,7 +409,7 @@ def pick_instance_sticky(
if idx < len(instances):
return instances[idx], idx
best_idx = min(range(len(instances)),
key=lambda i: instances[i].num_requests)
key=lambda i: instances[i].eff_num_requests())
if session_id:
affinity[session_id] = best_idx
return instances[best_idx], best_idx
@@ -378,8 +430,8 @@ def pick_instance_lmetric(instances: list[InstanceState], token_ids: list[int] |
for i, inst in enumerate(instances):
cache_hit = inst.estimate_cache_hit(token_ids)
new_prefill = max(0, input_length - cache_hit)
p_tokens = inst.pending_prefill_tokens + new_prefill
bs = inst.num_requests
p_tokens = inst.eff_pending_prefill() + new_prefill
bs = inst.eff_num_requests()
score = p_tokens * bs
if score < best_score:
best_score = score
@@ -416,7 +468,7 @@ def pick_instance_unified_hybrid(
"""
global _unified_fallback_rr_counter
n = len(instances)
avg_reqs = max(sum(i.num_requests for i in instances) / n, 1.0)
avg_reqs = max(sum(i.eff_num_requests() for i in instances) / n, 1.0)
decision: dict = {
"decision": "lmetric_fallback",
@@ -439,9 +491,9 @@ def pick_instance_unified_hybrid(
decision["affinity_idx"] = a_idx
decision["affinity_cache_hit"] = a_hit
decision["affinity_cache_ratio"] = a_ratio
decision["affinity_num_requests"] = a_inst.num_requests
decision["affinity_num_requests"] = a_inst.eff_num_requests()
if (a_ratio > 0.5
and a_inst.num_requests <= avg_reqs * SETTINGS.overload_factor):
and a_inst.eff_num_requests() <= avg_reqs * SETTINGS.overload_factor):
decision["decision"] = "affinity"
decision["chosen_idx"] = a_idx
return a_inst, a_idx, decision
@@ -460,9 +512,10 @@ def pick_instance_unified_hybrid(
for i, inst in enumerate(instances):
cache_hit = inst.estimate_cache_hit(token_ids)
new_prefill = max(0, input_length - cache_hit)
p_tokens = inst.pending_prefill_tokens + new_prefill
decode_pen = SETTINGS.lmetric_decode_weight * inst.ongoing_decode_tokens
bs = inst.num_requests
# P2: effective (real-or-shadow) load signals.
p_tokens = inst.eff_pending_prefill() + new_prefill
decode_pen = SETTINGS.lmetric_decode_weight * inst.eff_ongoing_decode()
bs = inst.eff_num_requests()
score = (p_tokens + decode_pen) * max(bs, 1)
keys.append((score, new_prefill, bs, i))
@@ -669,10 +722,10 @@ def pick_instance_unified_v3(
# Gate 2: affinity host must be busy with concurrent decodes — that's
# what migrating decode-traffic-away buys us. If the host is idle
# there's no point.
if prefill_host.ongoing_decode_tokens < SETTINGS.v3_min_prefill_decode_busy:
if prefill_host.eff_ongoing_decode() < SETTINGS.v3_min_prefill_decode_busy:
decision["v3_reason"] = (
f"prefill_host_not_busy "
f"(ongoing_decode_tokens={prefill_host.ongoing_decode_tokens} < "
f"(ongoing_decode_tokens={prefill_host.eff_ongoing_decode()} < "
f"{SETTINGS.v3_min_prefill_decode_busy})"
)
return prefill_host, prefill_idx, decision, None
@@ -683,12 +736,8 @@ def pick_instance_unified_v3(
cutoff = now_mono - SETTINGS.v3_recent_mig_window_s
def _real_load(inst):
# P2: prefer REAL engine state (running+waiting) over the proxy's
# 30s-stale shadow num_requests, when the engine-state feed is fresh.
rs = getattr(inst, "real_state", None)
if rs is not None:
return rs.get("num_running", 0) + rs.get("num_waiting", 0)
return inst.num_requests
# P2: effective (real-or-shadow) request load; see eff_num_requests.
return inst.eff_num_requests()
def effective_load(inst):
# Drop expired entries lazily.