proxy: Settings dataclass + cache-ratio gate + P-pick offload penalty (B4, M2, M3, D5)

- Replace mutable module constants (HEAVY_THRESHOLD/OVERLOAD_FACTOR/
  MAX_OFFLOAD_INFLIGHT/PREFILL_THROUGHPUT/RDMA_OVERHEAD_S/
  CACHE_CAPACITY_BLOCKS) with a Settings dataclass + SETTINGS singleton.
  __main__ now mutates SETTINGS so CLI overrides survive even when the
  module is imported as a library (e.g. by tests/) (D5).
- Add --max-offload-inflight CLI flag (M3) and read it from SETTINGS.
- Add --cache-gate-ratio CLI flag and a real gate before the cost-model
  branch: if cache_hit/input_length < ratio, mark cache_gate_REASON and
  fall back to colocated. cache_ratio is no longer a write-only field
  (B4).
- P candidate selection penalises instances already running offloaded
  HEAVY prefills, so back-to-back HEAVY requests don't pile onto the
  same P (M2).
- bench.sh forwards --max-offload-inflight / --cache-gate-ratio to the
  proxy.
- Tests cover SETTINGS knobs + the heavy_threshold-driven P-offload
  penalty.

tests/test_proxy_pick.py

@@ -95,8 +95,8 @@ def _make_inst(proxy, url: str, ongoing_tokens: int = 0,
 def test_record_prefix_evicts_oldest_block(proxy):
     """LRU bound on cached_blocks must evict the oldest entry once full."""
     inst = proxy.InstanceState("http://x")
-    saved = proxy.CACHE_CAPACITY_BLOCKS
-    proxy.CACHE_CAPACITY_BLOCKS = 2
+    saved = proxy.SETTINGS.cache_capacity_blocks
+    proxy.SETTINGS.cache_capacity_blocks = 2
     try:
         block_size = proxy.BLOCK_SIZE
         # Three distinct one-block prefixes; first must be evicted.
@@ -112,14 +112,14 @@ def test_record_prefix_evicts_oldest_block(proxy):
         assert inst.estimate_cache_hit(prefix_b) == block_size
         assert inst.estimate_cache_hit(prefix_c) == block_size
     finally:
-        proxy.CACHE_CAPACITY_BLOCKS = saved
+        proxy.SETTINGS.cache_capacity_blocks = saved
 
 
 def test_estimate_cache_hit_touches_lru(proxy):
     """A cache hit must move the block to the MRU position."""
     inst = proxy.InstanceState("http://x")
-    saved = proxy.CACHE_CAPACITY_BLOCKS
-    proxy.CACHE_CAPACITY_BLOCKS = 2
+    saved = proxy.SETTINGS.cache_capacity_blocks
+    proxy.SETTINGS.cache_capacity_blocks = 2
     try:
         block_size = proxy.BLOCK_SIZE
         a = [1] * block_size
@@ -134,7 +134,7 @@ def test_estimate_cache_hit_touches_lru(proxy):
         assert inst.estimate_cache_hit(a) == block_size
         assert inst.estimate_cache_hit(b) == 0
     finally:
-        proxy.CACHE_CAPACITY_BLOCKS = saved
+        proxy.SETTINGS.cache_capacity_blocks = saved
 
 
 def test_pick_instance_session_affinity_sticks(proxy):
@@ -178,3 +178,37 @@ def test_pick_instance_lmetric_picks_lowest_score(proxy):
     chosen, idx = proxy.pick_instance_lmetric(insts, None, None, 1000, {})
     # Empty instance has score = 1000 * 0 = 0; busy one has (5000+1000)*4.
     assert idx == 0 and chosen is insts[0]
+
+
+def test_settings_has_runtime_knobs(proxy):
+    """D5/B4/M3: Settings dataclass exposes the previously-hardcoded knobs."""
+    s = proxy.SETTINGS
+    for field in (
+        "heavy_threshold",
+        "overload_factor",
+        "max_offload_inflight",
+        "cache_gate_ratio",
+        "prefill_throughput",
+        "rdma_overhead_s",
+        "cache_capacity_blocks",
+    ):
+        assert hasattr(s, field), f"SETTINGS missing {field}"
+    # Runtime mutability matters for tests + __main__ override.
+    saved = s.cache_gate_ratio
+    s.cache_gate_ratio = 0.55
+    assert proxy.SETTINGS.cache_gate_ratio == 0.55
+    s.cache_gate_ratio = saved
+
+
+def test_p_offload_penalty_uses_settings_heavy_threshold(proxy):
+    """M2: tweaking SETTINGS.heavy_threshold changes the P-offload penalty."""
+    inst = proxy.InstanceState("http://x")
+    inst.active_p_offloads = 3
+    saved = proxy.SETTINGS.heavy_threshold
+    try:
+        proxy.SETTINGS.heavy_threshold = 10000
+        assert proxy._p_offload_penalty(inst) == 30000
+        proxy.SETTINGS.heavy_threshold = 50000
+        assert proxy._p_offload_penalty(inst) == 150000
+    finally:
+        proxy.SETTINGS.heavy_threshold = saved