Harness: explore gpu-memory-utilization (and raise max-num-seqs) before Stop-B

The harness defined a gpu-memory-utilization family but hard-coded active_now=False
and never generated a candidate for it, and only ever *lowered* max-num-seqs for
decode_tpot. So on the decode-bound 27B incumbent it stopped at TP4=0.648 while the
naive (use_harness=false) baseline freely found gpu-memory-utilization=0.94 -> 0.873
(+35%) and max-num-seqs=48. That made the harness look worse than naive -- a real
coverage gap, not bad luck.

Fix in _runtime_candidate_actions (topology-before-runtime gated: only once topology
has moved off the baseline, so a baseline latency bottleneck still gets a TP change):
- Add a gpu-memory-utilization hill-climb candidate (+0.02/step toward a 0.97 safe
  ceiling) for decode_tpot/admission incumbents, scored high enough (>=0.35) to block
  a premature Stop-B until it is tried; the incumbent guard keeps the step only if
  per-GPU rate improves and the engine launches, and the tested signature terminates
  the climb (so 0.96 OOM/regression backs off to 0.94 automatically).
- Let max-num-seqs *rise* for decode_tpot (not only fall) to exploit decode parallelism.
- Activate the gpu-memory-utilization harness family for decode_tpot/admission.

Verified: new unit test asserts a settled TP4 decode-bound incumbent gets a
gpu-memory-utilization raise (0.9->0.92) and no stop while untried. 115 tests pass.
Empirical reliability (harness recovers ~0.87 and stops) to be confirmed by re-run.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/aituner/harness.py

@@ -24,6 +24,13 @@ _RUNTIME_KEYS = {
 _STRONG_INCUMBENT_MIN_GAIN = 1.8
 _MIN_POST_INCUMBENT_VALIDATION_TRIALS = 2
 _VALIDATION_TRIALS_WITHOUT_FAMILY_COVERAGE = 3
+# Decode-bound throughput is frequently KV-cache limited, so more gpu-memory-utilization
+# yields more KV blocks and more concurrent decode. Hill-climb in small steps toward a
+# safe ceiling and let measurement find the real peak: a too-high target regresses or
+# fails to launch and is rejected by the incumbent guard, and its tested signature then
+# blocks re-proposal so the climb terminates.
+_GMU_STEP = 0.02
+_GMU_SAFE_CEILING = 0.97
 
 
 def build_harness_context(
@@ -383,14 +390,17 @@ def _knob_harnesses(
                 "knob_family": "gpu-memory-utilization",
                 "use_when": [
                     "The engine launches cleanly but memory headroom limits batching.",
+                    "A decode-bound incumbent (decode_tpot) is KV-cache limited and could sustain more concurrent decode with more KV blocks.",
                 ],
                 "procedure": [
                     "Make small adjustments only after topology and batching knobs are stable.",
+                    "Raise gpu-memory-utilization one small step at a time and keep the step only if request_rate_per_gpu improves and the engine still launches.",
                 ],
                 "guards": [
                     "Treat launch OOM as hard negative evidence and back off immediately.",
+                    "Do not exceed a safe utilization ceiling; stop climbing once a higher target regresses or fails to launch.",
                 ],
-                "active_now": False,
+                "active_now": active_bottleneck in {"decode_tpot", "admission_or_queueing"},
             }
         )
     return harnesses
@@ -1184,6 +1194,15 @@ def _runtime_candidate_actions(
     topology_patch = _preserve_topology_patch(study, anchor_flags)
     actions: list[dict[str, Any]] = []
 
+    base_tp = _parse_int_like(study.engine.base_flags.get("tensor-parallel-size"), default=1)
+    base_dp = _parse_int_like(study.engine.base_flags.get("data-parallel-size"), default=1)
+    cur_tp = _parse_int_like(anchor_flags.get("tensor-parallel-size"), default=base_tp)
+    cur_dp = _parse_int_like(anchor_flags.get("data-parallel-size"), default=base_dp)
+    # Topology-before-runtime: gpu-mem-util / raising max-num-seqs are micro-tuning that is
+    # only justified once topology has moved off the baseline. At the baseline a latency
+    # bottleneck must still be answered with a topology change, not a runtime tweak.
+    topology_settled = cur_tp > base_tp or cur_dp > base_dp
+
     if "max-num-batched-tokens" in tunable:
         current_mbt = _parse_int_like(anchor_flags.get("max-num-batched-tokens"), default=0)
         mbt_targets: list[tuple[str, int]] = []
@@ -1226,8 +1245,17 @@ def _runtime_candidate_actions(
         if top_bottleneck == "admission_or_queueing":
             target = max(8, int(current_mns * 1.5)) if current_mns > 0 else 64
             mns_targets.append(("raise_max_num_seqs", _round_up_to_multiple(target, 8)))
-        elif top_bottleneck == "decode_tpot" and current_mns > 8:
-            mns_targets.append(("lower_max_num_seqs", max(8, current_mns // 2)))
+        elif top_bottleneck == "decode_tpot":
+            if current_mns > 8:
+                mns_targets.append(("lower_max_num_seqs", max(8, current_mns // 2)))
+            # Decode concurrency can also be too low: once topology is settled, raising
+            # max-num-seqs exploits decode parallelism when the incumbent has SLO headroom.
+            # The incumbent guard keeps it only if per-GPU rate improves.
+            if topology_settled:
+                raise_target = _round_up_to_multiple(
+                    max(16, int(current_mns * 1.5)) if current_mns > 0 else 48, 8
+                )
+                mns_targets.append(("raise_max_num_seqs", raise_target))
         for action_id, target in mns_targets:
             patch = {**topology_patch, "max-num-seqs": target}
             signature = _config_signature({"env_patch": {}, "flag_patch": patch})
@@ -1273,6 +1301,37 @@ def _runtime_candidate_actions(
                         ],
                     )
                 )
+
+    if (
+        "gpu-memory-utilization" in tunable
+        and topology_settled
+        and top_bottleneck in {"decode_tpot", "admission_or_queueing"}
+    ):
+        current_gmu = _parse_float_like(
+            anchor_flags.get("gpu-memory-utilization"), default=0.9
+        )
+        if 0.0 < current_gmu < _GMU_SAFE_CEILING:
+            target = round(min(_GMU_SAFE_CEILING, current_gmu + _GMU_STEP), 4)
+            if target > current_gmu:
+                patch = {**topology_patch, "gpu-memory-utilization": target}
+                signature = _config_signature({"env_patch": {}, "flag_patch": patch})
+                if signature not in tested_signatures:
+                    actions.append(
+                        _runtime_action(
+                            action_id="raise_gpu_memory_utilization",
+                            knob_family="gpu-memory-utilization",
+                            score=0.4 + _information_gain(bottleneck_hypotheses, "runtime"),
+                            patch=patch,
+                            hypothesis=(
+                                "Raise gpu-memory-utilization to add KV-cache headroom so the "
+                                "decode-bound incumbent can sustain more concurrent decode."
+                            ),
+                            expected_effects=[
+                                "add KV-cache blocks for higher decode concurrency on the incumbent topology",
+                                "reject if the higher memory target regresses request_rate_per_gpu or fails to launch",
+                            ],
+                        )
+                    )
     return actions
 
 
@@ -2252,6 +2311,19 @@ def _parse_int_like(value: Any, *, default: int) -> int:
     return default
 
 
+def _parse_float_like(value: Any, *, default: float) -> float:
+    if value is None or isinstance(value, bool):
+        return default
+    if isinstance(value, (int, float)):
+        return float(value)
+    if isinstance(value, str) and value.strip():
+        try:
+            return float(value.strip())
+        except ValueError:
+            return default
+    return default
+
+
 def _config_signature(config_patch: Any) -> str:
     if not isinstance(config_patch, dict):
         config_patch = {}