Add bad-start harness recovery planning

src/aituner/harness.py

@@ -29,7 +29,10 @@ _VALIDATION_TRIALS_WITHOUT_FAMILY_COVERAGE = 3
 # safe ceiling and let measurement find the real peak: a too-high target regresses or
 # fails to launch and is rejected by the incumbent guard, and its tested signature then
 # blocks re-proposal so the climb terminates.
+# Pathological starts below the nominal floor jump back into the normal operating range
+# before this small-step climb begins.
 _GMU_STEP = 0.02
+_GMU_NOMINAL_FLOOR = 0.9
 _GMU_SAFE_CEILING = 0.97
 
 
@@ -1147,6 +1150,23 @@ def _topology_candidate_actions(
         current_ep=current_ep,
         current_enable_ep=current_enable_ep,
     )
+    adjacent_lower_tp = None
+    if (
+        study.trace.request_mode != "decode_only"
+        and not _anchor_has_topology_patch(anchor)
+        and not _has_unmeasured_higher_tp_candidate(
+            study,
+            legal,
+            current_tp=current_tp,
+            current_dp=current_dp,
+            tested_signatures=tested_signatures,
+        )
+    ):
+        adjacent_lower_tp = _adjacent_lower_tp_candidate(
+            legal,
+            current_tp=current_tp,
+            current_dp=current_dp,
+        )
     actions: list[dict[str, Any]] = []
     for point in legal:
         if point["tensor-parallel-size"] == current_tp and point["data-parallel-size"] == current_dp:
@@ -1164,6 +1184,14 @@ def _topology_candidate_actions(
             candidate_tp=point["tensor-parallel-size"],
             candidate_dp=point["data-parallel-size"],
         )
+        if (
+            adjacent_lower_tp is not None
+            and current_tp > 2
+            and point["tensor-parallel-size"] == adjacent_lower_tp
+            and point["data-parallel-size"] == current_dp
+        ):
+            score = max(score, 0.74)
+            factors["bad_start_topology_bracket"] = 0.74
         if score <= 0:
             continue
         action_id = _topology_action_id(current_tp, current_dp, point)
@@ -1457,7 +1485,11 @@ def _next_gpu_memory_utilization_target(
         elif item.get("status") == "failed":
             failed_gmus.append(gmu)
     climb_from = max(successful_gmus)
-    target = round(min(_GMU_SAFE_CEILING, climb_from + _GMU_STEP), 4)
+    if climb_from < _GMU_NOMINAL_FLOOR:
+        target = min(_GMU_SAFE_CEILING, _GMU_NOMINAL_FLOOR)
+    else:
+        target = min(_GMU_SAFE_CEILING, climb_from + _GMU_STEP)
+    target = round(target, 4)
     if target <= climb_from:
         return None
     if any(failed <= target + EPSILON for failed in failed_gmus):
@@ -1490,6 +1522,53 @@ def _runtime_action(
     }
 
 
+def _anchor_has_topology_patch(anchor: dict[str, Any]) -> bool:
+    patch = anchor.get("config_patch")
+    if not isinstance(patch, dict):
+        return False
+    flag_patch = patch.get("flag_patch")
+    if not isinstance(flag_patch, dict):
+        return False
+    return any(key in flag_patch for key in _TOPOLOGY_KEYS)
+
+
+def _has_unmeasured_higher_tp_candidate(
+    study: StudySpec,
+    legal: list[dict[str, Any]],
+    *,
+    current_tp: int,
+    current_dp: int,
+    tested_signatures: set[str],
+) -> bool:
+    for point in legal:
+        if (
+            point["data-parallel-size"] != current_dp
+            or point["tensor-parallel-size"] <= current_tp
+        ):
+            continue
+        signature = _config_signature(
+            {"env_patch": {}, "flag_patch": _topology_patch(study, point)}
+        )
+        if signature not in tested_signatures:
+            return True
+    return False
+
+
+def _adjacent_lower_tp_candidate(
+    legal: list[dict[str, Any]],
+    *,
+    current_tp: int,
+    current_dp: int,
+) -> int | None:
+    lower_tps = {
+        int(point["tensor-parallel-size"])
+        for point in legal
+        if point["data-parallel-size"] == current_dp
+        and point["tensor-parallel-size"] < current_tp
+    }
+    return max(lower_tps) if lower_tps else None
+
+
 def _legal_topology_points(
     study: StudySpec,
     *,