Harness: explore gpu-memory-utilization (and raise max-num-seqs) before Stop-B
The harness defined a gpu-memory-utilization family but hard-coded active_now=False and never generated a candidate for it, and only ever *lowered* max-num-seqs for decode_tpot. So on the decode-bound 27B incumbent it stopped at TP4=0.648 while the naive (use_harness=false) baseline freely found gpu-memory-utilization=0.94 -> 0.873 (+35%) and max-num-seqs=48. That made the harness look worse than naive -- a real coverage gap, not bad luck. Fix in _runtime_candidate_actions (topology-before-runtime gated: only once topology has moved off the baseline, so a baseline latency bottleneck still gets a TP change): - Add a gpu-memory-utilization hill-climb candidate (+0.02/step toward a 0.97 safe ceiling) for decode_tpot/admission incumbents, scored high enough (>=0.35) to block a premature Stop-B until it is tried; the incumbent guard keeps the step only if per-GPU rate improves and the engine launches, and the tested signature terminates the climb (so 0.96 OOM/regression backs off to 0.94 automatically). - Let max-num-seqs *rise* for decode_tpot (not only fall) to exploit decode parallelism. - Activate the gpu-memory-utilization harness family for decode_tpot/admission. Verified: new unit test asserts a settled TP4 decode-bound incumbent gets a gpu-memory-utilization raise (0.9->0.92) and no stop while untried. 115 tests pass. Empirical reliability (harness recovers ~0.87 and stops) to be confirmed by re-run. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1318,6 +1318,113 @@ class CoreFlowTests(unittest.TestCase):
|
||||
},
|
||||
)
|
||||
|
||||
def test_harness_raises_gpu_mem_util_on_settled_decode_bound_incumbent(self) -> None:
|
||||
"""Regression for the coverage gap that let the naive baseline beat the harness:
|
||||
a settled TP incumbent that is decode_tpot-bound must get a gpu-memory-utilization
|
||||
raise (KV-cache headroom) before the harness is allowed to stop."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
study_path = _write_study_assets(
|
||||
tmp_path,
|
||||
slo_overrides={
|
||||
"ttft_rule": {"kind": "fixed_ms", "threshold_ms": 4000},
|
||||
"tpot_rule": {"kind": "fixed_ms", "threshold_ms": 50},
|
||||
},
|
||||
engine_overrides={
|
||||
"tunable_flags": [
|
||||
"tensor-parallel-size",
|
||||
"gpu-memory-utilization",
|
||||
],
|
||||
"topology_constraints": {
|
||||
"allowed_tensor_parallel_sizes": [1, 2, 4],
|
||||
"allowed_data_parallel_sizes": [1],
|
||||
"allowed_tp_dp_products": [1, 2, 4],
|
||||
},
|
||||
},
|
||||
)
|
||||
study = load_study_spec(study_path)
|
||||
result_path = tmp_path / "trial-0002.json"
|
||||
result_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"best_sampling_u": 0.074,
|
||||
"best_request_rate": 2.6,
|
||||
"best_pass_rate": 0.97,
|
||||
"probes": [
|
||||
{
|
||||
"threshold": 0.074,
|
||||
"feasible": True,
|
||||
"payload": {
|
||||
"request_count": 300,
|
||||
"pass_rate": 0.97,
|
||||
"request_rate": 2.6,
|
||||
"latency_summary": {"failed_reason_counts": {}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"threshold": 0.09,
|
||||
"feasible": False,
|
||||
"payload": {
|
||||
"request_count": 300,
|
||||
"pass_rate": 0.6,
|
||||
"request_rate": 3.2,
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"latency_summary": {
|
||||
"failed_reason_counts": {"tpot_ms>50.0": 90}
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
state = StudyState(
|
||||
study_id=study.study_id,
|
||||
best_trial_id="trial-0002",
|
||||
best_request_rate=2.6,
|
||||
best_request_rate_per_gpu=0.65,
|
||||
trials=[
|
||||
TrialSummary(
|
||||
trial_id="trial-0001",
|
||||
status="completed",
|
||||
best_request_rate=1.1,
|
||||
best_request_rate_per_gpu=0.275,
|
||||
config_patch={"env_patch": {}, "flag_patch": {"tensor-parallel-size": 2}},
|
||||
),
|
||||
TrialSummary(
|
||||
trial_id="trial-0002",
|
||||
status="completed",
|
||||
best_request_rate=2.6,
|
||||
best_request_rate_per_gpu=0.65,
|
||||
result_path=str(result_path),
|
||||
config_patch={
|
||||
"env_patch": {},
|
||||
"flag_patch": {
|
||||
"tensor-parallel-size": 4,
|
||||
"gpu-memory-utilization": 0.9,
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
context = build_harness_context(
|
||||
study=study, window_summary={"prompt_tokens_p95": 1500}, state=state
|
||||
)
|
||||
proposal = build_harness_guided_proposal(context)
|
||||
self.assertIsNotNone(proposal)
|
||||
self.assertFalse(proposal.should_stop)
|
||||
# TP4 preserved; gpu-memory-utilization hill-climbed one step (0.9 -> 0.92).
|
||||
self.assertEqual(
|
||||
proposal.config_patch.flag_patch.get("tensor-parallel-size"), 4
|
||||
)
|
||||
self.assertEqual(
|
||||
proposal.config_patch.flag_patch.get("gpu-memory-utilization"), 0.92
|
||||
)
|
||||
# And the harness must NOT authorize a stop while that knob is untried.
|
||||
self.assertIsNone(build_harness_stop_proposal(context))
|
||||
|
||||
def test_harness_validates_unmeasured_tp_frontier_before_runtime_refinement(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
|
||||
Reference in New Issue
Block a user