Add bad-start harness recovery planning
This commit is contained in:
@@ -1904,6 +1904,212 @@ class CoreFlowTests(unittest.TestCase):
|
||||
)
|
||||
self.assertNotIn("gpu-memory-utilization", proposal.config_patch.flag_patch)
|
||||
|
||||
def test_harness_brackets_down_from_bad_high_tp_start_before_runtime_tuning(self) -> None:
|
||||
"""A no-LLM run that starts at the max TP should validate the adjacent lower
|
||||
topology before spending trials on runtime micro-tuning."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
study_path = _write_study_assets(
|
||||
tmp_path,
|
||||
slo_overrides={
|
||||
"ttft_rule": {"kind": "fixed_ms", "threshold_ms": 4000},
|
||||
"tpot_rule": {"kind": "fixed_ms", "threshold_ms": 50},
|
||||
},
|
||||
engine_overrides={
|
||||
"base_flags": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8000,
|
||||
"tensor-parallel-size": 8,
|
||||
"data-parallel-size": 1,
|
||||
"gpu-memory-utilization": 0.5,
|
||||
"max-num-seqs": 8,
|
||||
},
|
||||
"tunable_flags": [
|
||||
"tensor-parallel-size",
|
||||
"data-parallel-size",
|
||||
"gpu-memory-utilization",
|
||||
"max-num-seqs",
|
||||
],
|
||||
"topology_constraints": {
|
||||
"allowed_tensor_parallel_sizes": [1, 2, 4, 8],
|
||||
"allowed_data_parallel_sizes": [1],
|
||||
"allowed_tp_dp_products": [1, 2, 4, 8],
|
||||
},
|
||||
},
|
||||
)
|
||||
study = load_study_spec(study_path)
|
||||
result_path = tmp_path / "trial-0001.json"
|
||||
result_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"best_sampling_u": 0.05,
|
||||
"best_request_rate": 8.0,
|
||||
"best_pass_rate": 0.96,
|
||||
"probes": [
|
||||
{
|
||||
"threshold": 0.05,
|
||||
"feasible": True,
|
||||
"payload": {
|
||||
"request_count": 300,
|
||||
"pass_rate": 0.96,
|
||||
"request_rate": 8.0,
|
||||
"latency_summary": {"failed_reason_counts": {}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"threshold": 0.08,
|
||||
"feasible": False,
|
||||
"payload": {
|
||||
"request_count": 300,
|
||||
"pass_rate": 0.5,
|
||||
"request_rate": 10.0,
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"latency_summary": {
|
||||
"failed_reason_counts": {"ttft_ms>4000.0": 120}
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
state = StudyState(
|
||||
study_id=study.study_id,
|
||||
best_trial_id="trial-0001",
|
||||
best_request_rate=8.0,
|
||||
best_request_rate_per_gpu=1.0,
|
||||
trials=[
|
||||
TrialSummary(
|
||||
trial_id="trial-0001",
|
||||
status="completed",
|
||||
best_request_rate=8.0,
|
||||
best_request_rate_per_gpu=1.0,
|
||||
result_path=str(result_path),
|
||||
config_patch={"env_patch": {}, "flag_patch": {}},
|
||||
),
|
||||
],
|
||||
)
|
||||
context = build_harness_context(
|
||||
study=study,
|
||||
window_summary={"prompt_tokens_p95": 6500},
|
||||
state=state,
|
||||
)
|
||||
proposal = build_harness_guided_proposal(context)
|
||||
self.assertIsNotNone(proposal)
|
||||
self.assertEqual(
|
||||
proposal.config_patch.flag_patch.get("tensor-parallel-size"), 4
|
||||
)
|
||||
self.assertNotIn("gpu-memory-utilization", proposal.config_patch.flag_patch)
|
||||
self.assertNotIn("max-num-seqs", proposal.config_patch.flag_patch)
|
||||
|
||||
def test_harness_jumps_low_gpu_mem_util_to_nominal_floor_after_topology_settles(self) -> None:
|
||||
"""A pathological gmu=0.5 start should jump to the normal operating floor
|
||||
after topology is bracketed instead of wasting many 0.02 hill-climb trials."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
study_path = _write_study_assets(
|
||||
tmp_path,
|
||||
slo_overrides={
|
||||
"ttft_rule": {"kind": "fixed_ms", "threshold_ms": 4000},
|
||||
"tpot_rule": {"kind": "fixed_ms", "threshold_ms": 50},
|
||||
},
|
||||
engine_overrides={
|
||||
"base_flags": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8000,
|
||||
"tensor-parallel-size": 2,
|
||||
"data-parallel-size": 1,
|
||||
"gpu-memory-utilization": 0.5,
|
||||
},
|
||||
"tunable_flags": [
|
||||
"tensor-parallel-size",
|
||||
"gpu-memory-utilization",
|
||||
],
|
||||
"topology_constraints": {
|
||||
"allowed_tensor_parallel_sizes": [1, 2, 4],
|
||||
"allowed_data_parallel_sizes": [1],
|
||||
"allowed_tp_dp_products": [1, 2, 4],
|
||||
},
|
||||
},
|
||||
)
|
||||
study = load_study_spec(study_path)
|
||||
result_path = tmp_path / "trial-0001.json"
|
||||
result_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"best_sampling_u": 0.07,
|
||||
"best_request_rate": 2.4,
|
||||
"best_pass_rate": 0.97,
|
||||
"probes": [
|
||||
{
|
||||
"threshold": 0.07,
|
||||
"feasible": True,
|
||||
"payload": {
|
||||
"request_count": 300,
|
||||
"pass_rate": 0.97,
|
||||
"request_rate": 2.4,
|
||||
"latency_summary": {"failed_reason_counts": {}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"threshold": 0.1,
|
||||
"feasible": False,
|
||||
"payload": {
|
||||
"request_count": 300,
|
||||
"pass_rate": 0.55,
|
||||
"request_rate": 3.1,
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"latency_summary": {
|
||||
"failed_reason_counts": {"tpot_ms>50.0": 90}
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
state = StudyState(
|
||||
study_id=study.study_id,
|
||||
best_trial_id="trial-0001",
|
||||
best_request_rate=2.4,
|
||||
best_request_rate_per_gpu=1.2,
|
||||
trials=[
|
||||
TrialSummary(
|
||||
trial_id="trial-0001",
|
||||
status="completed",
|
||||
best_request_rate=2.4,
|
||||
best_request_rate_per_gpu=1.2,
|
||||
result_path=str(result_path),
|
||||
config_patch={"env_patch": {}, "flag_patch": {}},
|
||||
),
|
||||
TrialSummary(
|
||||
trial_id="trial-0002",
|
||||
status="completed",
|
||||
best_request_rate=2.2,
|
||||
best_request_rate_per_gpu=0.55,
|
||||
config_patch={
|
||||
"env_patch": {},
|
||||
"flag_patch": {"tensor-parallel-size": 4},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
context = build_harness_context(
|
||||
study=study,
|
||||
window_summary={"prompt_tokens_p95": 1500},
|
||||
state=state,
|
||||
)
|
||||
proposal = build_harness_guided_proposal(context)
|
||||
self.assertIsNotNone(proposal)
|
||||
self.assertEqual(
|
||||
proposal.config_patch.flag_patch.get("gpu-memory-utilization"), 0.9
|
||||
)
|
||||
self.assertNotIn("tensor-parallel-size", proposal.config_patch.flag_patch)
|
||||
|
||||
def test_harness_continues_gpu_mem_util_after_tied_same_topology_probe(self) -> None:
|
||||
"""After adjacent topology validation, gpu-memory-utilization should hill-climb
|
||||
on the incumbent topology even if an earlier gmu step tied the incumbent and
|
||||
|
||||
Reference in New Issue
Block a user