Use normalized full config signatures
This commit is contained in:
@@ -25,6 +25,7 @@ from aituner.http_client import (
|
||||
)
|
||||
from aituner.job import append_job, build_trial_job
|
||||
from aituner.harness import (
|
||||
_effective_config_signature,
|
||||
build_harness_context,
|
||||
build_harness_guided_proposal,
|
||||
build_harness_stop_proposal,
|
||||
@@ -358,6 +359,36 @@ class CoreFlowTests(unittest.TestCase):
|
||||
"search_high_lowered_to_trace_ceiling",
|
||||
)
|
||||
|
||||
def test_effective_config_signature_treats_noop_patch_as_baseline(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
study_path = _write_study_assets(
|
||||
Path(tmp),
|
||||
engine_overrides={
|
||||
"base_flags": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8000,
|
||||
"tensor-parallel-size": 8,
|
||||
"data-parallel-size": 1,
|
||||
"gpu-memory-utilization": 0.5,
|
||||
"max-num-seqs": 8,
|
||||
},
|
||||
},
|
||||
)
|
||||
study = load_study_spec(study_path)
|
||||
|
||||
baseline = _effective_config_signature(study, {"env_patch": {}, "flag_patch": {}})
|
||||
noop_tp = _effective_config_signature(
|
||||
study,
|
||||
{"env_patch": {}, "flag_patch": {"tensor-parallel-size": 8}},
|
||||
)
|
||||
changed_tp = _effective_config_signature(
|
||||
study,
|
||||
{"env_patch": {}, "flag_patch": {"tensor-parallel-size": 4}},
|
||||
)
|
||||
|
||||
self.assertEqual(baseline, noop_tp)
|
||||
self.assertNotEqual(baseline, changed_tp)
|
||||
|
||||
def test_lca_workload_profile_uses_standard_10d_features(self) -> None:
|
||||
window = WindowRecord(
|
||||
window_id="w1",
|
||||
@@ -1639,6 +1670,153 @@ class CoreFlowTests(unittest.TestCase):
|
||||
"search_high_saturation_requires_parallel_size_evidence",
|
||||
)
|
||||
|
||||
def test_harness_does_not_repropose_noop_topology_equivalent_to_baseline(
|
||||
self,
|
||||
) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
study_path = _write_study_assets(
|
||||
tmp_path,
|
||||
engine_overrides={
|
||||
"base_flags": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8000,
|
||||
"tensor-parallel-size": 8,
|
||||
"data-parallel-size": 1,
|
||||
"gpu-memory-utilization": 0.5,
|
||||
"max-num-seqs": 8,
|
||||
},
|
||||
"tunable_flags": ["tensor-parallel-size", "max-num-seqs"],
|
||||
"topology_constraints": {
|
||||
"allowed_tensor_parallel_sizes": [1, 2, 4, 8],
|
||||
"allowed_tp_dp_products": [1, 2, 4, 8],
|
||||
},
|
||||
},
|
||||
)
|
||||
study = load_study_spec(study_path)
|
||||
trial1_result = tmp_path / "trial-0001.json"
|
||||
trial1_result.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"best_sampling_u": 0.935616858887,
|
||||
"best_request_rate": 8.0,
|
||||
"best_pass_rate": 1.0,
|
||||
"probes": [
|
||||
{
|
||||
"threshold": 0.935616858887,
|
||||
"feasible": True,
|
||||
"payload": {
|
||||
"request_count": 480,
|
||||
"pass_rate": 1.0,
|
||||
"request_rate": 8.0,
|
||||
"early_stopped": False,
|
||||
"early_stop_reason": "",
|
||||
"latency_summary": {"failed_reason_counts": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
trial2_result = tmp_path / "trial-0002.json"
|
||||
trial2_result.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"best_sampling_u": 0.810867944369,
|
||||
"best_request_rate": 6.95,
|
||||
"best_pass_rate": 0.9784,
|
||||
"probes": [
|
||||
{
|
||||
"threshold": 0.873242401628,
|
||||
"feasible": False,
|
||||
"payload": {
|
||||
"request_count": 450,
|
||||
"pass_rate": 0.7844,
|
||||
"request_rate": 7.5,
|
||||
"early_stopped": True,
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"latency_summary": {
|
||||
"failed_reason_counts": {
|
||||
"ttft_ms>2000.0": 42,
|
||||
"slo_pass_rate_unrecoverable": 49,
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"threshold": 0.810867944369,
|
||||
"feasible": True,
|
||||
"payload": {
|
||||
"request_count": 417,
|
||||
"pass_rate": 0.9784,
|
||||
"request_rate": 6.95,
|
||||
"early_stopped": False,
|
||||
"early_stop_reason": "",
|
||||
"latency_summary": {
|
||||
"failed_reason_counts": {"ttft_ms>2000.0": 9}
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
state = StudyState(
|
||||
study_id=study.study_id,
|
||||
best_trial_id="trial-0002",
|
||||
best_parallel_size=4,
|
||||
best_sampling_u=0.810867944369,
|
||||
best_request_rate=6.95,
|
||||
best_request_rate_per_gpu=1.7375,
|
||||
next_trial_index=3,
|
||||
trials=[
|
||||
TrialSummary(
|
||||
trial_id="trial-0001",
|
||||
status="completed",
|
||||
parallel_size=8,
|
||||
best_request_rate=8.0,
|
||||
best_request_rate_per_gpu=1.0,
|
||||
result_path=str(trial1_result),
|
||||
config_patch={"env_patch": {}, "flag_patch": {}},
|
||||
),
|
||||
TrialSummary(
|
||||
trial_id="trial-0002",
|
||||
status="completed",
|
||||
parallel_size=4,
|
||||
best_request_rate=6.95,
|
||||
best_request_rate_per_gpu=1.7375,
|
||||
result_path=str(trial2_result),
|
||||
config_patch={
|
||||
"env_patch": {},
|
||||
"flag_patch": {"tensor-parallel-size": 4},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
context = build_harness_context(
|
||||
study=study,
|
||||
window_summary={"prompt_tokens_p95": 2048},
|
||||
state=state,
|
||||
)
|
||||
actions = context["experiment_plan"]["candidate_actions"]
|
||||
self.assertFalse(
|
||||
any(
|
||||
action.get("config_patch", {}).get("flag_patch")
|
||||
== {"tensor-parallel-size": 8}
|
||||
for action in actions
|
||||
)
|
||||
)
|
||||
proposal = build_harness_guided_proposal(context)
|
||||
self.assertTrue(
|
||||
proposal is None
|
||||
or proposal.config_patch.flag_patch != {"tensor-parallel-size": 8}
|
||||
)
|
||||
|
||||
def test_harness_guided_first_tp_probe_for_latency_bottleneck(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
|
||||
Reference in New Issue
Block a user