gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add frontier delta projection harness candidates

Browse Source
This commit is contained in:
Gahow Wang
2026-06-29 16:15:06 +08:00
parent 6c84dc91d7
commit 8dd9ada194
2 changed files with 553 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
tests/test_core_flow.py
View File

@@ -2720,6 +2720,157 @@ class CoreFlowTests(unittest.TestCase):
],
)
def test_harness_projects_measured_runtime_delta_to_other_frontier_anchor(self) -> None:
"""A runtime improvement found on one topology must be tested on other
Pareto anchors before the harness can keep micro-tuning the source topology."""
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
study_path = _write_study_assets(
tmp_path,
slo_overrides={
"ttft_rule": {"kind": "fixed_ms", "threshold_ms": 4000},
"tpot_rule": {"kind": "fixed_ms", "threshold_ms": 50},
},
engine_overrides={
"base_flags": {
"host": "127.0.0.1",
"port": 8000,
"tensor-parallel-size": 2,
"data-parallel-size": 1,
"gpu-memory-utilization": 0.5,
"max-num-seqs": 8,
},
"tunable_flags": [
"tensor-parallel-size",
"data-parallel-size",
"gpu-memory-utilization",
"max-num-seqs",
],
"topology_constraints": {
"allowed_tensor_parallel_sizes": [2, 4, 8],
"allowed_data_parallel_sizes": [1],
"allowed_tp_dp_products": [2, 4, 8],
},
},
)
study = load_study_spec(study_path)
latest_result_path = tmp_path / "trial-0005.json"
latest_result_path.write_text(
json.dumps(
{
"status": "completed",
"best_sampling_u": 0.1,
"best_request_rate": 8.0,
"best_pass_rate": 0.96,
"probes": [
{
"threshold": 0.1,
"feasible": True,
"payload": {
"request_count": 300,
"pass_rate": 0.96,
"request_rate": 8.0,
"latency_summary": {"failed_reason_counts": {}},
},
},
{
"threshold": 0.12,
"feasible": False,
"payload": {
"request_count": 300,
"pass_rate": 0.6,
"request_rate": 9.0,
"early_stop_reason": "slo_pass_rate_unrecoverable",
"latency_summary": {
"failed_reason_counts": {"ttft_ms>4000.0": 100}
},
},
},
],
}
),
encoding="utf-8",
)
state = StudyState(
study_id=study.study_id,
best_trial_id="trial-0005",
best_request_rate=8.0,
best_request_rate_per_gpu=2.0,
trials=[
TrialSummary(
trial_id="trial-0001",
status="completed",
parallel_size=2,
best_request_rate=3.466,
best_request_rate_per_gpu=1.733,
config_patch={"env_patch": {}, "flag_patch": {}},
),
TrialSummary(
trial_id="trial-0002",
status="completed",
parallel_size=4,
best_request_rate=6.95,
best_request_rate_per_gpu=1.7375,
config_patch={
"env_patch": {},
"flag_patch": {"tensor-parallel-size": 4},
},
),
TrialSummary(
trial_id="trial-0003",
status="completed",
parallel_size=8,
best_request_rate=8.0,
best_request_rate_per_gpu=1.0,
config_patch={
"env_patch": {},
"flag_patch": {"tensor-parallel-size": 8},
},
),
TrialSummary(
trial_id="trial-0004",
status="completed",
parallel_size=4,
best_request_rate=6.95,
best_request_rate_per_gpu=1.7375,
config_patch={
"env_patch": {},
"flag_patch": {
"tensor-parallel-size": 4,
"max-num-seqs": 16,
},
},
),
TrialSummary(
trial_id="trial-0005",
status="completed",
parallel_size=4,
best_request_rate=8.0,
best_request_rate_per_gpu=2.0,
result_path=str(latest_result_path),
config_patch={
"env_patch": {},
"flag_patch": {
"tensor-parallel-size": 4,
"gpu-memory-utilization": 0.9,
},
},
),
],
)
context = build_harness_context(
study=study,
window_summary={"prompt_tokens_p95": 6500},
state=state,
)
next_action = context["experiment_plan"]["next_action"]
self.assertEqual(next_action["knob_family"], "frontier-delta-projection")
self.assertEqual(
next_action["config_patch"]["flag_patch"],
{"gpu-memory-utilization": 0.9},
)
self.assertIsNone(build_harness_stop_proposal(context))
def test_harness_validates_unmeasured_tp_frontier_before_runtime_refinement(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)