Stop after strong incumbent harness gains
This commit is contained in:
@@ -365,6 +365,58 @@ class CoreFlowTests(unittest.TestCase):
|
||||
]
|
||||
)
|
||||
|
||||
def test_harness_strong_incumbent_guard_after_large_gain(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
study_path = _write_study_assets(tmp_path)
|
||||
study = load_study_spec(study_path)
|
||||
state = StudyState(
|
||||
study_id=study.study_id,
|
||||
best_trial_id="trial-0002",
|
||||
best_request_rate_per_gpu=0.21,
|
||||
trials=[
|
||||
TrialSummary(
|
||||
trial_id="trial-0001",
|
||||
status="completed",
|
||||
parallel_size=1,
|
||||
best_request_rate=0.035,
|
||||
best_request_rate_per_gpu=0.035,
|
||||
config_patch={"env_patch": {}, "flag_patch": {}},
|
||||
),
|
||||
TrialSummary(
|
||||
trial_id="trial-0002",
|
||||
status="completed",
|
||||
parallel_size=2,
|
||||
best_request_rate=0.42,
|
||||
best_request_rate_per_gpu=0.21,
|
||||
config_patch={
|
||||
"env_patch": {},
|
||||
"flag_patch": {
|
||||
"tensor-parallel-size": 2,
|
||||
"data-parallel-size": 1,
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
context = build_harness_context(
|
||||
study=study,
|
||||
window_summary={
|
||||
"prompt_tokens_p95": 7628,
|
||||
"prompt_tokens_p99": 8102,
|
||||
"prompt_tail_ratio_p95_p50": 3.83,
|
||||
},
|
||||
state=state,
|
||||
)
|
||||
guard = context["convergence_guard"]["strong_incumbent"]
|
||||
self.assertTrue(guard["guard_active"])
|
||||
self.assertGreaterEqual(guard["incumbent_gain_vs_baseline"], 3.0)
|
||||
self.assertTrue(
|
||||
context["convergence_guard"][
|
||||
"should_stop_if_no_harness_can_justify_a_new_adjacent_probe"
|
||||
]
|
||||
)
|
||||
|
||||
def test_trace_input_length_filter_keeps_only_matching_rows(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
|
||||
Reference in New Issue
Block a user