gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harness stop uses full state baseline

Browse Source
This commit is contained in:
Gahow Wang
2026-06-20 22:48:27 +08:00
parent a9d237bbfd
commit 426151bc9f
2 changed files with 139 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/aituner/harness.py
View File

@@ -607,6 +607,15 @@ def stateful_history_limit() -> int:
return 8
def _state_completed_trials_with_rates(state: StudyState) -> list[TrialSummary]:
return [
trial
for trial in state.trials
if trial.status == "completed"
and isinstance(trial.best_request_rate_per_gpu, (int, float))
]
def _load_result(trial: TrialSummary) -> dict[str, Any] | None:
if not trial.result_path:
return None
@@ -1960,15 +1969,19 @@ def _validation_exhausted_guard(
}
if not state.best_trial_id or not isinstance(state.best_request_rate_per_gpu, (int, float)):
return default
completed = [
item
for item in recent_diagnostics
if item.get("status") == "completed"
and isinstance(item.get("best_request_rate_per_gpu"), (int, float))
]
if not completed:
return default
baseline_rate = _as_float(completed[0].get("best_request_rate_per_gpu"))
state_completed = _state_completed_trials_with_rates(state)
if state_completed:
baseline_rate = float(state_completed[0].best_request_rate_per_gpu)
else:
completed = [
item
for item in recent_diagnostics
if item.get("status") == "completed"
and isinstance(item.get("best_request_rate_per_gpu"), (int, float))
]
if not completed:
return default
baseline_rate = _as_float(completed[0].get("best_request_rate_per_gpu"))
incumbent_rate = _as_float(state.best_request_rate_per_gpu)
if baseline_rate <= 0 or incumbent_rate <= 0:
return default
@@ -2084,16 +2097,11 @@ def _strong_incumbent_guard(
}
if state.best_trial_id is None or state.best_request_rate_per_gpu is None:
return default
completed = [
item
for item in recent_diagnostics
if item.get("status") == "completed"
and isinstance(item.get("best_request_rate_per_gpu"), (int, float))
]
completed = _state_completed_trials_with_rates(state)
if len(completed) < 2:
return default
baseline = completed[0]
baseline_rate = float(baseline["best_request_rate_per_gpu"])
baseline_rate = float(baseline.best_request_rate_per_gpu)
incumbent_rate = float(state.best_request_rate_per_gpu)
if baseline_rate <= 0:
return default
@@ -2103,7 +2111,7 @@ def _strong_incumbent_guard(
return {
"guard_active": True,
"reason": "incumbent_exceeds_baseline_by_1_8x_and_latest_trial_is_best_enter_validation_phase",
"baseline_trial_id": baseline.get("trial_id"),
"baseline_trial_id": baseline.trial_id,
"baseline_request_rate_per_gpu": baseline_rate,
"incumbent_gain_vs_baseline": gain,
"recommended_next_action": (
@@ -2113,7 +2121,7 @@ def _strong_incumbent_guard(
}
return {
**default,
"baseline_trial_id": baseline.get("trial_id"),
"baseline_trial_id": baseline.trial_id,
"baseline_request_rate_per_gpu": baseline_rate,
"incumbent_gain_vs_baseline": gain,
"reason": "need_more_evidence_before_strong_incumbent_stop",