gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Measure lower-range performance for infeasible trials

Browse Source
This commit is contained in:
Gahow Wang
2026-05-10 14:30:34 +08:00
parent bf7c02e721
commit 14259fcec9
4 changed files with 157 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
src/aituner/worker.py
View File

@@ -539,41 +539,88 @@ def run_trial(trial_spec_path: Path) -> dict[str, Any]:
payload=payload,
)
search = binary_search_max_feasible(
primary_search = binary_search_max_feasible(
low=trial.search.low,
high=trial.search.high,
tolerance=trial.search.tolerance,
max_probes=trial.search.max_probes,
evaluator=evaluator,
)
best = search.best_feasible_payload
search_for_best = primary_search
best = primary_search.best_feasible_payload
best_source = "primary_search"
fallback_search = None
original_search_low = float(study.search.low)
inherited_search_floor = float(trial.search.low)
if best is None and inherited_search_floor > original_search_low:
fallback_search = binary_search_max_feasible(
low=original_search_low,
high=inherited_search_floor,
tolerance=trial.search.tolerance,
max_probes=trial.search.max_probes,
evaluator=evaluator,
)
if fallback_search.best_feasible_payload is not None:
search_for_best = fallback_search
best = fallback_search.best_feasible_payload
best_source = "lower_range_fallback"
def serialize_probe(probe: ThresholdProbe[ProbePayload]) -> dict[str, Any]:
return {
"threshold": probe.threshold,
"feasible": probe.feasible,
"payload": {
"request_count": probe.payload.request_count,
"pass_rate": probe.payload.pass_rate,
"request_rate": probe.payload.request_rate,
"early_stopped": probe.payload.early_stopped,
"early_stop_reason": probe.payload.early_stop_reason,
"latency_summary": probe.payload.latency_summary,
},
}
all_probes = [
*primary_search.probes,
*((fallback_search.probes if fallback_search is not None else [])),
]
result = {
"study_id": trial.study_id,
"trial_id": trial.trial_id,
"status": "completed",
"config_patch": to_jsonable(trial.config_patch),
"best_sampling_u": search.best_threshold if best is not None else None,
"best_source": best_source,
"best_sampling_u": search_for_best.best_threshold if best is not None else None,
"best_request_rate": best.request_rate if best is not None else None,
"best_pass_rate": best.pass_rate if best is not None else None,
"best_request_count": best.request_count if best is not None else None,
"probes": [
{
"threshold": probe.threshold,
"feasible": probe.feasible,
"payload": {
"request_count": probe.payload.request_count,
"pass_rate": probe.payload.pass_rate,
"request_rate": probe.payload.request_rate,
"early_stopped": probe.payload.early_stopped,
"early_stop_reason": probe.payload.early_stop_reason,
"latency_summary": probe.payload.latency_summary,
},
}
for probe in search.probes
],
"probes": [serialize_probe(probe) for probe in all_probes],
}
if best is None and search.probes:
last_probe = search.probes[-1]
if fallback_search is not None:
result["primary_search"] = {
"low": inherited_search_floor,
"high": trial.search.high,
"best_sampling_u": primary_search.best_threshold
if primary_search.best_feasible_payload is not None
else None,
"best_request_rate": primary_search.best_feasible_payload.request_rate
if primary_search.best_feasible_payload is not None
else None,
"probes": [serialize_probe(probe) for probe in primary_search.probes],
}
result["lower_range_fallback"] = {
"triggered": True,
"low": original_search_low,
"high": inherited_search_floor,
"best_sampling_u": fallback_search.best_threshold
if fallback_search.best_feasible_payload is not None
else None,
"best_request_rate": fallback_search.best_feasible_payload.request_rate
if fallback_search.best_feasible_payload is not None
else None,
"probes": [serialize_probe(probe) for probe in fallback_search.probes],
}
if best is None and all_probes:
last_probe = all_probes[-1]
result["all_infeasible_diagnostics"] = {
"threshold": last_probe.threshold,
"request_count": last_probe.payload.request_count,