gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prevent prefill convergence stop before seq probe

Browse Source
This commit is contained in:
Gahow Wang
2026-06-22 14:43:55 +08:00
parent 4607711bb5
commit fd94ab9f3b
2 changed files with 173 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/aituner/harness.py
View File

@@ -1103,6 +1103,7 @@ def _candidate_actions(
anchor,
top_bottleneck,
bottleneck_hypotheses,
recent_diagnostics,
tested_signatures,
)
)
@@ -1196,6 +1197,7 @@ def _runtime_candidate_actions(
anchor: dict[str, Any],
top_bottleneck: str,
bottleneck_hypotheses: list[dict[str, Any]],
recent_diagnostics: list[dict[str, Any]],
tested_signatures: set[str],
) -> list[dict[str, Any]]:
tunable = set(study.engine.tunable_flags)
@@ -1258,6 +1260,14 @@ def _runtime_candidate_actions(
if "max-num-seqs" in tunable:
current_mns = _parse_int_like(anchor_flags.get("max-num-seqs"), default=0)
max_num_seqs_tested = any(
"max-num-seqs" in (
((item.get("config_patch") or {}).get("flag_patch") or {})
if isinstance(item.get("config_patch"), dict)
else {}
)
for item in recent_diagnostics
)
mns_targets: list[tuple[str, int]] = []
if top_bottleneck == "admission_or_queueing":
target = max(8, int(current_mns * 1.5)) if current_mns > 0 else 64
@@ -1273,12 +1283,25 @@ def _runtime_candidate_actions(
max(16, int(current_mns * 1.5)) if current_mns > 0 else 48, 8
)
mns_targets.append(("raise_max_num_seqs", raise_target))
elif top_bottleneck == "ttft_prefill" and topology_settled and not max_num_seqs_tested:
# Prefill-heavy TTFT can still be admission/concurrency limited after TP and
# max-num-batched-tokens probes settle. Try a modest same-topology seq cap
# increase before letting convergence guards declare the incumbent final.
target = _round_up_to_multiple(
max(16, int(current_mns * 1.5)) if current_mns > 0 else 64, 8
)
mns_targets.append(("raise_max_num_seqs", target))
for action_id, target in mns_targets:
patch = {**runtime_base_patch, "max-num-seqs": target}
signature = _config_signature({"env_patch": {}, "flag_patch": patch})
if signature in tested_signatures:
continue
relief = 0.25 if top_bottleneck in {"decode_tpot", "admission_or_queueing"} else 0.08
if top_bottleneck in {"decode_tpot", "admission_or_queueing"}:
relief = 0.25
elif top_bottleneck == "ttft_prefill":
relief = 0.3
else:
relief = 0.08
actions.append(
_runtime_action(
action_id=action_id,
@@ -1286,12 +1309,12 @@ def _runtime_candidate_actions(
score=relief + _information_gain(bottleneck_hypotheses, "runtime"),
patch=patch,
hypothesis=(
"Adjust max-num-seqs to test whether concurrency pressure is the "
"limiting factor under the configured SLO."
"Adjust max-num-seqs to test whether concurrency/admission pressure "
"is the limiting factor under the configured SLO."
),
expected_effects=[
"change decode/admission concurrency on the incumbent topology",
"confirm if TPOT or queueing pressure is caused by sequence concurrency",
"change prefill/decode admission concurrency on the incumbent topology",
"confirm if latency or queueing pressure is caused by sequence concurrency",
],
)
)