Add decode-only study mode support

This commit is contained in:
2026-04-09 11:23:17 +08:00
parent 96140b79bb
commit c158807fac
6 changed files with 282 additions and 1 deletions

View File

@@ -15,6 +15,15 @@ def build_prompt(
state: StudyState,
capability_profile: dict[str, Any] | None,
) -> str:
objective_notes: list[str] = []
if study.trace.request_mode == "decode_only":
objective_notes.append(
"This study is decode-only. The engine uses a KV decode benchmark connector, so TTFT is informational only unless an explicit TTFT rule is configured."
)
if study.slo.ttft_rule is None:
objective_notes.append("There is no TTFT SLO for this study.")
if study.slo.tpot_rule is None:
objective_notes.append("There is no TPOT SLO for this study.")
history = []
for trial in state.trials[-study.llm.max_history_trials :]:
history.append(
@@ -57,6 +66,7 @@ def build_prompt(
},
"trace": {
"window_id": study.trace.window_id,
"request_mode": study.trace.request_mode,
"input_length_filter": (
{
"min_input_tokens": study.trace.input_length_filter.min_input_tokens,
@@ -88,6 +98,7 @@ def build_prompt(
"target_pass_rate": study.slo.target_pass_rate,
"ttft_rule": study.slo.ttft_rule,
"tpot_rule": study.slo.tpot_rule,
"objective_notes": objective_notes,
},
default=lambda value: value.__dict__,
ensure_ascii=False,

View File

@@ -237,6 +237,7 @@ class TraceSpec:
windows_path: str
window_id: str
trace_file_override: str | None
request_mode: str
u_field: str
timestamp_field: str
max_concurrency: int
@@ -251,12 +252,16 @@ class TraceSpec:
def from_dict(cls, data: Mapping[str, Any]) -> "TraceSpec":
max_requests = data.get("max_requests_per_probe")
synthetic_prompt_cap = data.get("synthetic_prompt_cap_tokens")
request_mode = str(data.get("request_mode") or "chat").strip().lower()
if request_mode not in {"chat", "decode_only"}:
raise SpecError("trace.request_mode must be one of: chat, decode_only.")
return cls(
windows_path=_require_str(data.get("windows_path"), context="trace.windows_path"),
window_id=_require_str(data.get("window_id"), context="trace.window_id"),
trace_file_override=str(data["trace_file_override"]).strip()
if data.get("trace_file_override")
else None,
request_mode=request_mode,
u_field=str(data.get("u_field") or "sampling_u").strip(),
timestamp_field=str(data.get("timestamp_field") or "timestamp").strip(),
max_concurrency=_require_int(

View File

@@ -71,6 +71,7 @@ def _latency_summary(
tpot_values = [float(item.tpot_ms) for item in outcomes if item.tpot_ms is not None]
return {
"observed_request_count": len(outcomes),
"request_mode": study.trace.request_mode,
"ttft_ms": _metric_summary(ttft_values),
"tpot_ms": _metric_summary(tpot_values),
"failed_reason_counts": _reason_counts(evaluations),