Add decode-only study mode support
This commit is contained in:
@@ -15,6 +15,15 @@ def build_prompt(
|
||||
state: StudyState,
|
||||
capability_profile: dict[str, Any] | None,
|
||||
) -> str:
|
||||
objective_notes: list[str] = []
|
||||
if study.trace.request_mode == "decode_only":
|
||||
objective_notes.append(
|
||||
"This study is decode-only. The engine uses a KV decode benchmark connector, so TTFT is informational only unless an explicit TTFT rule is configured."
|
||||
)
|
||||
if study.slo.ttft_rule is None:
|
||||
objective_notes.append("There is no TTFT SLO for this study.")
|
||||
if study.slo.tpot_rule is None:
|
||||
objective_notes.append("There is no TPOT SLO for this study.")
|
||||
history = []
|
||||
for trial in state.trials[-study.llm.max_history_trials :]:
|
||||
history.append(
|
||||
@@ -57,6 +66,7 @@ def build_prompt(
|
||||
},
|
||||
"trace": {
|
||||
"window_id": study.trace.window_id,
|
||||
"request_mode": study.trace.request_mode,
|
||||
"input_length_filter": (
|
||||
{
|
||||
"min_input_tokens": study.trace.input_length_filter.min_input_tokens,
|
||||
@@ -88,6 +98,7 @@ def build_prompt(
|
||||
"target_pass_rate": study.slo.target_pass_rate,
|
||||
"ttft_rule": study.slo.ttft_rule,
|
||||
"tpot_rule": study.slo.tpot_rule,
|
||||
"objective_notes": objective_notes,
|
||||
},
|
||||
default=lambda value: value.__dict__,
|
||||
ensure_ascii=False,
|
||||
|
||||
@@ -237,6 +237,7 @@ class TraceSpec:
|
||||
windows_path: str
|
||||
window_id: str
|
||||
trace_file_override: str | None
|
||||
request_mode: str
|
||||
u_field: str
|
||||
timestamp_field: str
|
||||
max_concurrency: int
|
||||
@@ -251,12 +252,16 @@ class TraceSpec:
|
||||
def from_dict(cls, data: Mapping[str, Any]) -> "TraceSpec":
|
||||
max_requests = data.get("max_requests_per_probe")
|
||||
synthetic_prompt_cap = data.get("synthetic_prompt_cap_tokens")
|
||||
request_mode = str(data.get("request_mode") or "chat").strip().lower()
|
||||
if request_mode not in {"chat", "decode_only"}:
|
||||
raise SpecError("trace.request_mode must be one of: chat, decode_only.")
|
||||
return cls(
|
||||
windows_path=_require_str(data.get("windows_path"), context="trace.windows_path"),
|
||||
window_id=_require_str(data.get("window_id"), context="trace.window_id"),
|
||||
trace_file_override=str(data["trace_file_override"]).strip()
|
||||
if data.get("trace_file_override")
|
||||
else None,
|
||||
request_mode=request_mode,
|
||||
u_field=str(data.get("u_field") or "sampling_u").strip(),
|
||||
timestamp_field=str(data.get("timestamp_field") or "timestamp").strip(),
|
||||
max_concurrency=_require_int(
|
||||
|
||||
@@ -71,6 +71,7 @@ def _latency_summary(
|
||||
tpot_values = [float(item.tpot_ms) for item in outcomes if item.tpot_ms is not None]
|
||||
return {
|
||||
"observed_request_count": len(outcomes),
|
||||
"request_mode": study.trace.request_mode,
|
||||
"ttft_ms": _metric_summary(ttft_values),
|
||||
"tpot_ms": _metric_summary(tpot_values),
|
||||
"failed_reason_counts": _reason_counts(evaluations),
|
||||
|
||||
Reference in New Issue
Block a user