Add qwen235b prefill-only tuning support
This commit is contained in:
@@ -182,6 +182,11 @@ def build_prompt(
|
||||
objective_notes.append(
|
||||
"This study is decode-only. The engine uses a KV decode benchmark connector, so TTFT is informational only unless an explicit TTFT rule is configured."
|
||||
)
|
||||
if study.trace.completion_tokens_override is not None:
|
||||
objective_notes.append(
|
||||
"The trace replay overrides completion length for every request. "
|
||||
f"min_tokens=max_tokens={study.trace.completion_tokens_override}."
|
||||
)
|
||||
if study.slo.ttft_rule is None:
|
||||
objective_notes.append("There is no TTFT SLO for this study.")
|
||||
if study.slo.tpot_rule is None:
|
||||
@@ -243,6 +248,7 @@ def build_prompt(
|
||||
"trace": {
|
||||
"window_id": study.trace.window_id,
|
||||
"request_mode": study.trace.request_mode,
|
||||
"completion_tokens_override": study.trace.completion_tokens_override,
|
||||
"input_length_filter": (
|
||||
{
|
||||
"min_input_tokens": study.trace.input_length_filter.min_input_tokens,
|
||||
|
||||
@@ -317,6 +317,7 @@ class TraceSpec:
|
||||
window_id: str
|
||||
trace_file_override: str | None
|
||||
request_mode: str
|
||||
completion_tokens_override: int | None
|
||||
u_field: str
|
||||
timestamp_field: str
|
||||
max_concurrency: int
|
||||
@@ -331,9 +332,17 @@ class TraceSpec:
|
||||
def from_dict(cls, data: Mapping[str, Any]) -> "TraceSpec":
|
||||
max_requests = data.get("max_requests_per_probe")
|
||||
synthetic_prompt_cap = data.get("synthetic_prompt_cap_tokens")
|
||||
completion_tokens_override = data.get("completion_tokens_override")
|
||||
request_mode = str(data.get("request_mode") or "chat").strip().lower()
|
||||
if request_mode not in {"chat", "decode_only"}:
|
||||
raise SpecError("trace.request_mode must be one of: chat, decode_only.")
|
||||
if completion_tokens_override is not None:
|
||||
completion_tokens_override = _require_int(
|
||||
completion_tokens_override,
|
||||
context="trace.completion_tokens_override",
|
||||
)
|
||||
if completion_tokens_override < 0:
|
||||
raise SpecError("trace.completion_tokens_override must be >= 0.")
|
||||
return cls(
|
||||
windows_path=_require_str(data.get("windows_path"), context="trace.windows_path"),
|
||||
window_id=_require_str(data.get("window_id"), context="trace.window_id"),
|
||||
@@ -341,6 +350,7 @@ class TraceSpec:
|
||||
if data.get("trace_file_override")
|
||||
else None,
|
||||
request_mode=request_mode,
|
||||
completion_tokens_override=completion_tokens_override,
|
||||
u_field=str(data.get("u_field") or "sampling_u").strip(),
|
||||
timestamp_field=str(data.get("timestamp_field") or "timestamp").strip(),
|
||||
max_concurrency=_require_int(
|
||||
|
||||
@@ -204,7 +204,11 @@ def load_trace_requests(study: StudySpec, *, study_spec_path: Path) -> tuple[Win
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
completion_tokens = _coerce_completion_tokens(row)
|
||||
completion_tokens = (
|
||||
study.trace.completion_tokens_override
|
||||
if study.trace.completion_tokens_override is not None
|
||||
else _coerce_completion_tokens(row)
|
||||
)
|
||||
if completion_tokens is not None:
|
||||
body["min_tokens"] = completion_tokens
|
||||
body["max_tokens"] = completion_tokens
|
||||
|
||||
Reference in New Issue
Block a user