Generalize Qwen30 fixed-shape real runner
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Open-loop fixed-shape prefill-only workload for one real offered-load anchor."""
|
||||
"""Open-loop fixed-shape workload for one real offered-load anchor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -14,7 +14,6 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
TTFT_SLO_MS = 1256.0
|
||||
TARGET_PASS_RATE = 0.95
|
||||
|
||||
|
||||
@@ -27,6 +26,9 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--rate", type=float, required=True)
|
||||
parser.add_argument("--requests", type=int, default=64)
|
||||
parser.add_argument("--input-tokens", type=int, default=2048)
|
||||
parser.add_argument("--output-tokens", type=int, default=1)
|
||||
parser.add_argument("--ttft-slo-ms", type=float)
|
||||
parser.add_argument("--tpot-slo-ms", type=float, default=150.0)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=900.0)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
@@ -40,6 +42,10 @@ def percentile(values: list[float], fraction: float) -> float | None:
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def ttft_slo_ms(input_tokens: int) -> float:
|
||||
return 1000.0 + 1000.0 * input_tokens / 8000.0
|
||||
|
||||
|
||||
def run_request(
|
||||
*,
|
||||
request_index: int,
|
||||
@@ -63,8 +69,8 @@ def run_request(
|
||||
body = {
|
||||
"model": args.served_model,
|
||||
"prompt": prompt_ids,
|
||||
"min_tokens": 1,
|
||||
"max_tokens": 1,
|
||||
"min_tokens": args.output_tokens,
|
||||
"max_tokens": args.output_tokens,
|
||||
"ignore_eos": True,
|
||||
"temperature": 0,
|
||||
"stream": True,
|
||||
@@ -85,6 +91,7 @@ def run_request(
|
||||
f"HTTP {response.status}: {response.read().decode(errors='replace')}"
|
||||
)
|
||||
first_token_at = None
|
||||
last_token_at = None
|
||||
streamed_tokens = 0
|
||||
usage = None
|
||||
while True:
|
||||
@@ -105,16 +112,23 @@ def run_request(
|
||||
token_ids = choice.get("token_ids") or []
|
||||
emitted += len(token_ids) if token_ids else int(bool(choice.get("text")))
|
||||
if emitted:
|
||||
first_token_at = first_token_at or time.perf_counter()
|
||||
now = time.perf_counter()
|
||||
first_token_at = first_token_at or now
|
||||
last_token_at = now
|
||||
streamed_tokens += emitted
|
||||
finished = time.perf_counter()
|
||||
if first_token_at is None or usage is None:
|
||||
raise RuntimeError("missing streaming token or usage")
|
||||
prompt_tokens = int(usage["prompt_tokens"])
|
||||
completion_tokens = int(usage["completion_tokens"])
|
||||
if prompt_tokens != args.input_tokens or completion_tokens != 1:
|
||||
if prompt_tokens != args.input_tokens or completion_tokens != args.output_tokens:
|
||||
raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}")
|
||||
ttft = (first_token_at - started) * 1000.0
|
||||
tpot = (
|
||||
(last_token_at - first_token_at) * 1000.0 / (completion_tokens - 1)
|
||||
if completion_tokens > 1
|
||||
else None
|
||||
)
|
||||
record.update(
|
||||
{
|
||||
"success": True,
|
||||
@@ -122,8 +136,10 @@ def run_request(
|
||||
"completion_tokens": completion_tokens,
|
||||
"streamed_token_count": streamed_tokens,
|
||||
"ttft_ms": ttft,
|
||||
"tpot_ms": tpot,
|
||||
"e2e_ms": (finished - started) * 1000.0,
|
||||
"slo_pass": ttft <= TTFT_SLO_MS,
|
||||
"slo_pass": ttft <= args.ttft_slo_ms
|
||||
and (tpot is None or tpot <= args.tpot_slo_ms),
|
||||
}
|
||||
)
|
||||
except Exception as error: # Failed requests remain in the SLO denominator.
|
||||
@@ -136,8 +152,18 @@ def run_request(
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.rate <= 0 or args.requests <= 0 or args.input_tokens <= 0:
|
||||
raise ValueError("rate, requests, and input tokens must be positive")
|
||||
if min(
|
||||
args.rate,
|
||||
args.requests,
|
||||
args.input_tokens,
|
||||
args.output_tokens,
|
||||
args.tpot_slo_ms,
|
||||
) <= 0:
|
||||
raise ValueError("rate, requests, tokens, and SLO must be positive")
|
||||
if args.ttft_slo_ms is None:
|
||||
args.ttft_slo_ms = ttft_slo_ms(args.input_tokens)
|
||||
if args.ttft_slo_ms <= 0:
|
||||
raise ValueError("TTFT SLO must be positive")
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
@@ -174,14 +200,19 @@ def main() -> None:
|
||||
completed = [row for row in requests if row["success"]]
|
||||
passed = sum(bool(row["slo_pass"]) for row in requests)
|
||||
ttfts = [float(row["ttft_ms"]) for row in completed]
|
||||
tpots = [
|
||||
float(row["tpot_ms"])
|
||||
for row in completed
|
||||
if row["tpot_ms"] is not None
|
||||
]
|
||||
pass_rate = passed / len(requests)
|
||||
payload = {
|
||||
"schema": "qwen30-prefill-rate-anchor-v1",
|
||||
"schema": "qwen30-fixed-rate-anchor-v2",
|
||||
"workload": {
|
||||
"offered_request_rate": args.rate,
|
||||
"request_count": args.requests,
|
||||
"input_tokens": args.input_tokens,
|
||||
"output_tokens": 1,
|
||||
"output_tokens": args.output_tokens,
|
||||
"prefix_caching": False,
|
||||
"arrival": "open_loop_uniform",
|
||||
"last_scheduled_arrival_s": (args.requests - 1) / args.rate,
|
||||
@@ -193,11 +224,17 @@ def main() -> None:
|
||||
"ttft_p50_ms": percentile(ttfts, 0.50),
|
||||
"ttft_p95_ms": percentile(ttfts, 0.95),
|
||||
"ttft_max_ms": max(ttfts) if ttfts else None,
|
||||
"tpot_p50_ms": percentile(tpots, 0.50),
|
||||
"tpot_p95_ms": percentile(tpots, 0.95),
|
||||
"tpot_max_ms": max(tpots) if tpots else None,
|
||||
"admission_lag_max_ms": max(
|
||||
float(row["admission_lag_ms"]) for row in requests
|
||||
),
|
||||
"slo": {
|
||||
"ttft_threshold_ms": TTFT_SLO_MS,
|
||||
"ttft_threshold_ms": args.ttft_slo_ms,
|
||||
"tpot_threshold_ms": (
|
||||
args.tpot_slo_ms if args.output_tokens > 1 else None
|
||||
),
|
||||
"passed": passed,
|
||||
"pass_rate": pass_rate,
|
||||
"feasible": pass_rate >= TARGET_PASS_RATE,
|
||||
|
||||
Reference in New Issue
Block a user