Gahow Wang
8a6b22c11c
Adds `--dispatch-mode {tracets,thinktime}` to the replayer and documents that
agentic serving should be benchmarked with `thinktime` (the faithful load).
- `tracets` (old default): turn-k at the absolute trace timestamp, i.e.
max(prev_finished, trace_ts) -- collapses inter-turn think-time to ~0 when the
system is behind, manufacturing request bursts.
- `thinktime`: turn-1 at trace arrival; turn-k at prev_finished +
time_to_parent_chat (real production gap). scripts/add_time_to_parent.py
annotates a trace with that gap from the raw trace's request_ready/end_ms.
exp(c) ablation (v2/exp_c_dispatch_ablation/): at N=8 (capacity slack) thinktime
beats tracets -- E2E p90 -28% (73.5 vs 102.8s), TTFT p90 -29%, TPS +7%, because
tracets' bursts spike concurrency -> KV pressure -> preemption. At N=6
(saturated) they converge. So tracets makes the system look ~30% worse on tail
latency than realistic agent pacing. Root README.md carries the headline
guidance; raw per-request metrics gitignored (perf_summary.json kept).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
71 lines
3.1 KiB
Python
71 lines
3.1 KiB
Python
"""CLI entry point: python -m replayer replay ..."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from .replay import ReplayConfig, replay_trace
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser(description="Trace replayer for vLLM benchmarking")
|
|
p.add_argument("--trace", type=Path, required=True, help="Sampled trace JSONL")
|
|
p.add_argument("--output", type=Path, required=True, help="Output metrics JSONL")
|
|
p.add_argument("--endpoint", type=str, required=True,
|
|
help="vLLM server URL (e.g. http://localhost:8000)")
|
|
p.add_argument("--model", type=str, default="default", help="Model name for API")
|
|
p.add_argument("--concurrency-limit", type=int, default=2000,
|
|
help="Max concurrent HTTP requests (safety limit)")
|
|
_env_inflight = os.environ.get("REPLAY_MAX_INFLIGHT")
|
|
p.add_argument("--max-inflight-sessions", type=int,
|
|
default=int(_env_inflight) if _env_inflight else None,
|
|
help="Cap on concurrent sessions (None = unlimited; "
|
|
"trace-driven dispatch otherwise). Env: REPLAY_MAX_INFLIGHT")
|
|
_env_think = os.environ.get("REPLAY_INTER_TURN_THINK_S")
|
|
p.add_argument("--inter-turn-think", type=float,
|
|
default=float(_env_think) if _env_think else None,
|
|
help="Closed-loop think-time (s) after each turn completes; "
|
|
"ignore absolute trace schedule. Env: REPLAY_INTER_TURN_THINK_S")
|
|
p.add_argument("--dispatch-mode", choices=["tracets", "thinktime"],
|
|
default=os.environ.get("REPLAY_DISPATCH_MODE", "tracets"),
|
|
help="tracets (Mode 1): absolute trace ts = max(prev_finished, ts). "
|
|
"thinktime (Mode 2): turn-k at prev_finished + "
|
|
"time_to_parent_chat. Env: REPLAY_DISPATCH_MODE")
|
|
p.add_argument("--request-timeout", type=float, default=600.0)
|
|
p.add_argument("--request-limit", type=int, default=None,
|
|
help="Limit number of requests to replay")
|
|
p.add_argument("-v", "--verbose", action="store_true")
|
|
args = p.parse_args()
|
|
|
|
logging.basicConfig(
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
|
|
config = ReplayConfig(
|
|
trace_path=args.trace,
|
|
output_path=args.output,
|
|
endpoint_url=args.endpoint.rstrip("/"),
|
|
model_name=args.model,
|
|
concurrency_limit=args.concurrency_limit,
|
|
request_timeout_s=args.request_timeout,
|
|
request_limit=args.request_limit,
|
|
max_inflight_sessions=args.max_inflight_sessions,
|
|
inter_turn_think_s=args.inter_turn_think,
|
|
dispatch_mode=args.dispatch_mode,
|
|
)
|
|
|
|
results = asyncio.run(replay_trace(config))
|
|
succeeded = sum(1 for r in results if r.error is None)
|
|
print(f"\nDone: {succeeded}/{len(results)} requests succeeded")
|
|
print(f"Metrics: {args.output}")
|
|
print(f"Summary: {args.output.with_suffix('.summary.json')}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|