Gahow Wang
4089ffd63f
The replayer was artificially limiting concurrency with --max-inflight-sessions (semaphore) and --time-scale (time compression), producing unrealistically low 1 req/GPU load that masked prefill-decode interference. Replayer changes: - Remove session_sem and time_scale entirely - Each request dispatched at its trace timestamp exactly - Sessions still sequential (turn N+1 waits for turn N completion) - If turn completes late, next turn fires immediately Sampler changes: - Add --sample-ratio for GPU-proportional session sampling - Keep --target-requests for backwards compat - No time compression (preserve original arrival pattern) bench.sh: remove --time-scale and --max-inflight-sessions args Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
52 lines
1.9 KiB
Python
52 lines
1.9 KiB
Python
"""CLI entry point: python -m replayer replay ..."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
from .replay import ReplayConfig, replay_trace
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser(description="Trace replayer for vLLM benchmarking")
|
|
p.add_argument("--trace", type=Path, required=True, help="Sampled trace JSONL")
|
|
p.add_argument("--output", type=Path, required=True, help="Output metrics JSONL")
|
|
p.add_argument("--endpoint", type=str, required=True,
|
|
help="vLLM server URL (e.g. http://localhost:8000)")
|
|
p.add_argument("--model", type=str, default="default", help="Model name for API")
|
|
p.add_argument("--concurrency-limit", type=int, default=2000,
|
|
help="Max concurrent HTTP requests (safety limit)")
|
|
p.add_argument("--request-timeout", type=float, default=600.0)
|
|
p.add_argument("--request-limit", type=int, default=None,
|
|
help="Limit number of requests to replay")
|
|
p.add_argument("-v", "--verbose", action="store_true")
|
|
args = p.parse_args()
|
|
|
|
logging.basicConfig(
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
|
|
config = ReplayConfig(
|
|
trace_path=args.trace,
|
|
output_path=args.output,
|
|
endpoint_url=args.endpoint.rstrip("/"),
|
|
model_name=args.model,
|
|
concurrency_limit=args.concurrency_limit,
|
|
request_timeout_s=args.request_timeout,
|
|
request_limit=args.request_limit,
|
|
)
|
|
|
|
results = asyncio.run(replay_trace(config))
|
|
succeeded = sum(1 for r in results if r.error is None)
|
|
print(f"\nDone: {succeeded}/{len(results)} requests succeeded")
|
|
print(f"Metrics: {args.output}")
|
|
print(f"Summary: {args.output.with_suffix('.summary.json')}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|