diff --git a/runs/frontier-s3-real-v0/qwen30_exact_trace_client.py b/runs/frontier-s3-real-v0/qwen30_exact_trace_client.py index 0dcec9d..67d190e 100644 --- a/runs/frontier-s3-real-v0/qwen30_exact_trace_client.py +++ b/runs/frontier-s3-real-v0/qwen30_exact_trace_client.py @@ -13,6 +13,7 @@ from typing import Any TARGET_PASS_RATE = 0.95 +SSE_READ_BUFFER_BYTES = 8 * 1024 * 1024 def parse_args() -> argparse.Namespace: @@ -177,6 +178,10 @@ async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[d base_url=f"http://{args.host}:{args.port}", timeout=timeout, connector=connector, + # return_token_ids can put the full long prompt token vector in one + # SSE event. aiohttp's readline limit is 2x read_bufsize; the default + # 512 KiB limit rejects valid 100k+ context responses. + read_bufsize=SSE_READ_BUFFER_BYTES, ) as session: tasks = [ asyncio.create_task( @@ -230,6 +235,7 @@ def main() -> None: "input_output_prompt": "exact_source_values", "served_model_alias": args.served_model, "http_connection_reuse": False, + "sse_read_buffer_bytes": SSE_READ_BUFFER_BYTES, "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", "tpot_slo_ms": args.tpot_slo_ms, "target_pass_rate": TARGET_PASS_RATE,