diff --git a/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py b/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py index de40676..cff055e 100644 --- a/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py +++ b/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Open-loop fixed-shape prefill-only workload for one real offered-load anchor.""" +"""Open-loop fixed-shape workload for one real offered-load anchor.""" from __future__ import annotations @@ -14,7 +14,6 @@ from pathlib import Path from typing import Any -TTFT_SLO_MS = 1256.0 TARGET_PASS_RATE = 0.95 @@ -27,6 +26,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--rate", type=float, required=True) parser.add_argument("--requests", type=int, default=64) parser.add_argument("--input-tokens", type=int, default=2048) + parser.add_argument("--output-tokens", type=int, default=1) + parser.add_argument("--ttft-slo-ms", type=float) + parser.add_argument("--tpot-slo-ms", type=float, default=150.0) parser.add_argument("--timeout-seconds", type=float, default=900.0) parser.add_argument("--output", type=Path, required=True) return parser.parse_args() @@ -40,6 +42,10 @@ def percentile(values: list[float], fraction: float) -> float | None: return ordered[index] +def ttft_slo_ms(input_tokens: int) -> float: + return 1000.0 + 1000.0 * input_tokens / 8000.0 + + def run_request( *, request_index: int, @@ -63,8 +69,8 @@ def run_request( body = { "model": args.served_model, "prompt": prompt_ids, - "min_tokens": 1, - "max_tokens": 1, + "min_tokens": args.output_tokens, + "max_tokens": args.output_tokens, "ignore_eos": True, "temperature": 0, "stream": True, @@ -85,6 +91,7 @@ def run_request( f"HTTP {response.status}: {response.read().decode(errors='replace')}" ) first_token_at = None + last_token_at = None streamed_tokens = 0 usage = None while True: @@ -105,16 +112,23 @@ def run_request( token_ids = choice.get("token_ids") or [] emitted += len(token_ids) if token_ids else int(bool(choice.get("text"))) if emitted: - first_token_at = first_token_at or time.perf_counter() + now = time.perf_counter() + first_token_at = first_token_at or now + last_token_at = now streamed_tokens += emitted finished = time.perf_counter() if first_token_at is None or usage is None: raise RuntimeError("missing streaming token or usage") prompt_tokens = int(usage["prompt_tokens"]) completion_tokens = int(usage["completion_tokens"]) - if prompt_tokens != args.input_tokens or completion_tokens != 1: + if prompt_tokens != args.input_tokens or completion_tokens != args.output_tokens: raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}") ttft = (first_token_at - started) * 1000.0 + tpot = ( + (last_token_at - first_token_at) * 1000.0 / (completion_tokens - 1) + if completion_tokens > 1 + else None + ) record.update( { "success": True, @@ -122,8 +136,10 @@ def run_request( "completion_tokens": completion_tokens, "streamed_token_count": streamed_tokens, "ttft_ms": ttft, + "tpot_ms": tpot, "e2e_ms": (finished - started) * 1000.0, - "slo_pass": ttft <= TTFT_SLO_MS, + "slo_pass": ttft <= args.ttft_slo_ms + and (tpot is None or tpot <= args.tpot_slo_ms), } ) except Exception as error: # Failed requests remain in the SLO denominator. @@ -136,8 +152,18 @@ def run_request( def main() -> None: args = parse_args() - if args.rate <= 0 or args.requests <= 0 or args.input_tokens <= 0: - raise ValueError("rate, requests, and input tokens must be positive") + if min( + args.rate, + args.requests, + args.input_tokens, + args.output_tokens, + args.tpot_slo_ms, + ) <= 0: + raise ValueError("rate, requests, tokens, and SLO must be positive") + if args.ttft_slo_ms is None: + args.ttft_slo_ms = ttft_slo_ms(args.input_tokens) + if args.ttft_slo_ms <= 0: + raise ValueError("TTFT SLO must be positive") from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) @@ -174,14 +200,19 @@ def main() -> None: completed = [row for row in requests if row["success"]] passed = sum(bool(row["slo_pass"]) for row in requests) ttfts = [float(row["ttft_ms"]) for row in completed] + tpots = [ + float(row["tpot_ms"]) + for row in completed + if row["tpot_ms"] is not None + ] pass_rate = passed / len(requests) payload = { - "schema": "qwen30-prefill-rate-anchor-v1", + "schema": "qwen30-fixed-rate-anchor-v2", "workload": { "offered_request_rate": args.rate, "request_count": args.requests, "input_tokens": args.input_tokens, - "output_tokens": 1, + "output_tokens": args.output_tokens, "prefix_caching": False, "arrival": "open_loop_uniform", "last_scheduled_arrival_s": (args.requests - 1) / args.rate, @@ -193,11 +224,17 @@ def main() -> None: "ttft_p50_ms": percentile(ttfts, 0.50), "ttft_p95_ms": percentile(ttfts, 0.95), "ttft_max_ms": max(ttfts) if ttfts else None, + "tpot_p50_ms": percentile(tpots, 0.50), + "tpot_p95_ms": percentile(tpots, 0.95), + "tpot_max_ms": max(tpots) if tpots else None, "admission_lag_max_ms": max( float(row["admission_lag_ms"]) for row in requests ), "slo": { - "ttft_threshold_ms": TTFT_SLO_MS, + "ttft_threshold_ms": args.ttft_slo_ms, + "tpot_threshold_ms": ( + args.tpot_slo_ms if args.output_tokens > 1 else None + ), "passed": passed, "pass_rate": pass_rate, "feasible": pass_rate >= TARGET_PASS_RATE, diff --git a/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh b/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh index a406d51..2861c21 100644 --- a/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh +++ b/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh @@ -6,6 +6,10 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" TP="${TP:?TP is required}" MNS="${MNS:?MNS is required}" RATES="${RATES:-4 8 16 32 64}" +INPUT_TOKENS="${INPUT_TOKENS:-2048}" +OUTPUT_TOKENS="${OUTPUT_TOKENS:-1}" +TPOT_SLO_MS="${TPOT_SLO_MS:-150}" +WARMUP_SECONDS="${WARMUP_SECONDS:-2}" SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" @@ -35,18 +39,19 @@ if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then fi read -r -a RATE_ARRAY <<< "${RATES}" -echo "QWEN30_PREFILL_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL2048_OSL1 arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}" +echo "QWEN30_FIXED_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL${INPUT_TOKENS}_OSL${OUTPUT_TOKENS} ttft_slo=1000+1000*ISL/8000ms tpot_slo=${TPOT_SLO_MS}ms warmup_seconds=${WARMUP_SECONDS} arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum run_qwen30_prefill_real_config.sh qwen30_prefill_client.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" -"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" \ +"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" "${INPUT_TOKENS}" "${OUTPUT_TOKENS}" "${TPOT_SLO_MS}" "${WARMUP_SECONDS}" \ > "${OUTPUT_ROOT}/provenance/contract.json" <<'PY' import importlib.metadata as metadata import json import platform import sys -tp, mns, rates = sys.argv[1:] +tp, mns, rates, input_tokens, output_tokens, tpot_slo_ms, warmup_seconds = sys.argv[1:] +input_tokens = int(input_tokens) print(json.dumps({ "python": platform.python_version(), "torch": metadata.version("torch"), @@ -57,10 +62,11 @@ print(json.dumps({ "rounds": 2, "requests_per_anchor": 64, "anchor_isolation": "fresh_server_per_rate_per_round", - "target_rate_warmup_requests": "min(32, max(4, ceil(rate * 2)))", - "input_tokens": 2048, - "output_tokens": 1, - "ttft_slo_ms": 1256.0, + "target_rate_warmup_requests": f"min(32, max(4, ceil(rate * {warmup_seconds})))", + "input_tokens": input_tokens, + "output_tokens": int(output_tokens), + "ttft_slo_ms": 1000.0 + 1000.0 * input_tokens / 8000.0, + "tpot_slo_ms": float(tpot_slo_ms) if int(output_tokens) > 1 else None, "target_pass_rate": 0.95, }, indent=2, sort_keys=True)) PY @@ -112,19 +118,22 @@ for ROUND in 1 2; do exit 1 fi - WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY' + WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" "${WARMUP_SECONDS}" <<'PY' import math import sys -print(min(32, max(4, math.ceil(float(sys.argv[1]) * 2.0)))) +print(min(32, max(4, math.ceil(float(sys.argv[1]) * float(sys.argv[2]))))) PY )" "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ - --requests "${WARMUP_REQUESTS}" \ + --requests "${WARMUP_REQUESTS}" --input-tokens "${INPUT_TOKENS}" \ + --output-tokens "${OUTPUT_TOKENS}" --tpot-slo-ms "${TPOT_SLO_MS}" \ --output "${ROUND_ROOT}/results/warmup_${KEY}.json" "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ - --requests 64 --output "${ROUND_ROOT}/results/${KEY}.json" + --requests 64 --input-tokens "${INPUT_TOKENS}" \ + --output-tokens "${OUTPUT_TOKENS}" --tpot-slo-ms "${TPOT_SLO_MS}" \ + --output "${ROUND_ROOT}/results/${KEY}.json" cleanup done done diff --git a/runs/frontier-phase-factorial-v0/test_phase_factorial.py b/runs/frontier-phase-factorial-v0/test_phase_factorial.py index 0ceb170..07293da 100644 --- a/runs/frontier-phase-factorial-v0/test_phase_factorial.py +++ b/runs/frontier-phase-factorial-v0/test_phase_factorial.py @@ -25,6 +25,8 @@ def test_percentile() -> None: assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.50) == 2.0 assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.95) == 4.0 assert client.percentile([], 0.95) is None + assert client.ttft_slo_ms(512) == 1064.0 + assert client.ttft_slo_ms(2048) == 1256.0 def test_grid_and_trace(tmp_path: Path) -> None: