gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generalize Qwen30 fixed-shape real runner

Browse Source
This commit is contained in:
Gahow Wang
2026-07-17 10:17:21 +08:00
parent 6e8704d525
commit a2cf361ffe
3 changed files with 71 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
runs/frontier-phase-factorial-v0/qwen30_prefill_client.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Open-loop fixed-shape prefill-only workload for one real offered-load anchor.""" """Open-loop fixed-shape workload for one real offered-load anchor."""
from __future__ import annotations from __future__ import annotations
@@ -14,7 +14,6 @@ from pathlib import Path
from typing import Any from typing import Any
TTFT_SLO_MS = 1256.0
TARGET_PASS_RATE = 0.95 TARGET_PASS_RATE = 0.95
@@ -27,6 +26,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--rate", type=float, required=True) parser.add_argument("--rate", type=float, required=True)
parser.add_argument("--requests", type=int, default=64) parser.add_argument("--requests", type=int, default=64)
parser.add_argument("--input-tokens", type=int, default=2048) parser.add_argument("--input-tokens", type=int, default=2048)
parser.add_argument("--output-tokens", type=int, default=1)
parser.add_argument("--ttft-slo-ms", type=float)
parser.add_argument("--tpot-slo-ms", type=float, default=150.0)
parser.add_argument("--timeout-seconds", type=float, default=900.0) parser.add_argument("--timeout-seconds", type=float, default=900.0)
parser.add_argument("--output", type=Path, required=True) parser.add_argument("--output", type=Path, required=True)
return parser.parse_args() return parser.parse_args()
@@ -40,6 +42,10 @@ def percentile(values: list[float], fraction: float) -> float | None:
return ordered[index] return ordered[index]
def ttft_slo_ms(input_tokens: int) -> float:
return 1000.0 + 1000.0 * input_tokens / 8000.0
def run_request( def run_request(
*, *,
request_index: int, request_index: int,
@@ -63,8 +69,8 @@ def run_request(
body = { body = {
"model": args.served_model, "model": args.served_model,
"prompt": prompt_ids, "prompt": prompt_ids,
"min_tokens": 1, "min_tokens": args.output_tokens,
"max_tokens": 1, "max_tokens": args.output_tokens,
"ignore_eos": True, "ignore_eos": True,
"temperature": 0, "temperature": 0,
"stream": True, "stream": True,
@@ -85,6 +91,7 @@ def run_request(
f"HTTP {response.status}: {response.read().decode(errors='replace')}" f"HTTP {response.status}: {response.read().decode(errors='replace')}"
) )
first_token_at = None first_token_at = None
last_token_at = None
streamed_tokens = 0 streamed_tokens = 0
usage = None usage = None
while True: while True:
@@ -105,16 +112,23 @@ def run_request(
token_ids = choice.get("token_ids") or [] token_ids = choice.get("token_ids") or []
emitted += len(token_ids) if token_ids else int(bool(choice.get("text"))) emitted += len(token_ids) if token_ids else int(bool(choice.get("text")))
if emitted: if emitted:
first_token_at = first_token_at or time.perf_counter() now = time.perf_counter()
first_token_at = first_token_at or now
last_token_at = now
streamed_tokens += emitted streamed_tokens += emitted
finished = time.perf_counter() finished = time.perf_counter()
if first_token_at is None or usage is None: if first_token_at is None or usage is None:
raise RuntimeError("missing streaming token or usage") raise RuntimeError("missing streaming token or usage")
prompt_tokens = int(usage["prompt_tokens"]) prompt_tokens = int(usage["prompt_tokens"])
completion_tokens = int(usage["completion_tokens"]) completion_tokens = int(usage["completion_tokens"])
if prompt_tokens != args.input_tokens or completion_tokens != 1: if prompt_tokens != args.input_tokens or completion_tokens != args.output_tokens:
raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}") raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}")
ttft = (first_token_at - started) * 1000.0 ttft = (first_token_at - started) * 1000.0
tpot = (
(last_token_at - first_token_at) * 1000.0 / (completion_tokens - 1)
if completion_tokens > 1
else None
)
record.update( record.update(
{ {
"success": True, "success": True,
@@ -122,8 +136,10 @@ def run_request(
"completion_tokens": completion_tokens, "completion_tokens": completion_tokens,
"streamed_token_count": streamed_tokens, "streamed_token_count": streamed_tokens,
"ttft_ms": ttft, "ttft_ms": ttft,
"tpot_ms": tpot,
"e2e_ms": (finished - started) * 1000.0, "e2e_ms": (finished - started) * 1000.0,
"slo_pass": ttft <= TTFT_SLO_MS, "slo_pass": ttft <= args.ttft_slo_ms
and (tpot is None or tpot <= args.tpot_slo_ms),
} }
) )
except Exception as error: # Failed requests remain in the SLO denominator. except Exception as error: # Failed requests remain in the SLO denominator.
@@ -136,8 +152,18 @@ def run_request(
def main() -> None: def main() -> None:
args = parse_args() args = parse_args()
if args.rate <= 0 or args.requests <= 0 or args.input_tokens <= 0: if min(
raise ValueError("rate, requests, and input tokens must be positive") args.rate,
args.requests,
args.input_tokens,
args.output_tokens,
args.tpot_slo_ms,
) <= 0:
raise ValueError("rate, requests, tokens, and SLO must be positive")
if args.ttft_slo_ms is None:
args.ttft_slo_ms = ttft_slo_ms(args.input_tokens)
if args.ttft_slo_ms <= 0:
raise ValueError("TTFT SLO must be positive")
from transformers import AutoTokenizer from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
@@ -174,14 +200,19 @@ def main() -> None:
completed = [row for row in requests if row["success"]] completed = [row for row in requests if row["success"]]
passed = sum(bool(row["slo_pass"]) for row in requests) passed = sum(bool(row["slo_pass"]) for row in requests)
ttfts = [float(row["ttft_ms"]) for row in completed] ttfts = [float(row["ttft_ms"]) for row in completed]
tpots = [
float(row["tpot_ms"])
for row in completed
if row["tpot_ms"] is not None
]
pass_rate = passed / len(requests) pass_rate = passed / len(requests)
payload = { payload = {
"schema": "qwen30-prefill-rate-anchor-v1", "schema": "qwen30-fixed-rate-anchor-v2",
"workload": { "workload": {
"offered_request_rate": args.rate, "offered_request_rate": args.rate,
"request_count": args.requests, "request_count": args.requests,
"input_tokens": args.input_tokens, "input_tokens": args.input_tokens,
"output_tokens": 1, "output_tokens": args.output_tokens,
"prefix_caching": False, "prefix_caching": False,
"arrival": "open_loop_uniform", "arrival": "open_loop_uniform",
"last_scheduled_arrival_s": (args.requests - 1) / args.rate, "last_scheduled_arrival_s": (args.requests - 1) / args.rate,
@@ -193,11 +224,17 @@ def main() -> None:
"ttft_p50_ms": percentile(ttfts, 0.50), "ttft_p50_ms": percentile(ttfts, 0.50),
"ttft_p95_ms": percentile(ttfts, 0.95), "ttft_p95_ms": percentile(ttfts, 0.95),
"ttft_max_ms": max(ttfts) if ttfts else None, "ttft_max_ms": max(ttfts) if ttfts else None,
"tpot_p50_ms": percentile(tpots, 0.50),
"tpot_p95_ms": percentile(tpots, 0.95),
"tpot_max_ms": max(tpots) if tpots else None,
"admission_lag_max_ms": max( "admission_lag_max_ms": max(
float(row["admission_lag_ms"]) for row in requests float(row["admission_lag_ms"]) for row in requests
), ),
"slo": { "slo": {
"ttft_threshold_ms": TTFT_SLO_MS, "ttft_threshold_ms": args.ttft_slo_ms,
"tpot_threshold_ms": (
args.tpot_slo_ms if args.output_tokens > 1 else None
),
"passed": passed, "passed": passed,
"pass_rate": pass_rate, "pass_rate": pass_rate,
"feasible": pass_rate >= TARGET_PASS_RATE, "feasible": pass_rate >= TARGET_PASS_RATE,

31
runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh
View File

@@ -6,6 +6,10 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
TP="${TP:?TP is required}" TP="${TP:?TP is required}"
MNS="${MNS:?MNS is required}" MNS="${MNS:?MNS is required}"
RATES="${RATES:-4 8 16 32 64}" RATES="${RATES:-4 8 16 32 64}"
INPUT_TOKENS="${INPUT_TOKENS:-2048}"
OUTPUT_TOKENS="${OUTPUT_TOKENS:-1}"
TPOT_SLO_MS="${TPOT_SLO_MS:-150}"
WARMUP_SECONDS="${WARMUP_SECONDS:-2}"
SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}"
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
@@ -35,18 +39,19 @@ if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then
fi fi
read -r -a RATE_ARRAY <<< "${RATES}" read -r -a RATE_ARRAY <<< "${RATES}"
echo "QWEN30_PREFILL_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL2048_OSL1 arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}" echo "QWEN30_FIXED_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL${INPUT_TOKENS}_OSL${OUTPUT_TOKENS} ttft_slo=1000+1000*ISL/8000ms tpot_slo=${TPOT_SLO_MS}ms warmup_seconds=${WARMUP_SECONDS} arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}"
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
sha256sum run_qwen30_prefill_real_config.sh qwen30_prefill_client.py \ sha256sum run_qwen30_prefill_real_config.sh qwen30_prefill_client.py \
> "${OUTPUT_ROOT}/provenance/source.sha256" > "${OUTPUT_ROOT}/provenance/source.sha256"
"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" \ "${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" "${INPUT_TOKENS}" "${OUTPUT_TOKENS}" "${TPOT_SLO_MS}" "${WARMUP_SECONDS}" \
> "${OUTPUT_ROOT}/provenance/contract.json" <<'PY' > "${OUTPUT_ROOT}/provenance/contract.json" <<'PY'
import importlib.metadata as metadata import importlib.metadata as metadata
import json import json
import platform import platform
import sys import sys
tp, mns, rates = sys.argv[1:] tp, mns, rates, input_tokens, output_tokens, tpot_slo_ms, warmup_seconds = sys.argv[1:]
input_tokens = int(input_tokens)
print(json.dumps({ print(json.dumps({
"python": platform.python_version(), "python": platform.python_version(),
"torch": metadata.version("torch"), "torch": metadata.version("torch"),
@@ -57,10 +62,11 @@ print(json.dumps({
"rounds": 2, "rounds": 2,
"requests_per_anchor": 64, "requests_per_anchor": 64,
"anchor_isolation": "fresh_server_per_rate_per_round", "anchor_isolation": "fresh_server_per_rate_per_round",
"target_rate_warmup_requests": "min(32, max(4, ceil(rate * 2)))", "target_rate_warmup_requests": f"min(32, max(4, ceil(rate * {warmup_seconds})))",
"input_tokens": 2048, "input_tokens": input_tokens,
"output_tokens": 1, "output_tokens": int(output_tokens),
"ttft_slo_ms": 1256.0, "ttft_slo_ms": 1000.0 + 1000.0 * input_tokens / 8000.0,
"tpot_slo_ms": float(tpot_slo_ms) if int(output_tokens) > 1 else None,
"target_pass_rate": 0.95, "target_pass_rate": 0.95,
}, indent=2, sort_keys=True)) }, indent=2, sort_keys=True))
PY PY
@@ -112,19 +118,22 @@ for ROUND in 1 2; do
exit 1 exit 1
fi fi
WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY' WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" "${WARMUP_SECONDS}" <<'PY'
import math import math
import sys import sys
print(min(32, max(4, math.ceil(float(sys.argv[1]) * 2.0)))) print(min(32, max(4, math.ceil(float(sys.argv[1]) * float(sys.argv[2])))))
PY PY
)" )"
"${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \
--served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
--requests "${WARMUP_REQUESTS}" \ --requests "${WARMUP_REQUESTS}" --input-tokens "${INPUT_TOKENS}" \
--output-tokens "${OUTPUT_TOKENS}" --tpot-slo-ms "${TPOT_SLO_MS}" \
--output "${ROUND_ROOT}/results/warmup_${KEY}.json" --output "${ROUND_ROOT}/results/warmup_${KEY}.json"
"${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \
--served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \
--requests 64 --output "${ROUND_ROOT}/results/${KEY}.json" --requests 64 --input-tokens "${INPUT_TOKENS}" \
--output-tokens "${OUTPUT_TOKENS}" --tpot-slo-ms "${TPOT_SLO_MS}" \
--output "${ROUND_ROOT}/results/${KEY}.json"
cleanup cleanup
done done
done done

2
runs/frontier-phase-factorial-v0/test_phase_factorial.py
View File

@@ -25,6 +25,8 @@ def test_percentile() -> None:
assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.50) == 2.0 assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.50) == 2.0
assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.95) == 4.0 assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.95) == 4.0
assert client.percentile([], 0.95) is None assert client.percentile([], 0.95) is None
assert client.ttft_slo_ms(512) == 1064.0
assert client.ttft_slo_ms(2048) == 1256.0
def test_grid_and_trace(tmp_path: Path) -> None: def test_grid_and_trace(tmp_path: Path) -> None: