Gahow Wang
d5dcf1a5ab
runner/servers: add --pp for both engines (xserv --pp N; llama.cpp -sm layer over N GPUs). New drivers: pp_final.sh (sequential latency + per-GPU VRAM + byte-exact correctness), pp_diag.sh (single x2 vs pp4 x2 determinism control), pp_quality_full.sh / pp_llama_47.sh (AIME+GSM8K matrix, xserv on 0-3 || llama on 4-7), summarize_pp/summarize_fullq, pp_time.py latency probe. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
45 lines
1.3 KiB
Python
45 lines
1.3 KiB
Python
"""Tiny single-stream latency probe over the OpenAI HTTP API.
|
|
|
|
Usage: python3 pp_time.py BASE_URL "PROMPT"
|
|
Prints: TTFT_ms=.. TPOT_ms=.. tok_full=.. tok_s=..
|
|
|
|
TTFT ~ wall time of a max_tokens=1 request (prefill + 1 token).
|
|
TPOT ~ (t_full - t_1) / (tokens_full - tokens_1), using the server's reported
|
|
completion_tokens so it is exact even if generation stops early.
|
|
"""
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
base = sys.argv[1].rstrip("/")
|
|
prompt = sys.argv[2]
|
|
|
|
|
|
def req(max_tokens):
|
|
body = json.dumps({
|
|
"model": "qwen3-8b",
|
|
"messages": [{"role": "user", "content": prompt}],
|
|
"max_tokens": max_tokens,
|
|
"temperature": 0,
|
|
"stream": False,
|
|
}).encode()
|
|
r = urllib.request.Request(base + "/v1/chat/completions", body,
|
|
{"Content-Type": "application/json"})
|
|
t = time.time()
|
|
d = json.load(urllib.request.urlopen(r, timeout=600))
|
|
dt = time.time() - t
|
|
ct = d.get("usage", {}).get("completion_tokens")
|
|
return dt, ct
|
|
|
|
|
|
t1, c1 = req(1)
|
|
tF, cF = req(160)
|
|
ttft = t1 * 1000.0
|
|
denom = (cF - c1) if (cF and c1 and cF > c1) else None
|
|
if denom:
|
|
tpot = (tF - t1) / denom * 1000.0
|
|
print(f"TTFT_ms={ttft:.1f} TPOT_ms={tpot:.2f} tok_full={cF} tok_s={1000.0/tpot:.1f}")
|
|
else:
|
|
print(f"TTFT_ms={ttft:.1f} TPOT_ms=nan tok_full={cF} tok_s=nan")
|