gahow/xserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bench: PP harness (xserv --pp vs llama.cpp -sm layer)

Browse Source 
runner/servers: add --pp for both engines (xserv --pp N; llama.cpp
-sm layer over N GPUs). New drivers: pp_final.sh (sequential latency +
per-GPU VRAM + byte-exact correctness), pp_diag.sh (single x2 vs pp4 x2
determinism control), pp_quality_full.sh / pp_llama_47.sh (AIME+GSM8K
matrix, xserv on 0-3 || llama on 4-7), summarize_pp/summarize_fullq,
pp_time.py latency probe.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-29 18:45:59 +08:00
parent 824cc58daa
commit d5dcf1a5ab
12 changed files with 505 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
tools/bench/pp_clean_bench.sh Normal file
View File

@@ -0,0 +1,89 @@
#!/usr/bin/env bash
# Clean, strictly-sequential single-stream latency + per-GPU VRAM for PP.
# One server at a time. Readiness = first SUCCESSFUL generation (xserv's /health
# returns 200 before the model finishes loading, so we must not gate on it).
# Snapshots are therefore always post-load. Writes bench-out/PP_CLEAN.md.
#
# Env overrides: MODEL, GGUF, PPS (default "1 2 4"), LLAMA_BIN.
set -u
cd "$(dirname "$0")/../.."
export PATH=$HOME/.cargo/bin:/usr/local/cuda-12.9/bin:$PATH
export CUDA_HOME=${CUDA_HOME:-/usr/local/cuda-12.9}
MODEL=${MODEL:-/opt/wjh/models/qwen3-8b}
GGUF=${GGUF:-/opt/wjh/models/qwen3-8b/qwen3-8b-bf16.gguf}
LLAMA_BIN=${LLAMA_BIN:-third_party/llama.cpp/build/bin/llama-server}
XBIN=./target/release/xserv-server
PPS=${PPS:-1 2 4}
PROMPT='Write a detailed paragraph explaining how GPUs accelerate neural network training.'
OUT=bench-out/PP_CLEAN.md
mkdir -p bench-out
: > "$OUT"
echo "# PP clean single-stream latency + VRAM — $(date)" >> "$OUT"
echo "" >> "$OUT"
echo "| engine | PP | TTFT_ms | TPOT_ms | tok/s | per-GPU VRAM (MiB) |" >> "$OUT"
echo "|--------|----|---------|---------|-------|--------------------|" >> "$OUT"
killall_servers(){ pkill -9 -f xserv-server 2>/dev/null; pkill -9 -f llama-server 2>/dev/null; sleep 3; }
drain(){ # wait until GPUs $1 (csv) all < 1500 MiB, max 120s
for _ in $(seq 1 60); do
local hi=0
for g in ${1//,/ }; do
m=$(nvidia-smi -i "$g" --query-gpu=memory.used --format=csv,noheader,nounits)
[ "${m:-0}" -gt 1500 ] && hi=1
done
[ "$hi" -eq 0 ] && return 0; sleep 2
done
}
# probe_ready PORT PID -> 0 when a generation succeeds (deadline ~1200s)
probe_ready(){ local port=$1 pid=$2
for _ in $(seq 1 400); do
if curl -s -o /dev/null -w '%{http_code}' --max-time 8 \
"http://127.0.0.1:$port/v1/chat/completions" -H 'Content-Type: application/json' \
-d '{"model":"qwen3-8b","messages":[{"role":"user","content":"hi"}],"max_tokens":1,"temperature":0,"stream":false}' \
2>/dev/null | grep -q 200; then return 0; fi
kill -0 "$pid" 2>/dev/null || return 1
sleep 3
done; return 1
}
vram(){ local cvd=$1; local a b="" # stabilized snapshot of GPUs $cvd
for _ in $(seq 1 12); do
a=$(for g in ${cvd//,/ }; do nvidia-smi -i "$g" --query-gpu=memory.used --format=csv,noheader,nounits; done | paste -sd' ')
[ "$a" = "$b" ] && break; b=$a; sleep 2
done; echo "$a"
}
run_xserv(){ local pp=$1; local cvd; cvd=$(seq -s, 0 $((pp-1)))
killall_servers; drain "$cvd"
local extra=""; [ "$pp" -gt 1 ] && extra="--pp $pp"
XSERV_MAX_KV_BLOCKS=160 CUDA_VISIBLE_DEVICES=$cvd nohup $XBIN $MODEL --port 8090 --max-seq-len 2048 $extra >/tmp/x$pp.log 2>&1 &
local pid=$!
if ! probe_ready 8090 "$pid"; then echo "| xserv | $pp | FAILED (see /tmp/x$pp.log) | | | |" >> "$OUT"; kill -9 "$pid" 2>/dev/null; return; fi
local mib; mib=$(vram "$cvd")
local m; m=$(python3 tools/bench/pp_time.py http://127.0.0.1:8090 "$PROMPT")
local ttft tpot toks; ttft=$(echo "$m"|sed -n 's/.*TTFT_ms=\([0-9.]*\).*/\1/p'); tpot=$(echo "$m"|sed -n 's/.*TPOT_ms=\([0-9.a-z]*\).*/\1/p'); toks=$(echo "$m"|sed -n 's/.*tok_s=\([0-9.a-z]*\).*/\1/p')
echo "| xserv | $pp | $ttft | $tpot | $toks | $mib |" >> "$OUT"
kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; sleep 3
}
run_llama(){ local pp=$1; local cvd; cvd=$(seq -s, 0 $((pp-1)))
killall_servers; drain "$cvd"
local sm=(-sm none); [ "$pp" -gt 1 ] && sm=(-sm layer -ts "$(printf '1%.0s,' $(seq 1 $pp) | sed 's/,$//')")
CUDA_VISIBLE_DEVICES=$cvd nohup $LLAMA_BIN -m $GGUF --port 8090 --host 127.0.0.1 \
-c 2048 --parallel 1 -ngl 999 "${sm[@]}" >/tmp/l$pp.log 2>&1 &
local pid=$!
if ! probe_ready 8090 "$pid"; then echo "| llama | $pp | FAILED (see /tmp/l$pp.log) | | | |" >> "$OUT"; kill -9 "$pid" 2>/dev/null; return; fi
local mib; mib=$(vram "$cvd")
local m; m=$(python3 tools/bench/pp_time.py http://127.0.0.1:8090 "$PROMPT")
local ttft tpot toks; ttft=$(echo "$m"|sed -n 's/.*TTFT_ms=\([0-9.]*\).*/\1/p'); tpot=$(echo "$m"|sed -n 's/.*TPOT_ms=\([0-9.a-z]*\).*/\1/p'); toks=$(echo "$m"|sed -n 's/.*tok_s=\([0-9.a-z]*\).*/\1/p')
echo "| llama | $pp | $ttft | $tpot | $toks | $mib |" >> "$OUT"
kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; sleep 3
}
for pp in $PPS; do run_xserv "$pp"; done
for pp in $PPS; do run_llama "$pp"; done
killall_servers
echo "" >> "$OUT"
echo "PP_CLEAN_DONE" >> "$OUT"

44
tools/bench/pp_time.py Normal file
View File

@@ -0,0 +1,44 @@
"""Tiny single-stream latency probe over the OpenAI HTTP API.
Usage: python3 pp_time.py BASE_URL "PROMPT"
Prints: TTFT_ms=.. TPOT_ms=.. tok_full=.. tok_s=..
TTFT ~ wall time of a max_tokens=1 request (prefill + 1 token).
TPOT ~ (t_full - t_1) / (tokens_full - tokens_1), using the server's reported
completion_tokens so it is exact even if generation stops early.
"""
import json
import sys
import time
import urllib.request
base = sys.argv[1].rstrip("/")
prompt = sys.argv[2]
def req(max_tokens):
body = json.dumps({
"model": "qwen3-8b",
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": 0,
"stream": False,
}).encode()
r = urllib.request.Request(base + "/v1/chat/completions", body,
{"Content-Type": "application/json"})
t = time.time()
d = json.load(urllib.request.urlopen(r, timeout=600))
dt = time.time() - t
ct = d.get("usage", {}).get("completion_tokens")
return dt, ct
t1, c1 = req(1)
tF, cF = req(160)
ttft = t1 * 1000.0
denom = (cF - c1) if (cF and c1 and cF > c1) else None
if denom:
tpot = (tF - t1) / denom * 1000.0
print(f"TTFT_ms={ttft:.1f} TPOT_ms={tpot:.2f} tok_full={cF} tok_s={1000.0/tpot:.1f}")
else:
print(f"TTFT_ms={ttft:.1f} TPOT_ms=nan tok_full={cF} tok_s=nan")

42
tools/bench/run_pp_parallel.sh Normal file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# Run the PP=1/2/4 sweep with xserv and llama.cpp CONCURRENTLY on disjoint GPU
# groups: xserv (--pp) on GPUs 0..N-1, llama.cpp (-sm layer) on GPUs 4..4+N-1.
# The 8x5090 box is grouped 0-3 / 4-7 (PHB intra-group), so each engine's P2P
# stays intra-group and the two engines never contend for a GPU.
#
# xserv splits layers across N GPUs and hands off hidden states via NCCL P2P;
# llama.cpp's default `-sm layer` does the analogous layer-wise split.
#
# Run from the repo root on the GPU host. Produces bench-out/pp{1,2,4}-{xserv,llama}.
set -u
MODEL="${MODEL:-/opt/wjh/models/qwen3-8b}"
GGUF="${GGUF:-/opt/wjh/models/qwen3-8b/qwen3-8b-bf16.gguf}"
LIMIT="${LIMIT:-20}"
MAXSEQ="${MAXSEQ:-2048}"
PPS="${PPS:-1 2 4}"
TASKS="${TASKS:-gsm8k}"
for PP in $PPS; do
LD=$(seq -s, 4 $((3 + PP))) # llama GPUs: 4 / 4,5 / 4,5,6,7
echo "##### PP=$PP (xserv GPU 0..$((PP-1)) || llama GPU $LD) #####"
rm -rf "bench-out/pp$PP-xserv" "bench-out/pp$PP-llama"
python3 -u -m tools.bench.runner --systems xserv --pp "$PP" \
--xserv-bin ./target/release/xserv-server --xserv-model "$MODEL" \
--suite quality --quality-tasks "$TASKS" --quality-limit "$LIMIT" \
--max-batch 1 --max-seq-len "$MAXSEQ" \
--out-dir "bench-out/pp$PP-xserv" > "/tmp/pp$PP-xserv.log" 2>&1 &
XP=$!
python3 -u -m tools.bench.runner --systems llama.cpp --pp "$PP" --llama-devices "$LD" \
--llama-bin third_party/llama.cpp/build/bin/llama-server --llama-gguf "$GGUF" \
--suite quality --quality-tasks "$TASKS" --quality-limit "$LIMIT" \
--max-batch 1 --max-seq-len "$MAXSEQ" \
--out-dir "bench-out/pp$PP-llama" > "/tmp/pp$PP-llama.log" 2>&1 &
LP=$!
wait "$XP" "$LP"
echo "PP=$PP done"
done
echo ALL_DONE

13
tools/bench/runner.py
View File

@@ -72,6 +72,9 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--tp", type=int, default=1,
help="Tensor-parallel degree for BOTH engines (xserv --tp N; "
"llama.cpp --split-mode row over the first N GPUs).")
p.add_argument("--pp", type=int, default=1,
help="Pipeline-parallel degree for BOTH engines (xserv --pp N; "
"llama.cpp --split-mode layer over the first N GPUs).")
p.add_argument("--llama-devices", default=None,
help="Comma list of GPU ordinals for llama.cpp (first --tp used). "
"Lets llama run on a disjoint GPU group (e.g. 4,5,6,7) so it "
@@ -113,7 +116,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
model_id=args.xserv_model_id,
launch_cmd=xserv_launch_cmd(
args.xserv_bin, model_dir, args.xserv_port,
max_batch=args.max_batch, max_seq_len=args.max_seq_len, tp=args.tp,
max_batch=args.max_batch, max_seq_len=args.max_seq_len, tp=args.tp, pp=args.pp,
),
health_path="/health",
ready_timeout_s=1200.0,
@@ -140,10 +143,10 @@ def build_endpoints(args) -> list[SystemEndpoint]:
# so it can run concurrently with xserv on 0..N-1. --split-mode row
# then tensor-parallel-splits across exactly these devices.
if args.llama_devices:
devs = [d.strip() for d in args.llama_devices.split(",") if d.strip()][: max(args.tp, 1)]
devs = [d.strip() for d in args.llama_devices.split(",") if d.strip()][: max(args.tp, args.pp, 1)]
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(devs)}
elif args.tp > 1:
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(str(d) for d in range(args.tp))}
elif args.tp > 1 or args.pp > 1:
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(str(d) for d in range(max(args.tp, args.pp)))}
else:
llama_env = {}
eps.append(SystemEndpoint(
@@ -152,7 +155,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
model_id=args.llama_model_id,
launch_cmd=llama_cpp_launch_cmd(
args.llama_bin, gguf, args.llama_port,
n_parallel=args.max_batch, ctx_per_slot=args.max_seq_len, tp=args.tp,
n_parallel=args.max_batch, ctx_per_slot=args.max_seq_len, tp=args.tp, pp=args.pp,
),
launch_env=llama_env,
# llama-server's health endpoint also returns 200 only when model is loaded.

11
tools/bench/servers.py
View File

@@ -114,6 +114,7 @@ def xserv_launch_cmd(
max_batch: int,
max_seq_len: int,
tp: int = 1,
pp: int = 1,
) -> list[str]:
cmd = [
bin_path,
@@ -122,7 +123,9 @@ def xserv_launch_cmd(
"--max-batch", str(max_batch),
"--max-seq-len", str(max_seq_len),
]
if tp > 1:
if pp > 1:
cmd += ["--pp", str(pp)] # xserv binds stage s -> GPU s internally
elif tp > 1:
cmd += ["--tp", str(tp)] # xserv binds rank r -> GPU r internally
return cmd
@@ -136,6 +139,7 @@ def llama_cpp_launch_cmd(
ctx_per_slot: int,
n_gpu_layers: int = 99,
tp: int = 1,
pp: int = 1,
) -> list[str]:
# llama.cpp DIVIDES total -c across --parallel slots: per-slot context is
# n_ctx / n_parallel. xserv gives each sequence the full max_seq_len, so to
@@ -153,7 +157,10 @@ def llama_cpp_launch_cmd(
# NOTE: do NOT pass --log-disable; its startup log reports per-slot
# n_ctx, which is exactly the diagnostic that catches ctx misconfig.
]
if tp > 1:
if pp > 1:
# Pipeline / layer split across the visible GPUs (llama.cpp default).
cmd += ["--split-mode", "layer", "-ts", ",".join(["1"] * pp)]
elif tp > 1:
# Tensor-parallel split across the visible GPUs (caller restricts the
# set via CUDA_VISIBLE_DEVICES in launch_env). Row-split is llama.cpp's
# tensor-parallel mode (vs the default layer/pipeline split).

17
tools/bench/summarize_fullq.py Normal file
View File

@@ -0,0 +1,17 @@
"""Summarize the full quality matrix: bench-out/fullq-{xserv,llama}-pp{1,2,4}.
Prints one row per (engine, pp, task) with accuracy + latency."""
import glob, json, os, sys
base = sys.argv[1] if len(sys.argv) > 1 else "bench-out"
print("%-6s %-3s %-9s %-8s %6s %9s %9s %10s" %
("engine","PP","task","correct","acc%","mean_tok","TTFT_ms","TPOT_ms"))
for eng in ("xserv","llama"):
for pp in (1,2,4):
files = sorted(glob.glob(os.path.join(base, f"fullq-{eng}-pp{pp}", "comparison-*.json")))
if not files:
print(f"{eng:<6} {pp:<3} (no results)"); continue
d = json.load(open(files[-1]))
for r in d.get("quality",{}).get("summary",[]):
print("%-6s %-3d %-9s %-8s %5.1f%% %9.0f %9.1f %10.2f" % (
eng, pp, r["task"], f'{r["n_correct"]}/{r["n_total"]}',
r["accuracy"]*100, r.get("mean_completion_tokens",0),
r.get("mean_ttft_ms",0), r.get("mean_tpot_ms",0)))

24
tools/bench/summarize_pp.py Normal file
View File

@@ -0,0 +1,24 @@
"""Summarize the concurrent PP sweep: bench-out/pp{1,2,4}-{xserv,llama}."""
import glob
import json
import os
import sys
base = sys.argv[1] if len(sys.argv) > 1 else "bench-out"
rows = []
for pp in (1, 2, 4):
for sysname in ("xserv", "llama"):
files = sorted(glob.glob(os.path.join(base, f"pp{pp}-{sysname}", "comparison-*.json")))
if not files:
continue
d = json.load(open(files[-1]))
for r in d["quality"]["summary"]:
rows.append((pp, sysname, r["task"], r["n_correct"], r["n_total"],
r["accuracy"] * 100, r["mean_completion_tokens"],
r["mean_ttft_ms"], r["mean_tpot_ms"], r["wall_s"]))
print("%-3s %-7s %-9s %-9s %7s %9s %9s %10s %9s" %
("PP", "engine", "task", "correct", "acc%", "mean_tok", "TTFT_ms", "TPOT_ms", "wall_s"))
for (pp, s, task, nc, nt, acc, tok, ttft, tpot, wall) in rows:
print("%-3d %-7s %-9s %-9s %6.1f%% %9.0f %9.1f %10.2f %9.0f" %
(pp, s, task, f"{nc}/{nt}", acc, tok, ttft, tpot, wall))