gahow/xserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bench: PP harness (xserv --pp vs llama.cpp -sm layer)

Browse Source 
runner/servers: add --pp for both engines (xserv --pp N; llama.cpp
-sm layer over N GPUs). New drivers: pp_final.sh (sequential latency +
per-GPU VRAM + byte-exact correctness), pp_diag.sh (single x2 vs pp4 x2
determinism control), pp_quality_full.sh / pp_llama_47.sh (AIME+GSM8K
matrix, xserv on 0-3 || llama on 4-7), summarize_pp/summarize_fullq,
pp_time.py latency probe.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-29 18:45:59 +08:00
parent 824cc58daa
commit d5dcf1a5ab
12 changed files with 505 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tools/bench/runner.py
View File

@@ -72,6 +72,9 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--tp", type=int, default=1,
help="Tensor-parallel degree for BOTH engines (xserv --tp N; "
"llama.cpp --split-mode row over the first N GPUs).")
p.add_argument("--pp", type=int, default=1,
help="Pipeline-parallel degree for BOTH engines (xserv --pp N; "
"llama.cpp --split-mode layer over the first N GPUs).")
p.add_argument("--llama-devices", default=None,
help="Comma list of GPU ordinals for llama.cpp (first --tp used). "
"Lets llama run on a disjoint GPU group (e.g. 4,5,6,7) so it "
@@ -113,7 +116,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
model_id=args.xserv_model_id,
launch_cmd=xserv_launch_cmd(
args.xserv_bin, model_dir, args.xserv_port,
max_batch=args.max_batch, max_seq_len=args.max_seq_len, tp=args.tp,
max_batch=args.max_batch, max_seq_len=args.max_seq_len, tp=args.tp, pp=args.pp,
),
health_path="/health",
ready_timeout_s=1200.0,
@@ -140,10 +143,10 @@ def build_endpoints(args) -> list[SystemEndpoint]:
# so it can run concurrently with xserv on 0..N-1. --split-mode row
# then tensor-parallel-splits across exactly these devices.
if args.llama_devices:
devs = [d.strip() for d in args.llama_devices.split(",") if d.strip()][: max(args.tp, 1)]
devs = [d.strip() for d in args.llama_devices.split(",") if d.strip()][: max(args.tp, args.pp, 1)]
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(devs)}
elif args.tp > 1:
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(str(d) for d in range(args.tp))}
elif args.tp > 1 or args.pp > 1:
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(str(d) for d in range(max(args.tp, args.pp)))}
else:
llama_env = {}
eps.append(SystemEndpoint(
@@ -152,7 +155,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
model_id=args.llama_model_id,
launch_cmd=llama_cpp_launch_cmd(
args.llama_bin, gguf, args.llama_port,
n_parallel=args.max_batch, ctx_per_slot=args.max_seq_len, tp=args.tp,
n_parallel=args.max_batch, ctx_per_slot=args.max_seq_len, tp=args.tp, pp=args.pp,
),
launch_env=llama_env,
# llama-server's health endpoint also returns 200 only when model is loaded.