gahow/xserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bench: TP sweep harness (xserv --tp, llama row-split, concurrent groups)

Browse Source 
runner/servers gain --tp (xserv --tp N; llama.cpp --split-mode row) and
--llama-devices so llama can run on a disjoint GPU group. run_tp_parallel.sh
runs xserv (GPU 0..N-1) and llama.cpp (GPU 4..4+N-1) concurrently per TP,
matching the box's 0-3 / 4-7 PHB groups. summarize_tp.py tabulates the sweep.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-29 11:10:43 +08:00
parent 95eb61d639
commit a4a171d425
4 changed files with 98 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tools/bench/runner.py
View File

@@ -69,6 +69,13 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--max-seq-len", type=int, default=8192)
p.add_argument("--systems", default="xserv,llama.cpp",
help="Comma-separated subset to run, e.g. 'xserv' to skip llama.cpp")
p.add_argument("--tp", type=int, default=1,
help="Tensor-parallel degree for BOTH engines (xserv --tp N; "
"llama.cpp --split-mode row over the first N GPUs).")
p.add_argument("--llama-devices", default=None,
help="Comma list of GPU ordinals for llama.cpp (first --tp used). "
"Lets llama run on a disjoint GPU group (e.g. 4,5,6,7) so it "
"can run concurrently with xserv on 0..N-1.")
p.add_argument("--enable-thinking", action="store_true",
help="Enable Qwen3 thinking on llama.cpp. Default OFF to match "
"xserv, which hardcodes thinking off in its prompt builder.")
@@ -106,10 +113,10 @@ def build_endpoints(args) -> list[SystemEndpoint]:
model_id=args.xserv_model_id,
launch_cmd=xserv_launch_cmd(
args.xserv_bin, model_dir, args.xserv_port,
max_batch=args.max_batch, max_seq_len=args.max_seq_len,
max_batch=args.max_batch, max_seq_len=args.max_seq_len, tp=args.tp,
),
health_path="/health",
ready_timeout_s=900.0,
ready_timeout_s=1200.0,
))
# Match xserv's hardcoded thinking-OFF mode unless explicitly overridden.
@@ -128,17 +135,29 @@ def build_endpoints(args) -> list[SystemEndpoint]:
gguf = args.llama_gguf or os.environ.get("LLAMA_GGUF")
if not gguf:
raise SystemExit("--llama-gguf or LLAMA_GGUF required (or pass --llama-base-url)")
# Pick the GPUs llama.cpp runs on. Default is the first `tp` GPUs;
# pass --llama-devices to place it on a disjoint group (e.g. 4,5,6,7)
# so it can run concurrently with xserv on 0..N-1. --split-mode row
# then tensor-parallel-splits across exactly these devices.
if args.llama_devices:
devs = [d.strip() for d in args.llama_devices.split(",") if d.strip()][: max(args.tp, 1)]
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(devs)}
elif args.tp > 1:
llama_env = {"CUDA_VISIBLE_DEVICES": ",".join(str(d) for d in range(args.tp))}
else:
llama_env = {}
eps.append(SystemEndpoint(
name=SYSTEM_LLAMA_CPP,
base_url=f"http://127.0.0.1:{args.llama_port}",
model_id=args.llama_model_id,
launch_cmd=llama_cpp_launch_cmd(
args.llama_bin, gguf, args.llama_port,
n_parallel=args.max_batch, ctx_per_slot=args.max_seq_len,
n_parallel=args.max_batch, ctx_per_slot=args.max_seq_len, tp=args.tp,
),
launch_env=llama_env,
# llama-server's health endpoint also returns 200 only when model is loaded.
health_path="/health",
ready_timeout_s=900.0,
ready_timeout_s=1200.0,
extra_body=llama_extra_body,
))
return eps