gahow/xserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Qwen3.6 validation and benchmark coverage

Browse Source
This commit is contained in:
Gahow Wang
2026-07-13 20:24:57 +08:00
parent a2de146fb6
commit 0acaca34cb
12 changed files with 1657 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tools/bench/client.py
View File

@@ -110,12 +110,16 @@ async def chat_stream(
choice = choices[0]
delta = choice.get("delta") or {}
content = delta.get("content")
if content:
reasoning_content = delta.get("reasoning_content")
token_text = content if content is not None else reasoning_content
# Exclude the role announcement, but count every generated-token
# chunk, including an empty UTF-8 fragment, in TTFT/TPOT.
if token_text is not None and "role" not in delta:
now = time.perf_counter()
if res.ttft_s < 0:
res.ttft_s = now - t_start
res.chunk_times.append(now)
res.text += content
res.text += token_text
if choice.get("finish_reason"):
res.finish_reason = choice["finish_reason"]
except Exception as e: # noqa: BLE001 — surface any failure to the report

10
tools/bench/config.py
View File

@@ -25,10 +25,7 @@ class SystemEndpoint:
model_id: str # what to put in the request body's "model" field
api_key: str | None = None # llama-server doesn't need one; xserv ignores it
# Extra fields merged into every request body for this system. Used to keep
# the two engines in the SAME generation mode — xserv hardcodes Qwen3
# thinking OFF (empty <think></think> in its prompt builder), so we disable
# thinking on llama-server via chat_template_kwargs to match. Both engines
# ignore unknown fields, so this is safe.
# both engines in the same chat-template generation mode.
extra_body: dict | None = None
# Process supervision is optional — if base_url is already serving, we skip launch.
launch_cmd: list[str] | None = None
@@ -49,7 +46,12 @@ class BenchConfig:
quality_max_tokens_aime: int = 16384
quality_max_tokens_gsm8k: int = 2048
quality_limit: int | None = None # subsample for smoke tests; None = all
quality_seed: int | None = None
quality_temperature: float = 0.0
quality_top_k: int = 0
quality_top_p: float = 1.0
quality_presence_penalty: float = 0.0
quality_repetition_penalty: float = 1.0
request_timeout_s: float = 1800.0

25
tools/bench/quality.py
View File

@@ -14,6 +14,7 @@ extra moving parts aren't worth it for the first iteration.
from __future__ import annotations
import asyncio
import random
import statistics
import time
from dataclasses import asdict, dataclass
@@ -38,6 +39,7 @@ class QualityRow:
n_total: int
n_correct: int
n_errors: int
n_length: int
accuracy: float
mean_completion_tokens: float
mean_ttft_ms: float
@@ -58,7 +60,9 @@ class QualityCase:
tpot_ms: float
e2e_s: float
error: str | None
finish_reason: str | None
response_preview: str
response_text: str
async def _run_one_task(
@@ -66,7 +70,10 @@ async def _run_one_task(
) -> tuple[QualityRow, list[QualityCase]]:
problems = task_mod.load()
if cfg.quality_limit is not None:
problems = problems[: cfg.quality_limit]
if cfg.quality_seed is not None and cfg.quality_limit < len(problems):
problems = random.Random(cfg.quality_seed).sample(problems, cfg.quality_limit)
else:
problems = problems[: cfg.quality_limit]
print(f"[quality] {ep.name} / {task_name}: {len(problems)} problems "
f"(max_tokens={max_tokens})")
@@ -75,13 +82,20 @@ async def _run_one_task(
async with httpx.AsyncClient(timeout=cfg.request_timeout_s) as client:
for prob in problems:
messages = task_mod.make_messages(prob["problem"])
extra_body = dict(ep.extra_body or {})
extra_body.update({
"top_k": cfg.quality_top_k,
"top_p": cfg.quality_top_p,
"presence_penalty": cfg.quality_presence_penalty,
"repetition_penalty": cfg.quality_repetition_penalty,
})
r = await chat_stream(
client, ep.base_url, ep.model_id, messages,
max_tokens=max_tokens,
temperature=cfg.quality_temperature,
api_key=ep.api_key,
timeout=cfg.request_timeout_s,
extra_body=ep.extra_body,
extra_body=extra_body,
)
pred = task_mod.extract_answer(r.text) if r.error is None else None
correct = task_mod.score(pred, prob["answer"]) if r.error is None else False
@@ -91,8 +105,9 @@ async def _run_one_task(
correct=correct, completion_tokens=r.completion_tokens,
ttft_ms=r.ttft_s * 1000 if r.ttft_s > 0 else -1.0,
tpot_ms=r.tpot_s * 1000 if r.tpot_s > 0 else -1.0,
e2e_s=r.e2e_s, error=r.error,
e2e_s=r.e2e_s, error=r.error, finish_reason=r.finish_reason,
response_preview=(r.text or "")[:240].replace("\n", " "),
response_text=r.text or "",
))
mark = "✓" if correct else ("E" if r.error else "✗")
print(f" [{mark}] {prob['id']:>4s} gold={prob['answer']:>6s} "
@@ -109,7 +124,9 @@ async def _run_one_task(
n_total=len(cases),
n_correct=correct,
n_errors=errors,
accuracy=correct / max(len(cases) - errors, 1),
n_length=sum(1 for c in cases if c.finish_reason == "length"),
# Transport/runtime failures are incorrect attempts, not exclusions.
accuracy=correct / max(len(cases), 1),
mean_completion_tokens=statistics.mean(c.completion_tokens for c in ok) if ok else 0.0,
mean_ttft_ms=statistics.mean(c.ttft_ms for c in ok if c.ttft_ms > 0) if ok else -1.0,
mean_tpot_ms=statistics.mean(c.tpot_ms for c in ok if c.tpot_ms > 0) if ok else -1.0,

12
tools/bench/report.py
View File

@@ -32,7 +32,11 @@ def _speed_table(rows: list[dict[str, Any]]) -> str:
by = {(r["system"], r["scenario"]): r for r in rows}
out = []
out.append("| scenario | metric | " + " | ".join(systems) + " | speedup (xserv ÷ llama.cpp) |")
out.append(
"| scenario | metric | "
+ " | ".join(systems)
+ " | xserv relative performance (higher is better) |"
)
out.append("|---|---|" + "|".join(["---"] * (len(systems) + 1)) + "|")
metrics = [
@@ -68,13 +72,13 @@ def _quality_table(rows: list[dict[str, Any]]) -> str:
for r in rows:
by_task.setdefault(r["task"], []).append(r)
out: list[str] = []
out.append("| task | system | n | correct | accuracy | mean tokens | TTFT (ms) | TPOT (ms/tok) | wall (s) |")
out.append("|---|---|---|---|---|---|---|---|---|")
out.append("| task | system | n | correct | accuracy | length | mean tokens | TTFT (ms) | TPOT (ms/tok) | wall (s) |")
out.append("|---|---|---|---|---|---|---|---|---|---|")
for task, task_rows in by_task.items():
for r in task_rows:
out.append(
f"| {task} | {r['system']} | {r['n_total']} | {r['n_correct']} | "
f"{r['accuracy'] * 100:.1f}% | {r['mean_completion_tokens']:.0f} | "
f"{r['accuracy'] * 100:.1f}% | {r['n_length']} | {r['mean_completion_tokens']:.0f} | "
f"{_fmt(r['mean_ttft_ms'])} | {_fmt(r['mean_tpot_ms'], 2)} | {r['wall_s']:.1f} |"
)
return "\n".join(out) + "\n"

33
tools/bench/runner.py
View File

@@ -88,6 +88,15 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--quality-tasks", default="aime2025,gsm8k")
p.add_argument("--quality-limit", type=int, default=None,
help="Cap problems per task (smoke test). None = all problems.")
p.add_argument("--quality-seed", type=int, default=None,
help="Fixed seed for a random quality subset; default uses dataset prefix.")
p.add_argument("--quality-max-tokens-aime", type=int, default=16384)
p.add_argument("--quality-max-tokens-gsm8k", type=int, default=2048)
p.add_argument("--quality-temperature", type=float, default=0.0)
p.add_argument("--quality-top-k", type=int, default=0)
p.add_argument("--quality-top-p", type=float, default=1.0)
p.add_argument("--quality-presence-penalty", type=float, default=0.0)
p.add_argument("--quality-repetition-penalty", type=float, default=1.0)
p.add_argument("--speed-prompts", type=int, default=8)
p.add_argument("--speed-max-tokens", type=int, default=128)
p.add_argument("--speed-concurrency", default="1,2,4,8")
@@ -99,12 +108,16 @@ def parse_args() -> argparse.Namespace:
def build_endpoints(args) -> list[SystemEndpoint]:
wanted = set(s.strip() for s in args.systems.split(",") if s.strip())
eps: list[SystemEndpoint] = []
thinking_extra_body = None if args.enable_thinking else {
"chat_template_kwargs": {"enable_thinking": False}
}
if SYSTEM_XSERV in wanted:
if args.xserv_base_url:
eps.append(SystemEndpoint(
name=SYSTEM_XSERV, base_url=args.xserv_base_url,
model_id=args.xserv_model_id, launch_cmd=None,
extra_body=thinking_extra_body,
))
else:
model_dir = args.xserv_model or os.environ.get("XSERV_MODEL_DIR")
@@ -120,19 +133,15 @@ def build_endpoints(args) -> list[SystemEndpoint]:
),
health_path="/health",
ready_timeout_s=1200.0,
extra_body=thinking_extra_body,
))
# Match xserv's hardcoded thinking-OFF mode unless explicitly overridden.
llama_extra_body = None if args.enable_thinking else {
"chat_template_kwargs": {"enable_thinking": False}
}
if SYSTEM_LLAMA_CPP in wanted:
if args.llama_base_url:
eps.append(SystemEndpoint(
name=SYSTEM_LLAMA_CPP, base_url=args.llama_base_url,
model_id=args.llama_model_id, launch_cmd=None,
extra_body=llama_extra_body,
extra_body=thinking_extra_body,
))
else:
gguf = args.llama_gguf or os.environ.get("LLAMA_GGUF")
@@ -161,7 +170,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
# llama-server's health endpoint also returns 200 only when model is loaded.
health_path="/health",
ready_timeout_s=1200.0,
extra_body=llama_extra_body,
extra_body=thinking_extra_body,
))
return eps
@@ -194,7 +203,15 @@ def main() -> None:
speed_prompts=args.speed_prompts,
speed_max_tokens=args.speed_max_tokens,
speed_concurrency=tuple(int(c) for c in args.speed_concurrency.split(",") if c.strip()),
quality_max_tokens_aime=args.quality_max_tokens_aime,
quality_max_tokens_gsm8k=args.quality_max_tokens_gsm8k,
quality_limit=args.quality_limit,
quality_seed=args.quality_seed,
quality_temperature=args.quality_temperature,
quality_top_k=args.quality_top_k,
quality_top_p=args.quality_top_p,
quality_presence_penalty=args.quality_presence_penalty,
quality_repetition_penalty=args.quality_repetition_penalty,
)
os.makedirs(args.out_dir, exist_ok=True)
@@ -229,7 +246,7 @@ def main() -> None:
speed_raw=speed_raw,
quality_rows=q_rows_to_dicts(quality_rows) if quality_rows else [],
quality_cases=cases_to_dicts(quality_cases) if quality_cases else [],
env=collect_env(),
env={**collect_env(), "benchmark_args": vars(args)},
)

16
tools/bench/speed.py
View File

@@ -35,6 +35,7 @@ class SpeedRow:
scenario: str # e.g. "single/short", "concurrent-4"
requests: int
completion_tokens_total: int
prompt_tokens_mean: float
wall_s: float
ttft_ms_p50: float
ttft_ms_p95: float
@@ -64,6 +65,7 @@ def _summarize(system: str, scenario: str, results: list[StreamResult], wall_s:
scenario=scenario,
requests=len(results),
completion_tokens_total=total_tokens,
prompt_tokens_mean=(statistics.mean(r.prompt_tokens for r in ok) if ok else 0.0),
wall_s=wall_s,
ttft_ms_p50=_percentile(ttft_ms, 50),
ttft_ms_p95=_percentile(ttft_ms, 95),
@@ -82,6 +84,18 @@ async def run_single_stream(
rows: list[SpeedRow] = []
raw: list[dict[str, Any]] = []
for bucket, prompt in SPEED_PROMPTS.items():
# Prefill kernels/graphs can be shape-specific. Warm each prompt shape
# twice so p95 does not accidentally report one-time graph setup.
for _ in range(2):
await chat_concurrent(
ep.base_url, ep.model_id, [[{"role": "user", "content": prompt}]],
max_tokens=cfg.speed_max_tokens,
temperature=0.0,
api_key=ep.api_key,
timeout=cfg.request_timeout_s,
concurrency=1,
extra_body=ep.extra_body,
)
messages = [[{"role": "user", "content": prompt}]] * cfg.speed_prompts
results, wall = await chat_concurrent(
ep.base_url, ep.model_id, messages,
@@ -98,6 +112,7 @@ async def run_single_stream(
"system": ep.name, "scenario": f"single/{bucket}", "i": i,
"ttft_s": r.ttft_s, "tpot_s": r.tpot_s,
"completion_tokens": r.completion_tokens,
"prompt_tokens": r.prompt_tokens,
"e2e_s": r.e2e_s, "error": r.error,
"finish_reason": r.finish_reason,
})
@@ -131,6 +146,7 @@ async def run_concurrent(
"system": ep.name, "scenario": f"concurrent-{c}", "i": i,
"ttft_s": r.ttft_s, "tpot_s": r.tpot_s,
"completion_tokens": r.completion_tokens,
"prompt_tokens": r.prompt_tokens,
"e2e_s": r.e2e_s, "error": r.error,
"finish_reason": r.finish_reason,
})