gahow/xserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Qwen3.6 validation and benchmark coverage

Browse Source
This commit is contained in:
Gahow Wang
2026-07-13 20:24:57 +08:00
parent a2de146fb6
commit 0acaca34cb
12 changed files with 1657 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
tools/bench/runner.py
View File

@@ -88,6 +88,15 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--quality-tasks", default="aime2025,gsm8k")
p.add_argument("--quality-limit", type=int, default=None,
help="Cap problems per task (smoke test). None = all problems.")
p.add_argument("--quality-seed", type=int, default=None,
help="Fixed seed for a random quality subset; default uses dataset prefix.")
p.add_argument("--quality-max-tokens-aime", type=int, default=16384)
p.add_argument("--quality-max-tokens-gsm8k", type=int, default=2048)
p.add_argument("--quality-temperature", type=float, default=0.0)
p.add_argument("--quality-top-k", type=int, default=0)
p.add_argument("--quality-top-p", type=float, default=1.0)
p.add_argument("--quality-presence-penalty", type=float, default=0.0)
p.add_argument("--quality-repetition-penalty", type=float, default=1.0)
p.add_argument("--speed-prompts", type=int, default=8)
p.add_argument("--speed-max-tokens", type=int, default=128)
p.add_argument("--speed-concurrency", default="1,2,4,8")
@@ -99,12 +108,16 @@ def parse_args() -> argparse.Namespace:
def build_endpoints(args) -> list[SystemEndpoint]:
wanted = set(s.strip() for s in args.systems.split(",") if s.strip())
eps: list[SystemEndpoint] = []
thinking_extra_body = None if args.enable_thinking else {
"chat_template_kwargs": {"enable_thinking": False}
}
if SYSTEM_XSERV in wanted:
if args.xserv_base_url:
eps.append(SystemEndpoint(
name=SYSTEM_XSERV, base_url=args.xserv_base_url,
model_id=args.xserv_model_id, launch_cmd=None,
extra_body=thinking_extra_body,
))
else:
model_dir = args.xserv_model or os.environ.get("XSERV_MODEL_DIR")
@@ -120,19 +133,15 @@ def build_endpoints(args) -> list[SystemEndpoint]:
),
health_path="/health",
ready_timeout_s=1200.0,
extra_body=thinking_extra_body,
))
# Match xserv's hardcoded thinking-OFF mode unless explicitly overridden.
llama_extra_body = None if args.enable_thinking else {
"chat_template_kwargs": {"enable_thinking": False}
}
if SYSTEM_LLAMA_CPP in wanted:
if args.llama_base_url:
eps.append(SystemEndpoint(
name=SYSTEM_LLAMA_CPP, base_url=args.llama_base_url,
model_id=args.llama_model_id, launch_cmd=None,
extra_body=llama_extra_body,
extra_body=thinking_extra_body,
))
else:
gguf = args.llama_gguf or os.environ.get("LLAMA_GGUF")
@@ -161,7 +170,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
# llama-server's health endpoint also returns 200 only when model is loaded.
health_path="/health",
ready_timeout_s=1200.0,
extra_body=llama_extra_body,
extra_body=thinking_extra_body,
))
return eps
@@ -194,7 +203,15 @@ def main() -> None:
speed_prompts=args.speed_prompts,
speed_max_tokens=args.speed_max_tokens,
speed_concurrency=tuple(int(c) for c in args.speed_concurrency.split(",") if c.strip()),
quality_max_tokens_aime=args.quality_max_tokens_aime,
quality_max_tokens_gsm8k=args.quality_max_tokens_gsm8k,
quality_limit=args.quality_limit,
quality_seed=args.quality_seed,
quality_temperature=args.quality_temperature,
quality_top_k=args.quality_top_k,
quality_top_p=args.quality_top_p,
quality_presence_penalty=args.quality_presence_penalty,
quality_repetition_penalty=args.quality_repetition_penalty,
)
os.makedirs(args.out_dir, exist_ok=True)
@@ -229,7 +246,7 @@ def main() -> None:
speed_raw=speed_raw,
quality_rows=q_rows_to_dicts(quality_rows) if quality_rows else [],
quality_cases=cases_to_dicts(quality_cases) if quality_cases else [],
env=collect_env(),
env={**collect_env(), "benchmark_args": vars(args)},
)