bench: run one server at a time, match thinking mode, fix tools package

Refinements from end-to-end bring-up on the GPU host:

- Run each system start→suites→stop in sequence. Two BF16 8B models don't
  co-reside on one 32GB GPU, and a resident idle engine would distort the
  other's latency/throughput.
- Match generation mode: xserv hardcodes Qwen3 thinking off, so send
  chat_template_kwargs={enable_thinking:false} to llama.cpp via a per-endpoint
  extra_body. --enable-thinking opts back into thinking mode.
- Add tools/__init__.py so `python3 -m tools.bench.runner` resolves our package
  instead of a site-packages `tools` (nvfuser ships one that shadowed it).
- Document offline-GPU-host workflow, thinking-match, and the xserv 8192 OOM
  finding that the bench surfaced.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

tools/bench/runner.py

@@ -24,7 +24,6 @@ import os
 import platform
 import subprocess
 import sys
-from contextlib import ExitStack
 from typing import Any
 
 # Allow running as `python3 tools/bench/runner.py` from repo root.
@@ -35,7 +34,7 @@ from tools.bench.config import (
     BenchConfig, SystemEndpoint, SYSTEM_XSERV, SYSTEM_LLAMA_CPP,
 )
 from tools.bench.servers import (
-    ServerHandle, start_server, stop_server,
+    start_server, stop_server,
     xserv_launch_cmd, llama_cpp_launch_cmd,
 )
 from tools.bench.speed import run_speed, rows_to_dicts as speed_rows_to_dicts
@@ -70,6 +69,9 @@ def parse_args() -> argparse.Namespace:
     p.add_argument("--max-seq-len", type=int, default=8192)
     p.add_argument("--systems", default="xserv,llama.cpp",
                    help="Comma-separated subset to run, e.g. 'xserv' to skip llama.cpp")
+    p.add_argument("--enable-thinking", action="store_true",
+                   help="Enable Qwen3 thinking on llama.cpp. Default OFF to match "
+                        "xserv, which hardcodes thinking off in its prompt builder.")
 
     # Suites
     p.add_argument("--suite", choices=["speed", "quality", "all"], default="all")
@@ -110,11 +112,17 @@ def build_endpoints(args) -> list[SystemEndpoint]:
                 ready_timeout_s=900.0,
             ))
 
+    # Match xserv's hardcoded thinking-OFF mode unless explicitly overridden.
+    llama_extra_body = None if args.enable_thinking else {
+        "chat_template_kwargs": {"enable_thinking": False}
+    }
+
     if SYSTEM_LLAMA_CPP in wanted:
         if args.llama_base_url:
             eps.append(SystemEndpoint(
                 name=SYSTEM_LLAMA_CPP, base_url=args.llama_base_url,
                 model_id=args.llama_model_id, launch_cmd=None,
+                extra_body=llama_extra_body,
             ))
         else:
             gguf = args.llama_gguf or os.environ.get("LLAMA_GGUF")
@@ -131,6 +139,7 @@ def build_endpoints(args) -> list[SystemEndpoint]:
                 # llama-server's health endpoint also returns 200 only when model is loaded.
                 health_path="/health",
                 ready_timeout_s=900.0,
+                extra_body=llama_extra_body,
             ))
     return eps
 
@@ -169,24 +178,28 @@ def main() -> None:
     os.makedirs(args.out_dir, exist_ok=True)
     log_dir = os.path.join(args.out_dir, "logs")
 
-    handles: list[ServerHandle] = []
     speed_rows: list[Any] = []
     speed_raw: list[dict[str, Any]] = []
     quality_rows: list[Any] = []
     quality_cases: list[Any] = []
+    tasks = [t.strip() for t in args.quality_tasks.split(",") if t.strip()]
 
-    with ExitStack() as stack:
-        for ep in endpoints:
-            h = start_server(ep, log_dir)
-            handles.append(h)
-            stack.callback(stop_server, h)
-
-        if args.suite in ("speed", "all"):
-            speed_rows, speed_raw = run_speed(endpoints, cfg)
-
-        if args.suite in ("quality", "all"):
-            tasks = [t.strip() for t in args.quality_tasks.split(",") if t.strip()]
-            quality_rows, quality_cases = run_quality(endpoints, cfg, tasks)
+    # One server at a time. Two BF16 8B models (~16GB each) do not co-reside on a
+    # single 32GB GPU, and even if they did, a resident idle engine would distort
+    # the other's measurements. Start → run all suites → stop, then next system.
+    for ep in endpoints:
+        h = start_server(ep, log_dir)
+        try:
+            if args.suite in ("speed", "all"):
+                rows, raw = run_speed([ep], cfg)
+                speed_rows.extend(rows)
+                speed_raw.extend(raw)
+            if args.suite in ("quality", "all"):
+                rows, cases = run_quality([ep], cfg, tasks)
+                quality_rows.extend(rows)
+                quality_cases.extend(cases)
+        finally:
+            stop_server(h)
 
     write_report(
         out_dir=args.out_dir,