tools: add llama.cpp comparison baseline + standard benchmark suite
Vendor llama.cpp as a submodule pinned to b9371 and add a one-click benchmark driver that compares xserv against it on identical workloads: - setup-llama-cpp.sh: network-optional CUDA build (SM120); convert-to-gguf.sh converts the same safetensors to BF16 GGUF for an apples-to-apples baseline. - tools/bench/: black-box OpenAI-API driver measuring TTFT/TPOT/throughput (single-stream + concurrent) and response quality on AIME 2025 + GSM8K. - fetch_datasets.py pulls datasets to local JSON (GPU host has no network); task loaders prefer the local JSON. - sync-and-build.sh: `bench` subcommand transfers source + datasets to the GPU host via tar-over-ssh (no rsync there), builds, and runs the suite. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
46
tools/bench/tasks/__init__.py
Normal file
46
tools/bench/tasks/__init__.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Shared helpers for quality tasks.
|
||||
|
||||
Each task can be backed by a pre-fetched local JSON file (so the GPU host
|
||||
doesn't need network). The JSON is a list of records:
|
||||
[{"id": str, "problem": str, "answer": str, "source": str}, ...]
|
||||
|
||||
Use tools/bench/fetch_datasets.py on a networked machine to produce these
|
||||
files, then ship them to the GPU host (the bench sync does this automatically).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
|
||||
def data_dir() -> str:
|
||||
"""Directory holding pre-fetched dataset JSON. Override via BENCH_DATA_DIR."""
|
||||
return os.environ.get(
|
||||
"BENCH_DATA_DIR",
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data"),
|
||||
)
|
||||
|
||||
|
||||
def local_json_path(task_name: str) -> str:
|
||||
return os.path.normpath(os.path.join(data_dir(), f"{task_name}.json"))
|
||||
|
||||
|
||||
def load_local(task_name: str) -> list[dict[str, Any]] | None:
|
||||
"""Return records from the local JSON file if present, else None."""
|
||||
path = local_json_path(task_name)
|
||||
if not os.path.isfile(path):
|
||||
return None
|
||||
with open(path) as f:
|
||||
records = json.load(f)
|
||||
print(f"[tasks] loaded {len(records)} records from {path}")
|
||||
return records
|
||||
|
||||
|
||||
def save_local(task_name: str, records: list[dict[str, Any]]) -> str:
|
||||
path = local_json_path(task_name)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
json.dump(records, f, ensure_ascii=False, indent=1)
|
||||
return path
|
||||
114
tools/bench/tasks/aime.py
Normal file
114
tools/bench/tasks/aime.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""AIME 2025 — 30 problems, integer answers 0..999.
|
||||
|
||||
Scoring: exact-match of the integer in the last `\\boxed{...}` in the response,
|
||||
falling back to the last standalone integer in the response. Matches the
|
||||
convention used by most reasoning-model leaderboards.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from . import load_local
|
||||
|
||||
TASK_NAME = "aime2025"
|
||||
|
||||
# Tried in order; first one to load wins. These are the most-cited HF datasets
|
||||
# for AIME 2025 at time of writing; we don't depend on any one being present.
|
||||
DATASET_CANDIDATES = [
|
||||
("MathArena/aime_2025", None, "test"),
|
||||
("yentinglin/aime_2025", None, "train"),
|
||||
("opencompass/AIME2025", "AIME2025-I", "test"),
|
||||
]
|
||||
|
||||
|
||||
def load() -> list[dict[str, Any]]:
|
||||
# Prefer the pre-fetched local JSON (GPU host has no network).
|
||||
local = load_local(TASK_NAME)
|
||||
if local is not None:
|
||||
return local
|
||||
return load_remote()
|
||||
|
||||
|
||||
def load_remote() -> list[dict[str, Any]]:
|
||||
"""Fetch from HuggingFace. Requires network — used by fetch_datasets.py."""
|
||||
from datasets import load_dataset # noqa: PLC0415 — optional dep, see requirements.txt
|
||||
|
||||
last_err: Exception | None = None
|
||||
for repo, config, split in DATASET_CANDIDATES:
|
||||
try:
|
||||
ds = load_dataset(repo, config, split=split) if config else load_dataset(repo, split=split)
|
||||
except Exception as e: # noqa: BLE001 — try the next candidate
|
||||
last_err = e
|
||||
continue
|
||||
|
||||
problems: list[dict[str, Any]] = []
|
||||
for i, row in enumerate(ds):
|
||||
problem = row.get("problem") or row.get("question") or row.get("Problem")
|
||||
answer = row.get("answer") or row.get("Answer") or row.get("solution_int")
|
||||
if problem is None or answer is None:
|
||||
continue
|
||||
problems.append({
|
||||
"id": str(row.get("id") or row.get("ID") or i),
|
||||
"problem": problem,
|
||||
"answer": str(answer).strip(),
|
||||
"source": repo,
|
||||
})
|
||||
if problems:
|
||||
return problems
|
||||
|
||||
raise RuntimeError(
|
||||
f"Could not load AIME 2025 from any of {[c[0] for c in DATASET_CANDIDATES]} "
|
||||
f"(last error: {last_err!r}). Set HF_HOME / HF_TOKEN if needed."
|
||||
)
|
||||
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are a careful math problem solver. Solve the problem step by step. "
|
||||
"Put your final integer answer (an integer from 0 to 999) inside \\boxed{}."
|
||||
)
|
||||
|
||||
|
||||
def make_messages(problem: str) -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": problem},
|
||||
]
|
||||
|
||||
|
||||
_BOXED_RE = re.compile(r"\\boxed\s*\{([^{}]*)\}")
|
||||
_INT_RE = re.compile(r"-?\d+")
|
||||
|
||||
|
||||
def extract_answer(text: str) -> str | None:
|
||||
"""Return canonical integer string, or None if nothing parseable."""
|
||||
if not text:
|
||||
return None
|
||||
boxed = _BOXED_RE.findall(text)
|
||||
candidates: list[str] = []
|
||||
if boxed:
|
||||
# Inside the \boxed{} there may be extra latex; grab the last integer.
|
||||
ints = _INT_RE.findall(boxed[-1])
|
||||
if ints:
|
||||
candidates.append(ints[-1])
|
||||
# Fallback: the last integer anywhere in the response.
|
||||
if not candidates:
|
||||
ints = _INT_RE.findall(text)
|
||||
if ints:
|
||||
candidates.append(ints[-1])
|
||||
if not candidates:
|
||||
return None
|
||||
try:
|
||||
return str(int(candidates[-1]))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def score(pred: str | None, gold: str) -> bool:
|
||||
if pred is None:
|
||||
return False
|
||||
try:
|
||||
return int(pred) == int(gold)
|
||||
except ValueError:
|
||||
return False
|
||||
90
tools/bench/tasks/gsm8k.py
Normal file
90
tools/bench/tasks/gsm8k.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""GSM8K — 1319 grade-school math problems with integer/decimal answers.
|
||||
|
||||
Gold answers in the dataset are in the form `... #### 42`. We score by
|
||||
exact-match of the final number, with the same `\\boxed{}` / last-number
|
||||
extraction used for AIME, since for instruction-tuned models the response
|
||||
follows the prompt instructions, not the dataset's `####` convention.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from . import load_local
|
||||
|
||||
TASK_NAME = "gsm8k"
|
||||
|
||||
|
||||
def load() -> list[dict[str, Any]]:
|
||||
local = load_local(TASK_NAME)
|
||||
if local is not None:
|
||||
return local
|
||||
return load_remote()
|
||||
|
||||
|
||||
def load_remote() -> list[dict[str, Any]]:
|
||||
"""Fetch from HuggingFace. Requires network — used by fetch_datasets.py."""
|
||||
from datasets import load_dataset # noqa: PLC0415
|
||||
|
||||
ds = load_dataset("openai/gsm8k", "main", split="test")
|
||||
out: list[dict[str, Any]] = []
|
||||
for i, row in enumerate(ds):
|
||||
ans_full: str = row["answer"]
|
||||
# gold format: "<chain of thought>\n#### 42"
|
||||
gold = ans_full.split("####")[-1].strip().replace(",", "")
|
||||
out.append({
|
||||
"id": str(i),
|
||||
"problem": row["question"],
|
||||
"answer": gold,
|
||||
"source": "openai/gsm8k",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are a careful math problem solver. Solve the problem step by step. "
|
||||
"Put your final numeric answer inside \\boxed{}."
|
||||
)
|
||||
|
||||
|
||||
def make_messages(problem: str) -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": problem},
|
||||
]
|
||||
|
||||
|
||||
_BOXED_RE = re.compile(r"\\boxed\s*\{([^{}]*)\}")
|
||||
# Allow comma-grouped thousands (e.g. "3,500"); _normalize_num strips them.
|
||||
_NUM_RE = re.compile(r"-?\d+(?:,\d{3})*(?:\.\d+)?")
|
||||
|
||||
|
||||
def _normalize_num(s: str) -> str | None:
|
||||
s = s.replace(",", "").strip()
|
||||
try:
|
||||
f = float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
return str(int(f)) if f.is_integer() else f"{f:g}"
|
||||
|
||||
|
||||
def extract_answer(text: str) -> str | None:
|
||||
if not text:
|
||||
return None
|
||||
boxed = _BOXED_RE.findall(text)
|
||||
if boxed:
|
||||
nums = _NUM_RE.findall(boxed[-1])
|
||||
if nums:
|
||||
return _normalize_num(nums[-1])
|
||||
nums = _NUM_RE.findall(text)
|
||||
if nums:
|
||||
return _normalize_num(nums[-1])
|
||||
return None
|
||||
|
||||
|
||||
def score(pred: str | None, gold: str) -> bool:
|
||||
if pred is None:
|
||||
return False
|
||||
gold_norm = _normalize_num(gold)
|
||||
return gold_norm is not None and pred == gold_norm
|
||||
Reference in New Issue
Block a user