Track simulator fidelity experiment artifacts
This commit is contained in:
280
runs/frontier-multicase-sufficiency-v0/audit_ground_truth.py
Normal file
280
runs/frontier-multicase-sufficiency-v0/audit_ground_truth.py
Normal file
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit the Qwen235B real-machine surfaces before comparing Frontier.
|
||||
|
||||
This script intentionally does not consume simulator output. It establishes
|
||||
whether each real response surface is complete and discriminative enough to
|
||||
support a later claim about simulator config selection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCHEMA = "frontier-multicase-ground-truth-v0"
|
||||
EXPECTED_PROBES = 6
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
with path.open(encoding="utf-8") as source:
|
||||
value = json.load(source)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"expected JSON object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def config_from_result(result: dict[str, Any]) -> dict[str, int]:
|
||||
flags = result["config_patch"]["flag_patch"]
|
||||
tp = int(flags["tensor-parallel-size"])
|
||||
dp = int(flags.get("data-parallel-size", 1))
|
||||
return {
|
||||
"tp": tp,
|
||||
"dp": dp,
|
||||
"ep": int(flags.get("expert-parallel-size", 1)),
|
||||
"mns": int(flags["max-num-seqs"]),
|
||||
"mbt": int(flags["max-num-batched-tokens"]),
|
||||
"gpu_count": tp * dp,
|
||||
}
|
||||
|
||||
|
||||
def cell_id(config: dict[str, int]) -> str:
|
||||
topology = f"tp{config['tp']}"
|
||||
if config["dp"] != 1 or config["ep"] != 1:
|
||||
topology += f"_dp{config['dp']}_ep{config['ep']}"
|
||||
return f"{topology}_mns{config['mns']}_mbt{config['mbt']}"
|
||||
|
||||
|
||||
def trial_record(case: str, path: Path) -> dict[str, Any]:
|
||||
result = load_json(path)
|
||||
config = config_from_result(result)
|
||||
score = float(result["best_request_rate"]) / config["gpu_count"]
|
||||
probes = result.get("probes", [])
|
||||
infeasible_above = [
|
||||
float(probe.get("payload", probe)["request_rate"]) / config["gpu_count"]
|
||||
for probe in probes
|
||||
if not probe["feasible"]
|
||||
and float(probe.get("payload", probe)["request_rate"])
|
||||
/ config["gpu_count"]
|
||||
> score
|
||||
]
|
||||
upper_bound = min(infeasible_above) if infeasible_above else None
|
||||
probe_count = len(result.get("probes", []))
|
||||
primary_result = result.get("best_source") == "primary_search"
|
||||
no_probe_failure = not bool(result.get("completed_with_probe_failure", False))
|
||||
fully_valid = (
|
||||
result.get("status") == "completed"
|
||||
and probe_count == EXPECTED_PROBES
|
||||
and primary_result
|
||||
and no_probe_failure
|
||||
)
|
||||
return {
|
||||
"case": case,
|
||||
"cell_id": cell_id(config),
|
||||
**config,
|
||||
"score_req_s_per_gpu": score,
|
||||
"capacity_lower_bound_req_s_per_gpu": score,
|
||||
"capacity_upper_bound_req_s_per_gpu": upper_bound,
|
||||
"capacity_bracket_width_req_s_per_gpu": (
|
||||
upper_bound - score if upper_bound is not None else None
|
||||
),
|
||||
"best_request_rate_req_s": float(result["best_request_rate"]),
|
||||
"best_sampling_u": float(result["best_sampling_u"]),
|
||||
"best_pass_rate": float(result["best_pass_rate"]),
|
||||
"probe_count": probe_count,
|
||||
"best_source": result.get("best_source"),
|
||||
"completed_with_probe_failure": bool(
|
||||
result.get("completed_with_probe_failure", False)
|
||||
),
|
||||
"fully_valid": fully_valid,
|
||||
"result_path": str(path),
|
||||
"result_sha256": sha256_file(path),
|
||||
}
|
||||
|
||||
|
||||
def summarize_case(case: str, rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
if not rows:
|
||||
raise ValueError(f"no rows for case: {case}")
|
||||
ids = [row["cell_id"] for row in rows]
|
||||
if len(ids) != len(set(ids)):
|
||||
duplicates = sorted(cell for cell in set(ids) if ids.count(cell) > 1)
|
||||
raise ValueError(f"duplicate cells for {case}: {duplicates}")
|
||||
|
||||
best = max(row["score_req_s_per_gpu"] for row in rows)
|
||||
tolerance = max(1e-12, best * 1e-9)
|
||||
top = [
|
||||
row["cell_id"]
|
||||
for row in rows
|
||||
if math.isclose(row["score_req_s_per_gpu"], best, abs_tol=tolerance)
|
||||
]
|
||||
distinct_scores = []
|
||||
for score in sorted({row["score_req_s_per_gpu"] for row in rows}, reverse=True):
|
||||
if not any(math.isclose(score, seen, abs_tol=tolerance) for seen in distinct_scores):
|
||||
distinct_scores.append(score)
|
||||
|
||||
max_lower_bound = max(row["capacity_lower_bound_req_s_per_gpu"] for row in rows)
|
||||
possibly_optimal = [
|
||||
row["cell_id"]
|
||||
for row in rows
|
||||
if row["capacity_upper_bound_req_s_per_gpu"] is None
|
||||
or row["capacity_upper_bound_req_s_per_gpu"] + tolerance >= max_lower_bound
|
||||
]
|
||||
|
||||
total_pairs = len(rows) * (len(rows) - 1) // 2
|
||||
tied_pairs = sum(
|
||||
1
|
||||
for left_index, left in enumerate(rows)
|
||||
for right in rows[left_index + 1 :]
|
||||
if math.isclose(
|
||||
left["score_req_s_per_gpu"],
|
||||
right["score_req_s_per_gpu"],
|
||||
abs_tol=tolerance,
|
||||
)
|
||||
)
|
||||
return {
|
||||
"case": case,
|
||||
"cell_count": len(rows),
|
||||
"fully_valid_cell_count": sum(bool(row["fully_valid"]) for row in rows),
|
||||
"invalid_cells": [row["cell_id"] for row in rows if not row["fully_valid"]],
|
||||
"best_score_req_s_per_gpu": best,
|
||||
"top_set": sorted(top),
|
||||
"top_set_size": len(top),
|
||||
"random_top_set_hit_rate": len(top) / len(rows),
|
||||
"distinct_score_count": len(distinct_scores),
|
||||
"distinct_scores_req_s_per_gpu": distinct_scores,
|
||||
"possibly_optimal_set_from_search_brackets": sorted(possibly_optimal),
|
||||
"possibly_optimal_set_size": len(possibly_optimal),
|
||||
"pair_count": total_pairs,
|
||||
"tied_pair_count": tied_pairs,
|
||||
"informative_pair_count": total_pairs - tied_pairs,
|
||||
"informative_pair_fraction": (
|
||||
(total_pairs - tied_pairs) / total_pairs if total_pairs else 0.0
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def markdown_report(metrics: dict[str, Any], rows: list[dict[str, Any]]) -> str:
|
||||
lines = [
|
||||
"# Qwen235B ground-truth audit",
|
||||
"",
|
||||
"Objective: maximum SLO-feasible offered request throughput per GPU.",
|
||||
"This report contains real-machine data only; it makes no Frontier match claim.",
|
||||
"",
|
||||
"| case | valid cells | score levels | top-set size | random top-set hit | informative pairs |",
|
||||
"|---|---:|---:|---:|---:|---:|",
|
||||
]
|
||||
for case in metrics["cases"]:
|
||||
lines.append(
|
||||
f"| {case['case']} | {case['fully_valid_cell_count']}/{case['cell_count']} "
|
||||
f"| {case['distinct_score_count']} | {case['top_set_size']}/{case['cell_count']} "
|
||||
f"| {case['random_top_set_hit_rate']:.1%} "
|
||||
f"| {case['informative_pair_count']}/{case['pair_count']} "
|
||||
f"({case['informative_pair_fraction']:.1%}) |"
|
||||
)
|
||||
|
||||
lines.extend(["", "## Cells", ""])
|
||||
for case in metrics["cases"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {case['case']}",
|
||||
"",
|
||||
"| cell | capacity bracket (req/s/GPU) | valid | probes | source |",
|
||||
"|---|---:|---:|---:|---|",
|
||||
]
|
||||
)
|
||||
for row in sorted(
|
||||
(row for row in rows if row["case"] == case["case"]),
|
||||
key=lambda row: row["cell_id"],
|
||||
):
|
||||
upper = row["capacity_upper_bound_req_s_per_gpu"]
|
||||
bracket = (
|
||||
f"[{row['capacity_lower_bound_req_s_per_gpu']:.9f}, "
|
||||
f"{upper:.9f})"
|
||||
if upper is not None
|
||||
else f"[{row['capacity_lower_bound_req_s_per_gpu']:.9f}, +inf)"
|
||||
)
|
||||
lines.append(
|
||||
f"| {row['cell_id']} | {bracket} "
|
||||
f"| {'yes' if row['fully_valid'] else 'no'} | {row['probe_count']} "
|
||||
f"| {row['best_source']} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
f"Top set: `{', '.join(case['top_set'])}`.",
|
||||
f"Possibly optimal under binary-search brackets: "
|
||||
f"`{', '.join(case['possibly_optimal_set_from_search_brackets'])}`.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"## Interpretation guardrail",
|
||||
"",
|
||||
"A Frontier top-set hit is insufficient by itself because the surfaces contain "
|
||||
"large ties. The later comparison must report selected-config regret and "
|
||||
"tie-aware pairwise ranking, and must keep invalid real cells visible.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="") as target:
|
||||
writer = csv.DictWriter(target, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--prefill-root", action="append", type=Path, required=True)
|
||||
parser.add_argument("--decode-root", action="append", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
cases = {"qwen235b_prefill_only": args.prefill_root, "qwen235b_decode_only": args.decode_root}
|
||||
rows = []
|
||||
for case, roots in cases.items():
|
||||
for root in roots:
|
||||
paths = sorted(root.glob("store/*/trials/trial-*/result.json"))
|
||||
if not paths:
|
||||
raise ValueError(f"no result files below {root}")
|
||||
rows.extend(trial_record(case, path) for path in paths)
|
||||
|
||||
summaries = [
|
||||
summarize_case(case, [row for row in rows if row["case"] == case])
|
||||
for case in cases
|
||||
]
|
||||
metrics = {"schema": SCHEMA, "cases": summaries}
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
write_csv(args.output_dir / "cells.csv", rows)
|
||||
(args.output_dir / "metrics.json").write_text(
|
||||
json.dumps(metrics, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
(args.output_dir / "report.md").write_text(
|
||||
markdown_report(metrics, rows), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user