654 lines
27 KiB
Python
654 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
"""Re-evaluate Frontier and real replay with one SLO-feasible throughput objective.
|
|
|
|
The analysis is deliberately paired: a cell's capacity is the largest offered
|
|
load that satisfies the same request-level SLO among the anchors observed by
|
|
both systems. It does not extrapolate beyond the common anchor grid.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import sys
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
|
|
SCHEMA = "frontier-slo-alignment-v0"
|
|
MODE = "frozen-calibrated"
|
|
READING = "paired-grid-slo-feasible-max-offered-throughput"
|
|
EXPECTED_CELLS = {
|
|
f"tp{tp}_mns{mns}" for tp in (1, 2, 4) for mns in (8, 16, 32, 64)
|
|
}
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
with path.open(encoding="utf-8") as source:
|
|
value = json.load(source)
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"expected JSON object: {path}")
|
|
return value
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(1 << 20), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def git_revision(path: Path) -> dict[str, str]:
|
|
def run(*arguments: str) -> str:
|
|
return subprocess.run(
|
|
["git", *arguments],
|
|
cwd=path,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
|
|
return {"head": run("rev-parse", "HEAD"), "status_short": run("status", "--short")}
|
|
|
|
|
|
def load_real_cells(ground_truth_path: Path) -> dict[str, dict[str, Any]]:
|
|
ground_truth = load_json(ground_truth_path)
|
|
cells = {}
|
|
for cell in ground_truth["cells"]:
|
|
cell_id = str(cell["cell_id"])
|
|
if cell_id in cells:
|
|
raise ValueError(f"duplicate real cell: {cell_id}")
|
|
probes = {int(probe["probe_index"]): probe for probe in cell["probe_history"]}
|
|
if len(probes) != len(cell["probe_history"]):
|
|
raise ValueError(f"duplicate real probe index: {cell_id}")
|
|
cells[cell_id] = {**cell, "probes": probes}
|
|
if set(cells) != EXPECTED_CELLS:
|
|
raise ValueError(f"unexpected real cells: {sorted(cells)}")
|
|
return cells
|
|
|
|
|
|
def load_frontier_runs(results_dir: Path, mode: str = MODE) -> dict[tuple[str, int], dict[str, Any]]:
|
|
runs: dict[tuple[str, int], dict[str, Any]] = {}
|
|
manifests = sorted((results_dir / "raw").glob("*/trial-*/run_manifest.json"))
|
|
for manifest_path in manifests:
|
|
manifest = load_json(manifest_path)
|
|
run = manifest["run"]
|
|
if run["mode"] != mode:
|
|
continue
|
|
status_path = manifest_path.with_name("trial_status.json")
|
|
if not status_path.exists():
|
|
raise ValueError(f"missing trial status: {status_path}")
|
|
status = load_json(status_path)
|
|
if status["status"] != "pass":
|
|
continue
|
|
scorer_path = manifest_path.with_name("scorer_output.json")
|
|
if not scorer_path.exists():
|
|
raise ValueError(f"missing scorer output: {scorer_path}")
|
|
scorer = load_json(scorer_path)
|
|
key = (str(run["cell_id"]), int(run["probe_index"]))
|
|
if key in runs:
|
|
raise ValueError(f"duplicate Frontier run: {key}")
|
|
if int(scorer["total_requests"]) != int(run["request_count"]):
|
|
raise ValueError(f"request count mismatch: {key}")
|
|
runs[key] = {
|
|
"manifest_path": str(manifest_path),
|
|
"manifest": manifest,
|
|
"scorer_path": str(scorer_path),
|
|
"scorer": scorer,
|
|
}
|
|
return runs
|
|
|
|
|
|
def build_anchor_records(
|
|
real_cells: dict[str, dict[str, Any]],
|
|
frontier_runs: dict[tuple[str, int], dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
expected = {
|
|
(cell_id, probe_index)
|
|
for cell_id, cell in real_cells.items()
|
|
for probe_index in cell["probes"]
|
|
}
|
|
if set(frontier_runs) != expected:
|
|
missing = sorted(expected - set(frontier_runs))
|
|
extra = sorted(set(frontier_runs) - expected)
|
|
raise ValueError(f"Frontier/real anchor mismatch; missing={missing}, extra={extra}")
|
|
|
|
anchors = []
|
|
for cell_id, probe_index in sorted(expected):
|
|
cell = real_cells[cell_id]
|
|
probe = cell["probes"][probe_index]
|
|
frontier = frontier_runs[(cell_id, probe_index)]
|
|
run = frontier["manifest"]["run"]
|
|
scorer = frontier["scorer"]
|
|
if int(run["request_count"]) != int(probe["request_count"]):
|
|
raise ValueError(f"paired request count mismatch: {(cell_id, probe_index)}")
|
|
if not math.isclose(
|
|
float(run["sampling_u"]), float(probe["sampling_u"]), rel_tol=0.0, abs_tol=1e-15
|
|
):
|
|
raise ValueError(f"paired sampling_u mismatch: {(cell_id, probe_index)}")
|
|
rate = float(probe["request_rate_per_gpu_req_s_gpu"])
|
|
anchors.append(
|
|
{
|
|
"cell_id": cell_id,
|
|
"tp": int(cell["tensor_parallel_size"]),
|
|
"mns": int(cell["max_num_seqs"]),
|
|
"probe_index": probe_index,
|
|
"sampling_u": float(probe["sampling_u"]),
|
|
"request_count": int(probe["request_count"]),
|
|
"offered_req_s_per_gpu": rate,
|
|
"real_feasible": bool(probe["feasible"]),
|
|
"real_pass_rate": float(probe["pass_rate"]),
|
|
"frontier_feasible": bool(scorer["slo"]["feasible"]),
|
|
"frontier_pass_rate": float(scorer["slo"]["pass_rate"]),
|
|
"frontier_completed_req_s_per_gpu": float(
|
|
scorer["throughput_requests_per_second_per_gpu"]
|
|
),
|
|
}
|
|
)
|
|
return anchors
|
|
|
|
|
|
def monotonic_violations(rows: list[dict[str, Any]], field: str) -> list[dict[str, Any]]:
|
|
ordered = sorted(rows, key=lambda row: (row["offered_req_s_per_gpu"], row["probe_index"]))
|
|
violations = []
|
|
for lower_index, lower in enumerate(ordered):
|
|
for upper in ordered[lower_index + 1 :]:
|
|
if (
|
|
lower["offered_req_s_per_gpu"] < upper["offered_req_s_per_gpu"]
|
|
and not lower[field]
|
|
and upper[field]
|
|
):
|
|
violations.append(
|
|
{
|
|
"lower_probe": lower["probe_index"],
|
|
"lower_rate": lower["offered_req_s_per_gpu"],
|
|
"upper_probe": upper["probe_index"],
|
|
"upper_rate": upper["offered_req_s_per_gpu"],
|
|
}
|
|
)
|
|
return violations
|
|
|
|
|
|
def selected_anchor(rows: list[dict[str, Any]], field: str) -> dict[str, Any] | None:
|
|
feasible = [row for row in rows if row[field]]
|
|
if not feasible:
|
|
return None
|
|
return max(feasible, key=lambda row: (row["offered_req_s_per_gpu"], -row["probe_index"]))
|
|
|
|
|
|
def summarize_cells(anchors: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
grouped = {cell: [] for cell in sorted(EXPECTED_CELLS)}
|
|
for row in anchors:
|
|
grouped[row["cell_id"]].append(row)
|
|
|
|
summaries = {}
|
|
for cell_id, rows in grouped.items():
|
|
real_selected = selected_anchor(rows, "real_feasible")
|
|
frontier_selected = selected_anchor(rows, "frontier_feasible")
|
|
real_score = real_selected["offered_req_s_per_gpu"] if real_selected else 0.0
|
|
frontier_score = frontier_selected["offered_req_s_per_gpu"] if frontier_selected else 0.0
|
|
max_rate = max(row["offered_req_s_per_gpu"] for row in rows)
|
|
real_violations = monotonic_violations(rows, "real_feasible")
|
|
frontier_violations = monotonic_violations(rows, "frontier_feasible")
|
|
summaries[cell_id] = {
|
|
"tp": rows[0]["tp"],
|
|
"mns": rows[0]["mns"],
|
|
"anchor_count": len(rows),
|
|
"max_common_anchor_req_s_per_gpu": max_rate,
|
|
"real_feasible_anchor_count": sum(bool(row["real_feasible"]) for row in rows),
|
|
"frontier_feasible_anchor_count": sum(
|
|
bool(row["frontier_feasible"]) for row in rows
|
|
),
|
|
"real_score": real_score,
|
|
"frontier_score": frontier_score,
|
|
"absolute_error": frontier_score - real_score,
|
|
"relative_error": (frontier_score - real_score) / real_score if real_score else None,
|
|
"real_selected_probe": real_selected["probe_index"] if real_selected else None,
|
|
"frontier_selected_probe": frontier_selected["probe_index"] if frontier_selected else None,
|
|
"frontier_right_censored": bool(
|
|
frontier_selected and math.isclose(frontier_score, max_rate, abs_tol=1e-15)
|
|
),
|
|
"frontier_boundary_bracketed": bool(
|
|
frontier_selected
|
|
and any(
|
|
not row["frontier_feasible"]
|
|
and row["offered_req_s_per_gpu"] > frontier_score
|
|
for row in rows
|
|
)
|
|
),
|
|
"real_monotonic_violation_count": len(real_violations),
|
|
"frontier_monotonic_violation_count": len(frontier_violations),
|
|
"real_monotonic_violations": real_violations,
|
|
"frontier_monotonic_violations": frontier_violations,
|
|
}
|
|
return summaries
|
|
|
|
|
|
def score_buckets(scores: dict[str, float]) -> tuple[float, dict[str, int]]:
|
|
tolerance = max(1e-9, 1e-6 * max(abs(value) for value in scores.values()))
|
|
return tolerance, {cell: math.floor(value / tolerance) for cell, value in scores.items()}
|
|
|
|
|
|
def sign(value: int) -> int:
|
|
return (value > 0) - (value < 0)
|
|
|
|
|
|
def ranking_metrics(
|
|
real_scores: dict[str, float], frontier_scores: dict[str, float]
|
|
) -> dict[str, Any]:
|
|
if set(real_scores) != set(frontier_scores):
|
|
raise ValueError("ranking score cells differ")
|
|
real_tolerance, real_buckets = score_buckets(real_scores)
|
|
frontier_tolerance, frontier_buckets = score_buckets(frontier_scores)
|
|
cells = sorted(real_scores)
|
|
counts = Counter()
|
|
exact = 0
|
|
for left_index, left in enumerate(cells):
|
|
for right in cells[left_index + 1 :]:
|
|
real_sign = sign(real_buckets[left] - real_buckets[right])
|
|
frontier_sign = sign(frontier_buckets[left] - frontier_buckets[right])
|
|
exact += int(real_sign == frontier_sign)
|
|
if real_sign == 0 and frontier_sign == 0:
|
|
counts["both_tied"] += 1
|
|
elif real_sign == 0:
|
|
counts["real_only_tied"] += 1
|
|
elif frontier_sign == 0:
|
|
counts["frontier_only_tied"] += 1
|
|
elif real_sign == frontier_sign:
|
|
counts["concordant"] += 1
|
|
else:
|
|
counts["discordant"] += 1
|
|
numerator = counts["concordant"] - counts["discordant"]
|
|
denominator = math.sqrt(
|
|
(counts["concordant"] + counts["discordant"] + counts["real_only_tied"])
|
|
* (
|
|
counts["concordant"]
|
|
+ counts["discordant"]
|
|
+ counts["frontier_only_tied"]
|
|
)
|
|
)
|
|
tau = numerator / denominator if denominator else 0.0
|
|
top_bucket = max(frontier_buckets.values())
|
|
candidates = sorted(cell for cell, bucket in frontier_buckets.items() if bucket == top_bucket)
|
|
real_best = max(real_scores.values())
|
|
regrets = [(real_best - real_scores[cell]) / real_best for cell in candidates]
|
|
return {
|
|
"real_tolerance": real_tolerance,
|
|
"frontier_tolerance": frontier_tolerance,
|
|
"top1_candidate_cells": candidates,
|
|
"top1_effective_k": len(candidates),
|
|
"top1_optimistic_regret": min(regrets),
|
|
"top1_worst_case_regret": max(regrets),
|
|
"pair_count": len(cells) * (len(cells) - 1) // 2,
|
|
"pairwise_exact_sign_accuracy": exact / (len(cells) * (len(cells) - 1) // 2),
|
|
"kendall_tau_b": tau,
|
|
"kendall_counts": dict(counts),
|
|
}
|
|
|
|
|
|
def confusion_metrics(anchors: list[dict[str, Any]]) -> dict[str, Any]:
|
|
counts = Counter()
|
|
for row in anchors:
|
|
real = bool(row["real_feasible"])
|
|
frontier = bool(row["frontier_feasible"])
|
|
label = (
|
|
"true_feasible"
|
|
if real and frontier
|
|
else "true_infeasible"
|
|
if not real and not frontier
|
|
else "false_feasible"
|
|
if not real and frontier
|
|
else "false_infeasible"
|
|
)
|
|
counts[label] += 1
|
|
total = len(anchors)
|
|
positive = counts["true_feasible"] + counts["false_infeasible"]
|
|
negative = counts["true_infeasible"] + counts["false_feasible"]
|
|
return {
|
|
**{name: counts[name] for name in (
|
|
"true_feasible",
|
|
"true_infeasible",
|
|
"false_feasible",
|
|
"false_infeasible",
|
|
)},
|
|
"total": total,
|
|
"accuracy": (counts["true_feasible"] + counts["true_infeasible"]) / total,
|
|
"feasible_recall": counts["true_feasible"] / positive if positive else None,
|
|
"infeasible_recall": counts["true_infeasible"] / negative if negative else None,
|
|
"false_feasible_rate": counts["false_feasible"] / negative if negative else None,
|
|
"false_infeasible_rate": counts["false_infeasible"] / positive if positive else None,
|
|
}
|
|
|
|
|
|
def error_metrics(cell_summaries: dict[str, dict[str, Any]]) -> dict[str, float]:
|
|
errors = [row["absolute_error"] for row in cell_summaries.values()]
|
|
relative = [abs(row["relative_error"]) for row in cell_summaries.values()]
|
|
return {
|
|
"mean_absolute_error_req_s_per_gpu": sum(abs(value) for value in errors) / len(errors),
|
|
"root_mean_square_error_req_s_per_gpu": math.sqrt(
|
|
sum(value * value for value in errors) / len(errors)
|
|
),
|
|
"mean_absolute_percentage_error": sum(relative) / len(relative),
|
|
"right_censored_cell_count": sum(
|
|
bool(row["frontier_right_censored"]) for row in cell_summaries.values()
|
|
),
|
|
"bracketed_cell_count": sum(
|
|
bool(row["frontier_boundary_bracketed"]) for row in cell_summaries.values()
|
|
),
|
|
}
|
|
|
|
|
|
def grouped_error_metrics(
|
|
cell_summaries: dict[str, dict[str, Any]], field: str
|
|
) -> dict[str, dict[str, float]]:
|
|
groups: dict[int, list[dict[str, Any]]] = {}
|
|
for row in cell_summaries.values():
|
|
groups.setdefault(int(row[field]), []).append(row)
|
|
output = {}
|
|
for key, rows in sorted(groups.items()):
|
|
output[str(key)] = {
|
|
"cell_count": len(rows),
|
|
"mean_signed_error_req_s_per_gpu": sum(row["absolute_error"] for row in rows)
|
|
/ len(rows),
|
|
"mean_absolute_error_req_s_per_gpu": sum(
|
|
abs(row["absolute_error"]) for row in rows
|
|
)
|
|
/ len(rows),
|
|
"right_censored_cell_count": sum(bool(row["frontier_right_censored"]) for row in rows),
|
|
}
|
|
return output
|
|
|
|
|
|
def old_proxy_summary(previous_metrics_path: Path) -> dict[str, Any]:
|
|
previous = load_json(previous_metrics_path)
|
|
analysis = previous["analyses"]["frozen-calibrated/throughput-proxy"]
|
|
metrics = analysis["metrics"]
|
|
return {
|
|
"reading": "completed-throughput proxy (old, not SLO gated)",
|
|
"top1_candidate_cells": metrics["top1"]["candidate_cells"],
|
|
"top1_worst_case_regret": metrics["top1"]["worst_case_regret"],
|
|
"kendall_tau_b": metrics["kendall_tau_b"]["tau_b"],
|
|
"pairwise_exact_sign_accuracy": metrics["pairwise_direction"]["exact_sign_accuracy"],
|
|
}
|
|
|
|
|
|
def validate_previous_slo_reading(
|
|
previous_metrics_path: Path,
|
|
real_scores: dict[str, float],
|
|
frontier_scores: dict[str, float],
|
|
) -> dict[str, Any]:
|
|
previous = load_json(previous_metrics_path)
|
|
previous_real = {cell: float(value) for cell, value in previous["real_scores"].items()}
|
|
previous_frontier = {
|
|
cell: float(value)
|
|
for cell, value in previous["analyses"]["frozen-calibrated/SLO-gated"][
|
|
"simulated_scores"
|
|
].items()
|
|
}
|
|
real_difference = max(abs(real_scores[cell] - previous_real[cell]) for cell in real_scores)
|
|
frontier_difference = max(
|
|
abs(frontier_scores[cell] - previous_frontier[cell]) for cell in frontier_scores
|
|
)
|
|
if real_difference > 1e-12 or frontier_difference > 1e-12:
|
|
raise ValueError(
|
|
"independent aligned reconstruction disagrees with previous secondary reading: "
|
|
f"real={real_difference}, Frontier={frontier_difference}"
|
|
)
|
|
return {
|
|
"status": "PASS",
|
|
"previous_reading": "frozen-calibrated/SLO-gated",
|
|
"maximum_real_score_absolute_difference": real_difference,
|
|
"maximum_frontier_score_absolute_difference": frontier_difference,
|
|
}
|
|
|
|
|
|
def write_csv(path: Path, fieldnames: list[str], rows: Iterable[dict[str, Any]]) -> None:
|
|
with path.open("w", encoding="utf-8", newline="") as output:
|
|
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows({field: row.get(field) for field in fieldnames} for row in rows)
|
|
|
|
|
|
def render_report(metrics: dict[str, Any]) -> str:
|
|
rank = metrics["aligned_ranking"]
|
|
confusion = metrics["anchor_feasibility"]
|
|
error = metrics["score_error"]
|
|
grouped = metrics["grouped_score_error"]
|
|
old = metrics["objective_comparison"]["old_proxy"]
|
|
cells = metrics["cells"]
|
|
lines = [
|
|
"# Frontier SLO-aligned retrospective experiment",
|
|
"",
|
|
"## Result",
|
|
"",
|
|
(
|
|
"After aligning both systems to the same paired-grid objective—maximum offered "
|
|
"req/s/GPU with request-level SLO pass rate >= 0.95—the old 30.46% top-1 "
|
|
"regret does not reproduce. Frontier nominates `"
|
|
+ "`, `".join(rank["top1_candidate_cells"])
|
|
+ f"` (tie), with real-evaluated regret {100 * rank['top1_optimistic_regret']:.3f}%"
|
|
+ f"--{100 * rank['top1_worst_case_regret']:.3f}%."
|
|
),
|
|
"",
|
|
(
|
|
f"Ranking agreement is high on this fixed candidate grid: Kendall tau-b "
|
|
f"{rank['kendall_tau_b']:.4f}, pairwise exact-sign accuracy "
|
|
f"{100 * rank['pairwise_exact_sign_accuracy']:.2f}%. This is materially different "
|
|
f"from the old completed-throughput proxy (tau-b {old['kendall_tau_b']:.4f}, "
|
|
f"top-1 regret {100 * old['top1_worst_case_regret']:.2f}%)."
|
|
),
|
|
"",
|
|
"## Per-cell paired-grid capacity",
|
|
"",
|
|
"| Cell | Real | Frontier | Error | Frontier boundary | Real monotonic violations |",
|
|
"|---|---:|---:|---:|---|---:|",
|
|
]
|
|
for cell_id in sorted(cells, key=lambda cell: (cells[cell]["tp"], cells[cell]["mns"])):
|
|
row = cells[cell_id]
|
|
boundary = "right-censored" if row["frontier_right_censored"] else (
|
|
"bracketed" if row["frontier_boundary_bracketed"] else "unresolved"
|
|
)
|
|
lines.append(
|
|
f"| `{cell_id}` | {row['real_score']:.6f} | {row['frontier_score']:.6f} | "
|
|
f"{row['absolute_error']:+.6f} | {boundary} | "
|
|
f"{row['real_monotonic_violation_count']} |"
|
|
)
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"The paired-grid score MAE is "
|
|
f"{error['mean_absolute_error_req_s_per_gpu']:.4f} req/s/GPU and MAPE is "
|
|
f"{100 * error['mean_absolute_percentage_error']:.2f}%. These error aggregates "
|
|
"must be read as grid-clipped because "
|
|
f"{int(error['right_censored_cell_count'])}/12 Frontier cells remain feasible at "
|
|
"their highest common anchor.",
|
|
"",
|
|
"The error is configuration-dependent rather than a uniform scale offset. Mean "
|
|
f"signed error is {grouped['by_mns']['8']['mean_signed_error_req_s_per_gpu']:+.4f} "
|
|
"req/s/GPU for MNS=8, but rises to "
|
|
f"{grouped['by_mns']['32']['mean_signed_error_req_s_per_gpu']:+.4f} and "
|
|
f"{grouped['by_mns']['64']['mean_signed_error_req_s_per_gpu']:+.4f} for MNS=32/64. "
|
|
"The aligned score therefore still exposes a missing or distorted MNS response.",
|
|
"",
|
|
"## Anchor-level feasibility",
|
|
"",
|
|
f"Across {confusion['total']} paired anchors: true-feasible="
|
|
f"{confusion['true_feasible']}, true-infeasible={confusion['true_infeasible']}, "
|
|
f"false-feasible={confusion['false_feasible']}, false-infeasible="
|
|
f"{confusion['false_infeasible']}. Accuracy is {100 * confusion['accuracy']:.2f}%, "
|
|
f"but the false-feasible rate among real-infeasible anchors is "
|
|
f"{100 * confusion['false_feasible_rate']:.2f}%.",
|
|
"",
|
|
"Therefore the aligned experiment supports a narrower conclusion: Frontier can "
|
|
"recover the top configuration family on this frozen, ragged candidate grid, but "
|
|
"it is not yet a reliable SLO feasibility oracle. Good top-1 ranking is compatible "
|
|
"with many wrong boundary labels.",
|
|
"",
|
|
"## Validity limits",
|
|
"",
|
|
"- This is a retrospective single-run analysis; no confidence interval is valid.",
|
|
"- Real probes within a cell reused one vLLM process and prefix-cache history; "
|
|
"Frontier anchors were independent runs. History-dependent real pass-rate values "
|
|
"are retained rather than monotonicized; binary feasibility happens to remain "
|
|
"monotone on the observed per-cell grids.",
|
|
"- Right-censored cells expose only a lower bound on Frontier's own capacity. "
|
|
"They are valid for paired-grid selection, not for claiming an exact simulator knee.",
|
|
"- The frozen per-TP calibration and evaluation surface are not fully independent "
|
|
"across workload families, so this is not a generalization result.",
|
|
"",
|
|
"## Required prospective experiment",
|
|
"",
|
|
"Repeat a small set of boundary cells with one engine restart per anchor, explicit "
|
|
"cold-cache state, identical offered-load grid, and repeated trials. Extend the grid "
|
|
"until both real and Frontier have at least one feasible and one infeasible point. "
|
|
"Only that experiment can estimate boundary displacement and its uncertainty.",
|
|
"",
|
|
]
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def analyze(
|
|
ground_truth_path: Path,
|
|
results_dir: Path,
|
|
previous_metrics_path: Path,
|
|
repository_root: Path,
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
real_cells = load_real_cells(ground_truth_path)
|
|
frontier_runs = load_frontier_runs(results_dir)
|
|
anchors = build_anchor_records(real_cells, frontier_runs)
|
|
cells = summarize_cells(anchors)
|
|
real_scores = {cell: row["real_score"] for cell, row in cells.items()}
|
|
frontier_scores = {cell: row["frontier_score"] for cell, row in cells.items()}
|
|
frontier_revisions = {
|
|
run["manifest"]["frontier"]["git_head"] for run in frontier_runs.values()
|
|
}
|
|
if len(frontier_revisions) != 1:
|
|
raise ValueError(f"mixed Frontier revisions: {sorted(frontier_revisions)}")
|
|
metrics = {
|
|
"schema_version": SCHEMA,
|
|
"generated_utc": datetime.now(timezone.utc).isoformat(),
|
|
"status": "VALID_RETROSPECTIVE_ALIGNED_ANALYSIS",
|
|
"claim": (
|
|
"On the frozen common anchor grid, rank configurations by maximum offered "
|
|
"req/s/GPU whose request-level SLO pass rate is at least 0.95 in each system."
|
|
),
|
|
"estimand": READING,
|
|
"inputs": {
|
|
"ground_truth": {
|
|
"path": str(ground_truth_path),
|
|
"sha256": sha256_file(ground_truth_path),
|
|
},
|
|
"frontier_results": str(results_dir),
|
|
"previous_metrics": {
|
|
"path": str(previous_metrics_path),
|
|
"sha256": sha256_file(previous_metrics_path),
|
|
},
|
|
},
|
|
"provenance": {
|
|
"analysis_repository": git_revision(repository_root),
|
|
"frontier_git_head_recorded_by_runs": next(iter(frontier_revisions)),
|
|
"frontier_mode": MODE,
|
|
"real_engine_version": "vLLM 0.20.0",
|
|
"anchor_count": len(anchors),
|
|
"cell_count": len(cells),
|
|
},
|
|
"objective_comparison": {"old_proxy": old_proxy_summary(previous_metrics_path)},
|
|
"aligned_ranking": ranking_metrics(real_scores, frontier_scores),
|
|
"anchor_feasibility": confusion_metrics(anchors),
|
|
"score_error": error_metrics(cells),
|
|
"grouped_score_error": {
|
|
"by_tp": grouped_error_metrics(cells, "tp"),
|
|
"by_mns": grouped_error_metrics(cells, "mns"),
|
|
},
|
|
"independent_reconstruction_crosscheck": validate_previous_slo_reading(
|
|
previous_metrics_path, real_scores, frontier_scores
|
|
),
|
|
"cells": cells,
|
|
"statistics": {
|
|
"real_trials_per_anchor": 1,
|
|
"frontier_trials_per_anchor": 1,
|
|
"confidence_intervals": None,
|
|
"reason": "single retrospective observation per paired anchor",
|
|
},
|
|
"known_limits": [
|
|
"real probes reused a process and prefix-cache history within each cell",
|
|
"Frontier anchors were independent simulator runs",
|
|
"right-censored cells do not identify Frontier's exact capacity knee",
|
|
"the anchor grid is ragged across cells",
|
|
],
|
|
}
|
|
return metrics, anchors
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--ground-truth", type=Path, required=True)
|
|
parser.add_argument("--frontier-results", type=Path, required=True)
|
|
parser.add_argument("--previous-metrics", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
repository_root = Path(__file__).resolve().parents[2]
|
|
metrics, anchors = analyze(
|
|
args.ground_truth.resolve(),
|
|
args.frontier_results.resolve(),
|
|
args.previous_metrics.resolve(),
|
|
repository_root,
|
|
)
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
metrics_path = args.output_dir / "metrics.json"
|
|
metrics_path.write_text(json.dumps(metrics, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
write_csv(
|
|
args.output_dir / "cell_scores.csv",
|
|
[
|
|
"cell_id",
|
|
"tp",
|
|
"mns",
|
|
"anchor_count",
|
|
"real_score",
|
|
"frontier_score",
|
|
"absolute_error",
|
|
"relative_error",
|
|
"frontier_right_censored",
|
|
"frontier_boundary_bracketed",
|
|
"real_monotonic_violation_count",
|
|
"frontier_monotonic_violation_count",
|
|
],
|
|
({"cell_id": cell_id, **row} for cell_id, row in sorted(metrics["cells"].items())),
|
|
)
|
|
write_csv(
|
|
args.output_dir / "anchor_labels.csv",
|
|
[
|
|
"cell_id",
|
|
"tp",
|
|
"mns",
|
|
"probe_index",
|
|
"sampling_u",
|
|
"request_count",
|
|
"offered_req_s_per_gpu",
|
|
"real_feasible",
|
|
"real_pass_rate",
|
|
"frontier_feasible",
|
|
"frontier_pass_rate",
|
|
"frontier_completed_req_s_per_gpu",
|
|
],
|
|
anchors,
|
|
)
|
|
(args.output_dir / "report.md").write_text(render_report(metrics), encoding="utf-8")
|
|
print(json.dumps({"status": metrics["status"], "metrics": str(metrics_path)}, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|