Track simulator fidelity experiment artifacts
This commit is contained in:
653
runs/frontier-slo-alignment-v0/analyze.py
Normal file
653
runs/frontier-slo-alignment-v0/analyze.py
Normal file
@@ -0,0 +1,653 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-evaluate Frontier and real replay with one SLO-feasible throughput objective.
|
||||
|
||||
The analysis is deliberately paired: a cell's capacity is the largest offered
|
||||
load that satisfies the same request-level SLO among the anchors observed by
|
||||
both systems. It does not extrapolate beyond the common anchor grid.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
SCHEMA = "frontier-slo-alignment-v0"
|
||||
MODE = "frozen-calibrated"
|
||||
READING = "paired-grid-slo-feasible-max-offered-throughput"
|
||||
EXPECTED_CELLS = {
|
||||
f"tp{tp}_mns{mns}" for tp in (1, 2, 4) for mns in (8, 16, 32, 64)
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
with path.open(encoding="utf-8") as source:
|
||||
value = json.load(source)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"expected JSON object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def git_revision(path: Path) -> dict[str, str]:
|
||||
def run(*arguments: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", *arguments],
|
||||
cwd=path,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
|
||||
return {"head": run("rev-parse", "HEAD"), "status_short": run("status", "--short")}
|
||||
|
||||
|
||||
def load_real_cells(ground_truth_path: Path) -> dict[str, dict[str, Any]]:
|
||||
ground_truth = load_json(ground_truth_path)
|
||||
cells = {}
|
||||
for cell in ground_truth["cells"]:
|
||||
cell_id = str(cell["cell_id"])
|
||||
if cell_id in cells:
|
||||
raise ValueError(f"duplicate real cell: {cell_id}")
|
||||
probes = {int(probe["probe_index"]): probe for probe in cell["probe_history"]}
|
||||
if len(probes) != len(cell["probe_history"]):
|
||||
raise ValueError(f"duplicate real probe index: {cell_id}")
|
||||
cells[cell_id] = {**cell, "probes": probes}
|
||||
if set(cells) != EXPECTED_CELLS:
|
||||
raise ValueError(f"unexpected real cells: {sorted(cells)}")
|
||||
return cells
|
||||
|
||||
|
||||
def load_frontier_runs(results_dir: Path, mode: str = MODE) -> dict[tuple[str, int], dict[str, Any]]:
|
||||
runs: dict[tuple[str, int], dict[str, Any]] = {}
|
||||
manifests = sorted((results_dir / "raw").glob("*/trial-*/run_manifest.json"))
|
||||
for manifest_path in manifests:
|
||||
manifest = load_json(manifest_path)
|
||||
run = manifest["run"]
|
||||
if run["mode"] != mode:
|
||||
continue
|
||||
status_path = manifest_path.with_name("trial_status.json")
|
||||
if not status_path.exists():
|
||||
raise ValueError(f"missing trial status: {status_path}")
|
||||
status = load_json(status_path)
|
||||
if status["status"] != "pass":
|
||||
continue
|
||||
scorer_path = manifest_path.with_name("scorer_output.json")
|
||||
if not scorer_path.exists():
|
||||
raise ValueError(f"missing scorer output: {scorer_path}")
|
||||
scorer = load_json(scorer_path)
|
||||
key = (str(run["cell_id"]), int(run["probe_index"]))
|
||||
if key in runs:
|
||||
raise ValueError(f"duplicate Frontier run: {key}")
|
||||
if int(scorer["total_requests"]) != int(run["request_count"]):
|
||||
raise ValueError(f"request count mismatch: {key}")
|
||||
runs[key] = {
|
||||
"manifest_path": str(manifest_path),
|
||||
"manifest": manifest,
|
||||
"scorer_path": str(scorer_path),
|
||||
"scorer": scorer,
|
||||
}
|
||||
return runs
|
||||
|
||||
|
||||
def build_anchor_records(
|
||||
real_cells: dict[str, dict[str, Any]],
|
||||
frontier_runs: dict[tuple[str, int], dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
expected = {
|
||||
(cell_id, probe_index)
|
||||
for cell_id, cell in real_cells.items()
|
||||
for probe_index in cell["probes"]
|
||||
}
|
||||
if set(frontier_runs) != expected:
|
||||
missing = sorted(expected - set(frontier_runs))
|
||||
extra = sorted(set(frontier_runs) - expected)
|
||||
raise ValueError(f"Frontier/real anchor mismatch; missing={missing}, extra={extra}")
|
||||
|
||||
anchors = []
|
||||
for cell_id, probe_index in sorted(expected):
|
||||
cell = real_cells[cell_id]
|
||||
probe = cell["probes"][probe_index]
|
||||
frontier = frontier_runs[(cell_id, probe_index)]
|
||||
run = frontier["manifest"]["run"]
|
||||
scorer = frontier["scorer"]
|
||||
if int(run["request_count"]) != int(probe["request_count"]):
|
||||
raise ValueError(f"paired request count mismatch: {(cell_id, probe_index)}")
|
||||
if not math.isclose(
|
||||
float(run["sampling_u"]), float(probe["sampling_u"]), rel_tol=0.0, abs_tol=1e-15
|
||||
):
|
||||
raise ValueError(f"paired sampling_u mismatch: {(cell_id, probe_index)}")
|
||||
rate = float(probe["request_rate_per_gpu_req_s_gpu"])
|
||||
anchors.append(
|
||||
{
|
||||
"cell_id": cell_id,
|
||||
"tp": int(cell["tensor_parallel_size"]),
|
||||
"mns": int(cell["max_num_seqs"]),
|
||||
"probe_index": probe_index,
|
||||
"sampling_u": float(probe["sampling_u"]),
|
||||
"request_count": int(probe["request_count"]),
|
||||
"offered_req_s_per_gpu": rate,
|
||||
"real_feasible": bool(probe["feasible"]),
|
||||
"real_pass_rate": float(probe["pass_rate"]),
|
||||
"frontier_feasible": bool(scorer["slo"]["feasible"]),
|
||||
"frontier_pass_rate": float(scorer["slo"]["pass_rate"]),
|
||||
"frontier_completed_req_s_per_gpu": float(
|
||||
scorer["throughput_requests_per_second_per_gpu"]
|
||||
),
|
||||
}
|
||||
)
|
||||
return anchors
|
||||
|
||||
|
||||
def monotonic_violations(rows: list[dict[str, Any]], field: str) -> list[dict[str, Any]]:
|
||||
ordered = sorted(rows, key=lambda row: (row["offered_req_s_per_gpu"], row["probe_index"]))
|
||||
violations = []
|
||||
for lower_index, lower in enumerate(ordered):
|
||||
for upper in ordered[lower_index + 1 :]:
|
||||
if (
|
||||
lower["offered_req_s_per_gpu"] < upper["offered_req_s_per_gpu"]
|
||||
and not lower[field]
|
||||
and upper[field]
|
||||
):
|
||||
violations.append(
|
||||
{
|
||||
"lower_probe": lower["probe_index"],
|
||||
"lower_rate": lower["offered_req_s_per_gpu"],
|
||||
"upper_probe": upper["probe_index"],
|
||||
"upper_rate": upper["offered_req_s_per_gpu"],
|
||||
}
|
||||
)
|
||||
return violations
|
||||
|
||||
|
||||
def selected_anchor(rows: list[dict[str, Any]], field: str) -> dict[str, Any] | None:
|
||||
feasible = [row for row in rows if row[field]]
|
||||
if not feasible:
|
||||
return None
|
||||
return max(feasible, key=lambda row: (row["offered_req_s_per_gpu"], -row["probe_index"]))
|
||||
|
||||
|
||||
def summarize_cells(anchors: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
grouped = {cell: [] for cell in sorted(EXPECTED_CELLS)}
|
||||
for row in anchors:
|
||||
grouped[row["cell_id"]].append(row)
|
||||
|
||||
summaries = {}
|
||||
for cell_id, rows in grouped.items():
|
||||
real_selected = selected_anchor(rows, "real_feasible")
|
||||
frontier_selected = selected_anchor(rows, "frontier_feasible")
|
||||
real_score = real_selected["offered_req_s_per_gpu"] if real_selected else 0.0
|
||||
frontier_score = frontier_selected["offered_req_s_per_gpu"] if frontier_selected else 0.0
|
||||
max_rate = max(row["offered_req_s_per_gpu"] for row in rows)
|
||||
real_violations = monotonic_violations(rows, "real_feasible")
|
||||
frontier_violations = monotonic_violations(rows, "frontier_feasible")
|
||||
summaries[cell_id] = {
|
||||
"tp": rows[0]["tp"],
|
||||
"mns": rows[0]["mns"],
|
||||
"anchor_count": len(rows),
|
||||
"max_common_anchor_req_s_per_gpu": max_rate,
|
||||
"real_feasible_anchor_count": sum(bool(row["real_feasible"]) for row in rows),
|
||||
"frontier_feasible_anchor_count": sum(
|
||||
bool(row["frontier_feasible"]) for row in rows
|
||||
),
|
||||
"real_score": real_score,
|
||||
"frontier_score": frontier_score,
|
||||
"absolute_error": frontier_score - real_score,
|
||||
"relative_error": (frontier_score - real_score) / real_score if real_score else None,
|
||||
"real_selected_probe": real_selected["probe_index"] if real_selected else None,
|
||||
"frontier_selected_probe": frontier_selected["probe_index"] if frontier_selected else None,
|
||||
"frontier_right_censored": bool(
|
||||
frontier_selected and math.isclose(frontier_score, max_rate, abs_tol=1e-15)
|
||||
),
|
||||
"frontier_boundary_bracketed": bool(
|
||||
frontier_selected
|
||||
and any(
|
||||
not row["frontier_feasible"]
|
||||
and row["offered_req_s_per_gpu"] > frontier_score
|
||||
for row in rows
|
||||
)
|
||||
),
|
||||
"real_monotonic_violation_count": len(real_violations),
|
||||
"frontier_monotonic_violation_count": len(frontier_violations),
|
||||
"real_monotonic_violations": real_violations,
|
||||
"frontier_monotonic_violations": frontier_violations,
|
||||
}
|
||||
return summaries
|
||||
|
||||
|
||||
def score_buckets(scores: dict[str, float]) -> tuple[float, dict[str, int]]:
|
||||
tolerance = max(1e-9, 1e-6 * max(abs(value) for value in scores.values()))
|
||||
return tolerance, {cell: math.floor(value / tolerance) for cell, value in scores.items()}
|
||||
|
||||
|
||||
def sign(value: int) -> int:
|
||||
return (value > 0) - (value < 0)
|
||||
|
||||
|
||||
def ranking_metrics(
|
||||
real_scores: dict[str, float], frontier_scores: dict[str, float]
|
||||
) -> dict[str, Any]:
|
||||
if set(real_scores) != set(frontier_scores):
|
||||
raise ValueError("ranking score cells differ")
|
||||
real_tolerance, real_buckets = score_buckets(real_scores)
|
||||
frontier_tolerance, frontier_buckets = score_buckets(frontier_scores)
|
||||
cells = sorted(real_scores)
|
||||
counts = Counter()
|
||||
exact = 0
|
||||
for left_index, left in enumerate(cells):
|
||||
for right in cells[left_index + 1 :]:
|
||||
real_sign = sign(real_buckets[left] - real_buckets[right])
|
||||
frontier_sign = sign(frontier_buckets[left] - frontier_buckets[right])
|
||||
exact += int(real_sign == frontier_sign)
|
||||
if real_sign == 0 and frontier_sign == 0:
|
||||
counts["both_tied"] += 1
|
||||
elif real_sign == 0:
|
||||
counts["real_only_tied"] += 1
|
||||
elif frontier_sign == 0:
|
||||
counts["frontier_only_tied"] += 1
|
||||
elif real_sign == frontier_sign:
|
||||
counts["concordant"] += 1
|
||||
else:
|
||||
counts["discordant"] += 1
|
||||
numerator = counts["concordant"] - counts["discordant"]
|
||||
denominator = math.sqrt(
|
||||
(counts["concordant"] + counts["discordant"] + counts["real_only_tied"])
|
||||
* (
|
||||
counts["concordant"]
|
||||
+ counts["discordant"]
|
||||
+ counts["frontier_only_tied"]
|
||||
)
|
||||
)
|
||||
tau = numerator / denominator if denominator else 0.0
|
||||
top_bucket = max(frontier_buckets.values())
|
||||
candidates = sorted(cell for cell, bucket in frontier_buckets.items() if bucket == top_bucket)
|
||||
real_best = max(real_scores.values())
|
||||
regrets = [(real_best - real_scores[cell]) / real_best for cell in candidates]
|
||||
return {
|
||||
"real_tolerance": real_tolerance,
|
||||
"frontier_tolerance": frontier_tolerance,
|
||||
"top1_candidate_cells": candidates,
|
||||
"top1_effective_k": len(candidates),
|
||||
"top1_optimistic_regret": min(regrets),
|
||||
"top1_worst_case_regret": max(regrets),
|
||||
"pair_count": len(cells) * (len(cells) - 1) // 2,
|
||||
"pairwise_exact_sign_accuracy": exact / (len(cells) * (len(cells) - 1) // 2),
|
||||
"kendall_tau_b": tau,
|
||||
"kendall_counts": dict(counts),
|
||||
}
|
||||
|
||||
|
||||
def confusion_metrics(anchors: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
counts = Counter()
|
||||
for row in anchors:
|
||||
real = bool(row["real_feasible"])
|
||||
frontier = bool(row["frontier_feasible"])
|
||||
label = (
|
||||
"true_feasible"
|
||||
if real and frontier
|
||||
else "true_infeasible"
|
||||
if not real and not frontier
|
||||
else "false_feasible"
|
||||
if not real and frontier
|
||||
else "false_infeasible"
|
||||
)
|
||||
counts[label] += 1
|
||||
total = len(anchors)
|
||||
positive = counts["true_feasible"] + counts["false_infeasible"]
|
||||
negative = counts["true_infeasible"] + counts["false_feasible"]
|
||||
return {
|
||||
**{name: counts[name] for name in (
|
||||
"true_feasible",
|
||||
"true_infeasible",
|
||||
"false_feasible",
|
||||
"false_infeasible",
|
||||
)},
|
||||
"total": total,
|
||||
"accuracy": (counts["true_feasible"] + counts["true_infeasible"]) / total,
|
||||
"feasible_recall": counts["true_feasible"] / positive if positive else None,
|
||||
"infeasible_recall": counts["true_infeasible"] / negative if negative else None,
|
||||
"false_feasible_rate": counts["false_feasible"] / negative if negative else None,
|
||||
"false_infeasible_rate": counts["false_infeasible"] / positive if positive else None,
|
||||
}
|
||||
|
||||
|
||||
def error_metrics(cell_summaries: dict[str, dict[str, Any]]) -> dict[str, float]:
|
||||
errors = [row["absolute_error"] for row in cell_summaries.values()]
|
||||
relative = [abs(row["relative_error"]) for row in cell_summaries.values()]
|
||||
return {
|
||||
"mean_absolute_error_req_s_per_gpu": sum(abs(value) for value in errors) / len(errors),
|
||||
"root_mean_square_error_req_s_per_gpu": math.sqrt(
|
||||
sum(value * value for value in errors) / len(errors)
|
||||
),
|
||||
"mean_absolute_percentage_error": sum(relative) / len(relative),
|
||||
"right_censored_cell_count": sum(
|
||||
bool(row["frontier_right_censored"]) for row in cell_summaries.values()
|
||||
),
|
||||
"bracketed_cell_count": sum(
|
||||
bool(row["frontier_boundary_bracketed"]) for row in cell_summaries.values()
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def grouped_error_metrics(
|
||||
cell_summaries: dict[str, dict[str, Any]], field: str
|
||||
) -> dict[str, dict[str, float]]:
|
||||
groups: dict[int, list[dict[str, Any]]] = {}
|
||||
for row in cell_summaries.values():
|
||||
groups.setdefault(int(row[field]), []).append(row)
|
||||
output = {}
|
||||
for key, rows in sorted(groups.items()):
|
||||
output[str(key)] = {
|
||||
"cell_count": len(rows),
|
||||
"mean_signed_error_req_s_per_gpu": sum(row["absolute_error"] for row in rows)
|
||||
/ len(rows),
|
||||
"mean_absolute_error_req_s_per_gpu": sum(
|
||||
abs(row["absolute_error"]) for row in rows
|
||||
)
|
||||
/ len(rows),
|
||||
"right_censored_cell_count": sum(bool(row["frontier_right_censored"]) for row in rows),
|
||||
}
|
||||
return output
|
||||
|
||||
|
||||
def old_proxy_summary(previous_metrics_path: Path) -> dict[str, Any]:
|
||||
previous = load_json(previous_metrics_path)
|
||||
analysis = previous["analyses"]["frozen-calibrated/throughput-proxy"]
|
||||
metrics = analysis["metrics"]
|
||||
return {
|
||||
"reading": "completed-throughput proxy (old, not SLO gated)",
|
||||
"top1_candidate_cells": metrics["top1"]["candidate_cells"],
|
||||
"top1_worst_case_regret": metrics["top1"]["worst_case_regret"],
|
||||
"kendall_tau_b": metrics["kendall_tau_b"]["tau_b"],
|
||||
"pairwise_exact_sign_accuracy": metrics["pairwise_direction"]["exact_sign_accuracy"],
|
||||
}
|
||||
|
||||
|
||||
def validate_previous_slo_reading(
|
||||
previous_metrics_path: Path,
|
||||
real_scores: dict[str, float],
|
||||
frontier_scores: dict[str, float],
|
||||
) -> dict[str, Any]:
|
||||
previous = load_json(previous_metrics_path)
|
||||
previous_real = {cell: float(value) for cell, value in previous["real_scores"].items()}
|
||||
previous_frontier = {
|
||||
cell: float(value)
|
||||
for cell, value in previous["analyses"]["frozen-calibrated/SLO-gated"][
|
||||
"simulated_scores"
|
||||
].items()
|
||||
}
|
||||
real_difference = max(abs(real_scores[cell] - previous_real[cell]) for cell in real_scores)
|
||||
frontier_difference = max(
|
||||
abs(frontier_scores[cell] - previous_frontier[cell]) for cell in frontier_scores
|
||||
)
|
||||
if real_difference > 1e-12 or frontier_difference > 1e-12:
|
||||
raise ValueError(
|
||||
"independent aligned reconstruction disagrees with previous secondary reading: "
|
||||
f"real={real_difference}, Frontier={frontier_difference}"
|
||||
)
|
||||
return {
|
||||
"status": "PASS",
|
||||
"previous_reading": "frozen-calibrated/SLO-gated",
|
||||
"maximum_real_score_absolute_difference": real_difference,
|
||||
"maximum_frontier_score_absolute_difference": frontier_difference,
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path: Path, fieldnames: list[str], rows: Iterable[dict[str, Any]]) -> None:
|
||||
with path.open("w", encoding="utf-8", newline="") as output:
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows({field: row.get(field) for field in fieldnames} for row in rows)
|
||||
|
||||
|
||||
def render_report(metrics: dict[str, Any]) -> str:
|
||||
rank = metrics["aligned_ranking"]
|
||||
confusion = metrics["anchor_feasibility"]
|
||||
error = metrics["score_error"]
|
||||
grouped = metrics["grouped_score_error"]
|
||||
old = metrics["objective_comparison"]["old_proxy"]
|
||||
cells = metrics["cells"]
|
||||
lines = [
|
||||
"# Frontier SLO-aligned retrospective experiment",
|
||||
"",
|
||||
"## Result",
|
||||
"",
|
||||
(
|
||||
"After aligning both systems to the same paired-grid objective—maximum offered "
|
||||
"req/s/GPU with request-level SLO pass rate >= 0.95—the old 30.46% top-1 "
|
||||
"regret does not reproduce. Frontier nominates `"
|
||||
+ "`, `".join(rank["top1_candidate_cells"])
|
||||
+ f"` (tie), with real-evaluated regret {100 * rank['top1_optimistic_regret']:.3f}%"
|
||||
+ f"--{100 * rank['top1_worst_case_regret']:.3f}%."
|
||||
),
|
||||
"",
|
||||
(
|
||||
f"Ranking agreement is high on this fixed candidate grid: Kendall tau-b "
|
||||
f"{rank['kendall_tau_b']:.4f}, pairwise exact-sign accuracy "
|
||||
f"{100 * rank['pairwise_exact_sign_accuracy']:.2f}%. This is materially different "
|
||||
f"from the old completed-throughput proxy (tau-b {old['kendall_tau_b']:.4f}, "
|
||||
f"top-1 regret {100 * old['top1_worst_case_regret']:.2f}%)."
|
||||
),
|
||||
"",
|
||||
"## Per-cell paired-grid capacity",
|
||||
"",
|
||||
"| Cell | Real | Frontier | Error | Frontier boundary | Real monotonic violations |",
|
||||
"|---|---:|---:|---:|---|---:|",
|
||||
]
|
||||
for cell_id in sorted(cells, key=lambda cell: (cells[cell]["tp"], cells[cell]["mns"])):
|
||||
row = cells[cell_id]
|
||||
boundary = "right-censored" if row["frontier_right_censored"] else (
|
||||
"bracketed" if row["frontier_boundary_bracketed"] else "unresolved"
|
||||
)
|
||||
lines.append(
|
||||
f"| `{cell_id}` | {row['real_score']:.6f} | {row['frontier_score']:.6f} | "
|
||||
f"{row['absolute_error']:+.6f} | {boundary} | "
|
||||
f"{row['real_monotonic_violation_count']} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"The paired-grid score MAE is "
|
||||
f"{error['mean_absolute_error_req_s_per_gpu']:.4f} req/s/GPU and MAPE is "
|
||||
f"{100 * error['mean_absolute_percentage_error']:.2f}%. These error aggregates "
|
||||
"must be read as grid-clipped because "
|
||||
f"{int(error['right_censored_cell_count'])}/12 Frontier cells remain feasible at "
|
||||
"their highest common anchor.",
|
||||
"",
|
||||
"The error is configuration-dependent rather than a uniform scale offset. Mean "
|
||||
f"signed error is {grouped['by_mns']['8']['mean_signed_error_req_s_per_gpu']:+.4f} "
|
||||
"req/s/GPU for MNS=8, but rises to "
|
||||
f"{grouped['by_mns']['32']['mean_signed_error_req_s_per_gpu']:+.4f} and "
|
||||
f"{grouped['by_mns']['64']['mean_signed_error_req_s_per_gpu']:+.4f} for MNS=32/64. "
|
||||
"The aligned score therefore still exposes a missing or distorted MNS response.",
|
||||
"",
|
||||
"## Anchor-level feasibility",
|
||||
"",
|
||||
f"Across {confusion['total']} paired anchors: true-feasible="
|
||||
f"{confusion['true_feasible']}, true-infeasible={confusion['true_infeasible']}, "
|
||||
f"false-feasible={confusion['false_feasible']}, false-infeasible="
|
||||
f"{confusion['false_infeasible']}. Accuracy is {100 * confusion['accuracy']:.2f}%, "
|
||||
f"but the false-feasible rate among real-infeasible anchors is "
|
||||
f"{100 * confusion['false_feasible_rate']:.2f}%.",
|
||||
"",
|
||||
"Therefore the aligned experiment supports a narrower conclusion: Frontier can "
|
||||
"recover the top configuration family on this frozen, ragged candidate grid, but "
|
||||
"it is not yet a reliable SLO feasibility oracle. Good top-1 ranking is compatible "
|
||||
"with many wrong boundary labels.",
|
||||
"",
|
||||
"## Validity limits",
|
||||
"",
|
||||
"- This is a retrospective single-run analysis; no confidence interval is valid.",
|
||||
"- Real probes within a cell reused one vLLM process and prefix-cache history; "
|
||||
"Frontier anchors were independent runs. History-dependent real pass-rate values "
|
||||
"are retained rather than monotonicized; binary feasibility happens to remain "
|
||||
"monotone on the observed per-cell grids.",
|
||||
"- Right-censored cells expose only a lower bound on Frontier's own capacity. "
|
||||
"They are valid for paired-grid selection, not for claiming an exact simulator knee.",
|
||||
"- The frozen per-TP calibration and evaluation surface are not fully independent "
|
||||
"across workload families, so this is not a generalization result.",
|
||||
"",
|
||||
"## Required prospective experiment",
|
||||
"",
|
||||
"Repeat a small set of boundary cells with one engine restart per anchor, explicit "
|
||||
"cold-cache state, identical offered-load grid, and repeated trials. Extend the grid "
|
||||
"until both real and Frontier have at least one feasible and one infeasible point. "
|
||||
"Only that experiment can estimate boundary displacement and its uncertainty.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def analyze(
|
||||
ground_truth_path: Path,
|
||||
results_dir: Path,
|
||||
previous_metrics_path: Path,
|
||||
repository_root: Path,
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
real_cells = load_real_cells(ground_truth_path)
|
||||
frontier_runs = load_frontier_runs(results_dir)
|
||||
anchors = build_anchor_records(real_cells, frontier_runs)
|
||||
cells = summarize_cells(anchors)
|
||||
real_scores = {cell: row["real_score"] for cell, row in cells.items()}
|
||||
frontier_scores = {cell: row["frontier_score"] for cell, row in cells.items()}
|
||||
frontier_revisions = {
|
||||
run["manifest"]["frontier"]["git_head"] for run in frontier_runs.values()
|
||||
}
|
||||
if len(frontier_revisions) != 1:
|
||||
raise ValueError(f"mixed Frontier revisions: {sorted(frontier_revisions)}")
|
||||
metrics = {
|
||||
"schema_version": SCHEMA,
|
||||
"generated_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"status": "VALID_RETROSPECTIVE_ALIGNED_ANALYSIS",
|
||||
"claim": (
|
||||
"On the frozen common anchor grid, rank configurations by maximum offered "
|
||||
"req/s/GPU whose request-level SLO pass rate is at least 0.95 in each system."
|
||||
),
|
||||
"estimand": READING,
|
||||
"inputs": {
|
||||
"ground_truth": {
|
||||
"path": str(ground_truth_path),
|
||||
"sha256": sha256_file(ground_truth_path),
|
||||
},
|
||||
"frontier_results": str(results_dir),
|
||||
"previous_metrics": {
|
||||
"path": str(previous_metrics_path),
|
||||
"sha256": sha256_file(previous_metrics_path),
|
||||
},
|
||||
},
|
||||
"provenance": {
|
||||
"analysis_repository": git_revision(repository_root),
|
||||
"frontier_git_head_recorded_by_runs": next(iter(frontier_revisions)),
|
||||
"frontier_mode": MODE,
|
||||
"real_engine_version": "vLLM 0.20.0",
|
||||
"anchor_count": len(anchors),
|
||||
"cell_count": len(cells),
|
||||
},
|
||||
"objective_comparison": {"old_proxy": old_proxy_summary(previous_metrics_path)},
|
||||
"aligned_ranking": ranking_metrics(real_scores, frontier_scores),
|
||||
"anchor_feasibility": confusion_metrics(anchors),
|
||||
"score_error": error_metrics(cells),
|
||||
"grouped_score_error": {
|
||||
"by_tp": grouped_error_metrics(cells, "tp"),
|
||||
"by_mns": grouped_error_metrics(cells, "mns"),
|
||||
},
|
||||
"independent_reconstruction_crosscheck": validate_previous_slo_reading(
|
||||
previous_metrics_path, real_scores, frontier_scores
|
||||
),
|
||||
"cells": cells,
|
||||
"statistics": {
|
||||
"real_trials_per_anchor": 1,
|
||||
"frontier_trials_per_anchor": 1,
|
||||
"confidence_intervals": None,
|
||||
"reason": "single retrospective observation per paired anchor",
|
||||
},
|
||||
"known_limits": [
|
||||
"real probes reused a process and prefix-cache history within each cell",
|
||||
"Frontier anchors were independent simulator runs",
|
||||
"right-censored cells do not identify Frontier's exact capacity knee",
|
||||
"the anchor grid is ragged across cells",
|
||||
],
|
||||
}
|
||||
return metrics, anchors
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--ground-truth", type=Path, required=True)
|
||||
parser.add_argument("--frontier-results", type=Path, required=True)
|
||||
parser.add_argument("--previous-metrics", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
repository_root = Path(__file__).resolve().parents[2]
|
||||
metrics, anchors = analyze(
|
||||
args.ground_truth.resolve(),
|
||||
args.frontier_results.resolve(),
|
||||
args.previous_metrics.resolve(),
|
||||
repository_root,
|
||||
)
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
metrics_path = args.output_dir / "metrics.json"
|
||||
metrics_path.write_text(json.dumps(metrics, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
write_csv(
|
||||
args.output_dir / "cell_scores.csv",
|
||||
[
|
||||
"cell_id",
|
||||
"tp",
|
||||
"mns",
|
||||
"anchor_count",
|
||||
"real_score",
|
||||
"frontier_score",
|
||||
"absolute_error",
|
||||
"relative_error",
|
||||
"frontier_right_censored",
|
||||
"frontier_boundary_bracketed",
|
||||
"real_monotonic_violation_count",
|
||||
"frontier_monotonic_violation_count",
|
||||
],
|
||||
({"cell_id": cell_id, **row} for cell_id, row in sorted(metrics["cells"].items())),
|
||||
)
|
||||
write_csv(
|
||||
args.output_dir / "anchor_labels.csv",
|
||||
[
|
||||
"cell_id",
|
||||
"tp",
|
||||
"mns",
|
||||
"probe_index",
|
||||
"sampling_u",
|
||||
"request_count",
|
||||
"offered_req_s_per_gpu",
|
||||
"real_feasible",
|
||||
"real_pass_rate",
|
||||
"frontier_feasible",
|
||||
"frontier_pass_rate",
|
||||
"frontier_completed_req_s_per_gpu",
|
||||
],
|
||||
anchors,
|
||||
)
|
||||
(args.output_dir / "report.md").write_text(render_report(metrics), encoding="utf-8")
|
||||
print(json.dumps({"status": metrics["status"], "metrics": str(metrics_path)}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
93
runs/frontier-slo-alignment-v0/results/anchor_labels.csv
Normal file
93
runs/frontier-slo-alignment-v0/results/anchor_labels.csv
Normal file
@@ -0,0 +1,93 @@
|
||||
cell_id,tp,mns,probe_index,sampling_u,request_count,offered_req_s_per_gpu,real_feasible,real_pass_rate,frontier_feasible,frontier_pass_rate,frontier_completed_req_s_per_gpu
|
||||
tp1_mns16,1,16,0,0.5,276,4.6,False,0.09057971014492754,False,0.14855072463768115,3.2422237998507493
|
||||
tp1_mns16,1,16,1,0.25,143,2.3833333333333333,False,0.8461538461538461,True,1.0,2.2847815885743166
|
||||
tp1_mns16,1,16,2,0.125,66,1.1,True,1.0,True,1.0,1.0641771799103616
|
||||
tp1_mns16,1,16,3,0.1875,103,1.7166666666666666,True,1.0,True,1.0,1.6601410596627497
|
||||
tp1_mns16,1,16,4,0.21875,121,2.0166666666666666,True,1.0,True,1.0,1.9360040008720856
|
||||
tp1_mns16,1,16,5,0.234375,132,2.2,True,0.9924242424242424,True,1.0,2.1118607283809685
|
||||
tp1_mns16,1,16,6,0.2421875,137,2.283333333333333,True,0.9927007299270073,True,1.0,2.1914280693644232
|
||||
tp1_mns16,1,16,7,0.24609375,141,2.35,True,0.9574468085106383,True,1.0,2.2549713872507646
|
||||
tp1_mns32,1,32,0,0.5,276,4.6,False,0.014492753623188406,False,0.6666666666666666,4.297017390853387
|
||||
tp1_mns32,1,32,1,0.25,143,2.3833333333333333,False,0.7132867132867133,True,1.0,2.284832740403262
|
||||
tp1_mns32,1,32,2,0.125,66,1.1,True,1.0,True,1.0,1.0641771799103616
|
||||
tp1_mns32,1,32,3,0.1875,103,1.7166666666666666,True,1.0,True,1.0,1.6601410596627497
|
||||
tp1_mns32,1,32,4,0.21875,121,2.0166666666666666,True,1.0,True,1.0,1.9360040008720856
|
||||
tp1_mns32,1,32,5,0.234375,132,2.2,True,0.9621212121212122,True,1.0,2.11158903487612
|
||||
tp1_mns32,1,32,6,0.2421875,137,2.283333333333333,True,0.9635036496350365,True,1.0,2.1911815099759533
|
||||
tp1_mns32,1,32,7,0.24609375,141,2.35,False,0.723404255319149,True,1.0,2.2550534049233777
|
||||
tp1_mns64,1,64,0,0.5,276,4.6,False,0.057971014492753624,False,0.6666666666666666,4.356763578770651
|
||||
tp1_mns64,1,64,1,0.25,143,2.3833333333333333,False,0.7132867132867133,True,1.0,2.284832740403262
|
||||
tp1_mns64,1,64,2,0.125,66,1.1,True,1.0,True,1.0,1.0641771799103616
|
||||
tp1_mns64,1,64,3,0.1875,103,1.7166666666666666,True,1.0,True,1.0,1.6601410596627497
|
||||
tp1_mns64,1,64,4,0.21875,121,2.0166666666666666,True,1.0,True,1.0,1.9360040008720856
|
||||
tp1_mns64,1,64,5,0.234375,132,2.2,True,0.9621212121212122,True,1.0,2.11158903487612
|
||||
tp1_mns64,1,64,6,0.2421875,137,2.283333333333333,True,0.9635036496350365,True,1.0,2.1911815099759533
|
||||
tp1_mns64,1,64,7,0.24609375,141,2.35,False,0.723404255319149,True,1.0,2.2550534049233777
|
||||
tp1_mns8,1,8,0,0.5,276,4.6,False,0.06521739130434782,False,0.06884057971014493,2.1725065543636846
|
||||
tp1_mns8,1,8,1,0.25,143,2.3833333333333333,False,0.6993006993006993,False,0.6153846153846154,2.0743292183634
|
||||
tp1_mns8,1,8,2,0.125,66,1.1,True,1.0,True,1.0,1.0641771799103616
|
||||
tp1_mns8,1,8,3,0.1875,103,1.7166666666666666,True,1.0,True,1.0,1.660069637502782
|
||||
tp1_mns8,1,8,4,0.21875,121,2.0166666666666666,True,1.0,False,0.8677685950413223,1.8977642969225545
|
||||
tp1_mns8,1,8,5,0.234375,132,2.2,False,0.7121212121212122,False,0.7575757575757576,2.0012415074420837
|
||||
tp1_mns8,1,8,6,0.2265625,126,2.1,True,1.0,False,0.7936507936507936,1.942525444038986
|
||||
tp1_mns8,1,8,7,0.23046875,130,2.1666666666666665,False,0.8846153846153846,False,0.7846153846153846,1.9778490149228842
|
||||
tp2_mns16,2,16,0,0.5,276,2.3,False,0.9492753623188406,True,1.0,2.2439374417654148
|
||||
tp2_mns16,2,16,1,0.25,143,1.1916666666666667,True,1.0,True,1.0,1.1652366234427673
|
||||
tp2_mns16,2,16,2,0.375,209,1.7416666666666667,True,1.0,True,1.0,1.7024011425654029
|
||||
tp2_mns16,2,16,3,0.4375,243,2.025,True,1.0,True,1.0,1.978602236135322
|
||||
tp2_mns16,2,16,4,0.46875,256,2.1333333333333333,True,1.0,True,1.0,2.083276142822733
|
||||
tp2_mns16,2,16,5,0.484375,265,2.2083333333333335,True,1.0,True,1.0,2.15467356806669
|
||||
tp2_mns16,2,16,6,0.4921875,269,2.2416666666666667,True,1.0,True,1.0,2.187119822377907
|
||||
tp2_mns16,2,16,7,0.49609375,273,2.275,True,1.0,True,1.0,2.219545738132131
|
||||
tp2_mns32,2,32,0,0.5,276,2.3,True,0.9855072463768116,True,1.0,2.2439374417654148
|
||||
tp2_mns32,2,32,1,0.75,391,3.2583333333333333,True,1.0,True,1.0,3.171147514698164
|
||||
tp2_mns32,2,32,2,0.875,450,3.75,False,0.42,True,1.0,3.6496440319651886
|
||||
tp2_mns32,2,32,3,0.8125,417,3.475,False,0.7745803357314148,True,1.0,3.381833218203857
|
||||
tp2_mns32,2,32,4,0.78125,407,3.3916666666666666,False,0.20147420147420148,True,1.0,3.3006981169615908
|
||||
tp2_mns32,2,32,5,0.765625,400,3.3333333333333335,False,0.695,True,1.0,3.243915834454301
|
||||
tp2_mns32,2,32,6,0.7578125,396,3.3,False,0.8535353535353535,True,1.0,3.2118868660091686
|
||||
tp2_mns32,2,32,7,0.75390625,394,3.283333333333333,True,1.0,True,1.0,3.195580748135602
|
||||
tp2_mns64,2,64,0,0.5,276,2.3,True,0.9855072463768116,True,1.0,2.2439374417654148
|
||||
tp2_mns64,2,64,1,0.75,391,3.2583333333333333,True,1.0,True,1.0,3.171147514698164
|
||||
tp2_mns64,2,64,2,0.875,450,3.75,False,0.34,True,1.0,3.6496440319651886
|
||||
tp2_mns64,2,64,3,0.8125,417,3.475,False,0.6282973621103117,True,1.0,3.381833218203857
|
||||
tp2_mns64,2,64,4,0.78125,407,3.3916666666666666,False,0.8427518427518428,True,1.0,3.3006981169615908
|
||||
tp2_mns64,2,64,5,0.765625,400,3.3333333333333335,False,0.0775,True,1.0,3.243915834454301
|
||||
tp2_mns64,2,64,6,0.7578125,396,3.3,False,0.29292929292929293,True,1.0,3.2118868660091686
|
||||
tp2_mns64,2,64,7,0.75390625,394,3.283333333333333,False,0.6040609137055838,True,1.0,3.195580748135602
|
||||
tp2_mns8,2,8,0,0.5,276,2.3,False,0.2028985507246377,False,0.39492753623188404,2.037309149499862
|
||||
tp2_mns8,2,8,1,0.25,143,1.1916666666666667,True,1.0,True,1.0,1.1652509326668725
|
||||
tp2_mns8,2,8,2,0.375,209,1.7416666666666667,True,1.0,True,1.0,1.6993884308462768
|
||||
tp2_mns8,2,8,3,0.4375,243,2.025,True,1.0,False,0.9465020576131687,1.9636944417416549
|
||||
tp2_mns8,2,8,4,0.46875,256,2.1333333333333333,True,1.0,False,0.890625,2.0353351328293683
|
||||
tp2_mns8,2,8,5,0.484375,265,2.2083333333333335,True,1.0,False,0.6943396226415094,2.0340077558764986
|
||||
tp2_mns8,2,8,6,0.4921875,269,2.2416666666666667,True,1.0,False,0.5204460966542751,2.0394007508896705
|
||||
tp2_mns8,2,8,7,0.49609375,273,2.275,True,1.0,False,0.4358974358974359,2.0358394808879736
|
||||
tp4_mns16,4,16,0,0.034252608017,600,2.5,False,0.16,True,1.0,2.4366476386646814
|
||||
tp4_mns16,4,16,1,0.017126304009,317,1.3208333333333333,True,1.0,True,1.0,1.3046624956163406
|
||||
tp4_mns16,4,16,2,0.025689456013,453,1.8875,True,1.0,True,1.0,1.8463926715458043
|
||||
tp4_mns16,4,16,3,0.029971032015,516,2.15,True,1.0,True,1.0,2.0988716639818685
|
||||
tp4_mns16,4,16,4,0.032111820016,552,2.3,True,1.0,True,1.0,2.2421056884527815
|
||||
tp4_mns16,4,16,5,0.033182214016,575,2.3958333333333335,True,1.0,True,1.0,2.335243044645089
|
||||
tp4_mns16,4,16,6,0.033717411016,586,2.441666666666667,True,1.0,True,1.0,2.3799430022241697
|
||||
tp4_mns32,4,32,0,0.034252608017,600,2.5,False,0.3466666666666667,True,1.0,2.462327006841985
|
||||
tp4_mns32,4,32,1,0.017126304009,317,1.3208333333333333,True,1.0,True,1.0,1.3046624956163406
|
||||
tp4_mns32,4,32,2,0.025689456013,453,1.8875,True,1.0,True,1.0,1.859491597548273
|
||||
tp4_mns32,4,32,3,0.029971032015,516,2.15,True,1.0,True,1.0,2.117703981656738
|
||||
tp4_mns32,4,32,4,0.032111820016,552,2.3,True,1.0,True,1.0,2.2653938418034003
|
||||
tp4_mns32,4,32,5,0.033182214016,575,2.3958333333333335,True,1.0,True,1.0,2.359733397875389
|
||||
tp4_mns32,4,32,6,0.033717411016,586,2.441666666666667,True,1.0,True,1.0,2.4049469442687075
|
||||
tp4_mns64,4,64,0,0.034252608017,600,2.5,False,0.3466666666666667,True,1.0,2.462327006841985
|
||||
tp4_mns64,4,64,1,0.017126304009,317,1.3208333333333333,True,1.0,True,1.0,1.3046624956163406
|
||||
tp4_mns64,4,64,2,0.025689456013,453,1.8875,True,1.0,True,1.0,1.859491597548273
|
||||
tp4_mns64,4,64,3,0.029971032015,516,2.15,True,1.0,True,1.0,2.117703981656738
|
||||
tp4_mns64,4,64,4,0.032111820016,552,2.3,True,1.0,True,1.0,2.2653938418034003
|
||||
tp4_mns64,4,64,5,0.033182214016,575,2.3958333333333335,True,1.0,True,1.0,2.359733397875389
|
||||
tp4_mns64,4,64,6,0.033717411016,586,2.441666666666667,True,1.0,True,1.0,2.4049469442687075
|
||||
tp4_mns8,4,8,0,0.034252608017,600,2.5,False,0.056666666666666664,False,0.095,1.5449814460277083
|
||||
tp4_mns8,4,8,1,0.017126304009,317,1.3208333333333333,False,0.8517350157728707,True,1.0,1.277194040226129
|
||||
tp4_mns8,4,8,2,0.008563152005,159,0.6625,True,1.0,True,1.0,0.6553587315711599
|
||||
tp4_mns8,4,8,3,0.012844728007,243,1.0125,True,1.0,True,1.0,0.9892987097876506
|
||||
tp4_mns8,4,8,4,0.014985516008,274,1.1416666666666666,True,1.0,True,1.0,1.1153521862441735
|
||||
tp4_mns8,4,8,5,0.016055910008,301,1.2541666666666667,True,1.0,True,1.0,1.2252281710185793
|
||||
tp4_mns8,4,8,6,0.016591107009,308,1.2833333333333334,True,1.0,True,1.0,1.2459143726426762
|
||||
|
13
runs/frontier-slo-alignment-v0/results/cell_scores.csv
Normal file
13
runs/frontier-slo-alignment-v0/results/cell_scores.csv
Normal file
@@ -0,0 +1,13 @@
|
||||
cell_id,tp,mns,anchor_count,real_score,frontier_score,absolute_error,relative_error,frontier_right_censored,frontier_boundary_bracketed,real_monotonic_violation_count,frontier_monotonic_violation_count
|
||||
tp1_mns16,1,16,8,2.35,2.3833333333333333,0.033333333333333215,0.014184397163120517,False,True,0,0
|
||||
tp1_mns32,1,32,8,2.283333333333333,2.3833333333333333,0.10000000000000009,0.04379562043795625,False,True,0,0
|
||||
tp1_mns64,1,64,8,2.283333333333333,2.3833333333333333,0.10000000000000009,0.04379562043795625,False,True,0,0
|
||||
tp1_mns8,1,8,8,2.1,1.7166666666666666,-0.3833333333333335,-0.1825396825396826,False,True,0,0
|
||||
tp2_mns16,2,16,8,2.275,2.3,0.02499999999999991,0.01098901098901095,True,False,0,0
|
||||
tp2_mns32,2,32,8,3.283333333333333,3.75,0.4666666666666668,0.1421319796954315,True,False,0,0
|
||||
tp2_mns64,2,64,8,3.2583333333333333,3.75,0.4916666666666667,0.15089514066496165,True,False,0,0
|
||||
tp2_mns8,2,8,8,2.275,1.7416666666666667,-0.5333333333333332,-0.23443223443223438,False,True,0,0
|
||||
tp4_mns16,4,16,7,2.441666666666667,2.5,0.058333333333333126,0.02389078498293507,True,False,0,0
|
||||
tp4_mns32,4,32,7,2.441666666666667,2.5,0.058333333333333126,0.02389078498293507,True,False,0,0
|
||||
tp4_mns64,4,64,7,2.441666666666667,2.5,0.058333333333333126,0.02389078498293507,True,False,0,0
|
||||
tp4_mns8,4,8,7,1.2833333333333334,1.3208333333333333,0.03749999999999987,0.029220779220779116,False,True,0,0
|
||||
|
387
runs/frontier-slo-alignment-v0/results/metrics.json
Normal file
387
runs/frontier-slo-alignment-v0/results/metrics.json
Normal file
@@ -0,0 +1,387 @@
|
||||
{
|
||||
"aligned_ranking": {
|
||||
"frontier_tolerance": 3.7499999999999997e-06,
|
||||
"kendall_counts": {
|
||||
"both_tied": 4,
|
||||
"concordant": 58,
|
||||
"frontier_only_tied": 3,
|
||||
"real_only_tied": 1
|
||||
},
|
||||
"kendall_tau_b": 0.9668009539030813,
|
||||
"pair_count": 66,
|
||||
"pairwise_exact_sign_accuracy": 0.9393939393939394,
|
||||
"real_tolerance": 3.283333333333333e-06,
|
||||
"top1_candidate_cells": [
|
||||
"tp2_mns32",
|
||||
"tp2_mns64"
|
||||
],
|
||||
"top1_effective_k": 2,
|
||||
"top1_optimistic_regret": 0.0,
|
||||
"top1_worst_case_regret": 0.0076142131979695165
|
||||
},
|
||||
"anchor_feasibility": {
|
||||
"accuracy": 0.6956521739130435,
|
||||
"false_feasible": 21,
|
||||
"false_feasible_rate": 0.7,
|
||||
"false_infeasible": 7,
|
||||
"false_infeasible_rate": 0.11290322580645161,
|
||||
"feasible_recall": 0.8870967741935484,
|
||||
"infeasible_recall": 0.3,
|
||||
"total": 92,
|
||||
"true_feasible": 55,
|
||||
"true_infeasible": 9
|
||||
},
|
||||
"cells": {
|
||||
"tp1_mns16": {
|
||||
"absolute_error": 0.033333333333333215,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": true,
|
||||
"frontier_feasible_anchor_count": 7,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": false,
|
||||
"frontier_score": 2.3833333333333333,
|
||||
"frontier_selected_probe": 1,
|
||||
"max_common_anchor_req_s_per_gpu": 4.6,
|
||||
"mns": 16,
|
||||
"real_feasible_anchor_count": 6,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.35,
|
||||
"real_selected_probe": 7,
|
||||
"relative_error": 0.014184397163120517,
|
||||
"tp": 1
|
||||
},
|
||||
"tp1_mns32": {
|
||||
"absolute_error": 0.10000000000000009,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": true,
|
||||
"frontier_feasible_anchor_count": 7,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": false,
|
||||
"frontier_score": 2.3833333333333333,
|
||||
"frontier_selected_probe": 1,
|
||||
"max_common_anchor_req_s_per_gpu": 4.6,
|
||||
"mns": 32,
|
||||
"real_feasible_anchor_count": 5,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.283333333333333,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": 0.04379562043795625,
|
||||
"tp": 1
|
||||
},
|
||||
"tp1_mns64": {
|
||||
"absolute_error": 0.10000000000000009,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": true,
|
||||
"frontier_feasible_anchor_count": 7,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": false,
|
||||
"frontier_score": 2.3833333333333333,
|
||||
"frontier_selected_probe": 1,
|
||||
"max_common_anchor_req_s_per_gpu": 4.6,
|
||||
"mns": 64,
|
||||
"real_feasible_anchor_count": 5,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.283333333333333,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": 0.04379562043795625,
|
||||
"tp": 1
|
||||
},
|
||||
"tp1_mns8": {
|
||||
"absolute_error": -0.3833333333333335,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": true,
|
||||
"frontier_feasible_anchor_count": 2,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": false,
|
||||
"frontier_score": 1.7166666666666666,
|
||||
"frontier_selected_probe": 3,
|
||||
"max_common_anchor_req_s_per_gpu": 4.6,
|
||||
"mns": 8,
|
||||
"real_feasible_anchor_count": 4,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.1,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": -0.1825396825396826,
|
||||
"tp": 1
|
||||
},
|
||||
"tp2_mns16": {
|
||||
"absolute_error": 0.02499999999999991,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": false,
|
||||
"frontier_feasible_anchor_count": 8,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": true,
|
||||
"frontier_score": 2.3,
|
||||
"frontier_selected_probe": 0,
|
||||
"max_common_anchor_req_s_per_gpu": 2.3,
|
||||
"mns": 16,
|
||||
"real_feasible_anchor_count": 7,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.275,
|
||||
"real_selected_probe": 7,
|
||||
"relative_error": 0.01098901098901095,
|
||||
"tp": 2
|
||||
},
|
||||
"tp2_mns32": {
|
||||
"absolute_error": 0.4666666666666668,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": false,
|
||||
"frontier_feasible_anchor_count": 8,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": true,
|
||||
"frontier_score": 3.75,
|
||||
"frontier_selected_probe": 2,
|
||||
"max_common_anchor_req_s_per_gpu": 3.75,
|
||||
"mns": 32,
|
||||
"real_feasible_anchor_count": 3,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 3.283333333333333,
|
||||
"real_selected_probe": 7,
|
||||
"relative_error": 0.1421319796954315,
|
||||
"tp": 2
|
||||
},
|
||||
"tp2_mns64": {
|
||||
"absolute_error": 0.4916666666666667,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": false,
|
||||
"frontier_feasible_anchor_count": 8,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": true,
|
||||
"frontier_score": 3.75,
|
||||
"frontier_selected_probe": 2,
|
||||
"max_common_anchor_req_s_per_gpu": 3.75,
|
||||
"mns": 64,
|
||||
"real_feasible_anchor_count": 2,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 3.2583333333333333,
|
||||
"real_selected_probe": 1,
|
||||
"relative_error": 0.15089514066496165,
|
||||
"tp": 2
|
||||
},
|
||||
"tp2_mns8": {
|
||||
"absolute_error": -0.5333333333333332,
|
||||
"anchor_count": 8,
|
||||
"frontier_boundary_bracketed": true,
|
||||
"frontier_feasible_anchor_count": 2,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": false,
|
||||
"frontier_score": 1.7416666666666667,
|
||||
"frontier_selected_probe": 2,
|
||||
"max_common_anchor_req_s_per_gpu": 2.3,
|
||||
"mns": 8,
|
||||
"real_feasible_anchor_count": 7,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.275,
|
||||
"real_selected_probe": 7,
|
||||
"relative_error": -0.23443223443223438,
|
||||
"tp": 2
|
||||
},
|
||||
"tp4_mns16": {
|
||||
"absolute_error": 0.058333333333333126,
|
||||
"anchor_count": 7,
|
||||
"frontier_boundary_bracketed": false,
|
||||
"frontier_feasible_anchor_count": 7,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": true,
|
||||
"frontier_score": 2.5,
|
||||
"frontier_selected_probe": 0,
|
||||
"max_common_anchor_req_s_per_gpu": 2.5,
|
||||
"mns": 16,
|
||||
"real_feasible_anchor_count": 6,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.441666666666667,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": 0.02389078498293507,
|
||||
"tp": 4
|
||||
},
|
||||
"tp4_mns32": {
|
||||
"absolute_error": 0.058333333333333126,
|
||||
"anchor_count": 7,
|
||||
"frontier_boundary_bracketed": false,
|
||||
"frontier_feasible_anchor_count": 7,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": true,
|
||||
"frontier_score": 2.5,
|
||||
"frontier_selected_probe": 0,
|
||||
"max_common_anchor_req_s_per_gpu": 2.5,
|
||||
"mns": 32,
|
||||
"real_feasible_anchor_count": 6,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.441666666666667,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": 0.02389078498293507,
|
||||
"tp": 4
|
||||
},
|
||||
"tp4_mns64": {
|
||||
"absolute_error": 0.058333333333333126,
|
||||
"anchor_count": 7,
|
||||
"frontier_boundary_bracketed": false,
|
||||
"frontier_feasible_anchor_count": 7,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": true,
|
||||
"frontier_score": 2.5,
|
||||
"frontier_selected_probe": 0,
|
||||
"max_common_anchor_req_s_per_gpu": 2.5,
|
||||
"mns": 64,
|
||||
"real_feasible_anchor_count": 6,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 2.441666666666667,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": 0.02389078498293507,
|
||||
"tp": 4
|
||||
},
|
||||
"tp4_mns8": {
|
||||
"absolute_error": 0.03749999999999987,
|
||||
"anchor_count": 7,
|
||||
"frontier_boundary_bracketed": true,
|
||||
"frontier_feasible_anchor_count": 6,
|
||||
"frontier_monotonic_violation_count": 0,
|
||||
"frontier_monotonic_violations": [],
|
||||
"frontier_right_censored": false,
|
||||
"frontier_score": 1.3208333333333333,
|
||||
"frontier_selected_probe": 1,
|
||||
"max_common_anchor_req_s_per_gpu": 2.5,
|
||||
"mns": 8,
|
||||
"real_feasible_anchor_count": 5,
|
||||
"real_monotonic_violation_count": 0,
|
||||
"real_monotonic_violations": [],
|
||||
"real_score": 1.2833333333333334,
|
||||
"real_selected_probe": 6,
|
||||
"relative_error": 0.029220779220779116,
|
||||
"tp": 4
|
||||
}
|
||||
},
|
||||
"claim": "On the frozen common anchor grid, rank configurations by maximum offered req/s/GPU whose request-level SLO pass rate is at least 0.95 in each system.",
|
||||
"estimand": "paired-grid-slo-feasible-max-offered-throughput",
|
||||
"generated_utc": "2026-07-15T07:27:18.082402+00:00",
|
||||
"grouped_score_error": {
|
||||
"by_mns": {
|
||||
"16": {
|
||||
"cell_count": 3,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.03888888888888875,
|
||||
"mean_signed_error_req_s_per_gpu": 0.03888888888888875,
|
||||
"right_censored_cell_count": 2
|
||||
},
|
||||
"32": {
|
||||
"cell_count": 3,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.20833333333333334,
|
||||
"mean_signed_error_req_s_per_gpu": 0.20833333333333334,
|
||||
"right_censored_cell_count": 2
|
||||
},
|
||||
"64": {
|
||||
"cell_count": 3,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.21666666666666665,
|
||||
"mean_signed_error_req_s_per_gpu": 0.21666666666666665,
|
||||
"right_censored_cell_count": 2
|
||||
},
|
||||
"8": {
|
||||
"cell_count": 3,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.31805555555555554,
|
||||
"mean_signed_error_req_s_per_gpu": -0.2930555555555556,
|
||||
"right_censored_cell_count": 0
|
||||
}
|
||||
},
|
||||
"by_tp": {
|
||||
"1": {
|
||||
"cell_count": 4,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.15416666666666673,
|
||||
"mean_signed_error_req_s_per_gpu": -0.03750000000000003,
|
||||
"right_censored_cell_count": 0
|
||||
},
|
||||
"2": {
|
||||
"cell_count": 4,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.37916666666666665,
|
||||
"mean_signed_error_req_s_per_gpu": 0.11250000000000004,
|
||||
"right_censored_cell_count": 3
|
||||
},
|
||||
"4": {
|
||||
"cell_count": 4,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.05312499999999981,
|
||||
"mean_signed_error_req_s_per_gpu": 0.05312499999999981,
|
||||
"right_censored_cell_count": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"independent_reconstruction_crosscheck": {
|
||||
"maximum_frontier_score_absolute_difference": 0.0,
|
||||
"maximum_real_score_absolute_difference": 0.0,
|
||||
"previous_reading": "frozen-calibrated/SLO-gated",
|
||||
"status": "PASS"
|
||||
},
|
||||
"inputs": {
|
||||
"frontier_results": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results",
|
||||
"ground_truth": {
|
||||
"path": "/home/gahow/phd/replayserve/docs/assets/simfid_s2r/ground_truth.json",
|
||||
"sha256": "23ff3e6f6b88df8632bf37d37c0ff87bfef9d1676db81592948c08e898f7a670"
|
||||
},
|
||||
"previous_metrics": {
|
||||
"path": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
|
||||
"sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206"
|
||||
}
|
||||
},
|
||||
"known_limits": [
|
||||
"real probes reused a process and prefix-cache history within each cell",
|
||||
"Frontier anchors were independent simulator runs",
|
||||
"right-censored cells do not identify Frontier's exact capacity knee",
|
||||
"the anchor grid is ragged across cells"
|
||||
],
|
||||
"objective_comparison": {
|
||||
"old_proxy": {
|
||||
"kendall_tau_b": 0.44812907976513594,
|
||||
"pairwise_exact_sign_accuracy": 0.6818181818181818,
|
||||
"reading": "completed-throughput proxy (old, not SLO gated)",
|
||||
"top1_candidate_cells": [
|
||||
"tp1_mns64"
|
||||
],
|
||||
"top1_worst_case_regret": 0.30456852791878175
|
||||
}
|
||||
},
|
||||
"provenance": {
|
||||
"analysis_repository": {
|
||||
"head": "9c8570f36b1753d1b77c7cf1b0d16ecc360cad33",
|
||||
"status_short": "?? \"AITuner\\347\\263\\273\\347\\273\\237\\344\\274\\230\\345\\214\\226\\344\\270\\216\\346\\214\\221\\346\\210\\230.pdf\"\n?? runs/frontier-slo-alignment-v0/"
|
||||
},
|
||||
"anchor_count": 92,
|
||||
"cell_count": 12,
|
||||
"frontier_git_head_recorded_by_runs": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"frontier_mode": "frozen-calibrated",
|
||||
"real_engine_version": "vLLM 0.20.0"
|
||||
},
|
||||
"schema_version": "frontier-slo-alignment-v0",
|
||||
"score_error": {
|
||||
"bracketed_cell_count": 6,
|
||||
"mean_absolute_error_req_s_per_gpu": 0.19548611111111105,
|
||||
"mean_absolute_percentage_error": 0.07697140171082821,
|
||||
"right_censored_cell_count": 6,
|
||||
"root_mean_square_error_req_s_per_gpu": 0.2775267963371919
|
||||
},
|
||||
"statistics": {
|
||||
"confidence_intervals": null,
|
||||
"frontier_trials_per_anchor": 1,
|
||||
"real_trials_per_anchor": 1,
|
||||
"reason": "single retrospective observation per paired anchor"
|
||||
},
|
||||
"status": "VALID_RETROSPECTIVE_ALIGNED_ANALYSIS"
|
||||
}
|
||||
45
runs/frontier-slo-alignment-v0/results/report.md
Normal file
45
runs/frontier-slo-alignment-v0/results/report.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# Frontier SLO-aligned retrospective experiment
|
||||
|
||||
## Result
|
||||
|
||||
After aligning both systems to the same paired-grid objective—maximum offered req/s/GPU with request-level SLO pass rate >= 0.95—the old 30.46% top-1 regret does not reproduce. Frontier nominates `tp2_mns32`, `tp2_mns64` (tie), with real-evaluated regret 0.000%--0.761%.
|
||||
|
||||
Ranking agreement is high on this fixed candidate grid: Kendall tau-b 0.9668, pairwise exact-sign accuracy 93.94%. This is materially different from the old completed-throughput proxy (tau-b 0.4481, top-1 regret 30.46%).
|
||||
|
||||
## Per-cell paired-grid capacity
|
||||
|
||||
| Cell | Real | Frontier | Error | Frontier boundary | Real monotonic violations |
|
||||
|---|---:|---:|---:|---|---:|
|
||||
| `tp1_mns8` | 2.100000 | 1.716667 | -0.383333 | bracketed | 0 |
|
||||
| `tp1_mns16` | 2.350000 | 2.383333 | +0.033333 | bracketed | 0 |
|
||||
| `tp1_mns32` | 2.283333 | 2.383333 | +0.100000 | bracketed | 0 |
|
||||
| `tp1_mns64` | 2.283333 | 2.383333 | +0.100000 | bracketed | 0 |
|
||||
| `tp2_mns8` | 2.275000 | 1.741667 | -0.533333 | bracketed | 0 |
|
||||
| `tp2_mns16` | 2.275000 | 2.300000 | +0.025000 | right-censored | 0 |
|
||||
| `tp2_mns32` | 3.283333 | 3.750000 | +0.466667 | right-censored | 0 |
|
||||
| `tp2_mns64` | 3.258333 | 3.750000 | +0.491667 | right-censored | 0 |
|
||||
| `tp4_mns8` | 1.283333 | 1.320833 | +0.037500 | bracketed | 0 |
|
||||
| `tp4_mns16` | 2.441667 | 2.500000 | +0.058333 | right-censored | 0 |
|
||||
| `tp4_mns32` | 2.441667 | 2.500000 | +0.058333 | right-censored | 0 |
|
||||
| `tp4_mns64` | 2.441667 | 2.500000 | +0.058333 | right-censored | 0 |
|
||||
|
||||
The paired-grid score MAE is 0.1955 req/s/GPU and MAPE is 7.70%. These error aggregates must be read as grid-clipped because 6/12 Frontier cells remain feasible at their highest common anchor.
|
||||
|
||||
The error is configuration-dependent rather than a uniform scale offset. Mean signed error is -0.2931 req/s/GPU for MNS=8, but rises to +0.2083 and +0.2167 for MNS=32/64. The aligned score therefore still exposes a missing or distorted MNS response.
|
||||
|
||||
## Anchor-level feasibility
|
||||
|
||||
Across 92 paired anchors: true-feasible=55, true-infeasible=9, false-feasible=21, false-infeasible=7. Accuracy is 69.57%, but the false-feasible rate among real-infeasible anchors is 70.00%.
|
||||
|
||||
Therefore the aligned experiment supports a narrower conclusion: Frontier can recover the top configuration family on this frozen, ragged candidate grid, but it is not yet a reliable SLO feasibility oracle. Good top-1 ranking is compatible with many wrong boundary labels.
|
||||
|
||||
## Validity limits
|
||||
|
||||
- This is a retrospective single-run analysis; no confidence interval is valid.
|
||||
- Real probes within a cell reused one vLLM process and prefix-cache history; Frontier anchors were independent runs. History-dependent real pass-rate values are retained rather than monotonicized; binary feasibility happens to remain monotone on the observed per-cell grids.
|
||||
- Right-censored cells expose only a lower bound on Frontier's own capacity. They are valid for paired-grid selection, not for claiming an exact simulator knee.
|
||||
- The frozen per-TP calibration and evaluation surface are not fully independent across workload families, so this is not a generalization result.
|
||||
|
||||
## Required prospective experiment
|
||||
|
||||
Repeat a small set of boundary cells with one engine restart per anchor, explicit cold-cache state, identical offered-load grid, and repeated trials. Extend the grid until both real and Frontier have at least one feasible and one infeasible point. Only that experiment can estimate boundary displacement and its uncertainty.
|
||||
64
runs/frontier-slo-alignment-v0/test_analysis.py
Normal file
64
runs/frontier-slo-alignment-v0/test_analysis.py
Normal file
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def load_analysis():
|
||||
spec = importlib.util.spec_from_file_location("frontier_slo_alignment", HERE / "analyze.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def anchor(rate: float, real: bool, frontier: bool, probe: int) -> dict[str, object]:
|
||||
return {
|
||||
"cell_id": "tp1_mns8",
|
||||
"tp": 1,
|
||||
"mns": 8,
|
||||
"probe_index": probe,
|
||||
"offered_req_s_per_gpu": rate,
|
||||
"real_feasible": real,
|
||||
"frontier_feasible": frontier,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
analysis = load_analysis()
|
||||
rows = [
|
||||
anchor(1.0, True, True, 0),
|
||||
anchor(2.0, False, True, 1),
|
||||
anchor(3.0, True, False, 2),
|
||||
]
|
||||
real_violations = analysis.monotonic_violations(rows, "real_feasible")
|
||||
assert real_violations == [
|
||||
{"lower_probe": 1, "lower_rate": 2.0, "upper_probe": 2, "upper_rate": 3.0}
|
||||
]
|
||||
assert analysis.selected_anchor(rows, "real_feasible")["probe_index"] == 2
|
||||
confusion = analysis.confusion_metrics(rows)
|
||||
assert confusion["true_feasible"] == 1
|
||||
assert confusion["false_feasible"] == 1
|
||||
assert confusion["false_infeasible"] == 1
|
||||
assert math.isclose(confusion["accuracy"], 1.0 / 3.0)
|
||||
|
||||
rank = analysis.ranking_metrics(
|
||||
{"a": 3.0, "b": 2.0, "c": 1.0},
|
||||
{"a": 2.0, "b": 3.0, "c": 3.0},
|
||||
)
|
||||
assert rank["top1_candidate_cells"] == ["b", "c"]
|
||||
assert math.isclose(rank["top1_optimistic_regret"], 1.0 / 3.0)
|
||||
assert math.isclose(rank["top1_worst_case_regret"], 2.0 / 3.0)
|
||||
assert rank["pair_count"] == 3
|
||||
print("frontier SLO alignment analysis: PASS")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user