1445 lines
57 KiB
Python
1445 lines
57 KiB
Python
#!/usr/bin/env python3
|
||
"""Analyze frozen good/bad Frontier selection cases by margin and residual."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import math
|
||
import os
|
||
import re
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from statistics import fmean
|
||
from typing import Any
|
||
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
DEFAULT_INPUT_ROOT = HERE / "frozen-inputs"
|
||
DEFAULT_RESULTS_ROOT = HERE / "results"
|
||
|
||
OBJECTIVE_ORDER = {
|
||
"ttft_mean_ms": 0,
|
||
"ttft_p90_ms": 1,
|
||
"tpot_mean_ms": 2,
|
||
"tpot_p90_ms": 3,
|
||
"e2e_mean_ms": 4,
|
||
"e2e_p90_ms": 5,
|
||
}
|
||
|
||
CONFIG_PATTERN = re.compile(
|
||
r"^tp(?P<tp>[0-9]+)(?:_ep(?P<ep>[0-9]+))?_mns(?P<mns>[0-9]+)$"
|
||
)
|
||
INVERSION_AXES = ("tp-axis", "mns-axis", "mixed")
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Surface:
|
||
case: str
|
||
objective: str
|
||
values: dict[str, tuple[float, float]]
|
||
expected_configs: tuple[str, ...]
|
||
expected_sim_winner: str
|
||
expected_real_winner: str
|
||
expected_regret: float | None
|
||
sources: tuple[str, ...]
|
||
|
||
|
||
def load_json(path: Path) -> dict[str, Any]:
|
||
with path.open() as handle:
|
||
value = json.load(handle)
|
||
if not isinstance(value, dict):
|
||
raise ValueError(f"expected JSON object: {path}")
|
||
return value
|
||
|
||
|
||
def atomic_write(path: Path, text: str) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||
temporary.write_text(text)
|
||
os.replace(temporary, path)
|
||
|
||
|
||
def positive_float(value: Any, context: str) -> float:
|
||
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
||
raise ValueError(f"non-numeric value for {context}: {value!r}")
|
||
result = float(value)
|
||
if not math.isfinite(result) or result <= 0:
|
||
raise ValueError(f"non-positive or non-finite value for {context}: {value!r}")
|
||
return result
|
||
|
||
|
||
def normalized_objective(metric: str, statistic: str) -> str:
|
||
return f"{metric.removesuffix('_ms')}_{statistic}_ms"
|
||
|
||
|
||
def objective_key(objective: str) -> tuple[int, str]:
|
||
return OBJECTIVE_ORDER.get(objective, len(OBJECTIVE_ORDER)), objective
|
||
|
||
|
||
def config_dimensions(config: str) -> tuple[tuple[int, int | None], int]:
|
||
match = CONFIG_PATTERN.fullmatch(config)
|
||
if match is None:
|
||
raise ValueError(f"unrecognized config name: {config}")
|
||
ep = int(match.group("ep")) if match.group("ep") is not None else None
|
||
return (int(match.group("tp")), ep), int(match.group("mns"))
|
||
|
||
|
||
def pair_axis(left: str, right: str) -> str:
|
||
if left == right:
|
||
return "none"
|
||
left_plan, left_mns = config_dimensions(left)
|
||
right_plan, right_mns = config_dimensions(right)
|
||
plan_differs = left_plan != right_plan
|
||
mns_differs = left_mns != right_mns
|
||
if plan_differs and not mns_differs:
|
||
return "tp-axis"
|
||
if mns_differs and not plan_differs:
|
||
return "mns-axis"
|
||
if plan_differs and mns_differs:
|
||
return "mixed"
|
||
raise ValueError(f"distinct configs have identical dimensions: {left}, {right}")
|
||
|
||
|
||
def load_q30_trace_pd(input_root: Path) -> list[Surface]:
|
||
relative = Path("q30-trace-pd/comparison.json")
|
||
data = load_json(input_root / relative)
|
||
if data.get("schema") != "frontier-qwen30-piecewise-graph-comparison-v1":
|
||
raise ValueError(f"unexpected schema: {relative}")
|
||
configs = tuple(sorted(data["cells"]))
|
||
surfaces = []
|
||
for source_objective, selection in sorted(
|
||
data["selection"].items(),
|
||
key=lambda item: objective_key(
|
||
normalized_objective(*item[0].split(":"))
|
||
),
|
||
):
|
||
metric, statistic = source_objective.split(":")
|
||
objective = normalized_objective(metric, statistic)
|
||
values = {}
|
||
for config in configs:
|
||
cell = data["cells"][config]["metrics"][metric]
|
||
values[config] = (
|
||
positive_float(cell[f"sim_{statistic}_ms"], f"{relative}:{config}:sim"),
|
||
positive_float(cell[f"real_{statistic}_ms"], f"{relative}:{config}:real"),
|
||
)
|
||
surfaces.append(
|
||
Surface(
|
||
case="q30 trace-pd",
|
||
objective=objective,
|
||
values=values,
|
||
expected_configs=configs,
|
||
expected_sim_winner=selection["sim_winner"],
|
||
expected_real_winner=selection["real_winner"],
|
||
expected_regret=None,
|
||
sources=(str(relative),),
|
||
)
|
||
)
|
||
return surfaces
|
||
|
||
|
||
def load_q30_case(
|
||
input_root: Path,
|
||
case: str,
|
||
comparison_relative: Path,
|
||
audit_relative: Path,
|
||
) -> list[Surface]:
|
||
comparison = load_json(input_root / comparison_relative)
|
||
audit = load_json(input_root / audit_relative)
|
||
if comparison.get("schema") != "qwen30-latency-case-frontier-real-comparison-v1":
|
||
raise ValueError(f"unexpected schema: {comparison_relative}")
|
||
if audit.get("schema") != "qwen30-latency-case-real-audit-v1":
|
||
raise ValueError(f"unexpected schema: {audit_relative}")
|
||
configs = tuple(sorted(set(comparison["cells"]) | set(audit["configs"])))
|
||
surfaces = []
|
||
for source_objective, selection in sorted(
|
||
comparison["selection"].items(),
|
||
key=lambda item: objective_key(
|
||
normalized_objective(*item[0].split(":"))
|
||
),
|
||
):
|
||
metric, statistic = source_objective.split(":")
|
||
objective = normalized_objective(metric, statistic)
|
||
values = {}
|
||
for config in configs:
|
||
if config not in comparison["cells"] or config not in audit["configs"]:
|
||
continue
|
||
sim = comparison["cells"][config]["metrics"].get(metric, {}).get(statistic)
|
||
real = (
|
||
audit["configs"][config]["metrics"]
|
||
.get(metric, {})
|
||
.get(f"pooled_{statistic}_ms")
|
||
)
|
||
if sim is None or real is None:
|
||
continue
|
||
values[config] = (
|
||
positive_float(sim, f"{comparison_relative}:{config}:sim"),
|
||
positive_float(real, f"{audit_relative}:{config}:real"),
|
||
)
|
||
surfaces.append(
|
||
Surface(
|
||
case=case,
|
||
objective=objective,
|
||
values=values,
|
||
expected_configs=configs,
|
||
expected_sim_winner=selection["sim_winner"],
|
||
expected_real_winner=selection["real_winner"],
|
||
expected_regret=selection.get("selected_config_real_regret"),
|
||
sources=(str(comparison_relative), str(audit_relative)),
|
||
)
|
||
)
|
||
return surfaces
|
||
|
||
|
||
def load_q235_campaign(
|
||
input_root: Path, campaign: str, relative: Path
|
||
) -> list[Surface]:
|
||
data = load_json(input_root / relative)
|
||
if data.get("schema") != "qwen235-v020-simulator-real-comparison-v1":
|
||
raise ValueError(f"unexpected schema: {relative}")
|
||
surfaces = []
|
||
for case_name in ("fixed-pd", "fixed-po", "trace-pd", "trace-po"):
|
||
case = data["cases"][case_name]
|
||
configs = tuple(sorted(set(case["sim"]) | set(case["real"])))
|
||
for objective, selection in sorted(
|
||
case["comparison"].items(), key=lambda item: objective_key(item[0])
|
||
):
|
||
values = {}
|
||
for config in configs:
|
||
if config not in case["sim"] or config not in case["real"]:
|
||
continue
|
||
sim = case["sim"][config].get(objective)
|
||
real = case["real"][config].get(objective)
|
||
if sim is None or real is None:
|
||
continue
|
||
values[config] = (
|
||
positive_float(sim, f"{relative}:{case_name}:{config}:sim"),
|
||
positive_float(real, f"{relative}:{case_name}:{config}:real"),
|
||
)
|
||
surfaces.append(
|
||
Surface(
|
||
case=f"q235 {campaign} {case_name}",
|
||
objective=objective,
|
||
values=values,
|
||
expected_configs=configs,
|
||
expected_sim_winner=selection["sim_winner"],
|
||
expected_real_winner=selection["real_winner"],
|
||
expected_regret=selection.get("selected_real_regret"),
|
||
sources=(str(relative),),
|
||
)
|
||
)
|
||
return surfaces
|
||
|
||
|
||
def load_surfaces(input_root: Path) -> list[Surface]:
|
||
surfaces = load_q30_trace_pd(input_root)
|
||
q30_specs = (
|
||
(
|
||
"q30 fixed-pd high",
|
||
Path("q30-fixed-hi/fixed-pd/comparison.json"),
|
||
Path("q30-fixed-hi/fixed-pd/real-audit.json"),
|
||
),
|
||
(
|
||
"q30 fixed-po high",
|
||
Path("q30-fixed-hi/fixed-po/comparison.json"),
|
||
Path("q30-fixed-hi/fixed-po/real-audit.json"),
|
||
),
|
||
(
|
||
"q30 fixed-pd low",
|
||
Path("q30-expansion-lo/fixed-pd-comparison.json"),
|
||
Path("q30-expansion-lo/fixed-pd-real-audit.json"),
|
||
),
|
||
(
|
||
"q30 fixed-po low",
|
||
Path("q30-expansion-lo/fixed-po-comparison.json"),
|
||
Path("q30-expansion-lo/fixed-po-real-audit.json"),
|
||
),
|
||
(
|
||
"q30 trace-po low",
|
||
Path("q30-expansion-lo/trace-po-comparison.json"),
|
||
Path("q30-expansion-lo/trace-po-real-audit.json"),
|
||
),
|
||
)
|
||
for case, comparison, audit in q30_specs:
|
||
surfaces.extend(load_q30_case(input_root, case, comparison, audit))
|
||
surfaces.extend(
|
||
load_q235_campaign(
|
||
input_root, "A0", Path("q235-fourcase-a0/comparison.json")
|
||
)
|
||
)
|
||
surfaces.extend(
|
||
load_q235_campaign(
|
||
input_root, "A1", Path("q235-ablation-a1/comparison.json")
|
||
)
|
||
)
|
||
return surfaces
|
||
|
||
|
||
def analyze_surface(surface: Surface) -> tuple[dict[str, Any], list[str]]:
|
||
anomalies = []
|
||
paired_configs = tuple(sorted(surface.values))
|
||
missing_configs = sorted(set(surface.expected_configs) - set(paired_configs))
|
||
if missing_configs:
|
||
anomalies.append(
|
||
f"{surface.case} / {surface.objective}: missing paired sim+real data for "
|
||
+ ", ".join(missing_configs)
|
||
)
|
||
if len(paired_configs) < 2:
|
||
return (
|
||
{
|
||
"case": surface.case,
|
||
"objective": surface.objective,
|
||
"sim_winner": None,
|
||
"real_winner": None,
|
||
"regret": None,
|
||
"config_uniform_scale": None,
|
||
"log_ratio_spread_all": None,
|
||
"log_ratio_spread_real_top3": None,
|
||
"real_relative_margin_best_vs_second": None,
|
||
"sim_winner_real_relative_loss": None,
|
||
"failure": None,
|
||
"h_scale_condition_spread_gt_log_margin": None,
|
||
"h_scale_verdict": "N/A",
|
||
"winner_match": None,
|
||
"winner_deciding_axis": None,
|
||
"winner_pair_m_s_percent": None,
|
||
"winner_pair_delta_s_percent": None,
|
||
"winner_pair_strict_reversal": None,
|
||
"winner_pair_relation": "N/A",
|
||
"n_inv_tp": None,
|
||
"n_inv_mns": None,
|
||
"n_inv_mixed": None,
|
||
"n_tie_tp": None,
|
||
"n_tie_mns": None,
|
||
"n_tie_mixed": None,
|
||
"fragile_success": None,
|
||
"pairwise_vs_real_winner": [],
|
||
"paired_config_count": len(paired_configs),
|
||
"expected_config_count": len(surface.expected_configs),
|
||
"real_top3_configs": [],
|
||
"missing_configs": missing_configs,
|
||
"sources": list(surface.sources),
|
||
},
|
||
anomalies,
|
||
)
|
||
|
||
sim_ranking = sorted(paired_configs, key=lambda config: (surface.values[config][0], config))
|
||
real_ranking = sorted(paired_configs, key=lambda config: (surface.values[config][1], config))
|
||
sim_winner = sim_ranking[0]
|
||
real_winner = real_ranking[0]
|
||
real_best = surface.values[real_winner][1]
|
||
regret = surface.values[sim_winner][1] / real_best - 1.0
|
||
margin = surface.values[real_ranking[1]][1] / real_best - 1.0
|
||
log_ratios = {
|
||
config: math.log(surface.values[config][0] / surface.values[config][1])
|
||
for config in paired_configs
|
||
}
|
||
spread_all = max(log_ratios.values()) - min(log_ratios.values())
|
||
top3 = tuple(real_ranking[:3])
|
||
top3_ratios = [log_ratios[config] for config in top3]
|
||
spread_top3 = max(top3_ratios) - min(top3_ratios)
|
||
scale = math.exp(fmean(log_ratios.values()))
|
||
failure = regret > 0.05
|
||
residual_exceeds_log_margin = spread_top3 > math.log1p(margin)
|
||
h_scale_holds = failure == residual_exceeds_log_margin
|
||
|
||
pairwise = []
|
||
inversion_counts = {axis: 0 for axis in INVERSION_AXES}
|
||
sim_at_real_winner = surface.values[real_winner][0]
|
||
for config in paired_configs:
|
||
sim_value, real_value = surface.values[config]
|
||
real_pair_log = math.log(real_value / real_best)
|
||
sim_pair_log = math.log(sim_value / sim_at_real_winner)
|
||
pair_margin_percent = 100.0 * real_pair_log
|
||
delta_percent = 100.0 * (sim_pair_log - real_pair_log)
|
||
axis = pair_axis(real_winner, config)
|
||
inversion = real_value > real_best and sim_value < sim_at_real_winner
|
||
delta_inversion = (
|
||
pair_margin_percent > 0
|
||
and delta_percent < -pair_margin_percent
|
||
)
|
||
if inversion != delta_inversion:
|
||
anomalies.append(
|
||
f"{surface.case} / {surface.objective} / {config}: direct inversion "
|
||
f"test disagrees with delta < -margin"
|
||
)
|
||
if inversion:
|
||
inversion_counts[axis] += 1
|
||
pairwise.append(
|
||
{
|
||
"config": config,
|
||
"axis_vs_real_winner": axis,
|
||
"real_pair_margin_percent": pair_margin_percent,
|
||
"signed_pairwise_delta_percent": delta_percent,
|
||
"inversion": inversion,
|
||
}
|
||
)
|
||
|
||
winner_pair = next(item for item in pairwise if item["config"] == sim_winner)
|
||
winner_match = sim_winner == real_winner
|
||
if winner_match:
|
||
winner_pair_relation = "match"
|
||
elif winner_pair["inversion"]:
|
||
winner_pair_relation = "reversal"
|
||
elif surface.values[sim_winner][0] == sim_at_real_winner:
|
||
winner_pair_relation = "sim-tie"
|
||
else:
|
||
winner_pair_relation = "non-reversal"
|
||
tie_counts = {axis: 0 for axis in INVERSION_AXES}
|
||
if winner_pair_relation == "sim-tie":
|
||
tie_counts[winner_pair["axis_vs_real_winner"]] += 1
|
||
|
||
if sim_winner != surface.expected_sim_winner:
|
||
anomalies.append(
|
||
f"{surface.case} / {surface.objective}: recomputed sim winner "
|
||
f"{sim_winner} != frozen JSON {surface.expected_sim_winner}"
|
||
)
|
||
if real_winner != surface.expected_real_winner:
|
||
anomalies.append(
|
||
f"{surface.case} / {surface.objective}: recomputed real winner "
|
||
f"{real_winner} != frozen JSON {surface.expected_real_winner}"
|
||
)
|
||
if surface.expected_regret is not None and not math.isclose(
|
||
regret, float(surface.expected_regret), rel_tol=1e-12, abs_tol=1e-12
|
||
):
|
||
anomalies.append(
|
||
f"{surface.case} / {surface.objective}: recomputed regret "
|
||
f"{regret:.12g} != frozen JSON {float(surface.expected_regret):.12g}"
|
||
)
|
||
|
||
return (
|
||
{
|
||
"case": surface.case,
|
||
"objective": surface.objective,
|
||
"sim_winner": sim_winner,
|
||
"real_winner": real_winner,
|
||
"regret": regret,
|
||
"config_uniform_scale": scale,
|
||
"log_ratio_spread_all": spread_all,
|
||
"log_ratio_spread_real_top3": spread_top3,
|
||
"real_relative_margin_best_vs_second": margin,
|
||
"sim_winner_real_relative_loss": regret,
|
||
"failure": failure,
|
||
"h_scale_condition_spread_gt_log_margin": residual_exceeds_log_margin,
|
||
"h_scale_verdict": "HOLDS" if h_scale_holds else "COUNTEREXAMPLE",
|
||
"winner_match": winner_match,
|
||
"winner_deciding_axis": pair_axis(real_winner, sim_winner),
|
||
"winner_pair_m_s_percent": winner_pair["real_pair_margin_percent"],
|
||
"winner_pair_delta_s_percent": winner_pair["signed_pairwise_delta_percent"],
|
||
"winner_pair_strict_reversal": winner_pair["inversion"],
|
||
"winner_pair_relation": winner_pair_relation,
|
||
"n_inv_tp": inversion_counts["tp-axis"],
|
||
"n_inv_mns": inversion_counts["mns-axis"],
|
||
"n_inv_mixed": inversion_counts["mixed"],
|
||
"n_tie_tp": tie_counts["tp-axis"],
|
||
"n_tie_mns": tie_counts["mns-axis"],
|
||
"n_tie_mixed": tie_counts["mixed"],
|
||
"fragile_success": winner_match and margin < 0.01,
|
||
"pairwise_vs_real_winner": pairwise,
|
||
"paired_config_count": len(paired_configs),
|
||
"expected_config_count": len(surface.expected_configs),
|
||
"real_top3_configs": list(top3),
|
||
"missing_configs": missing_configs,
|
||
"sources": list(surface.sources),
|
||
},
|
||
anomalies,
|
||
)
|
||
|
||
|
||
def row_index(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, Any]]:
|
||
return {(row["case"], row["objective"]): row for row in rows}
|
||
|
||
|
||
def cross_checks(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[str]]:
|
||
by_key = row_index(rows)
|
||
checks = []
|
||
anomalies = []
|
||
|
||
def check(name: str, observed: float, expected_percent: float) -> None:
|
||
observed_percent = observed * 100.0
|
||
passed = round(observed_percent, 1) == expected_percent
|
||
checks.append(
|
||
{
|
||
"name": name,
|
||
"observed_percent": observed_percent,
|
||
"expected_percent": expected_percent,
|
||
"passed": passed,
|
||
}
|
||
)
|
||
if not passed:
|
||
anomalies.append(
|
||
f"hard cross-check failed: {name}: observed {observed_percent:.6f}% "
|
||
f"vs expected {expected_percent:.1f}%"
|
||
)
|
||
|
||
check(
|
||
"q30 high-pressure fixed-pd TTFT mean regret",
|
||
by_key[("q30 fixed-pd high", "ttft_mean_ms")]["regret"],
|
||
58.0,
|
||
)
|
||
check(
|
||
"q235 A0 fixed-pd TPOT mean regret",
|
||
by_key[("q235 A0 fixed-pd", "tpot_mean_ms")]["regret"],
|
||
33.0,
|
||
)
|
||
check(
|
||
"q235 A1 fixed-pd TPOT mean regret",
|
||
by_key[("q235 A1 fixed-pd", "tpot_mean_ms")]["regret"],
|
||
33.0,
|
||
)
|
||
trace_pd_rows = [row for row in rows if row["case"] == "q30 trace-pd"]
|
||
trace_passed = len(trace_pd_rows) == 6 and all(
|
||
round(row["regret"] * 100.0, 1) == 0.0 for row in trace_pd_rows
|
||
)
|
||
checks.append(
|
||
{
|
||
"name": "q30 trace-pd all-objective regret",
|
||
"observed_percent": [row["regret"] * 100.0 for row in trace_pd_rows],
|
||
"expected_percent": [0.0] * 6,
|
||
"passed": trace_passed,
|
||
}
|
||
)
|
||
if not trace_passed:
|
||
anomalies.append(
|
||
"hard cross-check failed: q30 trace-pd does not have six objectives all at 0.0% regret"
|
||
)
|
||
|
||
def check_sequence(
|
||
name: str, observed: list[float], expected_percent: list[float]
|
||
) -> None:
|
||
observed_percent = [value * 100.0 for value in observed]
|
||
passed = [round(value, 1) for value in observed_percent] == expected_percent
|
||
checks.append(
|
||
{
|
||
"name": name,
|
||
"observed_percent": observed_percent,
|
||
"expected_percent": expected_percent,
|
||
"passed": passed,
|
||
}
|
||
)
|
||
if not passed:
|
||
anomalies.append(
|
||
f"hard cross-check failed: {name}: observed "
|
||
f"{[round(value, 6) for value in observed_percent]} vs expected "
|
||
f"{expected_percent}"
|
||
)
|
||
|
||
check_sequence(
|
||
"q235 Trace-PO TTFT p90 A0 to A1 regret",
|
||
[
|
||
by_key[("q235 A0 trace-po", "ttft_p90_ms")]["regret"],
|
||
by_key[("q235 A1 trace-po", "ttft_p90_ms")]["regret"],
|
||
],
|
||
[21.2, 0.3],
|
||
)
|
||
fixed_pd_objectives = (
|
||
"tpot_mean_ms",
|
||
"tpot_p90_ms",
|
||
"e2e_mean_ms",
|
||
"e2e_p90_ms",
|
||
)
|
||
fixed_pd_expected = [33.0, 37.2, 30.7, 34.6]
|
||
check_sequence(
|
||
"q235 Fixed-PD TPOT/E2E regrets unchanged from A0 to A1",
|
||
[
|
||
by_key[(f"q235 {campaign} fixed-pd", objective)]["regret"]
|
||
for campaign in ("A0", "A1")
|
||
for objective in fixed_pd_objectives
|
||
],
|
||
fixed_pd_expected + fixed_pd_expected,
|
||
)
|
||
return checks, anomalies
|
||
|
||
|
||
def known_attribution(input_root: Path) -> dict[str, Any]:
|
||
q30_relative = Path("q30-admission-diag/result.json")
|
||
q235_relative = Path("q235-state-diag/state-diagnosis-exact.json")
|
||
q30 = load_json(input_root / q30_relative)
|
||
q235 = load_json(input_root / q235_relative)
|
||
if q30.get("schema") != "qwen30-fixed-pd-ttft-admission-diagnosis-v1":
|
||
raise ValueError(f"unexpected schema: {q30_relative}")
|
||
if q235.get("schema") != "qwen235-fixed-pd-state-diagnosis-v1":
|
||
raise ValueError(f"unexpected schema: {q235_relative}")
|
||
|
||
q30_cells = []
|
||
for config, cell in sorted(q30["cells"].items()):
|
||
q30_cells.append(
|
||
{
|
||
"config": config,
|
||
"sim_tpot_over_real": cell["tpot_overprediction_ratio"],
|
||
"sim_required_slots": cell["simulator"]["required_slots"],
|
||
"mns": cell["mns"],
|
||
"real_required_slots_upper_bound": cell["real"]["required_slots_upper_bound"],
|
||
"sim_first_scheduling_wait_ms": cell["simulator"]["first_scheduling_delay_ms"],
|
||
"sim_queue_free_ttft_ms": cell["simulator"]["queue_free_ttft_ms"],
|
||
"sim_ttft_ms": cell["simulator"]["ttft_ms"],
|
||
"real_ttft_ms": cell["real"]["ttft_ms"],
|
||
"real_waiting_max": cell["real"]["periodic_queue"]["waiting_max"],
|
||
}
|
||
)
|
||
|
||
q235_configs = []
|
||
for config in sorted(q235["simulator"]):
|
||
q235_configs.append(
|
||
{
|
||
"config": config,
|
||
"sim_own_decode_batch_mean": q235["simulator"][config]["decode_batch_size"]["mean"],
|
||
"real_exact_decode_batch_mean": q235["real_iteration_state"][config]["decode_batch_size"]["mean"],
|
||
"real_token_weighted_iteration_ms": q235["real_iteration_state"][config]["decode_token_weighted_iteration_elapsed_ms"],
|
||
"exact_state_matched_sim_ms": q235["exact_state_matched"]["configs"][config]["components_ms"]["total"],
|
||
"exact_state_coverage": q235["exact_state_matched"]["configs"][config]["coverage"],
|
||
}
|
||
)
|
||
|
||
component_order = (
|
||
"total",
|
||
"moe_compute",
|
||
"attention",
|
||
"tp_dp_communication",
|
||
"ep_communication",
|
||
"moe_routing",
|
||
"dense_mlp_compute",
|
||
"pipeline_communication",
|
||
"runtime_overhead",
|
||
)
|
||
q235_components = []
|
||
own = q235["simulator_internal_all_step_tp8_minus_tp4_ms"]
|
||
exact = q235["exact_state_matched"]["tp8_minus_tp4_ms"]
|
||
for component in component_order:
|
||
q235_components.append(
|
||
{
|
||
"component": component,
|
||
"own_composition_tp8_minus_tp4_ms": own[component],
|
||
"exact_state_tp8_minus_tp4_ms": exact[component],
|
||
}
|
||
)
|
||
return {
|
||
"q30_admission": {
|
||
"source": str(q30_relative),
|
||
"verdict": q30["verdict"],
|
||
"cells": q30_cells,
|
||
"contrasts": q30["contrasts"],
|
||
},
|
||
"q235_state": {
|
||
"source": str(q235_relative),
|
||
"verdict": q235["verdict"],
|
||
"configs": q235_configs,
|
||
"components": q235_components,
|
||
"contrast_decomposition": q235["contrast_decomposition"],
|
||
"reference": q235["reference"],
|
||
},
|
||
}
|
||
|
||
|
||
def directional_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||
case_order = list(dict.fromkeys(row["case"] for row in rows))
|
||
by_case = []
|
||
for case in case_order:
|
||
case_rows = [row for row in rows if row["case"] == case]
|
||
by_case.append(
|
||
{
|
||
"case": case,
|
||
"n_inv_tp": sum(row["n_inv_tp"] for row in case_rows),
|
||
"n_inv_mns": sum(row["n_inv_mns"] for row in case_rows),
|
||
"n_inv_mixed": sum(row["n_inv_mixed"] for row in case_rows),
|
||
"n_tie_tp": sum(row["n_tie_tp"] for row in case_rows),
|
||
"n_tie_mns": sum(row["n_tie_mns"] for row in case_rows),
|
||
"n_tie_mixed": sum(row["n_tie_mixed"] for row in case_rows),
|
||
}
|
||
)
|
||
fragile = [
|
||
{
|
||
"case": row["case"],
|
||
"objective": row["objective"],
|
||
"real_relative_margin_percent": 100.0
|
||
* row["real_relative_margin_best_vs_second"],
|
||
}
|
||
for row in rows
|
||
if row["fragile_success"]
|
||
]
|
||
material_failures = [
|
||
{
|
||
"case": row["case"],
|
||
"objective": row["objective"],
|
||
"winner_deciding_axis": row["winner_deciding_axis"],
|
||
"winner_pair_m_s_percent": row["winner_pair_m_s_percent"],
|
||
"winner_pair_delta_s_percent": row["winner_pair_delta_s_percent"],
|
||
"regret": row["regret"],
|
||
}
|
||
for row in rows
|
||
if row["failure"]
|
||
]
|
||
boundary_winner_ties = [
|
||
{
|
||
"case": row["case"],
|
||
"objective": row["objective"],
|
||
"real_winner": row["real_winner"],
|
||
"sim_winner": row["sim_winner"],
|
||
"winner_deciding_axis": row["winner_deciding_axis"],
|
||
}
|
||
for row in rows
|
||
if row["winner_pair_relation"] == "sim-tie"
|
||
]
|
||
return {
|
||
"axis_inversions_total": {
|
||
"tp-axis": sum(row["n_inv_tp"] for row in rows),
|
||
"mns-axis": sum(row["n_inv_mns"] for row in rows),
|
||
"mixed": sum(row["n_inv_mixed"] for row in rows),
|
||
},
|
||
"axis_inversions_by_case": by_case,
|
||
"axis_ties_total": {
|
||
"tp-axis": sum(row["n_tie_tp"] for row in rows),
|
||
"mns-axis": sum(row["n_tie_mns"] for row in rows),
|
||
"mixed": sum(row["n_tie_mixed"] for row in rows),
|
||
},
|
||
"strict_winner_reversal_count": sum(
|
||
row["winner_pair_relation"] == "reversal" for row in rows
|
||
),
|
||
"sim_tie_winner_mismatch_count": sum(
|
||
row["winner_pair_relation"] == "sim-tie" for row in rows
|
||
),
|
||
"boundary_winner_ties": boundary_winner_ties,
|
||
"material_failures": material_failures,
|
||
"margin_robust_success_count": sum(
|
||
row["winner_match"] and not row["fragile_success"] for row in rows
|
||
),
|
||
"fragile_success_count": len(fragile),
|
||
"fragile_successes": fragile,
|
||
}
|
||
|
||
|
||
def analyze(input_root: Path) -> dict[str, Any]:
|
||
surfaces = load_surfaces(input_root)
|
||
rows = []
|
||
anomalies = []
|
||
for surface in surfaces:
|
||
row, row_anomalies = analyze_surface(surface)
|
||
rows.append(row)
|
||
anomalies.extend(row_anomalies)
|
||
checks, check_anomalies = cross_checks(rows)
|
||
anomalies.extend(check_anomalies)
|
||
counterexamples = [
|
||
{"case": row["case"], "objective": row["objective"]}
|
||
for row in rows
|
||
if row["h_scale_verdict"] == "COUNTEREXAMPLE"
|
||
]
|
||
data_gaps = [
|
||
{
|
||
"case": row["case"],
|
||
"objective": row["objective"],
|
||
"missing_configs": row["missing_configs"],
|
||
}
|
||
for row in rows
|
||
if row["missing_configs"] or row["h_scale_verdict"] == "N/A"
|
||
]
|
||
directional = directional_summary(rows)
|
||
return {
|
||
"schema": "frontier-split-rootcause-decomposition-v2",
|
||
"definitions": {
|
||
"objective_direction": "all objectives are minimized",
|
||
"config_uniform_scale": "geomean_c(sim_c / real_c) over paired configs",
|
||
"log_ratio_spread_all": "max_c(log(sim_c / real_c)) - min_c(log(sim_c / real_c))",
|
||
"log_ratio_spread_real_top3": "log-ratio spread over the three configs with lowest real objective values",
|
||
"real_relative_margin_best_vs_second": "real_second / real_best - 1",
|
||
"regret": "real_sim_winner / real_best - 1, recomputed from per-config values",
|
||
"failure": "regret > 0.05",
|
||
"h_scale_verdict": "HOLDS iff failure equals (top-3 log-ratio spread > log1p(real relative margin))",
|
||
"signed_pairwise_delta_percent": "100 * (log(sim_c / sim_w) - log(real_c / real_w)); negative favors c relative to real",
|
||
"real_pair_margin_percent": "100 * log(real_c / real_w)",
|
||
"pair_inversion": "real_c > real_w and sim_c < sim_w, equivalently delta_c < -m_c",
|
||
"winner_pair_tie": "sim_s == sim_w with real_s > real_w; a decision-boundary winner-label mismatch, not a strict inversion",
|
||
"winner_deciding_axis": "axis difference between real winner w and sim winner s; none for a winner match",
|
||
"fragile_success": "winner match and real best-vs-second relative margin < 0.01",
|
||
},
|
||
"summary": {
|
||
"row_count": len(rows),
|
||
"case_count": len({row["case"] for row in rows}),
|
||
"failure_count": sum(row["failure"] is True for row in rows),
|
||
"winner_mismatch_count": sum(row["winner_match"] is False for row in rows),
|
||
"h_scale_counterexample_count": len(counterexamples),
|
||
"h_scale_counterexamples": counterexamples,
|
||
"data_gap_count": len(data_gaps),
|
||
},
|
||
"directional_summary": directional,
|
||
"rows": rows,
|
||
"cross_checks": checks,
|
||
"data_gaps": data_gaps,
|
||
"anomalies": anomalies,
|
||
"known_attribution": known_attribution(input_root),
|
||
}
|
||
|
||
|
||
def percent(value: float | None, digits: int = 1) -> str:
|
||
if value is None:
|
||
return "N/A"
|
||
return f"{value * 100.0:.{digits}f}%"
|
||
|
||
|
||
def multiple(value: float | None) -> str:
|
||
if value is None:
|
||
return "N/A"
|
||
return f"{value:.3f}x"
|
||
|
||
|
||
def percent_points(value: float | None, signed: bool = False) -> str:
|
||
if value is None:
|
||
return "N/A"
|
||
sign = "+" if signed else ""
|
||
return f"{value:{sign}.1f}%"
|
||
|
||
|
||
def markdown(result: dict[str, Any]) -> str:
|
||
summary = result["summary"]
|
||
lines = [
|
||
"# Frontier good/bad split decomposition",
|
||
"",
|
||
"All 70 rows are recomputed from frozen per-config JSON values. Lower is better for every objective. "
|
||
"`scale` is the geometric mean of `sim/real`; spreads are shown as `100 ×` log-space width, while `real margin` is the ordinary relative difference `100 × (real_second / real_best - 1)`. "
|
||
"H-SCALE compares the log-space spread with `log1p(real margin)`, not directly with the ordinary relative margin. "
|
||
"The real top-3 neighborhood is selected independently for every row. Directional `m_s` and `delta_s` are already expressed in log-percentage points.",
|
||
"",
|
||
f"- Rows / case surfaces: {summary['row_count']} / {summary['case_count']}.",
|
||
f"- Failures (`regret > 5%`): {summary['failure_count']}.",
|
||
f"- Exact winner mismatches: {summary['winner_mismatch_count']}.",
|
||
f"- H-SCALE counterexamples: {summary['h_scale_counterexample_count']}.",
|
||
"",
|
||
"## Unified decomposition",
|
||
"",
|
||
"| case | objective | sim winner | real winner | regret | scale | spread all | spread real top-3 | real margin (relative) | sim-winner real loss | failure | H-SCALE | deciding axis | m_s | delta_s | n_inv_tp | n_inv_mns | n_inv_mixed | fragile success |",
|
||
"|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---:|---:|---:|---:|---:|---|",
|
||
]
|
||
for row in result["rows"]:
|
||
lines.append(
|
||
"| {case} | {objective} | {sim} | {real} | {regret} | {scale} | "
|
||
"{spread_all} | {spread_top3} | {margin} | {loss} | {failure} | {verdict} | "
|
||
"{axis} | {pair_margin} | {delta} | {n_tp} | {n_mns} | {n_mixed} | {fragile} |".format(
|
||
case=row["case"],
|
||
objective=row["objective"],
|
||
sim=row["sim_winner"] or "N/A",
|
||
real=row["real_winner"] or "N/A",
|
||
regret=percent(row["regret"]),
|
||
scale=multiple(row["config_uniform_scale"]),
|
||
spread_all=percent(row["log_ratio_spread_all"]),
|
||
spread_top3=percent(row["log_ratio_spread_real_top3"]),
|
||
margin=percent(row["real_relative_margin_best_vs_second"]),
|
||
loss=percent(row["sim_winner_real_relative_loss"]),
|
||
failure=(
|
||
"yes" if row["failure"] is True else "no" if row["failure"] is False else "N/A"
|
||
),
|
||
verdict=row["h_scale_verdict"],
|
||
axis=row["winner_deciding_axis"] or "N/A",
|
||
pair_margin=percent_points(row["winner_pair_m_s_percent"]),
|
||
delta=percent_points(row["winner_pair_delta_s_percent"], signed=True),
|
||
n_tp=row["n_inv_tp"] if row["n_inv_tp"] is not None else "N/A",
|
||
n_mns=row["n_inv_mns"] if row["n_inv_mns"] is not None else "N/A",
|
||
n_mixed=(
|
||
row["n_inv_mixed"] if row["n_inv_mixed"] is not None else "N/A"
|
||
),
|
||
fragile=(
|
||
"yes"
|
||
if row["fragile_success"] is True
|
||
else "no"
|
||
if row["fragile_success"] is False
|
||
else "N/A"
|
||
),
|
||
)
|
||
)
|
||
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"## H-SCALE verdict",
|
||
"",
|
||
]
|
||
)
|
||
if summary["h_scale_counterexample_count"] == 0:
|
||
lines.append(
|
||
"H-SCALE holds on every available row: failure occurs exactly when the real-top-3 differential residual exceeds `log1p` of the real best-vs-second relative margin."
|
||
)
|
||
else:
|
||
failure_counterexamples = sum(
|
||
row["failure"] is True and row["h_scale_verdict"] == "COUNTEREXAMPLE"
|
||
for row in result["rows"]
|
||
)
|
||
nonfailure_counterexamples = summary["h_scale_counterexample_count"] - failure_counterexamples
|
||
lines.extend(
|
||
[
|
||
"H-SCALE is not sufficient as stated; the following rows are counterexamples to the biconditional:",
|
||
"",
|
||
f"All {nonfailure_counterexamples} observed counterexamples are non-failures with spread > `log1p(margin)`; "
|
||
f"there are {failure_counterexamples} failures with spread <= `log1p(margin)`. Thus the threshold is necessary for the observed failures but not sufficient to predict them.",
|
||
"",
|
||
"| case | objective | failure | spread real top-3 | real margin | counterexample type |",
|
||
"|---|---|---|---:|---:|---|",
|
||
]
|
||
)
|
||
for row in result["rows"]:
|
||
if row["h_scale_verdict"] != "COUNTEREXAMPLE":
|
||
continue
|
||
kind = (
|
||
"failure with spread <= log1p(margin)"
|
||
if row["failure"]
|
||
else "non-failure with spread > log1p(margin)"
|
||
)
|
||
lines.append(
|
||
f"| {row['case']} | {row['objective']} | "
|
||
f"{'yes' if row['failure'] else 'no'} | "
|
||
f"{percent(row['log_ratio_spread_real_top3'])} | "
|
||
f"{percent(row['real_relative_margin_best_vs_second'])} | {kind} |"
|
||
)
|
||
|
||
directional = result["directional_summary"]
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"## S0b directional conclusions",
|
||
"",
|
||
"Directional pair statistics remove the S0 range statistic's ambiguity by measuring every config against the real winner. "
|
||
"Strict inversions use the frozen-value test `real_c > real_w and sim_c < sim_w`; simulator ties are not counted as inversions.",
|
||
"",
|
||
"### (a) Which config axis decides each material failure?",
|
||
"",
|
||
"| case | objective | real winner → sim winner | deciding axis | m_s | delta_s | -delta_s | regret |",
|
||
"|---|---|---|---|---:|---:|---:|---:|",
|
||
]
|
||
)
|
||
for row in result["rows"]:
|
||
if not row["failure"]:
|
||
continue
|
||
lines.append(
|
||
f"| {row['case']} | {row['objective']} | {row['real_winner']} → "
|
||
f"{row['sim_winner']} | {row['winner_deciding_axis']} | "
|
||
f"{percent_points(row['winner_pair_m_s_percent'])} | "
|
||
f"{percent_points(row['winner_pair_delta_s_percent'], signed=True)} | "
|
||
f"{percent_points(-row['winner_pair_delta_s_percent'])} | "
|
||
f"{percent(row['regret'])} |"
|
||
)
|
||
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"### (b) Where do TP-axis and MNS-axis inversions and boundary ties concentrate?",
|
||
"",
|
||
"`n_inv_*` counts sum strict `(real winner, c)` pair inversions across all objectives in each case; a row can contain more than one inverted pair. "
|
||
"`n_tie_*` counts cover exact simulator ties only for the simulator-selected winner versus the real winner. Ties remain decision-boundary winner-label mismatches, not strict reversals.",
|
||
"",
|
||
"| case | n_inv_tp | n_inv_mns | n_inv_mixed | n_tie_tp | n_tie_mns | n_tie_mixed |",
|
||
"|---|---:|---:|---:|---:|---:|---:|",
|
||
]
|
||
)
|
||
for item in directional["axis_inversions_by_case"]:
|
||
lines.append(
|
||
f"| {item['case']} | {item['n_inv_tp']} | {item['n_inv_mns']} | "
|
||
f"{item['n_inv_mixed']} | {item['n_tie_tp']} | {item['n_tie_mns']} | "
|
||
f"{item['n_tie_mixed']} |"
|
||
)
|
||
totals = directional["axis_inversions_total"]
|
||
tie_totals = directional["axis_ties_total"]
|
||
lines.append(
|
||
f"| **total** | **{totals['tp-axis']}** | **{totals['mns-axis']}** | "
|
||
f"**{totals['mixed']}** | **{tie_totals['tp-axis']}** | "
|
||
f"**{tie_totals['mns-axis']}** | **{tie_totals['mixed']}** |"
|
||
)
|
||
|
||
def concentrated(axis_key: str) -> str:
|
||
ranked = sorted(
|
||
(
|
||
(item[axis_key], item["case"])
|
||
for item in directional["axis_inversions_by_case"]
|
||
if item[axis_key] > 0
|
||
),
|
||
key=lambda item: (-item[0], item[1]),
|
||
)
|
||
return ", ".join(f"{case} ({count})" for count, case in ranked[:5])
|
||
|
||
by_key = row_index(result["rows"])
|
||
q30_ttft_axes = [
|
||
by_key[("q30 fixed-pd high", objective)]["winner_deciding_axis"]
|
||
for objective in ("ttft_mean_ms", "ttft_p90_ms")
|
||
]
|
||
q235_fixed_pd_failures = [
|
||
row
|
||
for row in result["rows"]
|
||
if row["case"] in ("q235 A0 fixed-pd", "q235 A1 fixed-pd")
|
||
and row["failure"]
|
||
]
|
||
q235_fixed_pd_axes = {
|
||
row["winner_deciding_axis"] for row in q235_fixed_pd_failures
|
||
}
|
||
q235_consistency = (
|
||
"The eight Q235 A0/A1 Fixed-PD failures are all tp-axis. This does not conflict with the A0/MNS64 state-drift diagnosis, but does not constitute additional state-drift evidence for A1/MNS128."
|
||
if len(q235_fixed_pd_failures) == 8 and q235_fixed_pd_axes == {"tp-axis"}
|
||
else "The Q235 A0/A1 Fixed-PD failures do not match the expected eight tp-axis cases; no state-drift compatibility claim is made."
|
||
)
|
||
lines.extend(
|
||
[
|
||
"",
|
||
f"- Among strict inversions, TP-axis cases are most concentrated in: {concentrated('n_inv_tp')}.",
|
||
f"- Among strict inversions, MNS-axis cases are most concentrated in: {concentrated('n_inv_mns')}.",
|
||
f"- {q235_consistency}",
|
||
f"- Q30 admission evidence does not map one-to-one to an MNS-axis deciding pair: high-pressure Fixed-PD TTFT mean is **{q30_ttft_axes[0]}** and p90 is **{q30_ttft_axes[1]}**. "
|
||
"This is an axis/mechanism mismatch, not a direct causal contradiction: changing TP changes offered arrival pressure, while the diagnosed nonlinear event is crossing the MNS admission cap. It rules out the simpler claim that an MNS-threshold mechanism must appear as an mns-axis winner pair.",
|
||
f"- All {directional['sim_tie_winner_mismatch_count']} exact simulator winner ties are mns-axis. Including these boundary errors gives MNS **{totals['mns-axis']} + {tie_totals['mns-axis']} = {totals['mns-axis'] + tie_totals['mns-axis']}**, comparable to TP's **{totals['tp-axis']}** strict inversions.",
|
||
"",
|
||
"The exact ties below show that Frontier is completely insensitive to MNS changes within the corresponding case/objective cell:",
|
||
"",
|
||
"| case | objective | real winner ↔ simulator-selected tied winner (MNS pair) |",
|
||
"|---|---|---|",
|
||
]
|
||
)
|
||
for item in directional["boundary_winner_ties"]:
|
||
lines.append(
|
||
f"| {item['case']} | {item['objective']} | {item['real_winner']} ↔ "
|
||
f"{item['sim_winner']} |"
|
||
)
|
||
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"### (c) How many exact-winner successes are margin-robust?",
|
||
"",
|
||
f"Among exact winner matches, **{directional['margin_robust_success_count']}** are margin-robust (`margin >= 1%`) and "
|
||
f"**{directional['fragile_success_count']}** are fragile (`margin < 1%`). "
|
||
f"Separately, {directional['sim_tie_winner_mismatch_count']} winner-label mismatches come from exact simulator ties and sit on the decision boundary rather than being strict inversions.",
|
||
"",
|
||
"| fragile-success case | objective | real margin |",
|
||
"|---|---|---:|",
|
||
]
|
||
)
|
||
for item in directional["fragile_successes"]:
|
||
lines.append(
|
||
f"| {item['case']} | {item['objective']} | "
|
||
f"{item['real_relative_margin_percent']:.1f}% |"
|
||
)
|
||
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"## Cross-checks",
|
||
"",
|
||
"| check | observed | frozen expectation | result |",
|
||
"|---|---:|---:|---|",
|
||
]
|
||
)
|
||
for check in result["cross_checks"]:
|
||
observed = check["observed_percent"]
|
||
expected = check["expected_percent"]
|
||
if isinstance(observed, list):
|
||
observed_text = ", ".join(f"{value:.1f}%" for value in observed)
|
||
expected_text = ", ".join(f"{value:.1f}%" for value in expected)
|
||
else:
|
||
observed_text = f"{observed:.1f}%"
|
||
expected_text = f"{expected:.1f}%"
|
||
lines.append(
|
||
f"| {check['name']} | {observed_text} | {expected_text} | "
|
||
f"{'PASS' if check['passed'] else 'FAIL'} |"
|
||
)
|
||
|
||
lines.extend(["", "## Data gaps", ""])
|
||
if result["data_gaps"]:
|
||
for gap in result["data_gaps"]:
|
||
missing = ", ".join(gap["missing_configs"]) or "insufficient paired configs"
|
||
lines.append(f"- {gap['case']} / {gap['objective']}: N/A because {missing}.")
|
||
else:
|
||
lines.append("No per-config sim/real pairing gaps were found in the requested surfaces.")
|
||
|
||
lines.extend(["", "## 异常", ""])
|
||
if result["anomalies"]:
|
||
lines.extend(f"- {item}" for item in result["anomalies"])
|
||
else:
|
||
lines.append("- 无。重算 winner/regret 与 frozen JSON 一致，硬性交叉核对均通过。")
|
||
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"## Figure",
|
||
"",
|
||
"`margin-vs-residual.png` uses x = `100 × log1p(real best-vs-second relative margin)` and y = `100 ×` real-top-3 log-ratio spread. "
|
||
"Both axes are logarithmic because the real margins span six orders of magnitude. Green means exact winner match; red means winner mismatch. "
|
||
"The diagonal is the H-SCALE threshold, while failure itself is defined by regret, not by exact winner match.",
|
||
"",
|
||
"`decision-pair-axis.png` is a mechanism census, not a hypothesis test: for a strict winner reversal, `-delta_s > m_s` is algebraically equivalent to the mismatch. "
|
||
"The frozen surfaces also contain deterministic winner-label mismatches caused by exact simulator ties; those points lie on `-delta_s = m_s` and are not counted as strict inversions. "
|
||
"Color identifies the winner-deciding config axis and marker area is proportional to regret.",
|
||
"",
|
||
"## A0 vs A1 对照",
|
||
"",
|
||
"A1 repairs the Q235 Trace-PO p90 ranking error but leaves the four large Fixed-PD TPOT/E2E regrets unchanged.",
|
||
"",
|
||
"| case / objective | A0 regret | A1 regret |",
|
||
"|---|---:|---:|",
|
||
]
|
||
)
|
||
by_key = row_index(result["rows"])
|
||
a0_a1_rows = (
|
||
("trace-po / ttft_p90_ms", "trace-po", "ttft_p90_ms"),
|
||
("fixed-pd / tpot_mean_ms", "fixed-pd", "tpot_mean_ms"),
|
||
("fixed-pd / tpot_p90_ms", "fixed-pd", "tpot_p90_ms"),
|
||
("fixed-pd / e2e_mean_ms", "fixed-pd", "e2e_mean_ms"),
|
||
("fixed-pd / e2e_p90_ms", "fixed-pd", "e2e_p90_ms"),
|
||
)
|
||
for label, case, objective in a0_a1_rows:
|
||
a0 = by_key[(f"q235 A0 {case}", objective)]["regret"]
|
||
a1 = by_key[(f"q235 A1 {case}", objective)]["regret"]
|
||
lines.append(f"| {label} | {percent(a0)} | {percent(a1)} |")
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"Thus Trace-PO TTFT p90 changes **21.2% → 0.3%**, whereas Fixed-PD remains **33.0% / 37.2% / 30.7% / 34.6%** in both A0 and A1.",
|
||
"",
|
||
"## 已知分量归因摘要",
|
||
"",
|
||
"### Q30 high-pressure Fixed-PD TTFT: first-scheduling wait vs execution-side TTFT",
|
||
"",
|
||
"`queue-free TTFT` is the execution-side TTFT after removing first-scheduling wait; in the frozen diagnostic, simulated TTFT is their sum (up to floating-point rounding).",
|
||
"",
|
||
"| config | sim TPOT / real | sim slots / MNS | real slots upper / MNS | first-scheduling wait | queue-free TTFT | sim TTFT | real TTFT | real waiting max |",
|
||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|",
|
||
]
|
||
)
|
||
q30 = result["known_attribution"]["q30_admission"]
|
||
for cell in q30["cells"]:
|
||
lines.append(
|
||
f"| {cell['config']} | {cell['sim_tpot_over_real']:.2f}x | "
|
||
f"{cell['sim_required_slots']:.1f}/{cell['mns']} | "
|
||
f"{cell['real_required_slots_upper_bound']:.1f}/{cell['mns']} | "
|
||
f"{cell['sim_first_scheduling_wait_ms']:.1f} ms | "
|
||
f"{cell['sim_queue_free_ttft_ms']:.1f} ms | "
|
||
f"{cell['sim_ttft_ms']:.1f} ms | {cell['real_ttft_ms']:.1f} ms | "
|
||
f"{cell['real_waiting_max']} |"
|
||
)
|
||
contrasts = q30["contrasts"]
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"For TP4/MNS32 minus TP2/MNS64, the observed simulator contrast is "
|
||
f"**{contrasts['observed_sim_ttft_tp4_minus_tp2_ms']:+.1f} ms**; removing first-scheduling wait changes it to "
|
||
f"**{contrasts['queue_free_sim_ttft_tp4_minus_tp2_ms']:+.1f} ms**, aligned with the real contrast "
|
||
f"**{contrasts['real_ttft_tp4_minus_tp2_ms']:+.1f} ms**. This localizes the Q30 high-pressure TTFT reversal to admission queueing, while the upstream service-time overprediction remains unattributed in the frozen data.",
|
||
"",
|
||
"### Q235 Fixed-PD: own-composition vs exact-state contrast",
|
||
"",
|
||
"Positive TP8−TP4 means TP8 is slower and therefore agrees with the real TP4 winner.",
|
||
"",
|
||
"| config | sim own decode batch mean | real exact decode batch mean | real token-weighted iteration | exact-state matched sim | coverage |",
|
||
"|---|---:|---:|---:|---:|---:|",
|
||
]
|
||
)
|
||
q235 = result["known_attribution"]["q235_state"]
|
||
for config in q235["configs"]:
|
||
lines.append(
|
||
f"| {config['config']} | {config['sim_own_decode_batch_mean']:.3f} | "
|
||
f"{config['real_exact_decode_batch_mean']:.3f} | "
|
||
f"{config['real_token_weighted_iteration_ms']:.4f} ms | "
|
||
f"{config['exact_state_matched_sim_ms']:.4f} ms | "
|
||
f"{config['exact_state_coverage'] * 100.0:.1f}% |"
|
||
)
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"| component | own-composition TP8−TP4 | exact-state TP8−TP4 |",
|
||
"|---|---:|---:|",
|
||
]
|
||
)
|
||
for component in q235["components"]:
|
||
lines.append(
|
||
f"| {component['component']} | "
|
||
f"{component['own_composition_tp8_minus_tp4_ms']:+.4f} ms | "
|
||
f"{component['exact_state_tp8_minus_tp4_ms']:+.4f} ms |"
|
||
)
|
||
decomposition = q235["contrast_decomposition"]
|
||
reference = q235["reference"]
|
||
lines.extend(
|
||
[
|
||
"",
|
||
f"The total contrast flips from **{decomposition['simulator_internal_tp8_minus_tp4_ms']:+.4f} ms** on Frontier's own composition to "
|
||
f"**{decomposition['exact_state_matched_tp8_minus_tp4_ms']:+.4f} ms** at exact real state, a "
|
||
f"**{decomposition['state_composition_shift_ms']:+.4f} ms** state-composition shift. "
|
||
f"The exact real iteration contrast is **{reference['exact_iteration_tp8_minus_tp4_ms']:+.4f} ms** and the frozen real TPOT contrast is "
|
||
f"**{reference['observed_real_tp8_minus_tp4_ms']:+.4f} ms**. This supports closed-loop state/composition drift for Q235 Fixed-PD; it does not by itself attribute every remaining conditional execution residual.",
|
||
"",
|
||
]
|
||
)
|
||
return "\n".join(lines)
|
||
|
||
|
||
def render_plot(rows: list[dict[str, Any]], output: Path) -> None:
|
||
os.environ.setdefault(
|
||
"MPLCONFIGDIR", "/tmp/aituner-frontier-split-rootcause-matplotlib"
|
||
)
|
||
import matplotlib
|
||
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
|
||
usable = [
|
||
row
|
||
for row in rows
|
||
if row["real_relative_margin_best_vs_second"] is not None
|
||
and row["log_ratio_spread_real_top3"] is not None
|
||
]
|
||
if not usable:
|
||
raise ValueError("no complete rows available for margin-vs-residual plot")
|
||
|
||
plt.rcParams.update(
|
||
{
|
||
"font.family": "DejaVu Sans",
|
||
"font.size": 8,
|
||
"axes.titlesize": 10,
|
||
"axes.labelsize": 9,
|
||
}
|
||
)
|
||
fig, ax = plt.subplots(figsize=(13.0, 9.0), dpi=180)
|
||
colors = {True: "#2a9d8f", False: "#d1495b"}
|
||
labels = {True: "winner match", False: "winner mismatch"}
|
||
for match in (True, False):
|
||
selected = [row for row in usable if row["winner_match"] is match]
|
||
ax.scatter(
|
||
[
|
||
math.log1p(row["real_relative_margin_best_vs_second"]) * 100.0
|
||
for row in selected
|
||
],
|
||
[row["log_ratio_spread_real_top3"] * 100.0 for row in selected],
|
||
s=28,
|
||
color=colors[match],
|
||
edgecolor="white",
|
||
linewidth=0.4,
|
||
alpha=0.88,
|
||
label=labels[match],
|
||
zorder=3,
|
||
)
|
||
|
||
for index, row in enumerate(usable):
|
||
x = math.log1p(row["real_relative_margin_best_vs_second"]) * 100.0
|
||
y = row["log_ratio_spread_real_top3"] * 100.0
|
||
dx = 3 if index % 2 == 0 else -3
|
||
dy = 3 if (index // 2) % 2 == 0 else -5
|
||
ax.annotate(
|
||
row["case"],
|
||
(x, y),
|
||
xytext=(dx, dy),
|
||
textcoords="offset points",
|
||
ha="left" if dx > 0 else "right",
|
||
va="bottom" if dy > 0 else "top",
|
||
fontsize=4.5,
|
||
color="#303030",
|
||
alpha=0.82,
|
||
clip_on=True,
|
||
zorder=4,
|
||
)
|
||
|
||
max_value = max(
|
||
max(
|
||
math.log1p(row["real_relative_margin_best_vs_second"])
|
||
for row in usable
|
||
),
|
||
max(row["log_ratio_spread_real_top3"] for row in usable),
|
||
) * 100.0
|
||
min_value = min(
|
||
min(
|
||
math.log1p(row["real_relative_margin_best_vs_second"])
|
||
for row in usable
|
||
),
|
||
min(row["log_ratio_spread_real_top3"] for row in usable),
|
||
) * 100.0
|
||
lower = min_value / 5.0
|
||
upper = max_value * 2.0
|
||
ax.plot(
|
||
[lower, upper],
|
||
[lower, upper],
|
||
linestyle="--",
|
||
color="#555555",
|
||
linewidth=1.0,
|
||
label="spread = log1p(margin)",
|
||
zorder=2,
|
||
)
|
||
ax.set_xscale("log")
|
||
ax.set_yscale("log")
|
||
ax.set_xlim(left=lower, right=upper)
|
||
ax.set_ylim(bottom=lower, top=upper)
|
||
ax.set_xlabel("Real best-vs-second log margin: 100 × log1p(relative margin)")
|
||
ax.set_ylabel("Top-3 differential residual: 100 × log-ratio spread")
|
||
ax.set_title("Frontier selection: real margin vs config-differential residual")
|
||
ax.grid(True, color="#d8d8d8", linewidth=0.5, alpha=0.7, zorder=1)
|
||
ax.legend(frameon=False, loc="upper left")
|
||
fig.tight_layout()
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
temporary = output.with_suffix(output.suffix + ".tmp")
|
||
fig.savefig(
|
||
temporary,
|
||
format="png",
|
||
dpi=180,
|
||
metadata={"Software": "analyze_split_decomposition.py"},
|
||
)
|
||
plt.close(fig)
|
||
os.replace(temporary, output)
|
||
|
||
|
||
def render_decision_pair_axis_plot(
|
||
rows: list[dict[str, Any]], output: Path
|
||
) -> None:
|
||
os.environ.setdefault(
|
||
"MPLCONFIGDIR", "/tmp/aituner-frontier-split-rootcause-matplotlib"
|
||
)
|
||
import matplotlib
|
||
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.lines import Line2D
|
||
|
||
usable = [
|
||
row
|
||
for row in rows
|
||
if row["winner_match"] is False
|
||
and row["winner_pair_m_s_percent"] > 0
|
||
and -row["winner_pair_delta_s_percent"] > 0
|
||
]
|
||
if not usable:
|
||
raise ValueError("no winner mismatches available for decision-pair-axis plot")
|
||
|
||
plt.rcParams.update(
|
||
{
|
||
"font.family": "DejaVu Sans",
|
||
"font.size": 8,
|
||
"axes.titlesize": 10,
|
||
"axes.labelsize": 9,
|
||
}
|
||
)
|
||
colors = {
|
||
"tp-axis": "#3572b0",
|
||
"mns-axis": "#e07a1f",
|
||
"mixed": "#7b4ab5",
|
||
}
|
||
fig, ax = plt.subplots(figsize=(11.0, 8.0), dpi=180)
|
||
for axis in INVERSION_AXES:
|
||
selected = [row for row in usable if row["winner_deciding_axis"] == axis]
|
||
ax.scatter(
|
||
[row["winner_pair_m_s_percent"] for row in selected],
|
||
[-row["winner_pair_delta_s_percent"] for row in selected],
|
||
s=[1000.0 * row["regret"] for row in selected],
|
||
color=colors[axis],
|
||
edgecolor="white",
|
||
linewidth=0.6,
|
||
alpha=0.82,
|
||
label=axis,
|
||
zorder=3,
|
||
)
|
||
|
||
values = [row["winner_pair_m_s_percent"] for row in usable] + [
|
||
-row["winner_pair_delta_s_percent"] for row in usable
|
||
]
|
||
lower = min(values) / 1.8
|
||
upper = max(values) * 1.8
|
||
ax.plot(
|
||
[lower, upper],
|
||
[lower, upper],
|
||
linestyle="--",
|
||
color="#555555",
|
||
linewidth=1.0,
|
||
label="reversal boundary",
|
||
zorder=2,
|
||
)
|
||
ax.set_xscale("log")
|
||
ax.set_yscale("log")
|
||
ax.set_xlim(lower, upper)
|
||
ax.set_ylim(lower, upper)
|
||
ax.set_xlabel("Winner-pair real log margin $m_s$ (%)")
|
||
ax.set_ylabel("Simulator preference strength $-\\delta_s$ (%)")
|
||
ax.set_title("Winner-deciding pair census by config axis")
|
||
ax.grid(True, color="#d8d8d8", linewidth=0.5, alpha=0.7, zorder=1)
|
||
axis_handles = [
|
||
Line2D(
|
||
[0],
|
||
[0],
|
||
marker="o",
|
||
linestyle="none",
|
||
markerfacecolor=colors[axis],
|
||
markeredgecolor="white",
|
||
markersize=7,
|
||
label=axis,
|
||
)
|
||
for axis in INVERSION_AXES
|
||
]
|
||
axis_handles.append(
|
||
Line2D(
|
||
[0],
|
||
[0],
|
||
linestyle="--",
|
||
color="#555555",
|
||
linewidth=1.0,
|
||
label="reversal boundary",
|
||
)
|
||
)
|
||
axis_legend = ax.legend(
|
||
handles=axis_handles, frameon=False, loc="upper left", title="deciding axis"
|
||
)
|
||
ax.add_artist(axis_legend)
|
||
size_handles = [
|
||
ax.scatter(
|
||
[],
|
||
[],
|
||
s=10.0 * regret_percent,
|
||
facecolor="none",
|
||
edgecolor="#555555",
|
||
linewidth=0.7,
|
||
label=f"{regret_percent}%",
|
||
)
|
||
for regret_percent in (5, 20, 50)
|
||
]
|
||
ax.legend(
|
||
handles=size_handles,
|
||
frameon=False,
|
||
loc="lower right",
|
||
title="regret (marker area)",
|
||
)
|
||
fig.tight_layout()
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
temporary = output.with_suffix(output.suffix + ".tmp")
|
||
fig.savefig(
|
||
temporary,
|
||
format="png",
|
||
dpi=180,
|
||
metadata={"Software": "analyze_split_decomposition.py"},
|
||
)
|
||
plt.close(fig)
|
||
os.replace(temporary, output)
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--input-root", type=Path, default=DEFAULT_INPUT_ROOT)
|
||
parser.add_argument("--results-root", type=Path, default=DEFAULT_RESULTS_ROOT)
|
||
return parser.parse_args()
|
||
|
||
|
||
def main() -> None:
|
||
args = parse_args()
|
||
result = analyze(args.input_root.resolve())
|
||
results_root = args.results_root.resolve()
|
||
atomic_write(
|
||
results_root / "decomposition.json",
|
||
json.dumps(result, indent=2, sort_keys=True) + "\n",
|
||
)
|
||
atomic_write(results_root / "decomposition.md", markdown(result))
|
||
render_plot(result["rows"], results_root / "margin-vs-residual.png")
|
||
render_decision_pair_axis_plot(
|
||
result["rows"], results_root / "decision-pair-axis.png"
|
||
)
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"status": "PASS" if not result["anomalies"] else "ANOMALY",
|
||
"rows": result["summary"]["row_count"],
|
||
"h_scale_counterexamples": result["summary"]["h_scale_counterexample_count"],
|
||
"anomalies": len(result["anomalies"]),
|
||
},
|
||
sort_keys=True,
|
||
)
|
||
)
|
||
if result["anomalies"]:
|
||
raise SystemExit("analysis anomalies found; see results/decomposition.md")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|