92 lines
3.2 KiB
Python
92 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Retrospective point-estimate oracle bound from accepted Phase-3 cells."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from analyze import CONFIGS, PHASES, atomic_json, numeric
|
|
|
|
|
|
def build(metrics: dict[str, Any]) -> dict[str, Any]:
|
|
tables: dict[str, dict[str, dict[str, float]]] = {}
|
|
for load in ("saturation", "moderate"):
|
|
table: dict[str, dict[str, float]] = {}
|
|
for phase in PHASES:
|
|
row = {}
|
|
for config in CONFIGS:
|
|
run = metrics.get("runs", {}).get(f"{phase}-{config}-{load}")
|
|
if run is not None:
|
|
row[config] = float(run["clean"]["completed_throughput_rps"])
|
|
table[phase] = row
|
|
tables[load] = table
|
|
|
|
analyses = {}
|
|
all_values = []
|
|
for load, table in tables.items():
|
|
complete = all(set(row) == set(CONFIGS) for row in table.values())
|
|
if not complete:
|
|
raise RuntimeError(f"missing sentinel throughput cell for {load}: {table}")
|
|
per_phase = {}
|
|
c00_regrets = []
|
|
for phase, row in table.items():
|
|
oracle_config = max(row, key=row.get)
|
|
oracle = row[oracle_config]
|
|
regret = oracle / row["C00"] - 1.0
|
|
c00_regrets.append(regret)
|
|
per_phase[phase] = {
|
|
"throughput_rps": row,
|
|
"oracle_config": oracle_config,
|
|
"oracle_rps": oracle,
|
|
"c00_regret": regret,
|
|
}
|
|
all_values.extend(row.values())
|
|
equal_oracle = sum(item["oracle_rps"] for item in per_phase.values()) / len(PHASES)
|
|
static = {
|
|
config: sum(table[p][config] for p in PHASES) / len(PHASES)
|
|
for config in CONFIGS
|
|
}
|
|
analyses[load] = {
|
|
"per_phase": per_phase,
|
|
"universal_c00_point_bound": max(c00_regrets),
|
|
"equal_time_oracle_rps": equal_oracle,
|
|
"equal_time_best_static_config": max(static, key=static.get),
|
|
"equal_time_best_static_rps": max(static.values()),
|
|
"equal_time_gap": equal_oracle / max(static.values()) - 1.0,
|
|
"interpretation": (
|
|
"valid additive point-estimate precheck"
|
|
if load == "saturation"
|
|
else "diagnostic only: each config used its own offered rate"
|
|
),
|
|
}
|
|
return {
|
|
"schema": 1,
|
|
"scope": list(PHASES),
|
|
"analyses": analyses,
|
|
"sanity": {
|
|
"throughput_rps": numeric(all_values),
|
|
"invariants": {
|
|
"all_nonnegative": all(value >= 0 for value in all_values),
|
|
"all_cells_present": len(all_values) == 2 * len(PHASES) * len(CONFIGS),
|
|
"per_config_not_identical": len(set(all_values)) > len(CONFIGS),
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--metrics", required=True)
|
|
parser.add_argument("--out", required=True)
|
|
args = parser.parse_args()
|
|
result = build(json.loads(Path(args.metrics).read_text()))
|
|
atomic_json(Path(args.out), result)
|
|
print(json.dumps(result, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|