Add reproducible CollectiveSpec opportunity screen

scripts/collectivespec/summarize_static_k_pilot.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Summarize immutable artifacts from ``run_static_k_pilot.py``.
+
+The output intentionally distinguishes an absent/failed result from an
+SLO-infeasible probe, and includes a small data-sanity block for review.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from pathlib import Path
+from typing import Any
+
+
+def number(value: Any) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    value = float(value)
+    return value if math.isfinite(value) else None
+
+
+def first_result(store: Path) -> Path | None:
+    candidates = sorted(store.glob("*/trials/trial-*/result.json"))
+    return candidates[0] if candidates else None
+
+
+def summarize_result(k: int, path: Path | None) -> dict[str, Any]:
+    record: dict[str, Any] = {"k": k, "label": "NoSpec" if k == 0 else f"K={k}"}
+    if path is None:
+        record.update({"status": "missing_result"})
+        return record
+
+    payload = json.loads(path.read_text())
+    probes = payload.get("probes") if isinstance(payload.get("probes"), list) else []
+    feasible = [probe for probe in probes if isinstance(probe, dict) and probe.get("feasible")]
+    best_payload = feasible[-1].get("payload", {}) if feasible else {}
+    latency = best_payload.get("latency_summary", {}) if isinstance(best_payload, dict) else {}
+    tpot = latency.get("tpot_ms", {}) if isinstance(latency, dict) else {}
+    record.update(
+        {
+            "status": str(payload.get("status", "unknown")),
+            "result_path": str(path),
+            "best_sampling_u": number(payload.get("best_sampling_u")),
+            "best_request_rate": number(payload.get("best_request_rate")),
+            "best_pass_rate": number(payload.get("best_pass_rate")),
+            "best_request_count": number(payload.get("best_request_count")),
+            "probe_count": len(probes),
+            "feasible_probe_count": len(feasible),
+            "probe_thresholds": [number(probe.get("threshold")) for probe in probes if isinstance(probe, dict)],
+            "best_tpot_ms": {
+                metric: number(tpot.get(metric)) for metric in ("mean", "p50", "p90", "p95", "p99")
+            },
+        }
+    )
+    return record
+
+
+def sanity(records: list[dict[str, Any]]) -> dict[str, Any]:
+    completed = [r for r in records if r.get("status") == "completed"]
+    values = [r["best_sampling_u"] for r in completed if number(r.get("best_sampling_u")) is not None]
+    pass_rates = [r["best_pass_rate"] for r in completed if number(r.get("best_pass_rate")) is not None]
+    distinct = len(set(values))
+    return {
+        "n_configurations": len(records),
+        "n_completed": len(completed),
+        "sampling_u": {
+            "n": len(values),
+            "min": min(values) if values else None,
+            "max": max(values) if values else None,
+            "distinct_value_count": distinct,
+            "all_identical": len(values) > 1 and distinct == 1,
+            "non_negative": all(value >= 0 for value in values),
+        },
+        "pass_rate": {
+            "n": len(pass_rates),
+            "min": min(pass_rates) if pass_rates else None,
+            "max": max(pass_rates) if pass_rates else None,
+            "distinct_value_count": len(set(pass_rates)),
+            "within_0_1": all(0 <= value <= 1 for value in pass_rates),
+        },
+        "invariants": {
+            "one_result_per_k": len({r["k"] for r in records}) == len(records),
+            "all_best_request_rates_non_negative": all(
+                (value := number(r.get("best_request_rate"))) is None or value >= 0 for r in completed
+            ),
+        },
+    }
+
+
+def to_markdown(records: list[dict[str, Any]], checks: dict[str, Any]) -> str:
+    lines = [
+        "# CollectiveSpec static-K screening summary",
+        "",
+        "| configuration | status | max feasible sampling_u | request rate | pass rate | p95 TPOT (ms) | probes |",
+        "|---|---:|---:|---:|---:|---:|---:|",
+    ]
+    for r in records:
+        tpot = r.get("best_tpot_ms") or {}
+        fmt = lambda value: "—" if value is None else f"{value:.6g}"
+        lines.append(
+            "| {label} | {status} | {u} | {rate} | {pass_rate} | {p95} | {probes} |".format(
+                label=r["label"],
+                status=r["status"],
+                u=fmt(r.get("best_sampling_u")),
+                rate=fmt(r.get("best_request_rate")),
+                pass_rate=fmt(r.get("best_pass_rate")),
+                p95=fmt(tpot.get("p95")),
+                probes=r.get("probe_count", "—"),
+            )
+        )
+    lines.extend(["", "## Data sanity", "", "```json", json.dumps(checks, indent=2, sort_keys=True), "``", ""])
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--output-json", type=Path, required=True)
+    parser.add_argument("--output-md", type=Path, required=True)
+    args = parser.parse_args()
+
+    manifest = json.loads((args.root / "manifest.json").read_text())
+    k_values = [int(k) for k in manifest["k_values"]]
+    records = [summarize_result(k, first_result(args.root / "stores" / f"k{k}")) for k in sorted(k_values)]
+    checks = sanity(records)
+    output = {"manifest": manifest, "records": records, "data_sanity": checks}
+    args.output_json.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
+    args.output_md.write_text(to_markdown(records, checks))
+    print(json.dumps(output, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())