Report request-level SLO fidelity for Frontier

runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py

@@ -1565,6 +1565,97 @@ def _pass_rate_residuals(
     return variants
 
 
+def _request_slo_classification(
+    frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]]
+) -> dict[str, Any]:
+    def index(
+        results: list[dict[str, Any]],
+    ) -> dict[tuple[str, float, str], dict[str, Any]]:
+        return {
+            (
+                config["config"]["name"],
+                float(load["offered_request_rate"]),
+                str(request["request_id"]),
+            ): request
+            for config in results
+            for load in config["loads"]
+            for request in load["requests"]
+        }
+
+    def request_passes(request: dict[str, Any], variant: dict[str, Any]) -> bool:
+        return bool(
+            request["success"]
+            and request.get("ttft_ms") is not None
+            and float(request["ttft_ms"])
+            <= slo_threshold_ms(variant, int(request["input_tokens"]))
+        )
+
+    frontier_index = index(frontier_results)
+    real_index = index(real_results)
+    keys = sorted(set(frontier_index) & set(real_index))
+    variants = {}
+    for slo_name, variant in SLO_VARIANTS.items():
+        by_load: dict[tuple[str, float], dict[str, Any]] = {}
+        frontier_failures: set[tuple[str, float, str]] = set()
+        real_failures: set[tuple[str, float, str]] = set()
+        for key in keys:
+            frontier_passes = request_passes(frontier_index[key], variant)
+            real_passes = request_passes(real_index[key], variant)
+            if not frontier_passes:
+                frontier_failures.add(key)
+            if not real_passes:
+                real_failures.add(key)
+            group = by_load.setdefault(
+                key[:2],
+                {
+                    "matched_requests": 0,
+                    "matching_labels": 0,
+                    "false_slo_pass_request_ids": [],
+                    "false_slo_fail_request_ids": [],
+                },
+            )
+            group["matched_requests"] += 1
+            group["matching_labels"] += int(frontier_passes == real_passes)
+            if frontier_passes and not real_passes:
+                group["false_slo_pass_request_ids"].append(key[2])
+            elif not frontier_passes and real_passes:
+                group["false_slo_fail_request_ids"].append(key[2])
+        intersection = frontier_failures & real_failures
+        union = frontier_failures | real_failures
+        load_records = []
+        for (config, rate), record in sorted(by_load.items()):
+            matched = int(record["matched_requests"])
+            load_records.append(
+                {
+                    "config": config,
+                    "offered_request_rate": rate,
+                    **record,
+                    "label_accuracy": record["matching_labels"] / matched,
+                }
+            )
+        matching_labels = sum(item["matching_labels"] for item in by_load.values())
+        false_passes = sum(
+            len(item["false_slo_pass_request_ids"]) for item in by_load.values()
+        )
+        false_fails = sum(
+            len(item["false_slo_fail_request_ids"]) for item in by_load.values()
+        )
+        variants[slo_name] = {
+            "matched_requests": len(keys),
+            "matching_labels": matching_labels,
+            "label_accuracy": matching_labels / len(keys) if keys else None,
+            "false_slo_pass_requests": false_passes,
+            "false_slo_fail_requests": false_fails,
+            "frontier_failure_requests": len(frontier_failures),
+            "real_failure_requests": len(real_failures),
+            "failure_set_intersection": len(intersection),
+            "failure_set_union": len(union),
+            "failure_set_jaccard": len(intersection) / len(union) if union else 1.0,
+            "by_config_load": load_records,
+        }
+    return variants
+
+
 def compare(args: argparse.Namespace) -> Path:
     frontier = json.loads(args.frontier_freeze.read_text())
     community = json.loads(args.community_freeze.read_text())
@@ -1624,6 +1715,9 @@ def compare(args: argparse.Namespace) -> Path:
         "pass_rate_surface_residuals": _pass_rate_residuals(
             frontier["config_results"], community["config_results"]
         ),
+        "request_slo_classification": _request_slo_classification(
+            frontier["config_results"], community["config_results"]
+        ),
     }
     write_json(args.output, result)
     print(args.output)

runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py

@@ -258,3 +258,36 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() ->
         0.1,
         0.2,
     ]
+
+
+def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None:
+    def results(ttfts: list[float]) -> list[dict[str, object]]:
+        return [
+            {
+                "config": {"name": "a"},
+                "loads": [
+                    {
+                        "offered_request_rate": 0.1,
+                        "requests": [
+                            {
+                                "request_id": str(index),
+                                "input_tokens": 0,
+                                "success": True,
+                                "ttft_ms": ttft,
+                            }
+                            for index, ttft in enumerate(ttfts)
+                        ],
+                    }
+                ],
+            }
+        ]
+
+    classification = MODULE._request_slo_classification(
+        results([900.0, 1100.0]), results([1100.0, 900.0])
+    )[MODULE.PRIMARY_SLO]
+    assert classification["frontier_failure_requests"] == 1
+    assert classification["real_failure_requests"] == 1
+    assert classification["failure_set_jaccard"] == 0.0
+    assert classification["label_accuracy"] == 0.0
+    assert classification["false_slo_pass_requests"] == 1
+    assert classification["false_slo_fail_requests"] == 1