From f7052edd14be0329eb20eb06d440a1ae5b8efd1a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 01:42:13 +0800 Subject: [PATCH] Report request-level SLO fidelity for Frontier --- .../best_effort/fixed_cohort_rank.py | 94 +++++++++++++++++++ .../best_effort/test_fixed_cohort_rank.py | 33 +++++++ 2 files changed, 127 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 5b8feb4..ae9c64d 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -1565,6 +1565,97 @@ def _pass_rate_residuals( return variants +def _request_slo_classification( + frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]] +) -> dict[str, Any]: + def index( + results: list[dict[str, Any]], + ) -> dict[tuple[str, float, str], dict[str, Any]]: + return { + ( + config["config"]["name"], + float(load["offered_request_rate"]), + str(request["request_id"]), + ): request + for config in results + for load in config["loads"] + for request in load["requests"] + } + + def request_passes(request: dict[str, Any], variant: dict[str, Any]) -> bool: + return bool( + request["success"] + and request.get("ttft_ms") is not None + and float(request["ttft_ms"]) + <= slo_threshold_ms(variant, int(request["input_tokens"])) + ) + + frontier_index = index(frontier_results) + real_index = index(real_results) + keys = sorted(set(frontier_index) & set(real_index)) + variants = {} + for slo_name, variant in SLO_VARIANTS.items(): + by_load: dict[tuple[str, float], dict[str, Any]] = {} + frontier_failures: set[tuple[str, float, str]] = set() + real_failures: set[tuple[str, float, str]] = set() + for key in keys: + frontier_passes = request_passes(frontier_index[key], variant) + real_passes = request_passes(real_index[key], variant) + if not frontier_passes: + frontier_failures.add(key) + if not real_passes: + real_failures.add(key) + group = by_load.setdefault( + key[:2], + { + "matched_requests": 0, + "matching_labels": 0, + "false_slo_pass_request_ids": [], + "false_slo_fail_request_ids": [], + }, + ) + group["matched_requests"] += 1 + group["matching_labels"] += int(frontier_passes == real_passes) + if frontier_passes and not real_passes: + group["false_slo_pass_request_ids"].append(key[2]) + elif not frontier_passes and real_passes: + group["false_slo_fail_request_ids"].append(key[2]) + intersection = frontier_failures & real_failures + union = frontier_failures | real_failures + load_records = [] + for (config, rate), record in sorted(by_load.items()): + matched = int(record["matched_requests"]) + load_records.append( + { + "config": config, + "offered_request_rate": rate, + **record, + "label_accuracy": record["matching_labels"] / matched, + } + ) + matching_labels = sum(item["matching_labels"] for item in by_load.values()) + false_passes = sum( + len(item["false_slo_pass_request_ids"]) for item in by_load.values() + ) + false_fails = sum( + len(item["false_slo_fail_request_ids"]) for item in by_load.values() + ) + variants[slo_name] = { + "matched_requests": len(keys), + "matching_labels": matching_labels, + "label_accuracy": matching_labels / len(keys) if keys else None, + "false_slo_pass_requests": false_passes, + "false_slo_fail_requests": false_fails, + "frontier_failure_requests": len(frontier_failures), + "real_failure_requests": len(real_failures), + "failure_set_intersection": len(intersection), + "failure_set_union": len(union), + "failure_set_jaccard": len(intersection) / len(union) if union else 1.0, + "by_config_load": load_records, + } + return variants + + def compare(args: argparse.Namespace) -> Path: frontier = json.loads(args.frontier_freeze.read_text()) community = json.loads(args.community_freeze.read_text()) @@ -1624,6 +1715,9 @@ def compare(args: argparse.Namespace) -> Path: "pass_rate_surface_residuals": _pass_rate_residuals( frontier["config_results"], community["config_results"] ), + "request_slo_classification": _request_slo_classification( + frontier["config_results"], community["config_results"] + ), } write_json(args.output, result) print(args.output) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index cb9c030..f0e70d3 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -258,3 +258,36 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> 0.1, 0.2, ] + + +def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: + def results(ttfts: list[float]) -> list[dict[str, object]]: + return [ + { + "config": {"name": "a"}, + "loads": [ + { + "offered_request_rate": 0.1, + "requests": [ + { + "request_id": str(index), + "input_tokens": 0, + "success": True, + "ttft_ms": ttft, + } + for index, ttft in enumerate(ttfts) + ], + } + ], + } + ] + + classification = MODULE._request_slo_classification( + results([900.0, 1100.0]), results([1100.0, 900.0]) + )[MODULE.PRIMARY_SLO] + assert classification["frontier_failure_requests"] == 1 + assert classification["real_failure_requests"] == 1 + assert classification["failure_set_jaccard"] == 0.0 + assert classification["label_accuracy"] == 0.0 + assert classification["false_slo_pass_requests"] == 1 + assert classification["false_slo_fail_requests"] == 1