Compare Frontier rank under explicit objectives

runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py

@@ -1184,9 +1184,11 @@ def _rank_map(ranking: list[dict[str, Any]]) -> dict[str, int]:
     return {item["config"]["name"]: int(item["rank"]) for item in ranking}
 
 
-def _capacity_map(ranking: list[dict[str, Any]]) -> dict[str, float | None]:
+def _capacity_map(
+    ranking: list[dict[str, Any]], *, field: str
+) -> dict[str, float | None]:
     return {
-        item["config"]["name"]: item["maximum_tested_feasible_request_rate_per_gpu"]
+        item["config"]["name"]: item[field]
         for item in ranking
     }
 
@@ -1263,6 +1265,73 @@ def _pairwise_accuracy(
     }
 
 
+def _objective_fidelity(
+    frontier_ranking: list[dict[str, Any]],
+    real_ranking: list[dict[str, Any]],
+    *,
+    field: str,
+    only_names: set[str] | None = None,
+) -> dict[str, Any]:
+    frontier_capacity = _capacity_map(frontier_ranking, field=field)
+    real_capacity = _capacity_map(real_ranking, field=field)
+    if only_names is not None:
+        frontier_capacity = {
+            name: value for name, value in frontier_capacity.items() if name in only_names
+        }
+        real_capacity = {
+            name: value for name, value in real_capacity.items() if name in only_names
+        }
+    names = sorted(set(frontier_capacity) & set(real_capacity))
+    frontier_valid = [value for value in frontier_capacity.values() if value is not None]
+    real_valid = [value for value in real_capacity.values() if value is not None]
+    if not frontier_valid or not real_valid:
+        return {
+            "rankable": False,
+            "reason": "one_or_both_surfaces_have_no_feasible_config",
+            "frontier_capacity": frontier_capacity,
+            "real_capacity": real_capacity,
+        }
+    frontier_average_ranks = _average_ranks(frontier_capacity)
+    real_average_ranks = _average_ranks(real_capacity)
+    best_frontier = max(frontier_valid)
+    best_real = max(real_valid)
+    frontier_top_set = [
+        name for name, value in frontier_capacity.items() if value == best_frontier
+    ]
+    real_top_set = [name for name, value in real_capacity.items() if value == best_real]
+    selected_real_capacities = [
+        real_capacity[name]
+        for name in frontier_top_set
+        if real_capacity.get(name) is not None
+    ]
+    regret_values = [
+        (best_real - value) / best_real
+        for value in selected_real_capacities
+        if best_real != 0
+    ]
+    return {
+        "rankable": True,
+        "capacity_field": field,
+        "frontier_top1_set": frontier_top_set,
+        "real_top1_set": real_top_set,
+        "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)),
+        "top1_set_match": set(frontier_top_set) == set(real_top_set),
+        "top1_regret_fraction_best_tie_break": min(regret_values)
+        if regret_values
+        else None,
+        "top1_regret_fraction_worst_tie_break": max(regret_values)
+        if regret_values
+        else None,
+        "spearman_rank_correlation": _pearson(
+            [frontier_average_ranks[name] for name in names],
+            [real_average_ranks[name] for name in names],
+        ),
+        "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity),
+        "frontier_capacity": frontier_capacity,
+        "real_capacity": real_capacity,
+    }
+
+
 def _request_residuals(
     frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]]
 ) -> dict[str, Any]:
@@ -1322,10 +1391,11 @@ def _pass_rate_residuals(
 ) -> dict[str, Any]:
     def index(
         results: list[dict[str, Any]], slo_name: str
-    ) -> dict[tuple[str, float], float]:
+    ) -> dict[tuple[str, float], tuple[float, bool]]:
         return {
-            (config["config"]["name"], float(load["offered_request_rate"])): float(
+            (config["config"]["name"], float(load["offered_request_rate"])): (
-                load["scores"][slo_name]["slo_pass_rate"]
+                float(load["scores"][slo_name]["slo_pass_rate"]),
+                bool(load["scores"][slo_name]["feasible"]),
             )
             for config in results
             for load in config["loads"]
@@ -1336,12 +1406,24 @@ def _pass_rate_residuals(
         frontier_index = index(frontier_results, slo_name)
         real_index = index(real_results, slo_name)
         keys = sorted(set(frontier_index) & set(real_index))
-        errors = [frontier_index[key] - real_index[key] for key in keys]
+        errors = [frontier_index[key][0] - real_index[key][0] for key in keys]
+        label_matches = [frontier_index[key][1] == real_index[key][1] for key in keys]
+        false_positives = sum(
+            frontier_index[key][1] and not real_index[key][1] for key in keys
+        )
+        false_negatives = sum(
+            not frontier_index[key][1] and real_index[key][1] for key in keys
+        )
         variants[slo_name] = {
             "matched_config_load_points": len(errors),
             "signed_error_mean": statistics.fmean(errors) if errors else None,
             "mae": statistics.fmean(abs(value) for value in errors) if errors else None,
             "max_absolute_error": max((abs(value) for value in errors), default=None),
+            "feasibility_label_accuracy": (
+                sum(label_matches) / len(label_matches) if label_matches else None
+            ),
+            "false_feasible_points": false_positives,
+            "false_infeasible_points": false_negatives,
         }
     return variants
 
@@ -1359,51 +1441,30 @@ def compare(args: argparse.Namespace) -> Path:
     for slo_name in SLO_VARIANTS:
         frontier_ranking = frontier["rankings"][slo_name]
         real_ranking = community["rankings"][slo_name]
-        frontier_ranks = _rank_map(frontier_ranking)
-        real_ranks = _rank_map(real_ranking)
-        names = sorted(set(frontier_ranks) & set(real_ranks))
-        frontier_capacity = _capacity_map(frontier_ranking)
-        real_capacity = _capacity_map(real_ranking)
-        frontier_average_ranks = _average_ranks(frontier_capacity)
-        real_average_ranks = _average_ranks(real_capacity)
-        best_frontier = max(
-            (value for value in frontier_capacity.values() if value is not None),
-            default=None,
-        )
-        best_real = max(
-            (value for value in real_capacity.values() if value is not None),
-            default=None,
-        )
-        frontier_top_set = [
-            name for name, value in frontier_capacity.items() if value == best_frontier
-        ]
-        real_top_set = [name for name, value in real_capacity.items() if value == best_real]
-        selected_real_capacities = [
-            real_capacity[name]
-            for name in frontier_top_set
-            if real_capacity[name] is not None
-        ]
-        regret_values = [
-            (best_real - value) / best_real
-            for value in selected_real_capacities
-            if best_real not in (None, 0)
-        ]
         variants[slo_name] = {
-            "frontier_top1_set": frontier_top_set,
+            "per_gpu_efficiency": _objective_fidelity(
-            "real_top1_set": real_top_set,
+                frontier_ranking,
-            "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)),
+                real_ranking,
-            "top1_set_match": set(frontier_top_set) == set(real_top_set),
+                field="maximum_tested_feasible_request_rate_per_gpu",
-            "top1_regret_fraction_best_tie_break": min(regret_values)
-            if regret_values
-            else None,
-            "top1_regret_fraction_worst_tie_break": max(regret_values)
-            if regret_values
-            else None,
-            "spearman_rank_correlation": _pearson(
-                [frontier_average_ranks[name] for name in names],
-                [real_average_ranks[name] for name in names],
             ),
-            "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity),
+            "single_replica_raw_capacity": _objective_fidelity(
+                frontier_ranking,
+                real_ranking,
+                field="maximum_tested_feasible_request_rate",
+            ),
+            "within_topology_raw_capacity": {
+                f"tp{tp}": _objective_fidelity(
+                    frontier_ranking,
+                    real_ranking,
+                    field="maximum_tested_feasible_request_rate",
+                    only_names={
+                        item["config"]["name"]
+                        for item in frontier_ranking
+                        if int(item["config"]["tp"]) == tp
+                    },
+                )
+                for tp in (4, 8)
+            },
             "frontier_ranking": frontier_ranking,
             "real_ranking": real_ranking,
         }

runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py

@@ -169,3 +169,59 @@ def test_warmup_is_disjoint_and_covers_each_input_bin() -> None:
         range(len(MODULE.INPUT_BINS) - 1)
     )
     assert {request.arrival_s for request in selected} == {0.0}
+
+
+def test_objective_fidelity_separates_raw_and_per_gpu_rankings() -> None:
+    frontier = [
+        {
+            "config": {"name": "tp8", "tp": 8},
+            "maximum_tested_feasible_request_rate": 0.4,
+            "maximum_tested_feasible_request_rate_per_gpu": 0.05,
+        },
+        {
+            "config": {"name": "tp4", "tp": 4},
+            "maximum_tested_feasible_request_rate": 0.3,
+            "maximum_tested_feasible_request_rate_per_gpu": 0.075,
+        },
+    ]
+    real = [
+        {
+            "config": {"name": "tp8", "tp": 8},
+            "maximum_tested_feasible_request_rate": 0.2,
+            "maximum_tested_feasible_request_rate_per_gpu": 0.025,
+        },
+        {
+            "config": {"name": "tp4", "tp": 4},
+            "maximum_tested_feasible_request_rate": 0.3,
+            "maximum_tested_feasible_request_rate_per_gpu": 0.075,
+        },
+    ]
+    raw = MODULE._objective_fidelity(
+        frontier,
+        real,
+        field="maximum_tested_feasible_request_rate",
+    )
+    efficiency = MODULE._objective_fidelity(
+        frontier,
+        real,
+        field="maximum_tested_feasible_request_rate_per_gpu",
+    )
+    assert raw["frontier_top1_set"] == ["tp8"]
+    assert raw["real_top1_set"] == ["tp4"]
+    assert abs(raw["top1_regret_fraction_worst_tie_break"] - 1 / 3) < 1e-12
+    assert efficiency["frontier_top1_set"] == efficiency["real_top1_set"] == ["tp4"]
+
+
+def test_objective_fidelity_marks_null_capacity_unrankable() -> None:
+    ranking = [
+        {
+            "config": {"name": "a", "tp": 4},
+            "maximum_tested_feasible_request_rate": None,
+        }
+    ]
+    result = MODULE._objective_fidelity(
+        ranking,
+        ranking,
+        field="maximum_tested_feasible_request_rate",
+    )
+    assert result["rankable"] is False