From e47bbf3e760af6a5ed41b744c0c0f7caecdb5c88 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 23:32:07 +0800 Subject: [PATCH] Compare Frontier rank under explicit objectives --- .../best_effort/fixed_cohort_rank.py | 159 ++++++++++++------ .../best_effort/test_fixed_cohort_rank.py | 56 ++++++ 2 files changed, 166 insertions(+), 49 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index cc1cf73..2aa4300 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -1184,9 +1184,11 @@ def _rank_map(ranking: list[dict[str, Any]]) -> dict[str, int]: return {item["config"]["name"]: int(item["rank"]) for item in ranking} -def _capacity_map(ranking: list[dict[str, Any]]) -> dict[str, float | None]: +def _capacity_map( + ranking: list[dict[str, Any]], *, field: str +) -> dict[str, float | None]: return { - item["config"]["name"]: item["maximum_tested_feasible_request_rate_per_gpu"] + item["config"]["name"]: item[field] for item in ranking } @@ -1263,6 +1265,73 @@ def _pairwise_accuracy( } +def _objective_fidelity( + frontier_ranking: list[dict[str, Any]], + real_ranking: list[dict[str, Any]], + *, + field: str, + only_names: set[str] | None = None, +) -> dict[str, Any]: + frontier_capacity = _capacity_map(frontier_ranking, field=field) + real_capacity = _capacity_map(real_ranking, field=field) + if only_names is not None: + frontier_capacity = { + name: value for name, value in frontier_capacity.items() if name in only_names + } + real_capacity = { + name: value for name, value in real_capacity.items() if name in only_names + } + names = sorted(set(frontier_capacity) & set(real_capacity)) + frontier_valid = [value for value in frontier_capacity.values() if value is not None] + real_valid = [value for value in real_capacity.values() if value is not None] + if not frontier_valid or not real_valid: + return { + "rankable": False, + "reason": "one_or_both_surfaces_have_no_feasible_config", + "frontier_capacity": frontier_capacity, + "real_capacity": real_capacity, + } + frontier_average_ranks = _average_ranks(frontier_capacity) + real_average_ranks = _average_ranks(real_capacity) + best_frontier = max(frontier_valid) + best_real = max(real_valid) + frontier_top_set = [ + name for name, value in frontier_capacity.items() if value == best_frontier + ] + real_top_set = [name for name, value in real_capacity.items() if value == best_real] + selected_real_capacities = [ + real_capacity[name] + for name in frontier_top_set + if real_capacity.get(name) is not None + ] + regret_values = [ + (best_real - value) / best_real + for value in selected_real_capacities + if best_real != 0 + ] + return { + "rankable": True, + "capacity_field": field, + "frontier_top1_set": frontier_top_set, + "real_top1_set": real_top_set, + "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)), + "top1_set_match": set(frontier_top_set) == set(real_top_set), + "top1_regret_fraction_best_tie_break": min(regret_values) + if regret_values + else None, + "top1_regret_fraction_worst_tie_break": max(regret_values) + if regret_values + else None, + "spearman_rank_correlation": _pearson( + [frontier_average_ranks[name] for name in names], + [real_average_ranks[name] for name in names], + ), + "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity), + "frontier_capacity": frontier_capacity, + "real_capacity": real_capacity, + } + + def _request_residuals( frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]] ) -> dict[str, Any]: @@ -1322,10 +1391,11 @@ def _pass_rate_residuals( ) -> dict[str, Any]: def index( results: list[dict[str, Any]], slo_name: str - ) -> dict[tuple[str, float], float]: + ) -> dict[tuple[str, float], tuple[float, bool]]: return { - (config["config"]["name"], float(load["offered_request_rate"])): float( - load["scores"][slo_name]["slo_pass_rate"] + (config["config"]["name"], float(load["offered_request_rate"])): ( + float(load["scores"][slo_name]["slo_pass_rate"]), + bool(load["scores"][slo_name]["feasible"]), ) for config in results for load in config["loads"] @@ -1336,12 +1406,24 @@ def _pass_rate_residuals( frontier_index = index(frontier_results, slo_name) real_index = index(real_results, slo_name) keys = sorted(set(frontier_index) & set(real_index)) - errors = [frontier_index[key] - real_index[key] for key in keys] + errors = [frontier_index[key][0] - real_index[key][0] for key in keys] + label_matches = [frontier_index[key][1] == real_index[key][1] for key in keys] + false_positives = sum( + frontier_index[key][1] and not real_index[key][1] for key in keys + ) + false_negatives = sum( + not frontier_index[key][1] and real_index[key][1] for key in keys + ) variants[slo_name] = { "matched_config_load_points": len(errors), "signed_error_mean": statistics.fmean(errors) if errors else None, "mae": statistics.fmean(abs(value) for value in errors) if errors else None, "max_absolute_error": max((abs(value) for value in errors), default=None), + "feasibility_label_accuracy": ( + sum(label_matches) / len(label_matches) if label_matches else None + ), + "false_feasible_points": false_positives, + "false_infeasible_points": false_negatives, } return variants @@ -1359,51 +1441,30 @@ def compare(args: argparse.Namespace) -> Path: for slo_name in SLO_VARIANTS: frontier_ranking = frontier["rankings"][slo_name] real_ranking = community["rankings"][slo_name] - frontier_ranks = _rank_map(frontier_ranking) - real_ranks = _rank_map(real_ranking) - names = sorted(set(frontier_ranks) & set(real_ranks)) - frontier_capacity = _capacity_map(frontier_ranking) - real_capacity = _capacity_map(real_ranking) - frontier_average_ranks = _average_ranks(frontier_capacity) - real_average_ranks = _average_ranks(real_capacity) - best_frontier = max( - (value for value in frontier_capacity.values() if value is not None), - default=None, - ) - best_real = max( - (value for value in real_capacity.values() if value is not None), - default=None, - ) - frontier_top_set = [ - name for name, value in frontier_capacity.items() if value == best_frontier - ] - real_top_set = [name for name, value in real_capacity.items() if value == best_real] - selected_real_capacities = [ - real_capacity[name] - for name in frontier_top_set - if real_capacity[name] is not None - ] - regret_values = [ - (best_real - value) / best_real - for value in selected_real_capacities - if best_real not in (None, 0) - ] variants[slo_name] = { - "frontier_top1_set": frontier_top_set, - "real_top1_set": real_top_set, - "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)), - "top1_set_match": set(frontier_top_set) == set(real_top_set), - "top1_regret_fraction_best_tie_break": min(regret_values) - if regret_values - else None, - "top1_regret_fraction_worst_tie_break": max(regret_values) - if regret_values - else None, - "spearman_rank_correlation": _pearson( - [frontier_average_ranks[name] for name in names], - [real_average_ranks[name] for name in names], + "per_gpu_efficiency": _objective_fidelity( + frontier_ranking, + real_ranking, + field="maximum_tested_feasible_request_rate_per_gpu", ), - "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity), + "single_replica_raw_capacity": _objective_fidelity( + frontier_ranking, + real_ranking, + field="maximum_tested_feasible_request_rate", + ), + "within_topology_raw_capacity": { + f"tp{tp}": _objective_fidelity( + frontier_ranking, + real_ranking, + field="maximum_tested_feasible_request_rate", + only_names={ + item["config"]["name"] + for item in frontier_ranking + if int(item["config"]["tp"]) == tp + }, + ) + for tp in (4, 8) + }, "frontier_ranking": frontier_ranking, "real_ranking": real_ranking, } diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index 0dbbd11..182d081 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -169,3 +169,59 @@ def test_warmup_is_disjoint_and_covers_each_input_bin() -> None: range(len(MODULE.INPUT_BINS) - 1) ) assert {request.arrival_s for request in selected} == {0.0} + + +def test_objective_fidelity_separates_raw_and_per_gpu_rankings() -> None: + frontier = [ + { + "config": {"name": "tp8", "tp": 8}, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + }, + { + "config": {"name": "tp4", "tp": 4}, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + }, + ] + real = [ + { + "config": {"name": "tp8", "tp": 8}, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.025, + }, + { + "config": {"name": "tp4", "tp": 4}, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + }, + ] + raw = MODULE._objective_fidelity( + frontier, + real, + field="maximum_tested_feasible_request_rate", + ) + efficiency = MODULE._objective_fidelity( + frontier, + real, + field="maximum_tested_feasible_request_rate_per_gpu", + ) + assert raw["frontier_top1_set"] == ["tp8"] + assert raw["real_top1_set"] == ["tp4"] + assert abs(raw["top1_regret_fraction_worst_tie_break"] - 1 / 3) < 1e-12 + assert efficiency["frontier_top1_set"] == efficiency["real_top1_set"] == ["tp4"] + + +def test_objective_fidelity_marks_null_capacity_unrankable() -> None: + ranking = [ + { + "config": {"name": "a", "tp": 4}, + "maximum_tested_feasible_request_rate": None, + } + ] + result = MODULE._objective_fidelity( + ranking, + ranking, + field="maximum_tested_feasible_request_rate", + ) + assert result["rankable"] is False