gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare Frontier rank under explicit objectives

Browse Source
This commit is contained in:
Gahow Wang
2026-07-15 23:32:07 +08:00
parent ef5c17e6ec
commit e47bbf3e76
2 changed files with 166 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py
View File

@@ -1184,9 +1184,11 @@ def _rank_map(ranking: list[dict[str, Any]]) -> dict[str, int]:
return {item["config"]["name"]: int(item["rank"]) for item in ranking}
def _capacity_map(ranking: list[dict[str, Any]]) -> dict[str, float | None]:
def _capacity_map(
ranking: list[dict[str, Any]], *, field: str
) -> dict[str, float | None]:
return {
item["config"]["name"]: item["maximum_tested_feasible_request_rate_per_gpu"]
item["config"]["name"]: item[field]
for item in ranking
}
@@ -1263,6 +1265,73 @@ def _pairwise_accuracy(
}
def _objective_fidelity(
frontier_ranking: list[dict[str, Any]],
real_ranking: list[dict[str, Any]],
*,
field: str,
only_names: set[str] | None = None,
) -> dict[str, Any]:
frontier_capacity = _capacity_map(frontier_ranking, field=field)
real_capacity = _capacity_map(real_ranking, field=field)
if only_names is not None:
frontier_capacity = {
name: value for name, value in frontier_capacity.items() if name in only_names
}
real_capacity = {
name: value for name, value in real_capacity.items() if name in only_names
}
names = sorted(set(frontier_capacity) & set(real_capacity))
frontier_valid = [value for value in frontier_capacity.values() if value is not None]
real_valid = [value for value in real_capacity.values() if value is not None]
if not frontier_valid or not real_valid:
return {
"rankable": False,
"reason": "one_or_both_surfaces_have_no_feasible_config",
"frontier_capacity": frontier_capacity,
"real_capacity": real_capacity,
}
frontier_average_ranks = _average_ranks(frontier_capacity)
real_average_ranks = _average_ranks(real_capacity)
best_frontier = max(frontier_valid)
best_real = max(real_valid)
frontier_top_set = [
name for name, value in frontier_capacity.items() if value == best_frontier
]
real_top_set = [name for name, value in real_capacity.items() if value == best_real]
selected_real_capacities = [
real_capacity[name]
for name in frontier_top_set
if real_capacity.get(name) is not None
]
regret_values = [
(best_real - value) / best_real
for value in selected_real_capacities
if best_real != 0
]
return {
"rankable": True,
"capacity_field": field,
"frontier_top1_set": frontier_top_set,
"real_top1_set": real_top_set,
"top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)),
"top1_set_match": set(frontier_top_set) == set(real_top_set),
"top1_regret_fraction_best_tie_break": min(regret_values)
if regret_values
else None,
"top1_regret_fraction_worst_tie_break": max(regret_values)
if regret_values
else None,
"spearman_rank_correlation": _pearson(
[frontier_average_ranks[name] for name in names],
[real_average_ranks[name] for name in names],
),
"pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity),
"frontier_capacity": frontier_capacity,
"real_capacity": real_capacity,
}
def _request_residuals(
frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]]
) -> dict[str, Any]:
@@ -1322,10 +1391,11 @@ def _pass_rate_residuals(
) -> dict[str, Any]:
def index(
results: list[dict[str, Any]], slo_name: str
) -> dict[tuple[str, float], float]:
) -> dict[tuple[str, float], tuple[float, bool]]:
return {
(config["config"]["name"], float(load["offered_request_rate"])): float(
load["scores"][slo_name]["slo_pass_rate"]
(config["config"]["name"], float(load["offered_request_rate"])): (
float(load["scores"][slo_name]["slo_pass_rate"]),
bool(load["scores"][slo_name]["feasible"]),
)
for config in results
for load in config["loads"]
@@ -1336,12 +1406,24 @@ def _pass_rate_residuals(
frontier_index = index(frontier_results, slo_name)
real_index = index(real_results, slo_name)
keys = sorted(set(frontier_index) & set(real_index))
errors = [frontier_index[key] - real_index[key] for key in keys]
errors = [frontier_index[key][0] - real_index[key][0] for key in keys]
label_matches = [frontier_index[key][1] == real_index[key][1] for key in keys]
false_positives = sum(
frontier_index[key][1] and not real_index[key][1] for key in keys
)
false_negatives = sum(
not frontier_index[key][1] and real_index[key][1] for key in keys
)
variants[slo_name] = {
"matched_config_load_points": len(errors),
"signed_error_mean": statistics.fmean(errors) if errors else None,
"mae": statistics.fmean(abs(value) for value in errors) if errors else None,
"max_absolute_error": max((abs(value) for value in errors), default=None),
"feasibility_label_accuracy": (
sum(label_matches) / len(label_matches) if label_matches else None
),
"false_feasible_points": false_positives,
"false_infeasible_points": false_negatives,
}
return variants
@@ -1359,51 +1441,30 @@ def compare(args: argparse.Namespace) -> Path:
for slo_name in SLO_VARIANTS:
frontier_ranking = frontier["rankings"][slo_name]
real_ranking = community["rankings"][slo_name]
frontier_ranks = _rank_map(frontier_ranking)
real_ranks = _rank_map(real_ranking)
names = sorted(set(frontier_ranks) & set(real_ranks))
frontier_capacity = _capacity_map(frontier_ranking)
real_capacity = _capacity_map(real_ranking)
frontier_average_ranks = _average_ranks(frontier_capacity)
real_average_ranks = _average_ranks(real_capacity)
best_frontier = max(
(value for value in frontier_capacity.values() if value is not None),
default=None,
)
best_real = max(
(value for value in real_capacity.values() if value is not None),
default=None,
)
frontier_top_set = [
name for name, value in frontier_capacity.items() if value == best_frontier
]
real_top_set = [name for name, value in real_capacity.items() if value == best_real]
selected_real_capacities = [
real_capacity[name]
for name in frontier_top_set
if real_capacity[name] is not None
]
regret_values = [
(best_real - value) / best_real
for value in selected_real_capacities
if best_real not in (None, 0)
]
variants[slo_name] = {
"frontier_top1_set": frontier_top_set,
"real_top1_set": real_top_set,
"top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)),
"top1_set_match": set(frontier_top_set) == set(real_top_set),
"top1_regret_fraction_best_tie_break": min(regret_values)
if regret_values
else None,
"top1_regret_fraction_worst_tie_break": max(regret_values)
if regret_values
else None,
"spearman_rank_correlation": _pearson(
[frontier_average_ranks[name] for name in names],
[real_average_ranks[name] for name in names],
"per_gpu_efficiency": _objective_fidelity(
frontier_ranking,
real_ranking,
field="maximum_tested_feasible_request_rate_per_gpu",
),
"pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity),
"single_replica_raw_capacity": _objective_fidelity(
frontier_ranking,
real_ranking,
field="maximum_tested_feasible_request_rate",
),
"within_topology_raw_capacity": {
f"tp{tp}": _objective_fidelity(
frontier_ranking,
real_ranking,
field="maximum_tested_feasible_request_rate",
only_names={
item["config"]["name"]
for item in frontier_ranking
if int(item["config"]["tp"]) == tp
},
)
for tp in (4, 8)
},
"frontier_ranking": frontier_ranking,
"real_ranking": real_ranking,
}

56
runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py
View File

@@ -169,3 +169,59 @@ def test_warmup_is_disjoint_and_covers_each_input_bin() -> None:
range(len(MODULE.INPUT_BINS) - 1)
)
assert {request.arrival_s for request in selected} == {0.0}
def test_objective_fidelity_separates_raw_and_per_gpu_rankings() -> None:
frontier = [
{
"config": {"name": "tp8", "tp": 8},
"maximum_tested_feasible_request_rate": 0.4,
"maximum_tested_feasible_request_rate_per_gpu": 0.05,
},
{
"config": {"name": "tp4", "tp": 4},
"maximum_tested_feasible_request_rate": 0.3,
"maximum_tested_feasible_request_rate_per_gpu": 0.075,
},
]
real = [
{
"config": {"name": "tp8", "tp": 8},
"maximum_tested_feasible_request_rate": 0.2,
"maximum_tested_feasible_request_rate_per_gpu": 0.025,
},
{
"config": {"name": "tp4", "tp": 4},
"maximum_tested_feasible_request_rate": 0.3,
"maximum_tested_feasible_request_rate_per_gpu": 0.075,
},
]
raw = MODULE._objective_fidelity(
frontier,
real,
field="maximum_tested_feasible_request_rate",
)
efficiency = MODULE._objective_fidelity(
frontier,
real,
field="maximum_tested_feasible_request_rate_per_gpu",
)
assert raw["frontier_top1_set"] == ["tp8"]
assert raw["real_top1_set"] == ["tp4"]
assert abs(raw["top1_regret_fraction_worst_tie_break"] - 1 / 3) < 1e-12
assert efficiency["frontier_top1_set"] == efficiency["real_top1_set"] == ["tp4"]
def test_objective_fidelity_marks_null_capacity_unrankable() -> None:
ranking = [
{
"config": {"name": "a", "tp": 4},
"maximum_tested_feasible_request_rate": None,
}
]
result = MODULE._objective_fidelity(
ranking,
ranking,
field="maximum_tested_feasible_request_rate",
)
assert result["rankable"] is False