gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Skip null SLOs in boundary repeats

Browse Source
This commit is contained in:
Gahow Wang
2026-07-16 03:09:19 +08:00
parent 163fb3d9e8
commit 501ceb8171
2 changed files with 30 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py
View File

@@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1"
FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1"
COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1"
COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1"
REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v3" REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v4"
COHORT_SIZE = 64 COHORT_SIZE = 64
COHORT_SEED = 2026071501 COHORT_SEED = 2026071501
CONFIG_ORDER_SEED = 2026071502 CONFIG_ORDER_SEED = 2026071502
@@ -906,6 +906,13 @@ def _boundary_repeat_rates(
return sorted(selected) return sorted(selected)
def _slo_has_feasible_load(config_result: dict[str, Any], *, slo_name: str) -> bool:
return any(
bool(load["scores"][slo_name]["feasible"])
for load in config_result["loads"]
)
def prepare_community_repeat(args: argparse.Namespace) -> Path: def prepare_community_repeat(args: argparse.Namespace) -> Path:
"""Prepare a fresh-server reverse-order repeat of decision-boundary loads.""" """Prepare a fresh-server reverse-order repeat of decision-boundary loads."""
source_manifest = json.loads(args.source_manifest.read_text()) source_manifest = json.loads(args.source_manifest.read_text())
@@ -958,10 +965,21 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
reversed(first_execution_order), start=1 reversed(first_execution_order), start=1
): ):
name = source_record["config"]["name"] name = source_record["config"]["name"]
boundary_rates_by_slo = { excluded_unrankable_slos = [
slo_name: _boundary_repeat_rates( slo_name
for slo_name in SLO_VARIANTS
if not _slo_has_feasible_load(
first_results[name], slo_name=slo_name first_results[name], slo_name=slo_name
) )
]
boundary_rates_by_slo = {
slo_name: (
[]
if slo_name in excluded_unrankable_slos
else _boundary_repeat_rates(
first_results[name], slo_name=slo_name
)
)
for slo_name in SLO_VARIANTS for slo_name in SLO_VARIANTS
} }
boundary_rates = sorted( boundary_rates = sorted(
@@ -1003,6 +1021,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
selections.append( selections.append(
{ {
"config": name, "config": name,
"excluded_unrankable_slos": excluded_unrankable_slos,
"boundary_rates_by_slo": boundary_rates_by_slo, "boundary_rates_by_slo": boundary_rates_by_slo,
"boundary_rates": boundary_rates, "boundary_rates": boundary_rates,
"added_refinement_rates": refinement_rates, "added_refinement_rates": refinement_rates,
@@ -1044,6 +1063,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
"execution_contract": { "execution_contract": {
"schema": REPEAT_SELECTION_SCHEMA, "schema": REPEAT_SELECTION_SCHEMA,
"selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement",
"unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load",
"fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two",
"config_order": "reverse_of_first_pass", "config_order": "reverse_of_first_pass",
"rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates",

7
runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py
View File

@@ -265,6 +265,13 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() ->
assert MODULE._boundary_repeat_rates( assert MODULE._boundary_repeat_rates(
sensitivity, slo_name="linear_6k" sensitivity, slo_name="linear_6k"
) == [0.3, 0.4] ) == [0.3, 0.4]
assert MODULE._slo_has_feasible_load(
sensitivity, slo_name="linear_6k"
)
assert not MODULE._slo_has_feasible_load(
result([False, False, False, False]),
slo_name=MODULE.PRIMARY_SLO,
)
def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: