diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 77842c7..33a899f 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" -REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v3" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v4" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -906,6 +906,13 @@ def _boundary_repeat_rates( return sorted(selected) +def _slo_has_feasible_load(config_result: dict[str, Any], *, slo_name: str) -> bool: + return any( + bool(load["scores"][slo_name]["feasible"]) + for load in config_result["loads"] + ) + + def prepare_community_repeat(args: argparse.Namespace) -> Path: """Prepare a fresh-server reverse-order repeat of decision-boundary loads.""" source_manifest = json.loads(args.source_manifest.read_text()) @@ -958,10 +965,21 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: reversed(first_execution_order), start=1 ): name = source_record["config"]["name"] - boundary_rates_by_slo = { - slo_name: _boundary_repeat_rates( + excluded_unrankable_slos = [ + slo_name + for slo_name in SLO_VARIANTS + if not _slo_has_feasible_load( first_results[name], slo_name=slo_name ) + ] + boundary_rates_by_slo = { + slo_name: ( + [] + if slo_name in excluded_unrankable_slos + else _boundary_repeat_rates( + first_results[name], slo_name=slo_name + ) + ) for slo_name in SLO_VARIANTS } boundary_rates = sorted( @@ -1003,6 +1021,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: selections.append( { "config": name, + "excluded_unrankable_slos": excluded_unrankable_slos, "boundary_rates_by_slo": boundary_rates_by_slo, "boundary_rates": boundary_rates, "added_refinement_rates": refinement_rates, @@ -1044,6 +1063,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index de99b50..f764251 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -265,6 +265,13 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> assert MODULE._boundary_repeat_rates( sensitivity, slo_name="linear_6k" ) == [0.3, 0.4] + assert MODULE._slo_has_feasible_load( + sensitivity, slo_name="linear_6k" + ) + assert not MODULE._slo_has_feasible_load( + result([False, False, False, False]), + slo_name=MODULE.PRIMARY_SLO, + ) def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: