diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index cf61958..77842c7 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" -REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v2" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v3" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -879,8 +879,10 @@ def prepare_community(args: argparse.Namespace) -> Path: return path -def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]: - """Select every adjacent load pair whose primary feasibility label changes.""" +def _boundary_repeat_rates( + config_result: dict[str, Any], *, slo_name: str = PRIMARY_SLO +) -> list[float]: + """Select every adjacent load pair whose SLO feasibility label changes.""" loads = sorted( config_result["loads"], key=lambda item: float(item["offered_request_rate"]) ) @@ -889,7 +891,7 @@ def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]: raise ValueError(f"duplicate offered rate in {config_result['config']['name']}") if not rates: raise ValueError(f"no loads in {config_result['config']['name']}") - feasibility = [bool(item["scores"][PRIMARY_SLO]["feasible"]) for item in loads] + feasibility = [bool(item["scores"][slo_name]["feasible"]) for item in loads] selected: set[float] = set() for index in range(len(loads) - 1): if feasibility[index] != feasibility[index + 1]: @@ -956,11 +958,27 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: reversed(first_execution_order), start=1 ): name = source_record["config"]["name"] - boundary_rates = _boundary_repeat_rates(first_results[name]) + boundary_rates_by_slo = { + slo_name: _boundary_repeat_rates( + first_results[name], slo_name=slo_name + ) + for slo_name in SLO_VARIANTS + } + boundary_rates = sorted( + { + rate + for rates in boundary_rates_by_slo.values() + for rate in rates + } + ) refinement_rates = sorted( - rate - for rate in duration_by_rate - if min(boundary_rates) < rate < max(boundary_rates) + { + rate + for rates in boundary_rates_by_slo.values() + for low, high in zip(rates, rates[1:]) + for rate in duration_by_rate + if low < rate < high and rate not in boundary_rates + } ) selected_rates = sorted(set(boundary_rates) | set(refinement_rates)) source_relative_order = [ @@ -971,18 +989,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: if set(source_relative_order) != set(boundary_rates): raise ValueError(f"repeat rate is absent from source order: {name}") reversed_boundaries = list(reversed(source_relative_order)) - if len(reversed_boundaries) == 2: - refinement_order = sorted( - refinement_rates, - reverse=reversed_boundaries[0] > reversed_boundaries[1], - ) - repeat_order = [ - reversed_boundaries[0], - *refinement_order, - reversed_boundaries[1], - ] - else: - repeat_order = [*reversed_boundaries, *reversed(refinement_rates)] + repeat_order = [*reversed_boundaries, *reversed(refinement_rates)] records.append( { "execution_index": execution_index, @@ -996,6 +1003,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: selections.append( { "config": name, + "boundary_rates_by_slo": boundary_rates_by_slo, "boundary_rates": boundary_rates, "added_refinement_rates": refinement_rates, "selected_rates": selected_rates, @@ -1035,10 +1043,10 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: }, "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, - "selection": "all_adjacent_primary_slo_feasibility_transitions_plus_between_rate_refinement", + "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", - "rate_order": "reverse_first_pass_boundary_order_with_between_rate_refinement", + "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", "one_fresh_server_launch_per_config": True, "warmup": source_manifest["execution_contract"]["warmup"], "estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds, diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index b66758a..de99b50 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -259,6 +259,13 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> 0.2, ] + sensitivity = result([True, True, False, False]) + for load, feasible in zip(sensitivity["loads"], [True, True, True, False]): + load["scores"]["linear_6k"] = {"feasible": feasible} + assert MODULE._boundary_repeat_rates( + sensitivity, slo_name="linear_6k" + ) == [0.3, 0.4] + def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: def results(ttfts: list[float]) -> list[dict[str, object]]: