Repeat boundaries for every registered SLO
This commit is contained in:
@@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1"
|
|||||||
FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1"
|
FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1"
|
||||||
COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1"
|
COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1"
|
||||||
COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1"
|
COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1"
|
||||||
REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v2"
|
REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v3"
|
||||||
COHORT_SIZE = 64
|
COHORT_SIZE = 64
|
||||||
COHORT_SEED = 2026071501
|
COHORT_SEED = 2026071501
|
||||||
CONFIG_ORDER_SEED = 2026071502
|
CONFIG_ORDER_SEED = 2026071502
|
||||||
@@ -879,8 +879,10 @@ def prepare_community(args: argparse.Namespace) -> Path:
|
|||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]:
|
def _boundary_repeat_rates(
|
||||||
"""Select every adjacent load pair whose primary feasibility label changes."""
|
config_result: dict[str, Any], *, slo_name: str = PRIMARY_SLO
|
||||||
|
) -> list[float]:
|
||||||
|
"""Select every adjacent load pair whose SLO feasibility label changes."""
|
||||||
loads = sorted(
|
loads = sorted(
|
||||||
config_result["loads"], key=lambda item: float(item["offered_request_rate"])
|
config_result["loads"], key=lambda item: float(item["offered_request_rate"])
|
||||||
)
|
)
|
||||||
@@ -889,7 +891,7 @@ def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]:
|
|||||||
raise ValueError(f"duplicate offered rate in {config_result['config']['name']}")
|
raise ValueError(f"duplicate offered rate in {config_result['config']['name']}")
|
||||||
if not rates:
|
if not rates:
|
||||||
raise ValueError(f"no loads in {config_result['config']['name']}")
|
raise ValueError(f"no loads in {config_result['config']['name']}")
|
||||||
feasibility = [bool(item["scores"][PRIMARY_SLO]["feasible"]) for item in loads]
|
feasibility = [bool(item["scores"][slo_name]["feasible"]) for item in loads]
|
||||||
selected: set[float] = set()
|
selected: set[float] = set()
|
||||||
for index in range(len(loads) - 1):
|
for index in range(len(loads) - 1):
|
||||||
if feasibility[index] != feasibility[index + 1]:
|
if feasibility[index] != feasibility[index + 1]:
|
||||||
@@ -956,11 +958,27 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
|
|||||||
reversed(first_execution_order), start=1
|
reversed(first_execution_order), start=1
|
||||||
):
|
):
|
||||||
name = source_record["config"]["name"]
|
name = source_record["config"]["name"]
|
||||||
boundary_rates = _boundary_repeat_rates(first_results[name])
|
boundary_rates_by_slo = {
|
||||||
refinement_rates = sorted(
|
slo_name: _boundary_repeat_rates(
|
||||||
|
first_results[name], slo_name=slo_name
|
||||||
|
)
|
||||||
|
for slo_name in SLO_VARIANTS
|
||||||
|
}
|
||||||
|
boundary_rates = sorted(
|
||||||
|
{
|
||||||
rate
|
rate
|
||||||
|
for rates in boundary_rates_by_slo.values()
|
||||||
|
for rate in rates
|
||||||
|
}
|
||||||
|
)
|
||||||
|
refinement_rates = sorted(
|
||||||
|
{
|
||||||
|
rate
|
||||||
|
for rates in boundary_rates_by_slo.values()
|
||||||
|
for low, high in zip(rates, rates[1:])
|
||||||
for rate in duration_by_rate
|
for rate in duration_by_rate
|
||||||
if min(boundary_rates) < rate < max(boundary_rates)
|
if low < rate < high and rate not in boundary_rates
|
||||||
|
}
|
||||||
)
|
)
|
||||||
selected_rates = sorted(set(boundary_rates) | set(refinement_rates))
|
selected_rates = sorted(set(boundary_rates) | set(refinement_rates))
|
||||||
source_relative_order = [
|
source_relative_order = [
|
||||||
@@ -971,17 +989,6 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
|
|||||||
if set(source_relative_order) != set(boundary_rates):
|
if set(source_relative_order) != set(boundary_rates):
|
||||||
raise ValueError(f"repeat rate is absent from source order: {name}")
|
raise ValueError(f"repeat rate is absent from source order: {name}")
|
||||||
reversed_boundaries = list(reversed(source_relative_order))
|
reversed_boundaries = list(reversed(source_relative_order))
|
||||||
if len(reversed_boundaries) == 2:
|
|
||||||
refinement_order = sorted(
|
|
||||||
refinement_rates,
|
|
||||||
reverse=reversed_boundaries[0] > reversed_boundaries[1],
|
|
||||||
)
|
|
||||||
repeat_order = [
|
|
||||||
reversed_boundaries[0],
|
|
||||||
*refinement_order,
|
|
||||||
reversed_boundaries[1],
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
repeat_order = [*reversed_boundaries, *reversed(refinement_rates)]
|
repeat_order = [*reversed_boundaries, *reversed(refinement_rates)]
|
||||||
records.append(
|
records.append(
|
||||||
{
|
{
|
||||||
@@ -996,6 +1003,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
|
|||||||
selections.append(
|
selections.append(
|
||||||
{
|
{
|
||||||
"config": name,
|
"config": name,
|
||||||
|
"boundary_rates_by_slo": boundary_rates_by_slo,
|
||||||
"boundary_rates": boundary_rates,
|
"boundary_rates": boundary_rates,
|
||||||
"added_refinement_rates": refinement_rates,
|
"added_refinement_rates": refinement_rates,
|
||||||
"selected_rates": selected_rates,
|
"selected_rates": selected_rates,
|
||||||
@@ -1035,10 +1043,10 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
|
|||||||
},
|
},
|
||||||
"execution_contract": {
|
"execution_contract": {
|
||||||
"schema": REPEAT_SELECTION_SCHEMA,
|
"schema": REPEAT_SELECTION_SCHEMA,
|
||||||
"selection": "all_adjacent_primary_slo_feasibility_transitions_plus_between_rate_refinement",
|
"selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement",
|
||||||
"fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two",
|
"fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two",
|
||||||
"config_order": "reverse_of_first_pass",
|
"config_order": "reverse_of_first_pass",
|
||||||
"rate_order": "reverse_first_pass_boundary_order_with_between_rate_refinement",
|
"rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates",
|
||||||
"one_fresh_server_launch_per_config": True,
|
"one_fresh_server_launch_per_config": True,
|
||||||
"warmup": source_manifest["execution_contract"]["warmup"],
|
"warmup": source_manifest["execution_contract"]["warmup"],
|
||||||
"estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds,
|
"estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds,
|
||||||
|
|||||||
@@ -259,6 +259,13 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() ->
|
|||||||
0.2,
|
0.2,
|
||||||
]
|
]
|
||||||
|
|
||||||
|
sensitivity = result([True, True, False, False])
|
||||||
|
for load, feasible in zip(sensitivity["loads"], [True, True, True, False]):
|
||||||
|
load["scores"]["linear_6k"] = {"feasible": feasible}
|
||||||
|
assert MODULE._boundary_repeat_rates(
|
||||||
|
sensitivity, slo_name="linear_6k"
|
||||||
|
) == [0.3, 0.4]
|
||||||
|
|
||||||
|
|
||||||
def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None:
|
def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None:
|
||||||
def results(ttfts: list[float]) -> list[dict[str, object]]:
|
def results(ttfts: list[float]) -> list[dict[str, object]]:
|
||||||
|
|||||||
Reference in New Issue
Block a user