diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index ae9c64d..cf61958 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" -REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v1" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v2" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -768,7 +768,11 @@ def _validate_frontier_freeze(path: Path, protocol_manifest: Path) -> dict[str, raise ValueError("Frontier and community protocol manifests differ") if len(freeze.get("config_results") or []) != len(frontier_grid.GRID): raise ValueError("Frontier freeze is incomplete") - expected_rates = set(OFFERED_RATES) + protocol = json.loads(protocol_manifest.read_text()) + expected_rates = { + float(record["offered_request_rate"]) + for record in protocol["rates"].values() + } for item in freeze["config_results"]: rates = {float(load["offered_request_rate"]) for load in item["loads"]} if rates != expected_rates: @@ -913,9 +917,20 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: raise ValueError("unexpected/incomplete first-pass community freeze") if first_pass.get("run_manifest_sha256") != sha256(args.source_manifest): raise ValueError("first-pass freeze does not match the source manifest") - protocol_path = Path(source_manifest["protocol_manifest"]["path"]) - if sha256(protocol_path) != source_manifest["protocol_manifest"]["sha256"]: + source_protocol_path = Path(source_manifest["protocol_manifest"]["path"]) + if sha256(source_protocol_path) != source_manifest["protocol_manifest"]["sha256"]: raise ValueError("source protocol changed before repeat preparation") + protocol_path = ( + args.protocol_manifest.resolve() + if args.protocol_manifest is not None + else source_protocol_path + ) + frontier_freeze_path = ( + args.frontier_freeze.resolve() + if args.frontier_freeze is not None + else Path(source_manifest["frontier_freeze"]["path"]) + ) + _validate_frontier_freeze(frontier_freeze_path, protocol_path) protocol = json.loads(protocol_path.read_text()) duration_by_rate = { float(record["offered_request_rate"]): float(record["duration_s"]) @@ -941,15 +956,33 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: reversed(first_execution_order), start=1 ): name = source_record["config"]["name"] - selected_rates = _boundary_repeat_rates(first_results[name]) + boundary_rates = _boundary_repeat_rates(first_results[name]) + refinement_rates = sorted( + rate + for rate in duration_by_rate + if min(boundary_rates) < rate < max(boundary_rates) + ) + selected_rates = sorted(set(boundary_rates) | set(refinement_rates)) source_relative_order = [ float(rate) for rate in source_record["rate_order"] - if float(rate) in selected_rates + if float(rate) in boundary_rates ] - if set(source_relative_order) != set(selected_rates): + if set(source_relative_order) != set(boundary_rates): raise ValueError(f"repeat rate is absent from source order: {name}") - repeat_order = list(reversed(source_relative_order)) + reversed_boundaries = list(reversed(source_relative_order)) + if len(reversed_boundaries) == 2: + refinement_order = sorted( + refinement_rates, + reverse=reversed_boundaries[0] > reversed_boundaries[1], + ) + repeat_order = [ + reversed_boundaries[0], + *refinement_order, + reversed_boundaries[1], + ] + else: + repeat_order = [*reversed_boundaries, *reversed(refinement_rates)] records.append( { "execution_index": execution_index, @@ -963,6 +996,8 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: selections.append( { "config": name, + "boundary_rates": boundary_rates, + "added_refinement_rates": refinement_rates, "selected_rates": selected_rates, "source_relative_order": source_relative_order, "repeat_order": repeat_order, @@ -977,8 +1012,14 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: "schema": COMMUNITY_SCHEMA, "created_unix_s": time.time(), "repository": community_grid.git_fingerprint(args.repo.resolve()), - "protocol_manifest": source_manifest["protocol_manifest"], - "frontier_freeze": source_manifest["frontier_freeze"], + "protocol_manifest": { + "path": str(protocol_path), + "sha256": sha256(protocol_path), + }, + "frontier_freeze": { + "path": str(frontier_freeze_path), + "sha256": sha256(frontier_freeze_path), + }, "runtime": source_manifest["runtime"], "model": source_manifest["model"], "studies": source_manifest["studies"], @@ -994,10 +1035,10 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: }, "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, - "selection": "all_adjacent_primary_slo_feasibility_transitions", + "selection": "all_adjacent_primary_slo_feasibility_transitions_plus_between_rate_refinement", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", - "rate_order": "reverse_of_first_pass_relative_order_within_selection", + "rate_order": "reverse_first_pass_boundary_order_with_between_rate_refinement", "one_fresh_server_launch_per_config": True, "warmup": source_manifest["execution_contract"]["warmup"], "estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds, @@ -1774,6 +1815,8 @@ def parse_args() -> argparse.Namespace: repeat.add_argument("--repo", type=Path, required=True) repeat.add_argument("--source-manifest", type=Path, required=True) repeat.add_argument("--community-freeze", type=Path, required=True) + repeat.add_argument("--protocol-manifest", type=Path) + repeat.add_argument("--frontier-freeze", type=Path) repeat.add_argument("--output-root", type=Path, required=True) compare_parser = subparsers.add_parser("compare") diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index f0e70d3..b66758a 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -291,3 +291,36 @@ def test_request_slo_classification_exposes_aggregate_error_cancellation() -> No assert classification["label_accuracy"] == 0.0 assert classification["false_slo_pass_requests"] == 1 assert classification["false_slo_fail_requests"] == 1 + + +def test_frontier_freeze_validation_uses_manifest_rates(tmp_path: Path) -> None: + protocol_path = tmp_path / "protocol.json" + protocol = { + "rates": { + "low": {"offered_request_rate": 0.2}, + "high": {"offered_request_rate": 0.4}, + } + } + protocol_path.write_text(json.dumps(protocol)) + freeze_path = tmp_path / "freeze.json" + freeze_path.write_text( + json.dumps( + { + "schema": MODULE.FRONTIER_SCHEMA, + "status": "frozen_before_community_run", + "protocol_manifest_sha256": MODULE.sha256(protocol_path), + "config_results": [ + { + "config": {"name": config.name}, + "loads": [ + {"offered_request_rate": 0.2}, + {"offered_request_rate": 0.4}, + ], + } + for config in MODULE.frontier_grid.GRID + ], + } + ) + ) + validated = MODULE._validate_frontier_freeze(freeze_path, protocol_path) + assert validated["protocol_manifest_sha256"] == MODULE.sha256(protocol_path)