From 84584c719c84e6a0535e6f30183ab1d6274af0b7 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 00:52:13 +0800 Subject: [PATCH] Add reverse boundary repeats for Frontier comparison --- .../best_effort/fixed_cohort_rank.py | 145 ++++++++++++++++++ .../best_effort/test_fixed_cohort_rank.py | 33 ++++ 2 files changed, 178 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 2aa4300..5b8feb4 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,6 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v1" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -874,6 +875,142 @@ def prepare_community(args: argparse.Namespace) -> Path: return path +def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]: + """Select every adjacent load pair whose primary feasibility label changes.""" + loads = sorted( + config_result["loads"], key=lambda item: float(item["offered_request_rate"]) + ) + rates = [float(item["offered_request_rate"]) for item in loads] + if len(rates) != len(set(rates)): + raise ValueError(f"duplicate offered rate in {config_result['config']['name']}") + if not rates: + raise ValueError(f"no loads in {config_result['config']['name']}") + feasibility = [bool(item["scores"][PRIMARY_SLO]["feasible"]) for item in loads] + selected: set[float] = set() + for index in range(len(loads) - 1): + if feasibility[index] != feasibility[index + 1]: + selected.update((rates[index], rates[index + 1])) + if not selected: + if len(rates) == 1: + selected.add(rates[0]) + elif all(feasibility): + selected.update(rates[-2:]) + else: + selected.update(rates[:2]) + return sorted(selected) + + +def prepare_community_repeat(args: argparse.Namespace) -> Path: + """Prepare a fresh-server reverse-order repeat of decision-boundary loads.""" + source_manifest = json.loads(args.source_manifest.read_text()) + first_pass = json.loads(args.community_freeze.read_text()) + if source_manifest.get("schema") != COMMUNITY_SCHEMA: + raise ValueError("unexpected source community manifest") + if ( + first_pass.get("schema") != COMMUNITY_SCHEMA + or first_pass.get("status") != "completed" + ): + raise ValueError("unexpected/incomplete first-pass community freeze") + if first_pass.get("run_manifest_sha256") != sha256(args.source_manifest): + raise ValueError("first-pass freeze does not match the source manifest") + protocol_path = Path(source_manifest["protocol_manifest"]["path"]) + if sha256(protocol_path) != source_manifest["protocol_manifest"]["sha256"]: + raise ValueError("source protocol changed before repeat preparation") + protocol = json.loads(protocol_path.read_text()) + duration_by_rate = { + float(record["offered_request_rate"]): float(record["duration_s"]) + for record in protocol["rates"].values() + } + source_records = { + item["config"]["name"]: item for item in source_manifest["configs"] + } + first_results = { + item["config"]["name"]: item for item in first_pass["config_results"] + } + if set(source_records) != set(first_results): + raise ValueError("source manifest and first-pass config sets differ") + + output_root = args.output_root.resolve() + output_root.mkdir(parents=True, exist_ok=True) + first_execution_order = sorted( + source_manifest["configs"], key=lambda item: int(item["execution_index"]) + ) + records = [] + selections = [] + for execution_index, source_record in enumerate( + reversed(first_execution_order), start=1 + ): + name = source_record["config"]["name"] + selected_rates = _boundary_repeat_rates(first_results[name]) + source_relative_order = [ + float(rate) + for rate in source_record["rate_order"] + if float(rate) in selected_rates + ] + if set(source_relative_order) != set(selected_rates): + raise ValueError(f"repeat rate is absent from source order: {name}") + repeat_order = list(reversed(source_relative_order)) + records.append( + { + "execution_index": execution_index, + "config": source_record["config"], + "rate_order": repeat_order, + "study_path": source_record["study_path"], + "result_path": str(output_root / "results" / f"{name}.json"), + "engine_log_path": str(output_root / "runs" / name / "engine.log"), + } + ) + selections.append( + { + "config": name, + "selected_rates": selected_rates, + "source_relative_order": source_relative_order, + "repeat_order": repeat_order, + } + ) + total_replay_seconds = sum( + duration_by_rate[float(rate)] + for record in records + for rate in record["rate_order"] + ) + manifest = { + "schema": COMMUNITY_SCHEMA, + "created_unix_s": time.time(), + "repository": community_grid.git_fingerprint(args.repo.resolve()), + "protocol_manifest": source_manifest["protocol_manifest"], + "frontier_freeze": source_manifest["frontier_freeze"], + "runtime": source_manifest["runtime"], + "model": source_manifest["model"], + "studies": source_manifest["studies"], + "repeat_of": { + "source_manifest": { + "path": str(args.source_manifest.resolve()), + "sha256": sha256(args.source_manifest), + }, + "first_pass_freeze": { + "path": str(args.community_freeze.resolve()), + "sha256": sha256(args.community_freeze), + }, + }, + "execution_contract": { + "schema": REPEAT_SELECTION_SCHEMA, + "selection": "all_adjacent_primary_slo_feasibility_transitions", + "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", + "config_order": "reverse_of_first_pass", + "rate_order": "reverse_of_first_pass_relative_order_within_selection", + "one_fresh_server_launch_per_config": True, + "warmup": source_manifest["execution_contract"]["warmup"], + "estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds, + "selections": selections, + }, + "configs": records, + } + path = output_root / "run_manifest.json" + write_json(path, manifest) + print(path) + return path + + def _materialize_real_requests( *, all_requests: list[TraceRequest], @@ -1539,6 +1676,12 @@ def parse_args() -> argparse.Namespace: run_real = subparsers.add_parser("run-community") run_real.add_argument("--manifest", type=Path, required=True) + repeat = subparsers.add_parser("prepare-community-repeat") + repeat.add_argument("--repo", type=Path, required=True) + repeat.add_argument("--source-manifest", type=Path, required=True) + repeat.add_argument("--community-freeze", type=Path, required=True) + repeat.add_argument("--output-root", type=Path, required=True) + compare_parser = subparsers.add_parser("compare") compare_parser.add_argument("--frontier-freeze", type=Path, required=True) compare_parser.add_argument("--community-freeze", type=Path, required=True) @@ -1562,6 +1705,8 @@ def main() -> None: prepare_community(args) elif args.command == "run-community": run_community(args) + elif args.command == "prepare-community-repeat": + prepare_community_repeat(args) elif args.command == "compare": compare(args) else: diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index 182d081..cb9c030 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -225,3 +225,36 @@ def test_objective_fidelity_marks_null_capacity_unrankable() -> None: field="maximum_tested_feasible_request_rate", ) assert result["rankable"] is False + + +def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> None: + def result(feasibility: list[bool]) -> dict[str, object]: + return { + "config": {"name": "test"}, + "loads": [ + { + "offered_request_rate": rate, + "scores": {MODULE.PRIMARY_SLO: {"feasible": feasible}}, + } + for rate, feasible in zip((0.1, 0.2, 0.3, 0.4), feasibility) + ], + } + + assert MODULE._boundary_repeat_rates(result([True, True, False, False])) == [ + 0.2, + 0.3, + ] + assert MODULE._boundary_repeat_rates(result([True, False, True, False])) == [ + 0.1, + 0.2, + 0.3, + 0.4, + ] + assert MODULE._boundary_repeat_rates(result([True, True, True, True])) == [ + 0.3, + 0.4, + ] + assert MODULE._boundary_repeat_rates(result([False, False, False, False])) == [ + 0.1, + 0.2, + ]