gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refine Frontier rank boundaries before repeats

Browse Source
This commit is contained in:
Gahow Wang
2026-07-16 01:59:37 +08:00
parent f7052edd14
commit 993cce6143
2 changed files with 88 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py
View File

@@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1"
FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1"
COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1"
COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1"
REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v1"
REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v2"
COHORT_SIZE = 64
COHORT_SEED = 2026071501
CONFIG_ORDER_SEED = 2026071502
@@ -768,7 +768,11 @@ def _validate_frontier_freeze(path: Path, protocol_manifest: Path) -> dict[str,
raise ValueError("Frontier and community protocol manifests differ")
if len(freeze.get("config_results") or []) != len(frontier_grid.GRID):
raise ValueError("Frontier freeze is incomplete")
expected_rates = set(OFFERED_RATES)
protocol = json.loads(protocol_manifest.read_text())
expected_rates = {
float(record["offered_request_rate"])
for record in protocol["rates"].values()
}
for item in freeze["config_results"]:
rates = {float(load["offered_request_rate"]) for load in item["loads"]}
if rates != expected_rates:
@@ -913,9 +917,20 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
raise ValueError("unexpected/incomplete first-pass community freeze")
if first_pass.get("run_manifest_sha256") != sha256(args.source_manifest):
raise ValueError("first-pass freeze does not match the source manifest")
protocol_path = Path(source_manifest["protocol_manifest"]["path"])
if sha256(protocol_path) != source_manifest["protocol_manifest"]["sha256"]:
source_protocol_path = Path(source_manifest["protocol_manifest"]["path"])
if sha256(source_protocol_path) != source_manifest["protocol_manifest"]["sha256"]:
raise ValueError("source protocol changed before repeat preparation")
protocol_path = (
args.protocol_manifest.resolve()
if args.protocol_manifest is not None
else source_protocol_path
)
frontier_freeze_path = (
args.frontier_freeze.resolve()
if args.frontier_freeze is not None
else Path(source_manifest["frontier_freeze"]["path"])
)
_validate_frontier_freeze(frontier_freeze_path, protocol_path)
protocol = json.loads(protocol_path.read_text())
duration_by_rate = {
float(record["offered_request_rate"]): float(record["duration_s"])
@@ -941,15 +956,33 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
reversed(first_execution_order), start=1
):
name = source_record["config"]["name"]
selected_rates = _boundary_repeat_rates(first_results[name])
boundary_rates = _boundary_repeat_rates(first_results[name])
refinement_rates = sorted(
rate
for rate in duration_by_rate
if min(boundary_rates) < rate < max(boundary_rates)
)
selected_rates = sorted(set(boundary_rates) | set(refinement_rates))
source_relative_order = [
float(rate)
for rate in source_record["rate_order"]
if float(rate) in selected_rates
if float(rate) in boundary_rates
]
if set(source_relative_order) != set(selected_rates):
if set(source_relative_order) != set(boundary_rates):
raise ValueError(f"repeat rate is absent from source order: {name}")
repeat_order = list(reversed(source_relative_order))
reversed_boundaries = list(reversed(source_relative_order))
if len(reversed_boundaries) == 2:
refinement_order = sorted(
refinement_rates,
reverse=reversed_boundaries[0] > reversed_boundaries[1],
)
repeat_order = [
reversed_boundaries[0],
*refinement_order,
reversed_boundaries[1],
]
else:
repeat_order = [*reversed_boundaries, *reversed(refinement_rates)]
records.append(
{
"execution_index": execution_index,
@@ -963,6 +996,8 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
selections.append(
{
"config": name,
"boundary_rates": boundary_rates,
"added_refinement_rates": refinement_rates,
"selected_rates": selected_rates,
"source_relative_order": source_relative_order,
"repeat_order": repeat_order,
@@ -977,8 +1012,14 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
"schema": COMMUNITY_SCHEMA,
"created_unix_s": time.time(),
"repository": community_grid.git_fingerprint(args.repo.resolve()),
"protocol_manifest": source_manifest["protocol_manifest"],
"frontier_freeze": source_manifest["frontier_freeze"],
"protocol_manifest": {
"path": str(protocol_path),
"sha256": sha256(protocol_path),
},
"frontier_freeze": {
"path": str(frontier_freeze_path),
"sha256": sha256(frontier_freeze_path),
},
"runtime": source_manifest["runtime"],
"model": source_manifest["model"],
"studies": source_manifest["studies"],
@@ -994,10 +1035,10 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path:
},
"execution_contract": {
"schema": REPEAT_SELECTION_SCHEMA,
"selection": "all_adjacent_primary_slo_feasibility_transitions",
"selection": "all_adjacent_primary_slo_feasibility_transitions_plus_between_rate_refinement",
"fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two",
"config_order": "reverse_of_first_pass",
"rate_order": "reverse_of_first_pass_relative_order_within_selection",
"rate_order": "reverse_first_pass_boundary_order_with_between_rate_refinement",
"one_fresh_server_launch_per_config": True,
"warmup": source_manifest["execution_contract"]["warmup"],
"estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds,
@@ -1774,6 +1815,8 @@ def parse_args() -> argparse.Namespace:
repeat.add_argument("--repo", type=Path, required=True)
repeat.add_argument("--source-manifest", type=Path, required=True)
repeat.add_argument("--community-freeze", type=Path, required=True)
repeat.add_argument("--protocol-manifest", type=Path)
repeat.add_argument("--frontier-freeze", type=Path)
repeat.add_argument("--output-root", type=Path, required=True)
compare_parser = subparsers.add_parser("compare")

33
runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py
View File

@@ -291,3 +291,36 @@ def test_request_slo_classification_exposes_aggregate_error_cancellation() -> No
assert classification["label_accuracy"] == 0.0
assert classification["false_slo_pass_requests"] == 1
assert classification["false_slo_fail_requests"] == 1
def test_frontier_freeze_validation_uses_manifest_rates(tmp_path: Path) -> None:
protocol_path = tmp_path / "protocol.json"
protocol = {
"rates": {
"low": {"offered_request_rate": 0.2},
"high": {"offered_request_rate": 0.4},
}
}
protocol_path.write_text(json.dumps(protocol))
freeze_path = tmp_path / "freeze.json"
freeze_path.write_text(
json.dumps(
{
"schema": MODULE.FRONTIER_SCHEMA,
"status": "frozen_before_community_run",
"protocol_manifest_sha256": MODULE.sha256(protocol_path),
"config_results": [
{
"config": {"name": config.name},
"loads": [
{"offered_request_rate": 0.2},
{"offered_request_rate": 0.4},
],
}
for config in MODULE.frontier_grid.GRID
],
}
)
)
validated = MODULE._validate_frontier_freeze(freeze_path, protocol_path)
assert validated["protocol_manifest_sha256"] == MODULE.sha256(protocol_path)