gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add reverse boundary repeats for Frontier comparison

Browse Source
This commit is contained in:
Gahow Wang
2026-07-16 00:52:13 +08:00
parent e47bbf3e76
commit 84584c719c
2 changed files with 178 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py
View File

@@ -52,6 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1"
FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1"
COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1"
COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1"
REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v1"
COHORT_SIZE = 64
COHORT_SEED = 2026071501
CONFIG_ORDER_SEED = 2026071502
@@ -874,6 +875,142 @@ def prepare_community(args: argparse.Namespace) -> Path:
return path
def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]:
"""Select every adjacent load pair whose primary feasibility label changes."""
loads = sorted(
config_result["loads"], key=lambda item: float(item["offered_request_rate"])
)
rates = [float(item["offered_request_rate"]) for item in loads]
if len(rates) != len(set(rates)):
raise ValueError(f"duplicate offered rate in {config_result['config']['name']}")
if not rates:
raise ValueError(f"no loads in {config_result['config']['name']}")
feasibility = [bool(item["scores"][PRIMARY_SLO]["feasible"]) for item in loads]
selected: set[float] = set()
for index in range(len(loads) - 1):
if feasibility[index] != feasibility[index + 1]:
selected.update((rates[index], rates[index + 1]))
if not selected:
if len(rates) == 1:
selected.add(rates[0])
elif all(feasibility):
selected.update(rates[-2:])
else:
selected.update(rates[:2])
return sorted(selected)
def prepare_community_repeat(args: argparse.Namespace) -> Path:
"""Prepare a fresh-server reverse-order repeat of decision-boundary loads."""
source_manifest = json.loads(args.source_manifest.read_text())
first_pass = json.loads(args.community_freeze.read_text())
if source_manifest.get("schema") != COMMUNITY_SCHEMA:
raise ValueError("unexpected source community manifest")
if (
first_pass.get("schema") != COMMUNITY_SCHEMA
or first_pass.get("status") != "completed"
):
raise ValueError("unexpected/incomplete first-pass community freeze")
if first_pass.get("run_manifest_sha256") != sha256(args.source_manifest):
raise ValueError("first-pass freeze does not match the source manifest")
protocol_path = Path(source_manifest["protocol_manifest"]["path"])
if sha256(protocol_path) != source_manifest["protocol_manifest"]["sha256"]:
raise ValueError("source protocol changed before repeat preparation")
protocol = json.loads(protocol_path.read_text())
duration_by_rate = {
float(record["offered_request_rate"]): float(record["duration_s"])
for record in protocol["rates"].values()
}
source_records = {
item["config"]["name"]: item for item in source_manifest["configs"]
}
first_results = {
item["config"]["name"]: item for item in first_pass["config_results"]
}
if set(source_records) != set(first_results):
raise ValueError("source manifest and first-pass config sets differ")
output_root = args.output_root.resolve()
output_root.mkdir(parents=True, exist_ok=True)
first_execution_order = sorted(
source_manifest["configs"], key=lambda item: int(item["execution_index"])
)
records = []
selections = []
for execution_index, source_record in enumerate(
reversed(first_execution_order), start=1
):
name = source_record["config"]["name"]
selected_rates = _boundary_repeat_rates(first_results[name])
source_relative_order = [
float(rate)
for rate in source_record["rate_order"]
if float(rate) in selected_rates
]
if set(source_relative_order) != set(selected_rates):
raise ValueError(f"repeat rate is absent from source order: {name}")
repeat_order = list(reversed(source_relative_order))
records.append(
{
"execution_index": execution_index,
"config": source_record["config"],
"rate_order": repeat_order,
"study_path": source_record["study_path"],
"result_path": str(output_root / "results" / f"{name}.json"),
"engine_log_path": str(output_root / "runs" / name / "engine.log"),
}
)
selections.append(
{
"config": name,
"selected_rates": selected_rates,
"source_relative_order": source_relative_order,
"repeat_order": repeat_order,
}
)
total_replay_seconds = sum(
duration_by_rate[float(rate)]
for record in records
for rate in record["rate_order"]
)
manifest = {
"schema": COMMUNITY_SCHEMA,
"created_unix_s": time.time(),
"repository": community_grid.git_fingerprint(args.repo.resolve()),
"protocol_manifest": source_manifest["protocol_manifest"],
"frontier_freeze": source_manifest["frontier_freeze"],
"runtime": source_manifest["runtime"],
"model": source_manifest["model"],
"studies": source_manifest["studies"],
"repeat_of": {
"source_manifest": {
"path": str(args.source_manifest.resolve()),
"sha256": sha256(args.source_manifest),
},
"first_pass_freeze": {
"path": str(args.community_freeze.resolve()),
"sha256": sha256(args.community_freeze),
},
},
"execution_contract": {
"schema": REPEAT_SELECTION_SCHEMA,
"selection": "all_adjacent_primary_slo_feasibility_transitions",
"fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two",
"config_order": "reverse_of_first_pass",
"rate_order": "reverse_of_first_pass_relative_order_within_selection",
"one_fresh_server_launch_per_config": True,
"warmup": source_manifest["execution_contract"]["warmup"],
"estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds,
"selections": selections,
},
"configs": records,
}
path = output_root / "run_manifest.json"
write_json(path, manifest)
print(path)
return path
def _materialize_real_requests(
*,
all_requests: list[TraceRequest],
@@ -1539,6 +1676,12 @@ def parse_args() -> argparse.Namespace:
run_real = subparsers.add_parser("run-community")
run_real.add_argument("--manifest", type=Path, required=True)
repeat = subparsers.add_parser("prepare-community-repeat")
repeat.add_argument("--repo", type=Path, required=True)
repeat.add_argument("--source-manifest", type=Path, required=True)
repeat.add_argument("--community-freeze", type=Path, required=True)
repeat.add_argument("--output-root", type=Path, required=True)
compare_parser = subparsers.add_parser("compare")
compare_parser.add_argument("--frontier-freeze", type=Path, required=True)
compare_parser.add_argument("--community-freeze", type=Path, required=True)
@@ -1562,6 +1705,8 @@ def main() -> None:
prepare_community(args)
elif args.command == "run-community":
run_community(args)
elif args.command == "prepare-community-repeat":
prepare_community_repeat(args)
elif args.command == "compare":
compare(args)
else:

33
runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py
View File

@@ -225,3 +225,36 @@ def test_objective_fidelity_marks_null_capacity_unrankable() -> None:
field="maximum_tested_feasible_request_rate",
)
assert result["rankable"] is False
def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> None:
def result(feasibility: list[bool]) -> dict[str, object]:
return {
"config": {"name": "test"},
"loads": [
{
"offered_request_rate": rate,
"scores": {MODULE.PRIMARY_SLO: {"feasible": feasible}},
}
for rate, feasible in zip((0.1, 0.2, 0.3, 0.4), feasibility)
],
}
assert MODULE._boundary_repeat_rates(result([True, True, False, False])) == [
0.2,
0.3,
]
assert MODULE._boundary_repeat_rates(result([True, False, True, False])) == [
0.1,
0.2,
0.3,
0.4,
]
assert MODULE._boundary_repeat_rates(result([True, True, True, True])) == [
0.3,
0.4,
]
assert MODULE._boundary_repeat_rates(result([False, False, False, False])) == [
0.1,
0.2,
]