Evaluate baseline before LLM tuning

This commit is contained in:
2026-04-25 17:14:05 +08:00
parent 2d7ebe50ee
commit 6c04b9dbbc
3 changed files with 97 additions and 2 deletions

View File

@@ -113,7 +113,7 @@ def cmd_study_tune(args: argparse.Namespace) -> int:
study_root = store.init_study(spec_path=spec_path, study=study)
capability_profile = load_capability_profile(study, study_spec_path=spec_path)
proposal_files = [Path(item).resolve() for item in (args.proposal_file or [])]
max_trials = args.max_trials or (len(proposal_files) if proposal_files else 1)
max_trials = args.max_trials or (len(proposal_files) if proposal_files else 2)
if max_trials <= 0:
raise SpecError("max_trials must be positive")
if proposal_files and max_trials > len(proposal_files):
@@ -134,7 +134,29 @@ def cmd_study_tune(args: argparse.Namespace) -> int:
prompt_name = f"prompt-{state.next_trial_index:04d}"
store.write_prompt(study.study_id, prompt_name, prompt)
if proposal_files:
if (
not proposal_files
and not args.skip_baseline
and state.next_trial_index == 1
and not state.trials
):
proposal_source = None
proposal_name = "baseline-0001"
proposal_text = json.dumps(
{
"observation": "Evaluate the study's initial engine configuration before LLM-guided edits.",
"diagnosis": "Baseline trial aligned with the AITuner evaluate-then-search loop.",
"config_patch": {"env_patch": {}, "flag_patch": {}},
"expected_effects": [
"establish incumbent performance",
"provide bottleneck evidence for harness-guided proposals",
],
"why_not_previous_failures": "No config changes are applied.",
"should_stop": False,
},
ensure_ascii=False,
)
elif proposal_files:
proposal_source = proposal_files[idx]
proposal_text = proposal_source.read_text(encoding="utf-8")
proposal_name = proposal_source.stem
@@ -264,6 +286,11 @@ def build_parser() -> argparse.ArgumentParser:
tune.add_argument("--store-root")
tune.add_argument("--proposal-file", action="append")
tune.add_argument("--max-trials", type=int)
tune.add_argument(
"--skip-baseline",
action="store_true",
help="Do not automatically evaluate the initial config before LLM proposals.",
)
tune.set_defaults(func=cmd_study_tune)
worker = subparsers.add_parser("worker")