Evaluate baseline before LLM tuning
This commit is contained in:
@@ -113,7 +113,7 @@ def cmd_study_tune(args: argparse.Namespace) -> int:
|
||||
study_root = store.init_study(spec_path=spec_path, study=study)
|
||||
capability_profile = load_capability_profile(study, study_spec_path=spec_path)
|
||||
proposal_files = [Path(item).resolve() for item in (args.proposal_file or [])]
|
||||
max_trials = args.max_trials or (len(proposal_files) if proposal_files else 1)
|
||||
max_trials = args.max_trials or (len(proposal_files) if proposal_files else 2)
|
||||
if max_trials <= 0:
|
||||
raise SpecError("max_trials must be positive")
|
||||
if proposal_files and max_trials > len(proposal_files):
|
||||
@@ -134,7 +134,29 @@ def cmd_study_tune(args: argparse.Namespace) -> int:
|
||||
prompt_name = f"prompt-{state.next_trial_index:04d}"
|
||||
store.write_prompt(study.study_id, prompt_name, prompt)
|
||||
|
||||
if proposal_files:
|
||||
if (
|
||||
not proposal_files
|
||||
and not args.skip_baseline
|
||||
and state.next_trial_index == 1
|
||||
and not state.trials
|
||||
):
|
||||
proposal_source = None
|
||||
proposal_name = "baseline-0001"
|
||||
proposal_text = json.dumps(
|
||||
{
|
||||
"observation": "Evaluate the study's initial engine configuration before LLM-guided edits.",
|
||||
"diagnosis": "Baseline trial aligned with the AITuner evaluate-then-search loop.",
|
||||
"config_patch": {"env_patch": {}, "flag_patch": {}},
|
||||
"expected_effects": [
|
||||
"establish incumbent performance",
|
||||
"provide bottleneck evidence for harness-guided proposals",
|
||||
],
|
||||
"why_not_previous_failures": "No config changes are applied.",
|
||||
"should_stop": False,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
elif proposal_files:
|
||||
proposal_source = proposal_files[idx]
|
||||
proposal_text = proposal_source.read_text(encoding="utf-8")
|
||||
proposal_name = proposal_source.stem
|
||||
@@ -264,6 +286,11 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
tune.add_argument("--store-root")
|
||||
tune.add_argument("--proposal-file", action="append")
|
||||
tune.add_argument("--max-trials", type=int)
|
||||
tune.add_argument(
|
||||
"--skip-baseline",
|
||||
action="store_true",
|
||||
help="Do not automatically evaluate the initial config before LLM proposals.",
|
||||
)
|
||||
tune.set_defaults(func=cmd_study_tune)
|
||||
|
||||
worker = subparsers.add_parser("worker")
|
||||
|
||||
Reference in New Issue
Block a user