Persist harness candidate set snapshots

This commit is contained in:
2026-06-26 22:17:47 +08:00
parent 5080b50315
commit 2937539b49
4 changed files with 72 additions and 3 deletions

View File

@@ -68,6 +68,42 @@ def _reject_repeated_effective_config(
)
def _harness_snapshot_payload(
*,
study: StudySpec,
state: StudyState,
harness_context: dict[str, object],
) -> dict[str, object]:
experiment_plan = harness_context.get("experiment_plan")
if not isinstance(experiment_plan, dict):
experiment_plan = {}
candidate_set = experiment_plan.get("candidate_set")
if not isinstance(candidate_set, dict):
candidate_set = {}
return {
"schema_version": 1,
"study_id": study.study_id,
"iteration": state.next_trial_index,
"planner_version": experiment_plan.get("planner_version"),
"candidate_set_hash": candidate_set.get("candidate_set_hash"),
"state_ref": {
"best_trial_id": state.best_trial_id,
"best_parallel_size": state.best_parallel_size,
"best_request_rate": state.best_request_rate,
"best_request_rate_per_gpu": state.best_request_rate_per_gpu,
"next_trial_index": state.next_trial_index,
"trial_count": len(state.trials),
},
"candidate_set": candidate_set,
"decisions": {
"next_action": experiment_plan.get("next_action"),
"harness_proposal": harness_context.get("harness_proposal"),
"harness_stop": harness_context.get("harness_stop"),
"stop_authority": harness_context.get("stop_authority"),
},
}
def _latency_percentiles(summary: object, metric: str) -> dict[str, float]:
if not isinstance(summary, dict):
return {}
@@ -289,6 +325,16 @@ def cmd_study_tune(args: argparse.Namespace) -> int:
if study.llm.use_harness
else None
)
if harness_context is not None:
store.write_harness_snapshot(
study.study_id,
f"candidate-set-{state.next_trial_index:04d}",
_harness_snapshot_payload(
study=study,
state=state,
harness_context=harness_context,
),
)
prompt = build_prompt(
study=study,
window_summary=window_summary,

View File

@@ -27,7 +27,7 @@ class StudyStore:
def init_study(self, *, spec_path: Path, study: StudySpec) -> Path:
root = self.study_root(study.study_id)
for rel in ("prompts", "proposals", "trials", "results"):
for rel in ("prompts", "proposals", "trials", "results", "harness"):
(root / rel).mkdir(parents=True, exist_ok=True)
(root / "study_spec.source").write_text(str(spec_path.resolve()) + "\n", encoding="utf-8")
self.write_json(root / "study_spec.snapshot.json", to_jsonable(study))
@@ -70,6 +70,16 @@ class StudyStore:
self.write_json(path, to_jsonable(proposal))
return path
def write_harness_snapshot(
self,
study_id: str,
snapshot_name: str,
payload: dict[str, Any],
) -> Path:
path = self.study_root(study_id) / "harness" / f"{snapshot_name}.json"
self.write_json(path, payload)
return path
def materialize_trial(
self,
*,