Record failed trial context
This commit is contained in:
@@ -25,6 +25,8 @@ def build_prompt(
|
||||
"best_request_rate": trial.best_request_rate,
|
||||
"best_pass_rate": trial.best_pass_rate,
|
||||
"diagnosis": trial.diagnosis,
|
||||
"config_patch": trial.config_patch,
|
||||
"failure_reason": trial.failure_reason,
|
||||
}
|
||||
)
|
||||
sections = [
|
||||
@@ -34,6 +36,7 @@ def build_prompt(
|
||||
"expected_effects must be a JSON array of short strings, not an object.",
|
||||
"Only use allowed tunable env keys and allowed tunable flag keys.",
|
||||
"Do not wrap the JSON in markdown fences or any extra text.",
|
||||
"Do not repeat a config that previously failed to launch unless the new patch explicitly removes the failing knob.",
|
||||
"",
|
||||
"Study stack:",
|
||||
json.dumps(
|
||||
|
||||
@@ -446,6 +446,8 @@ class TrialSummary:
|
||||
best_pass_rate: float | None = None
|
||||
result_path: str | None = None
|
||||
diagnosis: str = ""
|
||||
config_patch: dict[str, Any] | None = None
|
||||
failure_reason: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -74,7 +74,12 @@ class StudyStore:
|
||||
self.write_json(trial_root / "trial_spec.json", to_jsonable(spec))
|
||||
next_state = replace(state, next_trial_index=state.next_trial_index + 1)
|
||||
next_state.trials.append(
|
||||
TrialSummary(trial_id=trial_id, status="queued", diagnosis=proposal.diagnosis)
|
||||
TrialSummary(
|
||||
trial_id=trial_id,
|
||||
status="queued",
|
||||
diagnosis=proposal.diagnosis,
|
||||
config_patch=to_jsonable(proposal.config_patch),
|
||||
)
|
||||
)
|
||||
self.save_state(next_state)
|
||||
return spec, next_state
|
||||
@@ -101,6 +106,7 @@ class StudyStore:
|
||||
summary.best_request_rate = payload.get("best_request_rate")
|
||||
summary.best_pass_rate = payload.get("best_pass_rate")
|
||||
summary.result_path = str(result_path)
|
||||
summary.failure_reason = str(payload.get("failure_reason") or "").strip()
|
||||
if (
|
||||
isinstance(summary.best_request_rate, (int, float))
|
||||
and (best_rate is None or summary.best_request_rate > best_rate)
|
||||
|
||||
@@ -183,6 +183,28 @@ def _replay_requests(
|
||||
return ordered, early_stopped, early_stop_reason
|
||||
|
||||
|
||||
def _wait_for_server_or_exit(
|
||||
process: subprocess.Popen[str],
|
||||
*,
|
||||
base_url: str,
|
||||
healthcheck_path: str,
|
||||
ready_timeout_s: float,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + ready_timeout_s
|
||||
last_error = "server_not_ready"
|
||||
while time.monotonic() < deadline:
|
||||
exit_code = process.poll()
|
||||
if exit_code is not None:
|
||||
raise RuntimeError(f"engine_process_exited_before_ready exit_code={exit_code}")
|
||||
try:
|
||||
wait_for_server(base_url, healthcheck_path, timeout_s=1.0)
|
||||
return
|
||||
except HttpClientError as exc:
|
||||
last_error = str(exc)
|
||||
time.sleep(1.0)
|
||||
raise HttpClientError(f"Timed out waiting for {base_url}{healthcheck_path}: {last_error}")
|
||||
|
||||
|
||||
def run_trial(trial_spec_path: Path) -> dict[str, Any]:
|
||||
from .store import StudyStore
|
||||
|
||||
@@ -203,9 +225,14 @@ def run_trial(trial_spec_path: Path) -> dict[str, Any]:
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
probe_history: list[dict[str, Any]] = []
|
||||
try:
|
||||
wait_for_server(recipe.base_url, recipe.healthcheck_path, recipe.ready_timeout_s)
|
||||
probe_history: list[dict[str, Any]] = []
|
||||
_wait_for_server_or_exit(
|
||||
process,
|
||||
base_url=recipe.base_url,
|
||||
healthcheck_path=recipe.healthcheck_path,
|
||||
ready_timeout_s=recipe.ready_timeout_s,
|
||||
)
|
||||
|
||||
def evaluator(threshold: float) -> ThresholdProbe[ProbePayload]:
|
||||
selected = select_requests_for_threshold(requests, threshold=threshold)
|
||||
@@ -297,10 +324,25 @@ def run_trial(trial_spec_path: Path) -> dict[str, Any]:
|
||||
}
|
||||
StudyStore.write_json(Path(trial.result_path), result)
|
||||
return result
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result = {
|
||||
"study_id": trial.study_id,
|
||||
"trial_id": trial.trial_id,
|
||||
"status": "failed",
|
||||
"best_sampling_u": None,
|
||||
"best_request_rate": None,
|
||||
"best_pass_rate": None,
|
||||
"best_request_count": None,
|
||||
"failure_reason": str(exc),
|
||||
"probes": probe_history,
|
||||
}
|
||||
StudyStore.write_json(Path(trial.result_path), result)
|
||||
return result
|
||||
finally:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=30)
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=30)
|
||||
|
||||
Reference in New Issue
Block a user