Add harness candidate set audit

This commit is contained in:
2026-06-26 22:02:09 +08:00
parent 42f75553a6
commit 825d3e03e9
3 changed files with 245 additions and 1 deletions

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import hashlib
import contextlib
import io
import math
@@ -1803,6 +1804,33 @@ class CoreFlowTests(unittest.TestCase):
window_summary={"prompt_tokens_p95": 2048},
state=state,
)
candidate_set = context["experiment_plan"]["candidate_set"]
self.assertEqual(candidate_set["version"], "candidate-set-v1")
self.assertIn("candidate_set_hash", candidate_set)
self.assertGreaterEqual(
candidate_set["blocked_reason_summary"].get(
"blocked_noop_or_repeat_effective_full_config",
0,
),
1,
)
baseline_fingerprint = hashlib.sha256(
_effective_config_signature(
study,
{"env_patch": {}, "flag_patch": {}},
).encode("utf-8")
).hexdigest()
blocked_baseline_equivalent = [
item
for item in candidate_set["blocked_candidates"]
if item.get("effective_config_fingerprint") == baseline_fingerprint
]
self.assertTrue(blocked_baseline_equivalent)
self.assertEqual(
blocked_baseline_equivalent[0]["blocked_reason"],
"blocked_noop_or_repeat_effective_full_config",
)
self.assertIn("effective_config_fingerprint", blocked_baseline_equivalent[0])
actions = context["experiment_plan"]["candidate_actions"]
self.assertFalse(
any(