Add harness candidate set audit
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import contextlib
|
||||
import io
|
||||
import math
|
||||
@@ -1803,6 +1804,33 @@ class CoreFlowTests(unittest.TestCase):
|
||||
window_summary={"prompt_tokens_p95": 2048},
|
||||
state=state,
|
||||
)
|
||||
candidate_set = context["experiment_plan"]["candidate_set"]
|
||||
self.assertEqual(candidate_set["version"], "candidate-set-v1")
|
||||
self.assertIn("candidate_set_hash", candidate_set)
|
||||
self.assertGreaterEqual(
|
||||
candidate_set["blocked_reason_summary"].get(
|
||||
"blocked_noop_or_repeat_effective_full_config",
|
||||
0,
|
||||
),
|
||||
1,
|
||||
)
|
||||
baseline_fingerprint = hashlib.sha256(
|
||||
_effective_config_signature(
|
||||
study,
|
||||
{"env_patch": {}, "flag_patch": {}},
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
blocked_baseline_equivalent = [
|
||||
item
|
||||
for item in candidate_set["blocked_candidates"]
|
||||
if item.get("effective_config_fingerprint") == baseline_fingerprint
|
||||
]
|
||||
self.assertTrue(blocked_baseline_equivalent)
|
||||
self.assertEqual(
|
||||
blocked_baseline_equivalent[0]["blocked_reason"],
|
||||
"blocked_noop_or_repeat_effective_full_config",
|
||||
)
|
||||
self.assertIn("effective_config_fingerprint", blocked_baseline_equivalent[0])
|
||||
actions = context["experiment_plan"]["candidate_actions"]
|
||||
self.assertFalse(
|
||||
any(
|
||||
|
||||
Reference in New Issue
Block a user