#!/usr/bin/env python3 """Train and audit outcome-only versus telemetry action-response policies.""" from __future__ import annotations import argparse import hashlib import importlib.util import json import math import os import sys from collections import defaultdict from pathlib import Path from typing import Any, Mapping, Sequence HERE = Path(__file__).resolve().parent def _load_model(): spec = importlib.util.spec_from_file_location( "active_intervention_model", HERE / "model.py" ) module = importlib.util.module_from_spec(spec) assert spec.loader is not None sys.modules[spec.name] = module spec.loader.exec_module(module) return module MODEL = _load_model() REGULARIZATION = 10.0 MINIMUM_MARGIN = 0.02 CONFIDENCE_Z = 1.0 def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def atomic_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text( json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) os.replace(temporary, path) def grouped( examples: Sequence[Mapping[str, Any]], key: str ) -> dict[str, list[Mapping[str, Any]]]: result: dict[str, list[Mapping[str, Any]]] = defaultdict(list) for example in examples: result[str(example[key])].append(example) return dict(result) def evaluate_grouped_cv( examples: Sequence[Mapping[str, Any]], *, include_telemetry: bool, holdout_key: str, ) -> dict[str, Any]: holdouts = grouped(examples, holdout_key) decision_rows = [] for held_out, test_examples in sorted(holdouts.items()): training = [example for example in examples if str(example[holdout_key]) != held_out] if len({str(example["decision_id"]) for example in training}) < 2: continue model = MODEL.fit_ridge( training, include_telemetry=include_telemetry, regularization=REGULARIZATION, ) for decision_id, candidates in sorted(grouped(test_examples, "decision_id").items()): predictions = [] for candidate in candidates: source = candidate["source"] action = candidate["action"] noop = ( int(action["target_mns"]) == int(source["mns"]) and int(action["target_mbbt"]) == int(source["mbbt"]) ) if noop: prediction = 0.0 else: names, vector = MODEL.feature_vector( candidate, include_telemetry=include_telemetry ) if tuple(names) != model.feature_names: raise ValueError("cross-validation feature schema mismatch") prediction = max(-1.0, min(1.0, model.predict(vector))) predictions.append( { "action_id": str(candidate["action"]["id"]), "prediction": prediction, "real": float(candidate["target_normalized_goodput"]), } ) predictions.sort(key=lambda row: (-row["prediction"], row["action_id"])) selected = predictions[0] oracle = max(row["real"] for row in predictions) regret = 1.0 - selected["real"] / oracle if oracle > 0 else 0.0 best_actions = { row["action_id"] for row in predictions if math.isclose(row["real"], oracle) } decision_rows.append( { "holdout": held_out, "decision_id": decision_id, "selected_action": selected["action_id"], "best_actions": sorted(best_actions), "correct": selected["action_id"] in best_actions, "selected_real": selected["real"], "oracle_real": oracle, "regret": regret, "predictions": predictions, } ) if not decision_rows: return {"status": "INSUFFICIENT_GROUPS", "decisions": []} regrets = [float(row["regret"]) for row in decision_rows] return { "status": "VALID", "holdout_key": holdout_key, "decision_n": len(decision_rows), "correct_n": sum(bool(row["correct"]) for row in decision_rows), "accuracy": sum(bool(row["correct"]) for row in decision_rows) / len(decision_rows), "mean_regret": sum(regrets) / len(regrets), "max_regret": max(regrets), "decisions": decision_rows, } def paired_delta(outcome: Mapping[str, Any], telemetry: Mapping[str, Any]) -> dict[str, Any]: if outcome.get("status") != "VALID" or telemetry.get("status") != "VALID": return {"status": "INSUFFICIENT_GROUPS"} outcome_by_id = {row["decision_id"]: row for row in outcome["decisions"]} telemetry_by_id = {row["decision_id"]: row for row in telemetry["decisions"]} common = sorted(set(outcome_by_id) & set(telemetry_by_id)) rows = [] for decision_id in common: before = outcome_by_id[decision_id] after = telemetry_by_id[decision_id] rows.append( { "decision_id": decision_id, "outcome_action": before["selected_action"], "telemetry_action": after["selected_action"], "action_changed": before["selected_action"] != after["selected_action"], "regret_delta": float(after["regret"]) - float(before["regret"]), "telemetry_corrected": (not before["correct"]) and bool(after["correct"]), "telemetry_harmed": bool(before["correct"]) and (not after["correct"]), } ) return { "status": "VALID", "decision_n": len(rows), "action_changed_n": sum(row["action_changed"] for row in rows), "corrected_n": sum(row["telemetry_corrected"] for row in rows), "harmed_n": sum(row["telemetry_harmed"] for row in rows), "mean_regret_delta": ( sum(float(row["regret_delta"]) for row in rows) / len(rows) if rows else 0.0 ), "rows": rows, } def build_policy(dataset_path: Path) -> dict[str, Any]: dataset = json.loads(dataset_path.read_text(encoding="utf-8")) if dataset.get("status") != "VALID" or dataset["sanity"]["red_flags"]: raise ValueError("training dataset is not valid") examples = dataset["examples"] phases = sorted({str(example["phase"]) for example in examples}, key=float) phase_results = {} incremental_candidates = [] for phase in phases: selected = [example for example in examples if str(example["phase"]) == phase] outcome_cv = evaluate_grouped_cv( selected, include_telemetry=False, holdout_key="repetition" ) telemetry_cv = evaluate_grouped_cv( selected, include_telemetry=True, holdout_key="repetition" ) outcome_regime = evaluate_grouped_cv( selected, include_telemetry=False, holdout_key="regime" ) telemetry_regime = evaluate_grouped_cv( selected, include_telemetry=True, holdout_key="regime" ) delta = paired_delta(outcome_cv, telemetry_cv) outcome_models = MODEL.fit_jackknife_ensemble( selected, include_telemetry=False, regularization=REGULARIZATION, ) telemetry_models = MODEL.fit_jackknife_ensemble( selected, include_telemetry=True, regularization=REGULARIZATION, ) incremental = bool( delta.get("status") == "VALID" and int(delta["corrected_n"]) >= 1 and int(delta["harmed_n"]) == 0 and float(delta["mean_regret_delta"]) < -1e-12 and float(telemetry_cv["max_regret"]) <= 0.05 ) if incremental: incremental_candidates.append(phase) phase_results[phase] = { "cutoff_s": float(selected[0]["cutoff_s"]), "outcome_only": { "leave_repetition_out": outcome_cv, "leave_regime_out": outcome_regime, "models": MODEL.models_to_json(outcome_models), }, "telemetry": { "leave_repetition_out": telemetry_cv, "leave_regime_out": telemetry_regime, "models": MODEL.models_to_json(telemetry_models), }, "paired_incremental": delta, "incremental_gate": incremental, } selected_phase = incremental_candidates[0] if incremental_candidates else phases[-1] status = ( "RETROSPECTIVE_INCREMENTAL_SIGNAL" if incremental_candidates else "NO_RETROSPECTIVE_INCREMENTAL_SIGNAL" ) target_values = [float(example["target_normalized_goodput"]) for example in examples] effect_values = [ float(example["target_delta_normalized_goodput"]) for example in examples ] invariants = { "four_phases": len(phases) == 4, "targets_bounded": all(0.0 <= value <= 1.0 for value in target_values), "targets_not_all_identical": len(set(target_values)) > 1, "effects_bounded": all(-1.0 <= value <= 1.0 for value in effect_values), "effects_not_all_identical": len(set(effect_values)) > 1, "models_present_every_phase": all( phase_results[phase][mode]["models"] for phase in phases for mode in ("outcome_only", "telemetry") ), } red_flags = [name for name, passed in invariants.items() if not passed] if red_flags: raise RuntimeError(f"policy sanity failed: {red_flags}") return { "schema": "active-intervention-policy-v0", "status": status, "training": { "dataset": str(dataset_path), "dataset_sha256": sha256_file(dataset_path), "examples": len(examples), "decisions": len({example["decision_id"] for example in examples}), "regularization": REGULARIZATION, "confidence_z": CONFIDENCE_Z, "minimum_margin": MINIMUM_MARGIN, }, "measurement_policy": { "selected_phase": selected_phase, "selected_cutoff_s": phase_results[selected_phase]["cutoff_s"], "selection_reason": ( "earliest phase passing the frozen incremental gate" if incremental_candidates else "no incremental phase; retain full measurement for exploratory held-out test" ), }, "phases": phase_results, "sanity": { "invariants": invariants, "red_flags": red_flags, "target_normalized_goodput": { "n": len(target_values), "min": min(target_values), "max": max(target_values), "distinct_n": len(set(target_values)), }, "target_delta_normalized_goodput": { "n": len(effect_values), "min": min(effect_values), "max": max(effect_values), "distinct_n": len(set(effect_values)), }, }, } def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--dataset", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() policy = build_policy(args.dataset) atomic_json(args.output, policy) print( json.dumps( { "status": policy["status"], "measurement_policy": policy["measurement_policy"], "sanity": policy["sanity"], }, sort_keys=True, ) ) if __name__ == "__main__": main()