#!/usr/bin/env python3 """Prospective-repeat confirmation of the intervention-response hypothesis. P1 contains three pre-arranged, disjoint request bands per cell/load. TP1 and TP4 use matched offered loads and request sequences across their MNS endpoints. This script asks both whether the MNS response exceeds prospective repeat noise and whether an early telemetry delta predicts full-run action efficacy beyond the corresponding external-outcome delta. """ from __future__ import annotations import argparse import hashlib import importlib.util import json import math import re import sys from collections import defaultdict from pathlib import Path from statistics import fmean from typing import Any, Iterable, Mapping HERE = Path(__file__).resolve().parent COMMON_STATE_DIR = HERE.parent / "telemetry-residual" sys.path.insert(0, str(COMMON_STATE_DIR)) from common_state import load_jsonl, summarize_engine # noqa: E402 def _load_v0(): spec = importlib.util.spec_from_file_location( "intervention_response_phase6_v0", HERE / "analyze_phase6.py" ) module = importlib.util.module_from_spec(spec) assert spec.loader is not None spec.loader.exec_module(module) return module V0 = _load_v0() SCHEMA = "intervention-response-p1-confirmation-v1" HORIZONS_S = V0.HORIZONS_S EXPECTED_ACTION_PAIRS = 12 EXPECTED_REPEAT_PAIRS = 24 MIN_EFFICACY_CLASS = 4 MIN_EFFICACY_BALANCED_ACCURACY = 0.75 MIN_EFFICACY_DELTA_OVER_OUTCOME = 0.15 OUTCOME_FEATURES = ( "admitted_fraction", "completed_over_admitted", "completed_pass_rate", "completed_fail_fraction_of_total", "outstanding_over_admitted", "ttft_max_over_slo_max", "ttft_mean_over_slo_max", "tpot_max_over_slo", "tpot_mean_over_slo", "admitted_input_tokens_mean_over_limit", ) RUN_PATTERN = re.compile(r"^(low|high)-rep([123])$") def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def _prefix_outcome( result: Mapping[str, Any], requests: list[dict[str, Any]], horizon_s: float, ) -> dict[str, float]: admitted = [request for request in requests if float(request["arrival_s"]) <= horizon_s] completed = [ request for request in requests if request.get("completed_elapsed_s") is not None and float(request["completed_elapsed_s"]) <= horizon_s ] if not admitted: raise ValueError("prefix contains no admitted request") admitted_ids = {str(request["request_id"]) for request in admitted} if any(str(request["request_id"]) not in admitted_ids for request in completed): raise ValueError("completed request was not admitted in the prefix") passed = sum(bool(request["slo_pass"]) for request in completed) ttft = [float(request["ttft_ms"]) for request in completed] tpot = [float(request["tpot_ms"]) for request in completed] total = int(result["selection"]["count"]) if total != len(requests): raise ValueError("request JSONL count does not match the result") return { "admitted_fraction": len(admitted) / total, "completed_over_admitted": len(completed) / len(admitted), "completed_pass_rate": passed / max(1, len(completed)), "completed_fail_fraction_of_total": (len(completed) - passed) / total, "outstanding_over_admitted": (len(admitted) - len(completed)) / len(admitted), "ttft_max_over_slo_max": max(ttft, default=0.0) / 6000.0, "ttft_mean_over_slo_max": fmean(ttft) / 6000.0 if ttft else 0.0, "tpot_max_over_slo": max(tpot, default=0.0) / 50.0, "tpot_mean_over_slo": fmean(tpot) / 50.0 if tpot else 0.0, "admitted_input_tokens_mean_over_limit": fmean( float(request["raw_input_tokens"]) for request in admitted ) / 8192.0, } def load_trials( run_root: Path, *, horizons_s: tuple[float, ...] = HORIZONS_S, ) -> tuple[dict[float, list[dict[str, Any]]], list[dict[str, Any]]]: by_horizon = {horizon: [] for horizon in horizons_s} streams = [] for cell_dir in sorted((run_root / "cells").iterdir()): if not cell_dir.is_dir(): continue stream_paths = sorted((cell_dir / "opprof").glob("*.jsonl")) if len(stream_paths) != 1: raise ValueError(f"{cell_dir}: expected one Layer-1 stream") stream_path = stream_paths[0] stream = load_jsonl(stream_path) streams.append( { "path": str(stream_path.resolve()), "sha256": sha256_file(stream_path), "bytes": stream_path.stat().st_size, } ) for run_dir in sorted(cell_dir.iterdir()): match = RUN_PATTERN.match(run_dir.name) if match is None: continue level, replicate_text = match.groups() replicate = int(replicate_text) result_path = run_dir / "result.json" requests_path = run_dir / "requests.jsonl" result = json.loads(result_path.read_text(encoding="utf-8")) requests = load_jsonl(requests_path) elapsed_s = float(result["interval"]["elapsed_s"]) start_ns = int(result["interval"]["start_mono_ns"]) for horizon_s in horizons_s: if elapsed_s < horizon_s: raise ValueError( f"{result_path}: elapsed {elapsed_s} shorter than {horizon_s}s" ) state = V0.flatten_state( summarize_engine( stream, start_ns=start_ns, end_ns=start_ns + int(horizon_s * 1e9), request_count=int(result["selection"]["count"]), ) ) by_horizon[horizon_s].append( { "trial_id": str(result_path.relative_to(run_root)), "cell": str(result["cell"]), "tp": int(result["tp"]), "mns": int(result["mns"]), "level": level, "replicate": replicate, "offered_rate_per_gpu": float( result["selection"]["offered_req_s_per_gpu"] ), "request_hash": str( result["selection"]["request_id_order_sha256"] ), "request_count": int(result["selection"]["count"]), "result_sha256": sha256_file(result_path), "requests_sha256": sha256_file(requests_path), "full_pass_rate": float(result["pass_rate"]), "full_feasible": bool(result["feasible"]), "early_stopped": bool(result["early_stopped"]), "state": state, "outcome": _prefix_outcome(result, requests, horizon_s), } ) return by_horizon, streams def validate_manifest( trials: list[dict[str, Any]], manifest_path: Path ) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest.get("schema") != "fidelity-prefix-pilot-manifest-v1": raise ValueError("unexpected P1 manifest schema") cells = manifest.get("cells") if not isinstance(cells, dict): raise ValueError("P1 manifest has no cell mapping") seen = set() for trial in trials: key = (trial["cell"], trial["level"], trial["replicate"]) if key in seen: raise ValueError(f"duplicate P1 trial identity: {key}") seen.add(key) try: cell = cells[trial["cell"]] selection = cell["targets"][trial["level"]]["selections"][ f"{trial['level']}{trial['replicate']}" ] except (KeyError, TypeError) as error: raise ValueError(f"trial is absent from P1 manifest: {key}") from error if int(cell["tp"]) != trial["tp"] or int(cell["mns"]) != trial["mns"]: raise ValueError(f"trial config disagrees with P1 manifest: {key}") if str(selection["request_id_order_sha256"]) != trial["request_hash"]: raise ValueError(f"trial request hash disagrees with P1 manifest: {key}") if int(selection["selected_count"]) != trial["request_count"]: raise ValueError(f"trial request count disagrees with P1 manifest: {key}") if not math.isclose( float(selection["offered_req_s_per_gpu"]), trial["offered_rate_per_gpu"], rel_tol=0.0, abs_tol=1e-12, ): raise ValueError(f"trial offered load disagrees with P1 manifest: {key}") expected = { (cell_name, level, replicate) for cell_name in cells for level in ("low", "high") for replicate in (1, 2, 3) } if seen != expected: missing = sorted(expected - seen) unexpected = sorted(seen - expected) raise ValueError( f"P1 trial/manifest coverage mismatch: missing={missing}, " f"unexpected={unexpected}" ) return { "schema": str(manifest["schema"]), "expected_trials": len(expected), "matched_trials": len(seen), } def _delta( source: Mapping[str, Any], target: Mapping[str, Any], features: Iterable[str], ) -> dict[str, float]: return { feature: float(target[feature]) - float(source[feature]) for feature in features } def _action_pair(source: Mapping[str, Any], target: Mapping[str, Any]) -> dict[str, Any]: if source["tp"] != target["tp"]: raise ValueError("action endpoints changed TP") if source["level"] != target["level"] or source["replicate"] != target["replicate"]: raise ValueError("action endpoints changed load role or repeat") if source["request_hash"] != target["request_hash"]: raise ValueError("action endpoints changed request sequence") if not math.isclose( source["offered_rate_per_gpu"], target["offered_rate_per_gpu"], rel_tol=0.0, abs_tol=1e-12, ): raise ValueError("action endpoints changed offered load") if source["mns"] >= target["mns"]: raise ValueError("action must increase MNS") beneficial = target["full_feasible"] and not source["full_feasible"] return { "kind": "matched_mns_increase", "group": { "tp": source["tp"], "level": source["level"], "replicate": source["replicate"], "request_hash": source["request_hash"], "offered_rate_per_gpu": source["offered_rate_per_gpu"], }, "source": { key: source[key] for key in ( "trial_id", "result_sha256", "requests_sha256", "cell", "mns", "full_pass_rate", "full_feasible", "early_stopped", ) }, "target": { key: target[key] for key in ( "trial_id", "result_sha256", "requests_sha256", "cell", "mns", "full_pass_rate", "full_feasible", "early_stopped", ) }, "delta_state": _delta(source["state"], target["state"], V0.ALL_FEATURES), "delta_outcome": _delta(source["outcome"], target["outcome"], OUTCOME_FEATURES), "full_action_efficacy": int(beneficial), "full_feasibility_transition": ( f"{str(source['full_feasible']).lower()}->" f"{str(target['full_feasible']).lower()}" ), } def _repeat_pair(source: Mapping[str, Any], target: Mapping[str, Any]) -> dict[str, Any]: if source["cell"] != target["cell"] or source["level"] != target["level"]: raise ValueError("repeat endpoints changed config or load role") if target["replicate"] != source["replicate"] + 1: raise ValueError("repeat endpoints are not consecutive pre-arranged bands") if not math.isclose( source["offered_rate_per_gpu"], target["offered_rate_per_gpu"], rel_tol=0.0, abs_tol=1e-12, ): raise ValueError("repeat endpoints changed offered load") return { "kind": "same_config_workload_repeat", "group": { "cell": source["cell"], "tp": source["tp"], "mns": source["mns"], "level": source["level"], "source_replicate": source["replicate"], "target_replicate": target["replicate"], }, "source": { key: source[key] for key in ("trial_id", "result_sha256", "requests_sha256") }, "target": { key: target[key] for key in ("trial_id", "result_sha256", "requests_sha256") }, "delta_state": _delta(source["state"], target["state"], V0.ALL_FEATURES), "delta_outcome": _delta(source["outcome"], target["outcome"], OUTCOME_FEATURES), } def build_pairs( trials: list[dict[str, Any]], ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: action_groups: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list) repeat_groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) for trial in trials: action_groups[ ( trial["tp"], trial["level"], trial["replicate"], trial["request_hash"], trial["offered_rate_per_gpu"], ) ].append(trial) repeat_groups[(trial["cell"], trial["level"])].append(trial) actions = [] for group in action_groups.values(): if len(group) != 2: continue source, target = sorted(group, key=lambda trial: trial["mns"]) actions.append(_action_pair(source, target)) repeats = [] for group in repeat_groups.values(): ordered = sorted(group, key=lambda trial: trial["replicate"]) if len(ordered) != 3: raise ValueError("each prospective repeat group must contain three runs") repeats.extend( _repeat_pair(source, target) for source, target in zip(ordered, ordered[1:], strict=False) ) return actions, repeats def _balanced_accuracy(labels: list[int], predictions: list[int]) -> float: positive = [prediction for label, prediction in zip(labels, predictions) if label == 1] negative = [prediction for label, prediction in zip(labels, predictions) if label == 0] if not positive or not negative: raise ValueError("balanced accuracy requires both classes") sensitivity = sum(prediction == 1 for prediction in positive) / len(positive) specificity = sum(prediction == 0 for prediction in negative) / len(negative) return (sensitivity + specificity) / 2.0 def _threshold_candidates(values: list[float]) -> list[float]: unique = sorted(set(values)) if len(unique) == 1: return [unique[0] - 1.0, unique[0], unique[0] + 1.0] scale = max(1.0, max(abs(value) for value in unique)) candidates = [unique[0] - scale * 1e-6] candidates.extend( (left + right) / 2.0 for left, right in zip(unique, unique[1:], strict=False) ) candidates.append(unique[-1] + scale * 1e-6) return candidates def _fit_threshold(values: list[float], labels: list[int]) -> tuple[float, int, float]: best: tuple[float, int, float, float] | None = None for threshold in _threshold_candidates(values): for direction in (-1, 1): predictions = [int(direction * (value - threshold) >= 0.0) for value in values] balanced = _balanced_accuracy(labels, predictions) accuracy = sum( prediction == label for prediction, label in zip(predictions, labels, strict=True) ) / len(labels) candidate = (balanced, accuracy, -abs(threshold), float(direction)) if best is None or candidate > best: best = candidate selected_threshold = threshold selected_direction = direction assert best is not None return selected_threshold, selected_direction, best[0] def one_feature_leave_repeat_out( actions: list[dict[str, Any]], *, delta_key: str, features: tuple[str, ...], ) -> dict[str, Any]: labels = [int(pair["full_action_efficacy"]) for pair in actions] results = {} for feature in features: predictions = [] held_out_labels = [] folds = [] for held_out in (1, 2, 3): train = [pair for pair in actions if pair["group"]["replicate"] != held_out] test = [pair for pair in actions if pair["group"]["replicate"] == held_out] train_values = [float(pair[delta_key][feature]) for pair in train] train_labels = [int(pair["full_action_efficacy"]) for pair in train] threshold, direction, train_balanced = _fit_threshold( train_values, train_labels ) test_values = [float(pair[delta_key][feature]) for pair in test] test_predictions = [ int(direction * (value - threshold) >= 0.0) for value in test_values ] test_labels = [int(pair["full_action_efficacy"]) for pair in test] predictions.extend(test_predictions) held_out_labels.extend(test_labels) folds.append( { "held_out_replicate": held_out, "threshold": threshold, "direction": direction, "train_balanced_accuracy": train_balanced, "test_labels": test_labels, "test_predictions": test_predictions, } ) balanced = _balanced_accuracy(held_out_labels, predictions) accuracy = sum( prediction == label for prediction, label in zip(predictions, held_out_labels, strict=True) ) / len(held_out_labels) results[feature] = { "balanced_accuracy": balanced, "accuracy": accuracy, "folds": folds, } best_feature = max( results, key=lambda feature: ( results[feature]["balanced_accuracy"], results[feature]["accuracy"], feature, ), ) return { "labels": V0.numeric(labels), "positive": sum(labels), "negative": len(labels) - sum(labels), "features": results, "best_feature": best_feature, "best_balanced_accuracy": results[best_feature]["balanced_accuracy"], "best_accuracy": results[best_feature]["accuracy"], } def analyze_horizon(trials: list[dict[str, Any]], horizon_s: float) -> dict[str, Any]: actions, repeats = build_pairs(trials) response = V0.response_statistics(actions, repeats) qualifying_response = sorted( feature for feature, item in response.items() if item["qualifies"] ) outcome_cv = one_feature_leave_repeat_out( actions, delta_key="delta_outcome", features=OUTCOME_FEATURES, ) telemetry_cv = one_feature_leave_repeat_out( actions, delta_key="delta_state", features=V0.GATE_FEATURES, ) outcome_best = float(outcome_cv["best_balanced_accuracy"]) efficacy_qualifying = sorted( feature for feature, item in telemetry_cv["features"].items() if item["balanced_accuracy"] >= MIN_EFFICACY_BALANCED_ACCURACY and item["balanced_accuracy"] >= outcome_best + MIN_EFFICACY_DELTA_OVER_OUTCOME ) action_hashes_match = all( pair["group"]["request_hash"] for pair in actions ) labels = [int(pair["full_action_efficacy"]) for pair in actions] invariants = { "expected_action_pair_count": len(actions) == EXPECTED_ACTION_PAIRS, "expected_repeat_pair_count": len(repeats) == EXPECTED_REPEAT_PAIRS, "matched_action_request_hashes": action_hashes_match, "efficacy_label_balance": ( sum(labels) >= MIN_EFFICACY_CLASS and len(labels) - sum(labels) >= MIN_EFFICACY_CLASS ), "finite_deltas": all( math.isfinite(value) for pair in [*actions, *repeats] for values in (pair["delta_state"], pair["delta_outcome"]) for value in values.values() ), "probabilities_bounded": all( 0.0 <= trial["outcome"][feature] <= 1.0 for trial in trials for feature in ( "admitted_fraction", "completed_over_admitted", "completed_pass_rate", "completed_fail_fraction_of_total", "outstanding_over_admitted", "admitted_input_tokens_mean_over_limit", ) ), } red_flags = [name for name, passed in invariants.items() if not passed] transitions = defaultdict(int) for pair in actions: transitions[pair["full_feasibility_transition"]] += 1 return { "horizon_s": horizon_s, "actions": actions, "repeats": repeats, "response_statistics": response, "qualifying_response_features": qualifying_response, "efficacy": { "outcome_delta": outcome_cv, "telemetry_delta": telemetry_cv, "telemetry_qualifying_features": efficacy_qualifying, "minimum_balanced_accuracy": MIN_EFFICACY_BALANCED_ACCURACY, "minimum_delta_over_best_outcome": MIN_EFFICACY_DELTA_OVER_OUTCOME, "feasibility_transitions": dict(sorted(transitions.items())), }, "sanity": { "trials": len(trials), "action_pairs": len(actions), "repeat_pairs": len(repeats), "invariants": invariants, "red_flags": red_flags, }, } def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: trials_by_horizon, streams = load_trials(run_root) manifest_validation = validate_manifest( trials_by_horizon[min(trials_by_horizon)], manifest_path ) horizons = { str(int(horizon)): analyze_horizon(trials, horizon) for horizon, trials in sorted(trials_by_horizon.items()) } red_flags = sorted( { flag for horizon in horizons.values() for flag in horizon["sanity"]["red_flags"] } ) stable_response = sorted( set.intersection( *( set(horizon["qualifying_response_features"]) for horizon in horizons.values() ) ) ) stable_efficacy = sorted( set.intersection( *( set(horizon["efficacy"]["telemetry_qualifying_features"]) for horizon in horizons.values() ) ) ) if red_flags: decision = "STOP_DATA_INVALID" elif len(stable_response) < V0.MIN_STABLE_FEATURES: decision = "STOP_NO_PROSPECTIVE_RESPONSE" elif not stable_efficacy: decision = "STOP_NO_INCREMENTAL_TUNING_SIGNAL" else: decision = "OPEN_MATCHED_GPU_PILOT" payload = { "schema": SCHEMA, "status": "COMPLETE", "decision": decision, "claim_boundary": ( "Development-only confirmation on an already-consumed P1 task. " "Passing can open a newly registered matched pilot but cannot be " "reported as held-out tuning evidence." ), "frozen_gate": { "response_thresholds_identical_to_phase6_v0": True, "expected_action_pairs": EXPECTED_ACTION_PAIRS, "expected_repeat_pairs": EXPECTED_REPEAT_PAIRS, "minimum_stable_response_features": V0.MIN_STABLE_FEATURES, "minimum_efficacy_class": MIN_EFFICACY_CLASS, "minimum_efficacy_balanced_accuracy": MIN_EFFICACY_BALANCED_ACCURACY, "minimum_efficacy_delta_over_best_outcome": ( MIN_EFFICACY_DELTA_OVER_OUTCOME ), }, "stable_response_features": stable_response, "stable_incremental_efficacy_features": stable_efficacy, "horizons": horizons, "provenance": { "analysis_script": str(Path(__file__).resolve()), "analysis_script_sha256": sha256_file(Path(__file__).resolve()), "phase6_v0_script_sha256": sha256_file(HERE / "analyze_phase6.py"), "run_root": str(run_root.resolve()), "manifest": str(manifest_path.resolve()), "manifest_sha256": sha256_file(manifest_path), "manifest_validation": manifest_validation, "streams": streams, }, "sanity": { "stream_count": len(streams), "stream_bytes": V0.numeric(item["bytes"] for item in streams), "red_flags": red_flags, }, } output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") return payload def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--run-root", type=Path, required=True) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() payload = audit( run_root=args.run_root, manifest_path=args.manifest, output_path=args.output, ) print( json.dumps( { "decision": payload["decision"], "stable_response_features": payload["stable_response_features"], "stable_incremental_efficacy_features": payload[ "stable_incremental_efficacy_features" ], "sanity": payload["sanity"], }, indent=2, sort_keys=True, ) ) if __name__ == "__main__": main()