#!/usr/bin/env python3 """Development-only cross-config telemetry transfer diagnostic for P1. Each example asks whether state observed at one source anchor helps predict the pass rate at a different target config. The hybrid branch predicts the real-minus-simulator residual; the direct branch never reads simulator state or outcomes. Folds exclude both the source and target config identities. The two offered-load anchors belong to one trace/SLO task, so this is a premise check rather than generalization evidence. """ from __future__ import annotations import argparse import json import math from pathlib import Path from typing import Any, Sequence import numpy as np REGULARIZATION = (0.1, 1.0, 10.0, 100.0) PRIOR_SHRINKAGE = (0.0, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0) def atomic_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") temporary.replace(path) def finite(value: Any, *, name: str) -> float: result = float(value) if not math.isfinite(result): raise ValueError(f"non-finite feature {name}={value!r}") return result def flatten_residual_state(example: dict[str, Any]) -> tuple[float, ...]: residual = example["state_residual"]["values"] values = [finite(residual[name], name=name) for name in sorted(residual)] engine_only = example["engine"]["engine_only"] values.extend( finite(engine_only[name], name=name) for name in sorted(engine_only) ) return tuple(values) def flatten_engine_state(example: dict[str, Any]) -> tuple[float, ...]: values = [] for name in sorted(example["engine"]["common"]): value = example["engine"]["common"][name] if isinstance(value, dict): values.extend( finite(value[statistic], name=f"{name}.{statistic}") for statistic in ("mean", "max", "cv") ) elif value is not None: values.append(finite(value, name=name)) engine_only = example["engine"]["engine_only"] values.extend( finite(engine_only[name], name=name) for name in sorted(engine_only) ) return tuple(values) def config_transition_features( source: dict[str, Any], target: dict[str, Any] ) -> tuple[float, ...]: source_rate = finite( source["offered_req_s_per_gpu"], name="source_offered_req_s_per_gpu" ) target_rate = finite( target["offered_req_s_per_gpu"], name="target_offered_req_s_per_gpu" ) return ( math.log2(float(source["tp"])), math.log2(float(source["mns"])), math.log2(float(target["tp"])), math.log2(float(target["mns"])), math.log2(float(target["tp"]) / float(source["tp"])), math.log2(float(target["mns"]) / float(source["mns"])), math.log2(source_rate), math.log2(target_rate), math.log2(target_rate / source_rate), ) def hybrid_base_features( source: dict[str, Any], target: dict[str, Any] ) -> tuple[float, ...]: return ( finite(source["pass_rate_residual"], name="source_pass_rate_residual"), finite(source["sim_pass_rate"], name="source_sim_pass_rate"), finite(target["sim_pass_rate"], name="target_sim_pass_rate"), ) + config_transition_features(source, target) def direct_base_features( source: dict[str, Any], target: dict[str, Any] ) -> tuple[float, ...]: return ( finite(source["real_pass_rate_rep1"], name="source_real_pass_rate"), ) + config_transition_features(source, target) def transitions(examples: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: rows = [] for source in examples: residual_state = flatten_residual_state(source) engine_state = flatten_engine_state(source) for target in examples: # Low/high are offered-load anchors inside one workload/SLO task. # Cross-load transitions are legal; same-cell transitions are # excluded because R0 asks about transfer to a new configuration. if source["cell"] == target["cell"]: continue hybrid_base = hybrid_base_features(source, target) direct_base = direct_base_features(source, target) rows.append( { "source_cell": source["cell"], "source_level": source["level"], "target_cell": target["cell"], "target_level": target["level"], "hybrid_base": hybrid_base, "hybrid_telemetry": hybrid_base + residual_state, "direct_base": direct_base, "direct_telemetry": direct_base + engine_state, "target_residual": finite( target["pass_rate_residual"], name="target_residual" ), "target_residual_delta": finite( target["pass_rate_residual"], name="target_residual" ) - finite(source["pass_rate_residual"], name="source_residual"), "target_real_pass_rate": finite( target["real_pass_rate_rep1"], name="target_real_pass_rate" ), "target_real_pass_rate_delta": finite( target["real_pass_rate_rep1"], name="target_real_pass_rate" ) - finite( source["real_pass_rate_rep1"], name="source_real_pass_rate" ), "source_residual": finite( source["pass_rate_residual"], name="source_residual" ), "source_real_pass_rate": finite( source["real_pass_rate_rep1"], name="source_real_pass_rate" ), "target_sim_pass_rate": finite( target["sim_pass_rate"], name="target_sim_pass_rate" ), "target_real_feasible": bool(target["real_feasible"]), "target_sim_feasible": bool(target["sim_feasible"]), } ) return rows def fit_predict( train_x: np.ndarray, train_y: np.ndarray, test_x: np.ndarray, regularization: float, ) -> np.ndarray: mean = train_x.mean(axis=0) std = train_x.std(axis=0) std[std == 0.0] = 1.0 train = (train_x - mean) / std test = (test_x - mean) / std train = np.column_stack((np.ones(len(train)), train)) test = np.column_stack((np.ones(len(test)), test)) penalty = np.eye(train.shape[1], dtype=np.float64) penalty[0, 0] = 0.0 weights = np.linalg.lstsq( train.T @ train + regularization * penalty, train.T @ train_y, rcond=None, )[0] return test @ weights def grouped_predictions( rows: Sequence[dict[str, Any]], *, feature_name: str, target_name: str, regularization: float, ) -> np.ndarray: predictions = np.zeros(len(rows), dtype=np.float64) groups = sorted({(row["source_cell"], row["target_cell"]) for row in rows}) for source_cell, target_cell in groups: held_out_cells = {source_cell, target_cell} test_indexes = [ index for index, row in enumerate(rows) if row["source_cell"] == source_cell and row["target_cell"] == target_cell ] train_indexes = [ index for index, row in enumerate(rows) if row["source_cell"] not in held_out_cells and row["target_cell"] not in held_out_cells ] if not test_indexes or not train_indexes: raise ValueError(f"empty grouped fold for {source_cell}->{target_cell}") train_x = np.asarray([rows[index][feature_name] for index in train_indexes]) train_y = np.asarray([rows[index][target_name] for index in train_indexes]) test_x = np.asarray([rows[index][feature_name] for index in test_indexes]) predictions[test_indexes] = fit_predict( train_x, train_y, test_x, regularization ) return predictions def metrics(rows: Sequence[dict[str, Any]], predicted_pass: np.ndarray) -> dict[str, Any]: truth = np.asarray( [row["target_real_pass_rate"] for row in rows], dtype=np.float64 ) real_feasible = np.asarray( [row["target_real_feasible"] for row in rows], dtype=bool ) sim_feasible = np.asarray( [row["target_sim_feasible"] for row in rows], dtype=bool ) clipped_pass = np.clip(predicted_pass, 0.0, 1.0) predicted_feasible = clipped_pass >= 0.95 baseline_correct = sim_feasible == real_feasible model_correct = predicted_feasible == real_feasible return { "rmse": float(np.sqrt(np.mean((predicted_pass - truth) ** 2))), "mae": float(np.mean(np.abs(predicted_pass - truth))), "feasibility_accuracy": float(np.mean(model_correct)), "false_feasible": int(np.sum(predicted_feasible & ~real_feasible)), "false_infeasible": int(np.sum(~predicted_feasible & real_feasible)), "simulator_errors_corrected": int(np.sum(~baseline_correct & model_correct)), "simulator_correct_corrupted": int(np.sum(baseline_correct & ~model_correct)), "predicted_pass_rate": { "n": len(clipped_pass), "min": float(clipped_pass.min()), "max": float(clipped_pass.max()), "distinct_n": len(set(float(value) for value in clipped_pass)), }, } def compare( rows: Sequence[dict[str, Any]], baseline_prediction: np.ndarray, telemetry_prediction: np.ndarray, baseline: dict[str, Any], telemetry: dict[str, Any], ) -> dict[str, Any]: truth = np.asarray([row["target_real_feasible"] for row in rows], dtype=bool) baseline_feasible = np.clip(baseline_prediction, 0.0, 1.0) >= 0.95 telemetry_feasible = np.clip(telemetry_prediction, 0.0, 1.0) >= 0.95 baseline_correct = baseline_feasible == truth telemetry_correct = telemetry_feasible == truth return { "delta_telemetry_minus_baseline": { "rmse": telemetry["rmse"] - baseline["rmse"], "mae": telemetry["mae"] - baseline["mae"], "feasibility_accuracy": telemetry["feasibility_accuracy"] - baseline["feasibility_accuracy"], }, "baseline_errors_corrected": int( np.sum(~baseline_correct & telemetry_correct) ), "baseline_correct_corrupted": int( np.sum(baseline_correct & ~telemetry_correct) ), } def execute(args: argparse.Namespace) -> dict[str, Any]: paired = json.loads(args.paired_state.read_text(encoding="utf-8")) if paired.get("status") != "PASS" or len(paired["examples"]) != 12: raise RuntimeError("paired P1 state evidence is incomplete") rows = transitions(paired["examples"]) residual_truth = np.asarray( [row["target_residual"] for row in rows], dtype=np.float64 ) pass_truth = np.asarray( [row["target_real_pass_rate"] for row in rows], dtype=np.float64 ) sim_pass = np.asarray( [row["target_sim_pass_rate"] for row in rows], dtype=np.float64 ) simulator = metrics(rows, sim_pass) sensitivity = {} for regularization in REGULARIZATION: hybrid_base_delta = grouped_predictions( rows, feature_name="hybrid_base", target_name="target_residual_delta", regularization=regularization, ) hybrid_telemetry_delta = grouped_predictions( rows, feature_name="hybrid_telemetry", target_name="target_residual_delta", regularization=regularization, ) source_residual = np.asarray( [row["source_residual"] for row in rows], dtype=np.float64 ) hybrid_base_correction = source_residual + hybrid_base_delta hybrid_telemetry_correction = source_residual + hybrid_telemetry_delta hybrid_base_prediction = sim_pass + hybrid_base_correction hybrid_telemetry_prediction = sim_pass + hybrid_telemetry_correction hybrid_base = metrics(rows, hybrid_base_prediction) hybrid_telemetry = metrics(rows, hybrid_telemetry_prediction) prior_shrinkage = {} for weight in PRIOR_SHRINKAGE: prior_shrinkage[str(weight)] = { "raw_simulator_prior": { "sim_plus_outcome": metrics( rows, sim_pass + weight * hybrid_base_correction ), "sim_plus_outcome_plus_telemetry": metrics( rows, sim_pass + weight * hybrid_telemetry_correction ), }, "anchor_offset_prior": { "sim_plus_outcome": metrics( rows, sim_pass + source_residual + weight * hybrid_base_delta, ), "sim_plus_outcome_plus_telemetry": metrics( rows, sim_pass + source_residual + weight * hybrid_telemetry_delta, ), }, } direct_base_prediction = grouped_predictions( rows, feature_name="direct_base", target_name="target_real_pass_rate_delta", regularization=regularization, ) direct_telemetry_prediction = grouped_predictions( rows, feature_name="direct_telemetry", target_name="target_real_pass_rate_delta", regularization=regularization, ) source_real_pass = np.asarray( [row["source_real_pass_rate"] for row in rows], dtype=np.float64 ) direct_base_prediction = source_real_pass + direct_base_prediction direct_telemetry_prediction = source_real_pass + direct_telemetry_prediction direct_base = metrics(rows, direct_base_prediction) direct_telemetry = metrics(rows, direct_telemetry_prediction) sensitivity[str(regularization)] = { "hybrid": { "sim_plus_outcome": hybrid_base, "sim_plus_outcome_plus_telemetry": hybrid_telemetry, "comparison": compare( rows, hybrid_base_prediction, hybrid_telemetry_prediction, hybrid_base, hybrid_telemetry, ), "prior_shrinkage": prior_shrinkage, }, "direct": { "real_outcome_only": direct_base, "telemetry_only": direct_telemetry, "comparison": compare( rows, direct_base_prediction, direct_telemetry_prediction, direct_base, direct_telemetry, ), }, } red_flags = [] if any(not math.isfinite(value) for value in residual_truth): red_flags.append("nonfinite_residual_target") if any(not math.isfinite(value) for value in pass_truth): red_flags.append("nonfinite_pass_rate_target") result = { "schema": "telemetry-residual-cross-config-diagnostic-v1", "status": "PASS" if not red_flags else "STOP", "scope": ( "single P1 trace/SLO-task development diagnostic; ordered transitions " "are not independent tasks and cannot support a generalization claim" ), "split": ( "hold out both ordered source config and target config identities; " "each fold contains both offered-load anchors" ), "features": { "sim_plus_outcome": len(rows[0]["hybrid_base"]), "sim_plus_outcome_plus_telemetry": len(rows[0]["hybrid_telemetry"]), "real_outcome_only": len(rows[0]["direct_base"]), "telemetry_only": len(rows[0]["direct_telemetry"]), }, "simulator": simulator, "regularization_sensitivity": sensitivity, "red_flags": red_flags, "sanity": { "transitions": len(rows), "load_levels": len( {row["source_level"] for row in rows} | {row["target_level"] for row in rows} ), "cross_load_transitions": sum( row["source_level"] != row["target_level"] for row in rows ), "ordered_cell_pairs": len( {(row["source_cell"], row["target_cell"]) for row in rows} ), "target_residual": { "n": len(residual_truth), "min": float(residual_truth.min()), "max": float(residual_truth.max()), "distinct_n": len(set(float(value) for value in residual_truth)), }, "target_real_pass_rate": { "n": len(pass_truth), "min": float(pass_truth.min()), "max": float(pass_truth.max()), "distinct_n": len(set(float(value) for value in pass_truth)), }, "invariants": { "finite_targets": not red_flags, "ratios_bounded": all( 0.0 <= row["target_sim_pass_rate"] <= 1.0 for row in rows ), "source_differs_from_target": all( row["source_cell"] != row["target_cell"] for row in rows ), "per_config_not_identical": len( set(float(value) for value in pass_truth) ) > 1, }, }, } atomic_json(args.output, result) if result["status"] != "PASS": raise RuntimeError(red_flags) return result def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser() result.add_argument("--paired-state", type=Path, required=True) result.add_argument("--output", type=Path, required=True) return result def main() -> None: result = execute(parser().parse_args()) print( json.dumps( { "status": result["status"], "transitions": result["sanity"]["transitions"], "sensitivity": result["regularization_sensitivity"], "sanity": result["sanity"], "red_flags": result["red_flags"], }, sort_keys=True, ) ) if __name__ == "__main__": main()