508 lines
19 KiB
Python
508 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Exploratory P1 audit against the strengthened simulator-aware baseline.
|
|
|
|
P1 was already running when the strong baseline was added, so this script is
|
|
not paper-facing prospective evidence. It trains only on the historical
|
|
Phase-6 task and evaluates the exact P1 primary probes. Both nested models
|
|
receive identical Frontier predictions; engine telemetry is the sole feature
|
|
difference.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from analyze_existing import (
|
|
DEFAULT_REGULARIZATION,
|
|
REGULARIZATION_SENSITIVITY,
|
|
_classification_metrics,
|
|
_fit_logistic,
|
|
_mcnemar_exact_p,
|
|
_sigmoid,
|
|
)
|
|
from analyze_pilot import build_pilot_examples, campaign_gpu_accounting
|
|
from analyze_prefixes import (
|
|
INSTRUMENTATION_FEATURES,
|
|
OUTCOME_FEATURES,
|
|
PrefixExample,
|
|
build_examples,
|
|
numeric,
|
|
policy_metrics,
|
|
sha256_file,
|
|
)
|
|
from analyze_strong_baseline import (
|
|
SIMULATOR_FEATURES,
|
|
load_simulator_features,
|
|
simulator_row,
|
|
)
|
|
|
|
|
|
AITUNER_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def git_capture(*arguments: str) -> str:
|
|
return subprocess.run(
|
|
["git", "-C", str(AITUNER_ROOT), *arguments],
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
).stdout
|
|
|
|
|
|
def load_pilot_simulator(
|
|
path: Path,
|
|
) -> tuple[dict[tuple[str, str], tuple[float, ...]], list[str]]:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
red_flags = []
|
|
if payload.get("status") != "PASS":
|
|
red_flags.append("pilot_simulator_not_pass")
|
|
features: dict[tuple[str, str], tuple[float, ...]] = {}
|
|
for item in payload.get("results", []):
|
|
key = (str(item["cell"]), str(item["role"]))
|
|
if key in features:
|
|
red_flags.append(f"duplicate_pilot_simulator_{key[0]}_{key[1]}")
|
|
continue
|
|
scorer = item["scorer"]
|
|
throughput = float(scorer["throughput_requests_per_second_per_gpu"])
|
|
pass_rate = float(scorer["slo"]["pass_rate"])
|
|
if throughput <= 0:
|
|
red_flags.append(f"nonpositive_pilot_simulator_throughput_{key[0]}_{key[1]}")
|
|
if not 0.0 <= pass_rate <= 1.0:
|
|
red_flags.append(f"pilot_simulator_ratio_out_of_range_{key[0]}_{key[1]}")
|
|
features[key] = (
|
|
math.log(throughput),
|
|
pass_rate,
|
|
float(bool(scorer["slo"]["feasible"])),
|
|
)
|
|
if len(features) != 12:
|
|
red_flags.append("pilot_simulator_entries_not_12")
|
|
return features, red_flags
|
|
|
|
|
|
def fit_model(
|
|
examples: list[PrefixExample],
|
|
simulator: list[tuple[float, ...]],
|
|
*,
|
|
instrumentation_aware: bool,
|
|
regularization: float,
|
|
) -> dict[str, Any]:
|
|
rows = []
|
|
for example, simulator_features in zip(examples, simulator):
|
|
values = example.outcome + simulator_features
|
|
if instrumentation_aware:
|
|
values += example.instrumentation
|
|
rows.append((1.0, *values))
|
|
matrix = np.asarray(rows, dtype=np.float64)
|
|
labels = np.asarray([example.feasible for example in examples], dtype=np.float64)
|
|
mean = matrix[:, 1:].mean(axis=0)
|
|
standard_deviation = matrix[:, 1:].std(axis=0)
|
|
standard_deviation[standard_deviation < 1e-8] = 1.0
|
|
standardized = matrix.copy()
|
|
standardized[:, 1:] = (standardized[:, 1:] - mean) / standard_deviation
|
|
weights = _fit_logistic(standardized, labels, regularization)
|
|
return {
|
|
"instrumentation_aware": instrumentation_aware,
|
|
"regularization": regularization,
|
|
"feature_mean": mean,
|
|
"feature_standard_deviation": standard_deviation,
|
|
"weights": weights,
|
|
}
|
|
|
|
|
|
def predict_model(
|
|
model: dict[str, Any],
|
|
examples: list[PrefixExample],
|
|
simulator: list[tuple[float, ...]],
|
|
) -> np.ndarray:
|
|
rows = []
|
|
for example, simulator_features in zip(examples, simulator):
|
|
values = example.outcome + simulator_features
|
|
if model["instrumentation_aware"]:
|
|
values += example.instrumentation
|
|
rows.append((1.0, *values))
|
|
matrix = np.asarray(rows, dtype=np.float64)
|
|
matrix[:, 1:] = (
|
|
matrix[:, 1:] - model["feature_mean"]
|
|
) / model["feature_standard_deviation"]
|
|
return _sigmoid(matrix @ model["weights"])
|
|
|
|
|
|
def covariate_shift(
|
|
training_examples: list[PrefixExample],
|
|
training_simulator: list[tuple[float, ...]],
|
|
pilot_examples: list[PrefixExample],
|
|
pilot_simulator: list[tuple[float, ...]],
|
|
*,
|
|
instrumentation_aware: bool,
|
|
) -> dict[str, Any]:
|
|
def matrix(
|
|
examples: list[PrefixExample], simulator: list[tuple[float, ...]]
|
|
) -> np.ndarray:
|
|
rows = []
|
|
for example, simulator_features in zip(examples, simulator):
|
|
values = example.outcome + simulator_features
|
|
if instrumentation_aware:
|
|
values += example.instrumentation
|
|
rows.append(values)
|
|
return np.asarray(rows, dtype=np.float64)
|
|
|
|
training = matrix(training_examples, training_simulator)
|
|
pilot = matrix(pilot_examples, pilot_simulator)
|
|
mean = training.mean(axis=0)
|
|
standard_deviation = training.std(axis=0)
|
|
standard_deviation[standard_deviation < 1e-8] = 1.0
|
|
absolute_z = np.abs((pilot - mean) / standard_deviation)
|
|
names = [*OUTCOME_FEATURES, *SIMULATOR_FEATURES]
|
|
if instrumentation_aware:
|
|
names.extend(INSTRUMENTATION_FEATURES)
|
|
return {
|
|
"values": numeric(absolute_z.ravel().tolist()),
|
|
"count_gt_3": int(np.sum(absolute_z > 3.0)),
|
|
"count_gt_5": int(np.sum(absolute_z > 5.0)),
|
|
"total_feature_values": int(absolute_z.size),
|
|
"per_feature_max_abs_z": {
|
|
name: float(value) for name, value in zip(names, absolute_z.max(axis=0))
|
|
},
|
|
}
|
|
|
|
|
|
def comparison(
|
|
training_examples: list[PrefixExample],
|
|
training_simulator: list[tuple[float, ...]],
|
|
pilot_examples: list[PrefixExample],
|
|
pilot_simulator: list[tuple[float, ...]],
|
|
regularization: float,
|
|
) -> dict[str, Any]:
|
|
labels = np.asarray([example.feasible for example in pilot_examples], dtype=np.int64)
|
|
baseline_model = fit_model(
|
|
training_examples,
|
|
training_simulator,
|
|
instrumentation_aware=False,
|
|
regularization=regularization,
|
|
)
|
|
instrument_model = fit_model(
|
|
training_examples,
|
|
training_simulator,
|
|
instrumentation_aware=True,
|
|
regularization=regularization,
|
|
)
|
|
baseline_probability = predict_model(
|
|
baseline_model, pilot_examples, pilot_simulator
|
|
)
|
|
instrument_probability = predict_model(
|
|
instrument_model, pilot_examples, pilot_simulator
|
|
)
|
|
baseline_correct = (baseline_probability >= 0.5) == labels
|
|
instrument_correct = (instrument_probability >= 0.5) == labels
|
|
paired = {
|
|
"both_correct": int(np.sum(baseline_correct & instrument_correct)),
|
|
"sim_outcome_only_correct": int(
|
|
np.sum(baseline_correct & ~instrument_correct)
|
|
),
|
|
"instrumentation_only_correct": int(
|
|
np.sum(~baseline_correct & instrument_correct)
|
|
),
|
|
"both_wrong": int(np.sum(~baseline_correct & ~instrument_correct)),
|
|
}
|
|
paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p(
|
|
paired["sim_outcome_only_correct"], paired["instrumentation_only_correct"]
|
|
)
|
|
return {
|
|
"sim_plus_outcome": {
|
|
"classification": _classification_metrics(labels, baseline_probability),
|
|
"policy_0p95": policy_metrics(
|
|
pilot_examples, labels, baseline_probability, 0.95
|
|
),
|
|
"probability": baseline_probability.tolist(),
|
|
},
|
|
"sim_plus_outcome_plus_instrumentation": {
|
|
"classification": _classification_metrics(labels, instrument_probability),
|
|
"policy_0p95": policy_metrics(
|
|
pilot_examples, labels, instrument_probability, 0.95
|
|
),
|
|
"probability": instrument_probability.tolist(),
|
|
},
|
|
"paired_correctness": paired,
|
|
}
|
|
|
|
|
|
def analyze(
|
|
phase6_path: Path,
|
|
phase6_raw_root: Path,
|
|
training_simulator_root: Path,
|
|
pilot_manifest_path: Path,
|
|
pilot_run_root: Path,
|
|
pilot_simulator_path: Path,
|
|
prior_state_paths: tuple[Path, ...] = (),
|
|
) -> dict[str, Any]:
|
|
phase6 = json.loads(phase6_path.read_text(encoding="utf-8"))
|
|
pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8"))
|
|
pilot_state_path = pilot_run_root / "controller-state.json"
|
|
pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8"))
|
|
gpu_accounting = campaign_gpu_accounting(
|
|
pilot_state_path, prior_state_paths
|
|
)
|
|
training_examples = build_examples(phase6, phase6_raw_root, 5.0)
|
|
training_simulator_map, training_simulator_sha256 = load_simulator_features(
|
|
training_simulator_root
|
|
)
|
|
training_simulator = [
|
|
simulator_row(example, training_simulator_map)
|
|
for example in training_examples
|
|
]
|
|
pilot_examples, pilot_details, red_flags = build_pilot_examples(
|
|
pilot_manifest, pilot_run_root, 5.0
|
|
)
|
|
pilot_simulator_map, simulator_red_flags = load_pilot_simulator(
|
|
pilot_simulator_path
|
|
)
|
|
red_flags.extend(simulator_red_flags)
|
|
pilot_simulator = []
|
|
for example, detail in zip(pilot_examples, pilot_details):
|
|
role = f"{detail['level']}1"
|
|
key = (example.cell, role)
|
|
if key not in pilot_simulator_map:
|
|
red_flags.append(f"missing_pilot_simulator_{example.cell}_{role}")
|
|
pilot_simulator.append((0.0, 0.0, 0.0))
|
|
else:
|
|
pilot_simulator.append(pilot_simulator_map[key])
|
|
|
|
sensitivity = {}
|
|
if not red_flags:
|
|
for regularization in REGULARIZATION_SENSITIVITY:
|
|
sensitivity[str(regularization)] = comparison(
|
|
training_examples,
|
|
training_simulator,
|
|
pilot_examples,
|
|
pilot_simulator,
|
|
regularization,
|
|
)
|
|
headline = sensitivity.get(str(DEFAULT_REGULARIZATION))
|
|
labels = [example.feasible for example in pilot_examples]
|
|
simulator_pass_rates = [row[1] for row in pilot_simulator]
|
|
simulator_labels = [int(row[2]) for row in pilot_simulator]
|
|
if len(training_examples) != 37:
|
|
red_flags.append("training_examples_not_37")
|
|
if len(pilot_examples) != 12:
|
|
red_flags.append("pilot_examples_not_12")
|
|
if len(set(labels)) != 2:
|
|
red_flags.append("pilot_single_label")
|
|
if len(set(simulator_pass_rates)) <= 1:
|
|
red_flags.append("pilot_simulator_results_identical")
|
|
if pilot_state.get("status") != "complete" or int(
|
|
pilot_state.get("completed_cells", 0)
|
|
) != 6:
|
|
red_flags.append("pilot_campaign_incomplete")
|
|
if any(detail["actual_timestamped_outcomes"] == 0 for detail in pilot_details):
|
|
red_flags.append("pilot_no_exact_request_timestamps")
|
|
all_cell_validations = all(
|
|
cell.get("validation") is not None
|
|
and all(cell["validation"]["invariants"].values())
|
|
for cell in pilot_state.get("cells", {}).values()
|
|
)
|
|
if not all_cell_validations:
|
|
red_flags.append("pilot_cell_validation_failed")
|
|
if not all(gpu_accounting["invariants"].values()):
|
|
red_flags.append("pilot_hard_cap_exceeded")
|
|
covariate_diagnostics = {
|
|
"sim_plus_outcome": covariate_shift(
|
|
training_examples,
|
|
training_simulator,
|
|
pilot_examples,
|
|
pilot_simulator,
|
|
instrumentation_aware=False,
|
|
),
|
|
"sim_plus_outcome_plus_instrumentation": covariate_shift(
|
|
training_examples,
|
|
training_simulator,
|
|
pilot_examples,
|
|
pilot_simulator,
|
|
instrumentation_aware=True,
|
|
),
|
|
}
|
|
|
|
if headline is None:
|
|
decision = {
|
|
"strong_incremental_gate": False,
|
|
"reason": "analysis red flag prevented nested comparison",
|
|
}
|
|
else:
|
|
baseline_policy = headline["sim_plus_outcome"]["policy_0p95"]
|
|
instrument_policy = headline[
|
|
"sim_plus_outcome_plus_instrumentation"
|
|
]["policy_0p95"]
|
|
baseline_errors = baseline_policy["false_accept"] + baseline_policy["false_reject"]
|
|
instrument_errors = (
|
|
instrument_policy["false_accept"] + instrument_policy["false_reject"]
|
|
)
|
|
baseline_reduction = baseline_policy["valid_cost_reduction_fraction"]
|
|
instrument_reduction = instrument_policy["valid_cost_reduction_fraction"]
|
|
reduction_delta = (
|
|
instrument_reduction - baseline_reduction
|
|
if baseline_reduction is not None and instrument_reduction is not None
|
|
else None
|
|
)
|
|
per_lambda_safe_and_better = []
|
|
for item in sensitivity.values():
|
|
baseline = item["sim_plus_outcome"]["policy_0p95"]
|
|
instrument = item["sim_plus_outcome_plus_instrumentation"]["policy_0p95"]
|
|
base_errors = baseline["false_accept"] + baseline["false_reject"]
|
|
inst_errors = instrument["false_accept"] + instrument["false_reject"]
|
|
base_reduction = baseline["valid_cost_reduction_fraction"]
|
|
inst_reduction = instrument["valid_cost_reduction_fraction"]
|
|
per_lambda_safe_and_better.append(
|
|
inst_errors == 0
|
|
and inst_errors <= base_errors
|
|
and base_reduction is not None
|
|
and inst_reduction is not None
|
|
and inst_reduction > base_reduction
|
|
)
|
|
decision = {
|
|
"strong_incremental_gate": bool(
|
|
not red_flags
|
|
and instrument_errors == 0
|
|
and instrument_errors <= baseline_errors
|
|
and reduction_delta is not None
|
|
and reduction_delta >= 0.15
|
|
),
|
|
"regularization_robust": all(per_lambda_safe_and_better),
|
|
"valid_cost_reduction_fraction_delta": reduction_delta,
|
|
"scope": "exploratory task; may choose P2 design but cannot establish contribution",
|
|
}
|
|
|
|
return {
|
|
"schema": "fidelity-strong-pilot-v1",
|
|
"status": "PASS" if not red_flags else "STOP",
|
|
"scope": (
|
|
"post-amendment exploratory P1 audit; strong model was not frozen before "
|
|
"partial P1 outcomes, so this is not prospective contribution evidence"
|
|
),
|
|
"features": {
|
|
"shared_outcome": list(OUTCOME_FEATURES),
|
|
"shared_simulator": list(SIMULATOR_FEATURES),
|
|
"instrumentation_only": list(INSTRUMENTATION_FEATURES),
|
|
},
|
|
"headline_regularization": DEFAULT_REGULARIZATION,
|
|
"headline": headline,
|
|
"regularization_sensitivity": sensitivity,
|
|
"simulator_only": {
|
|
"classification": _classification_metrics(
|
|
np.asarray(labels, dtype=np.int64),
|
|
np.asarray(simulator_labels, dtype=np.float64),
|
|
)
|
|
if labels
|
|
else None,
|
|
"predicted_feasible": simulator_labels,
|
|
},
|
|
"pilot_examples": [
|
|
{
|
|
**detail,
|
|
"sim_completed_throughput_per_gpu": math.exp(simulator[0]),
|
|
"sim_slo_pass_rate": simulator[1],
|
|
"sim_slo_feasible": bool(simulator[2]),
|
|
}
|
|
for detail, simulator in zip(pilot_details, pilot_simulator)
|
|
],
|
|
"covariate_shift_diagnostic": covariate_diagnostics,
|
|
"decision": decision,
|
|
"gpu": {
|
|
"primary_attempt_h20_hours": pilot_state["gpu_hours_total"],
|
|
**gpu_accounting,
|
|
},
|
|
"analysis": {
|
|
"script": str(Path(__file__).resolve()),
|
|
"script_sha256": sha256_file(Path(__file__).resolve()),
|
|
"aituner_git_head": git_capture("rev-parse", "HEAD").strip(),
|
|
"aituner_git_status_short": git_capture("status", "--short"),
|
|
},
|
|
"provenance": {
|
|
"phase6_metrics": str(phase6_path.resolve()),
|
|
"phase6_metrics_sha256": sha256_file(phase6_path),
|
|
"phase6_raw_root": str(phase6_raw_root.resolve()),
|
|
"training_simulator_root": str(training_simulator_root.resolve()),
|
|
"training_simulator_manifest_scorer_set_sha256": training_simulator_sha256,
|
|
"pilot_manifest": str(pilot_manifest_path.resolve()),
|
|
"pilot_manifest_sha256": sha256_file(pilot_manifest_path),
|
|
"pilot_run_root": str(pilot_run_root.resolve()),
|
|
"pilot_controller_state": str(pilot_state_path.resolve()),
|
|
"pilot_controller_state_sha256": sha256_file(pilot_state_path),
|
|
"pilot_simulator": str(pilot_simulator_path.resolve()),
|
|
"pilot_simulator_sha256": sha256_file(pilot_simulator_path),
|
|
},
|
|
"sanity": {
|
|
"red_flags": red_flags,
|
|
"training_examples": numeric([1 for _ in training_examples]),
|
|
"pilot_labels": {
|
|
**numeric(labels),
|
|
"positive": sum(labels),
|
|
"negative": len(labels) - sum(labels),
|
|
},
|
|
"pilot_simulator_pass_rate": numeric(simulator_pass_rates),
|
|
"invariants": {
|
|
"training_examples_37": len(training_examples) == 37,
|
|
"pilot_examples_12": len(pilot_examples) == 12,
|
|
"pilot_cells_6": len({example.cell for example in pilot_examples}) == 6,
|
|
"pilot_both_labels": len(set(labels)) == 2,
|
|
"simulator_ratios_bounded": all(
|
|
0.0 <= value <= 1.0 for value in simulator_pass_rates
|
|
),
|
|
"per_config_not_all_identical": len(set(simulator_pass_rates)) > 1,
|
|
"all_prefixes_exact_monotonic": all(
|
|
example.completion_time_source in {"exact_monotonic", "none_completed"}
|
|
for example in pilot_examples
|
|
),
|
|
"all_cell_validations": all_cell_validations,
|
|
"gpu_cost_nonnegative_below_cap": (
|
|
all(gpu_accounting["invariants"].values())
|
|
),
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--phase6-metrics", type=Path, required=True)
|
|
parser.add_argument("--phase6-raw-root", type=Path, required=True)
|
|
parser.add_argument("--training-simulator-root", type=Path, required=True)
|
|
parser.add_argument("--pilot-manifest", type=Path, required=True)
|
|
parser.add_argument("--pilot-run-root", type=Path, required=True)
|
|
parser.add_argument("--pilot-simulator", type=Path, required=True)
|
|
parser.add_argument("--prior-state", type=Path, action="append", default=[])
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
result = analyze(
|
|
args.phase6_metrics,
|
|
args.phase6_raw_root,
|
|
args.training_simulator_root,
|
|
args.pilot_manifest,
|
|
args.pilot_run_root,
|
|
args.pilot_simulator,
|
|
tuple(args.prior_state),
|
|
)
|
|
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": result["status"],
|
|
"red_flags": result["sanity"]["red_flags"],
|
|
"decision": result["decision"],
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
if result["status"] != "PASS":
|
|
raise RuntimeError(result["sanity"]["red_flags"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|