gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Strengthen fidelity calibration baseline

Browse Source
This commit is contained in:
Gahow Wang
2026-07-14 13:08:45 +08:00
parent 93daf291f6
commit 23142aa359
5 changed files with 881 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

298
runs/fidelity-headroom/analyze_strong_baseline.py Normal file
View File

@@ -0,0 +1,298 @@
#!/usr/bin/env python3
"""Audit telemetry against a simulator-aware outcome calibration baseline.
This is a retrospective headroom check. It strengthens the earlier
outcome-only baseline by giving both nested models the same per-anchor
Frontier throughput and SLO predictions. The only additional inputs to the
larger model are real engine Layer-1 features.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from pathlib import Path
from typing import Any
import numpy as np
from analyze_existing import (
DEFAULT_REGULARIZATION,
REGULARIZATION_SENSITIVITY,
_classification_metrics,
_fit_logistic,
_group_bootstrap_delta,
_mcnemar_exact_p,
_sigmoid,
)
from analyze_prefixes import (
INSTRUMENTATION_FEATURES,
OUTCOME_FEATURES,
PrefixExample,
build_examples,
numeric,
policy_metrics,
sha256_file,
)
SIMULATOR_FEATURES = (
"log_sim_completed_throughput_per_gpu",
"sim_slo_pass_rate",
"sim_slo_feasible",
)
def load_simulator_features(raw_root: Path) -> tuple[dict[tuple[str, float], tuple[float, ...]], str]:
features: dict[tuple[str, float], tuple[float, ...]] = {}
digest = hashlib.sha256()
paths = sorted(raw_root.glob("*/trial-0001/run_manifest.json"))
for manifest_path in paths:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
run = manifest["run"]
if run["mode"] != "frozen-calibrated":
continue
scorer_path = manifest_path.parent / "scorer_output.json"
scorer = json.loads(scorer_path.read_text(encoding="utf-8"))
key = (str(run["cell_id"]), float(run["sampling_u"]))
if key in features:
raise ValueError(f"duplicate frozen simulator run: {key}")
throughput = float(scorer["throughput_requests_per_second_per_gpu"])
pass_rate = float(scorer["slo"]["pass_rate"])
if throughput <= 0 or not 0.0 <= pass_rate <= 1.0:
raise ValueError(f"invalid simulator output: {key}")
features[key] = (
math.log(throughput),
pass_rate,
float(bool(scorer["slo"]["feasible"])),
)
for path in (manifest_path, scorer_path):
digest.update(str(path.relative_to(raw_root)).encode())
digest.update(path.read_bytes())
return features, digest.hexdigest()
def simulator_row(
example: PrefixExample,
features: dict[tuple[str, float], tuple[float, ...]],
) -> tuple[float, ...]:
matches = [
values
for (cell, anchor), values in features.items()
if cell == example.cell
and math.isclose(anchor, example.anchor, rel_tol=0.0, abs_tol=1e-12)
]
if len(matches) != 1:
raise ValueError(
f"expected one simulator match for {example.cell}/{example.anchor}: {len(matches)}"
)
return matches[0]
def grouped_predictions(
examples: list[PrefixExample],
simulator: dict[tuple[str, float], tuple[float, ...]],
*,
instrumentation_aware: bool,
regularization: float,
) -> tuple[np.ndarray, np.ndarray, list[str]]:
probabilities: list[float] = []
labels: list[int] = []
groups: list[str] = []
for held_out in sorted({example.cell for example in examples}):
train = [example for example in examples if example.cell != held_out]
test = [example for example in examples if example.cell == held_out]
def row(example: PrefixExample) -> np.ndarray:
values = example.outcome + simulator_row(example, simulator)
if instrumentation_aware:
values += example.instrumentation
return np.asarray((1.0, *values), dtype=np.float64)
x_train = np.stack([row(example) for example in train])
x_test = np.stack([row(example) for example in test])
y_train = np.asarray([example.feasible for example in train], dtype=np.float64)
mean = x_train[:, 1:].mean(axis=0)
standard_deviation = x_train[:, 1:].std(axis=0)
standard_deviation[standard_deviation < 1e-8] = 1.0
x_train[:, 1:] = (x_train[:, 1:] - mean) / standard_deviation
x_test[:, 1:] = (x_test[:, 1:] - mean) / standard_deviation
weights = _fit_logistic(x_train, y_train, regularization)
probabilities.extend(_sigmoid(x_test @ weights).tolist())
labels.extend(example.feasible for example in test)
groups.extend(held_out for _ in test)
return (
np.asarray(labels, dtype=np.int64),
np.asarray(probabilities, dtype=np.float64),
groups,
)
def analyze(
phase6_path: Path,
phase6_raw_root: Path,
simulator_raw_root: Path,
simulator_metrics_path: Path,
) -> dict[str, Any]:
phase6 = json.loads(phase6_path.read_text(encoding="utf-8"))
examples = build_examples(phase6, phase6_raw_root, 5.0)
simulator, simulator_raw_sha256 = load_simulator_features(simulator_raw_root)
red_flags = []
try:
matched = [simulator_row(example, simulator) for example in examples]
except ValueError as error:
matched = []
red_flags.append(str(error))
sensitivity = {}
if matched:
for regularization in REGULARIZATION_SENSITIVITY:
labels, baseline_probability, groups = grouped_predictions(
examples,
simulator,
instrumentation_aware=False,
regularization=regularization,
)
instrument_labels, instrument_probability, instrument_groups = grouped_predictions(
examples,
simulator,
instrumentation_aware=True,
regularization=regularization,
)
if not np.array_equal(labels, instrument_labels) or groups != instrument_groups:
raise AssertionError("nested baseline folds differ")
baseline_correct = (baseline_probability >= 0.5) == labels
instrument_correct = (instrument_probability >= 0.5) == labels
paired = {
"both_correct": int(np.sum(baseline_correct & instrument_correct)),
"sim_outcome_only_correct": int(
np.sum(baseline_correct & ~instrument_correct)
),
"instrumentation_only_correct": int(
np.sum(~baseline_correct & instrument_correct)
),
"both_wrong": int(np.sum(~baseline_correct & ~instrument_correct)),
}
paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p(
paired["sim_outcome_only_correct"],
paired["instrumentation_only_correct"],
)
sensitivity[str(regularization)] = {
"sim_plus_outcome": {
"classification": _classification_metrics(labels, baseline_probability),
"policy_0p95": policy_metrics(
examples, labels, baseline_probability, 0.95
),
},
"sim_plus_outcome_plus_instrumentation": {
"classification": _classification_metrics(labels, instrument_probability),
"policy_0p95": policy_metrics(
examples, labels, instrument_probability, 0.95
),
},
"paired_correctness": paired,
"group_bootstrap": _group_bootstrap_delta(
labels,
baseline_probability,
instrument_probability,
groups,
),
}
headline = sensitivity.get(str(DEFAULT_REGULARIZATION))
simulator_pass_rates = [row[1] for row in matched]
labels = [example.feasible for example in examples]
if len(examples) != 37:
red_flags.append("examples_not_37")
if len(simulator) != 92:
red_flags.append("frozen_simulator_runs_not_92")
if len(set(labels)) != 2:
red_flags.append("single_label")
if matched and not all(0.0 <= value <= 1.0 for value in simulator_pass_rates):
red_flags.append("simulator_pass_rate_out_of_range")
return {
"schema": "fidelity-strong-baseline-v1",
"status": "PASS" if not red_flags else "STOP",
"scope": "retrospective one-task headroom audit; not contribution evidence",
"comparison": (
"same 5-second prefix, folds, logistic family, regularization, and frozen "
"Frontier outputs; the only nested difference is real Layer-1 engine state"
),
"features": {
"shared_outcome": list(OUTCOME_FEATURES),
"shared_simulator": list(SIMULATOR_FEATURES),
"instrumentation_only": list(INSTRUMENTATION_FEATURES),
},
"headline_regularization": DEFAULT_REGULARIZATION,
"headline": headline,
"regularization_sensitivity": sensitivity,
"provenance": {
"phase6_metrics": str(phase6_path.resolve()),
"phase6_metrics_sha256": sha256_file(phase6_path),
"phase6_raw_root": str(phase6_raw_root.resolve()),
"simulator_metrics": str(simulator_metrics_path.resolve()),
"simulator_metrics_sha256": sha256_file(simulator_metrics_path),
"simulator_raw_root": str(simulator_raw_root.resolve()),
"frozen_simulator_manifest_scorer_set_sha256": simulator_raw_sha256,
},
"decision": {
"contribution_established": False,
"prospective_requirement": (
"repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks"
),
},
"sanity": {
"red_flags": red_flags,
"examples": numeric([1 for _ in examples]),
"labels": {
**numeric(labels),
"positive": sum(labels),
"negative": len(labels) - sum(labels),
},
"matched_simulator_pass_rate": numeric(simulator_pass_rates),
"frozen_simulator_runs": len(simulator),
"invariants": {
"all_examples_matched_once": len(matched) == len(examples),
"same_nested_folds": True,
"simulator_ratios_bounded": all(
0.0 <= value <= 1.0 for value in simulator_pass_rates
),
"labels_not_identical": len(set(labels)) == 2,
"per_config_results_not_all_identical": len(set(simulator_pass_rates)) > 1,
},
},
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--phase6-metrics", type=Path, required=True)
parser.add_argument("--phase6-raw-root", type=Path, required=True)
parser.add_argument("--simulator-raw-root", type=Path, required=True)
parser.add_argument("--simulator-metrics", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = analyze(
args.phase6_metrics,
args.phase6_raw_root,
args.simulator_raw_root,
args.simulator_metrics,
)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
print(
json.dumps(
{
"status": result["status"],
"output": str(args.output),
"red_flags": result["sanity"]["red_flags"],
},
sort_keys=True,
)
)
if __name__ == "__main__":
main()

477
runs/fidelity-headroom/strong-baseline-metrics.json Normal file
View File

@@ -0,0 +1,477 @@
{
"comparison": "same 5-second prefix, folds, logistic family, regularization, and frozen Frontier outputs; the only nested difference is real Layer-1 engine state",
"decision": {
"contribution_established": false,
"prospective_requirement": "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks"
},
"features": {
"instrumentation_only": [
"model_steps_per_second",
"waiting_mean",
"waiting_max",
"waiting_nonzero_share",
"running_mean",
"running_max",
"decode_batch_mean",
"decode_batch_max",
"decode_batch_cv",
"kv_usage_mean",
"kv_usage_max",
"kv_usage_end_minus_start",
"graph_none_share",
"graph_full_share",
"padding_fraction",
"prefill_token_fraction",
"preemptions"
],
"shared_outcome": [
"log_offered_rate_per_gpu",
"log2_tp",
"log2_max_num_seqs",
"admitted_fraction",
"completed_over_admitted",
"completed_pass_rate",
"completed_fail_fraction_of_total",
"outstanding_over_admitted",
"ttft_max_over_slo_max",
"ttft_mean_over_slo_max",
"tpot_max_over_slo",
"tpot_mean_over_slo",
"admitted_input_tokens_mean_over_limit"
],
"shared_simulator": [
"log_sim_completed_throughput_per_gpu",
"sim_slo_pass_rate",
"sim_slo_feasible"
]
},
"headline": {
"group_bootstrap": {
"accuracy_delta_instrumentation_minus_outcome": {
"ci95": [
0.0,
0.18181818181818188
],
"point": 0.08108108108108103
},
"brier_delta_instrumentation_minus_outcome": {
"ci95": [
-0.04292727744470806,
0.019924730979981074
],
"point": -0.010145365131402809
},
"replicates": 10000,
"seed": 20260714,
"semantics": "group bootstrap over cells; diagnostic confidence interval"
},
"paired_correctness": {
"both_correct": 30,
"both_wrong": 4,
"instrumentation_only_correct": 3,
"mcnemar_exact_two_sided_p": 0.25,
"sim_outcome_only_correct": 0
},
"sim_plus_outcome": {
"classification": {
"accuracy": 0.8108108108108109,
"balanced_accuracy": 0.7242063492063493,
"brier": 0.1058226346682949,
"confusion": {
"false_negative": 3,
"false_positive": 4,
"true_negative": 5,
"true_positive": 25
},
"log_loss": 0.3011048455679668
},
"policy_0p95": {
"abstain_continue_full": 17,
"correctly_saved_h20_hours": 0.5429431818208333,
"decision_coverage": 0.5405405405405406,
"early_accept": 16,
"early_reject": 4,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.5429431818208333,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.5088695307144538,
"valid_zero_error_policy": true
}
},
"sim_plus_outcome_plus_instrumentation": {
"classification": {
"accuracy": 0.8918918918918919,
"balanced_accuracy": 0.8154761904761905,
"brier": 0.0956772695368921,
"confusion": {
"false_negative": 1,
"false_positive": 3,
"true_negative": 6,
"true_positive": 27
},
"log_loss": 0.288823031828762
},
"policy_0p95": {
"abstain_continue_full": 12,
"correctly_saved_h20_hours": 0.7360063646722222,
"decision_coverage": 0.6756756756756757,
"early_accept": 20,
"early_reject": 5,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.7360063646722222,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.6898165884274738,
"valid_zero_error_policy": true
}
}
},
"headline_regularization": 1.0,
"provenance": {
"frozen_simulator_manifest_scorer_set_sha256": "833842d96ecaa0b059ef99852621752f7989e63d100118b6025425fb119b7a55",
"phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json",
"phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16",
"phase6_raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells",
"simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
"simulator_metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206",
"simulator_raw_root": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/raw"
},
"regularization_sensitivity": {
"0.1": {
"group_bootstrap": {
"accuracy_delta_instrumentation_minus_outcome": {
"ci95": [
-0.17500000000000004,
0.0
],
"point": -0.08108108108108103
},
"brier_delta_instrumentation_minus_outcome": {
"ci95": [
-0.026383192545085435,
0.0607951286646285
],
"point": 0.019228316404518567
},
"replicates": 10000,
"seed": 20260714,
"semantics": "group bootstrap over cells; diagnostic confidence interval"
},
"paired_correctness": {
"both_correct": 30,
"both_wrong": 4,
"instrumentation_only_correct": 0,
"mcnemar_exact_two_sided_p": 0.25,
"sim_outcome_only_correct": 3
},
"sim_plus_outcome": {
"classification": {
"accuracy": 0.8918918918918919,
"balanced_accuracy": 0.8154761904761905,
"brier": 0.10990776306815446,
"confusion": {
"false_negative": 1,
"false_positive": 3,
"true_negative": 6,
"true_positive": 27
},
"log_loss": 0.328357763455984
},
"policy_0p95": {
"abstain_continue_full": 12,
"correctly_saved_h20_hours": 0.7402314096841667,
"decision_coverage": 0.6756756756756757,
"early_accept": 20,
"early_reject": 5,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.7402314096841667,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.6937764809990414,
"valid_zero_error_policy": true
}
},
"sim_plus_outcome_plus_instrumentation": {
"classification": {
"accuracy": 0.8108108108108109,
"balanced_accuracy": 0.7619047619047619,
"brier": 0.12913607947267303,
"confusion": {
"false_negative": 4,
"false_positive": 3,
"true_negative": 6,
"true_positive": 24
},
"log_loss": 0.4373556318820343
},
"policy_0p95": {
"abstain_continue_full": 9,
"correctly_saved_h20_hours": 0.7469523484622221,
"decision_coverage": 0.7567567567567568,
"early_accept": 22,
"early_reject": 6,
"false_accept": 2,
"false_accept_examples": [
{
"anchor": 0.49609375,
"cell": "tp2_mns8",
"label_feasible": false,
"probability_feasible": 0.9869795738005246,
"remaining_h20_hours": 0.010117910306111111
},
{
"anchor": 0.033717411016,
"cell": "tp4_mns16",
"label_feasible": false,
"probability_feasible": 0.9855364057197005,
"remaining_h20_hours": 0.023106262014444445
}
],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.03322417232055556,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.7801765207827777,
"threshold": 0.95,
"valid_cost_reduction_fraction": null,
"valid_zero_error_policy": false
}
}
},
"1.0": {
"group_bootstrap": {
"accuracy_delta_instrumentation_minus_outcome": {
"ci95": [
0.0,
0.18181818181818188
],
"point": 0.08108108108108103
},
"brier_delta_instrumentation_minus_outcome": {
"ci95": [
-0.04292727744470806,
0.019924730979981074
],
"point": -0.010145365131402809
},
"replicates": 10000,
"seed": 20260714,
"semantics": "group bootstrap over cells; diagnostic confidence interval"
},
"paired_correctness": {
"both_correct": 30,
"both_wrong": 4,
"instrumentation_only_correct": 3,
"mcnemar_exact_two_sided_p": 0.25,
"sim_outcome_only_correct": 0
},
"sim_plus_outcome": {
"classification": {
"accuracy": 0.8108108108108109,
"balanced_accuracy": 0.7242063492063493,
"brier": 0.1058226346682949,
"confusion": {
"false_negative": 3,
"false_positive": 4,
"true_negative": 5,
"true_positive": 25
},
"log_loss": 0.3011048455679668
},
"policy_0p95": {
"abstain_continue_full": 17,
"correctly_saved_h20_hours": 0.5429431818208333,
"decision_coverage": 0.5405405405405406,
"early_accept": 16,
"early_reject": 4,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.5429431818208333,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.5088695307144538,
"valid_zero_error_policy": true
}
},
"sim_plus_outcome_plus_instrumentation": {
"classification": {
"accuracy": 0.8918918918918919,
"balanced_accuracy": 0.8154761904761905,
"brier": 0.0956772695368921,
"confusion": {
"false_negative": 1,
"false_positive": 3,
"true_negative": 6,
"true_positive": 27
},
"log_loss": 0.288823031828762
},
"policy_0p95": {
"abstain_continue_full": 12,
"correctly_saved_h20_hours": 0.7360063646722222,
"decision_coverage": 0.6756756756756757,
"early_accept": 20,
"early_reject": 5,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.7360063646722222,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.6898165884274738,
"valid_zero_error_policy": true
}
}
},
"10.0": {
"group_bootstrap": {
"accuracy_delta_instrumentation_minus_outcome": {
"ci95": [
-0.13333333333333341,
0.05555555555555558
],
"point": -0.027027027027027084
},
"brier_delta_instrumentation_minus_outcome": {
"ci95": [
-0.03091105649870874,
0.01684192005239855
],
"point": -0.007318433328714388
},
"replicates": 10000,
"seed": 20260714,
"semantics": "group bootstrap over cells; diagnostic confidence interval"
},
"paired_correctness": {
"both_correct": 30,
"both_wrong": 4,
"instrumentation_only_correct": 1,
"mcnemar_exact_two_sided_p": 1.0,
"sim_outcome_only_correct": 2
},
"sim_plus_outcome": {
"classification": {
"accuracy": 0.8648648648648649,
"balanced_accuracy": 0.7222222222222222,
"brier": 0.10613344425735322,
"confusion": {
"false_negative": 0,
"false_positive": 5,
"true_negative": 4,
"true_positive": 28
},
"log_loss": 0.3404203142465075
},
"policy_0p95": {
"abstain_continue_full": 32,
"correctly_saved_h20_hours": 0.21727432337249997,
"decision_coverage": 0.13513513513513514,
"early_accept": 5,
"early_reject": 0,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.21727432337249997,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.20363877229302757,
"valid_zero_error_policy": true
}
},
"sim_plus_outcome_plus_instrumentation": {
"classification": {
"accuracy": 0.8378378378378378,
"balanced_accuracy": 0.7420634920634921,
"brier": 0.09881501092863883,
"confusion": {
"false_negative": 2,
"false_positive": 4,
"true_negative": 5,
"true_positive": 26
},
"log_loss": 0.312914193285738
},
"policy_0p95": {
"abstain_continue_full": 30,
"correctly_saved_h20_hours": 0.2384080185036111,
"decision_coverage": 0.1891891891891892,
"early_accept": 6,
"early_reject": 1,
"false_accept": 0,
"false_accept_examples": [],
"false_reject": 0,
"false_reject_examples": [],
"full_trial_h20_hours": 1.0669595034675,
"invalidly_saved_h20_hours": 0.0,
"remaining_h20_hours_at_cutoff": 0.957237281245278,
"saved_h20_hours_if_decisions_used": 0.2384080185036111,
"threshold": 0.95,
"valid_cost_reduction_fraction": 0.22344617366339725,
"valid_zero_error_policy": true
}
}
}
},
"sanity": {
"examples": {
"distinct_n": 1,
"max": 1.0,
"min": 1.0,
"n": 37
},
"frozen_simulator_runs": 92,
"invariants": {
"all_examples_matched_once": true,
"labels_not_identical": true,
"per_config_results_not_all_identical": true,
"same_nested_folds": true,
"simulator_ratios_bounded": true
},
"labels": {
"distinct_n": 2,
"max": 1.0,
"min": 0.0,
"n": 37,
"negative": 9,
"positive": 28
},
"matched_simulator_pass_rate": {
"distinct_n": 12,
"max": 1.0,
"min": 0.06884057971014493,
"n": 37
},
"red_flags": []
},
"schema": "fidelity-strong-baseline-v1",
"scope": "retrospective one-task headroom audit; not contribution evidence",
"status": "PASS"
}

37
runs/fidelity-headroom/test_strong_baseline.py Normal file
View File

@@ -0,0 +1,37 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
from pathlib import Path
from analyze_strong_baseline import analyze
ROOT = Path(__file__).resolve().parents[2]
REPLAYSERVE = ROOT.parent / "replayserve"
def main() -> None:
result = analyze(
ROOT / "runs/opprof-phase6/phase6/metrics.json",
ROOT / "runs/opprof-phase6/phase6/solo-authoritative/cells",
REPLAYSERVE / "runs/simfid_s2rb/results/raw",
REPLAYSERVE / "runs/simfid_s2rb/results/metrics.json",
)
assert result["status"] == "PASS", json.dumps(result["sanity"], indent=2)
assert result["sanity"]["frozen_simulator_runs"] == 92
assert result["sanity"]["labels"]["n"] == 37
headline = result["headline"]
assert headline["sim_plus_outcome"]["policy_0p95"]["false_accept"] == 0
assert headline["sim_plus_outcome"]["policy_0p95"]["false_reject"] == 0
assert (
headline["sim_plus_outcome_plus_instrumentation"]["policy_0p95"][
"false_accept"
]
== 0
)
print("fidelity strong baseline: PASS")
if __name__ == "__main__":
main()