From 23142aa359e57a7e82be18427419de3ae69ca7f1 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:08:45 +0800 Subject: [PATCH] Strengthen fidelity calibration baseline --- ...idelity-aware-harness-headroom-20260714.md | 34 ++ ...idelity-aware-harness-protocol-20260714.md | 41 +- .../analyze_strong_baseline.py | 298 +++++++++++ .../strong-baseline-metrics.json | 477 ++++++++++++++++++ .../fidelity-headroom/test_strong_baseline.py | 37 ++ 5 files changed, 881 insertions(+), 6 deletions(-) create mode 100644 runs/fidelity-headroom/analyze_strong_baseline.py create mode 100644 runs/fidelity-headroom/strong-baseline-metrics.json create mode 100644 runs/fidelity-headroom/test_strong_baseline.py diff --git a/docs/fidelity-aware-harness-headroom-20260714.md b/docs/fidelity-aware-harness-headroom-20260714.md index a3cb832..8f11346 100644 --- a/docs/fidelity-aware-harness-headroom-20260714.md +++ b/docs/fidelity-aware-harness-headroom-20260714.md @@ -46,6 +46,35 @@ at 15 seconds it is 88.89% versus 91.67%; at 20 seconds it is 86.11% versus 91.67%, but both 0.95 policies make one false reject. Five seconds is therefore a training-selected operating point, not a test result. +## Strong simulator-aware calibration baseline + +The original nested comparison used the same simulator shortlist but did not +put Frontier's per-anchor prediction in either model. A stronger retrospective +audit now gives both models frozen-calibrated simulated throughput, simulated +SLO pass rate, and simulated feasibility. Under the same leave-one-cell-out +folds, 5-second cutoff, L2 logistic family, regularization 1.0, and threshold +0.95: + +| Metric | Sim + outcome | Sim + outcome + instrumentation | Delta | +|---|---:|---:|---:| +| Accuracy | 81.08% | 89.19% | +8.11 pp | +| Balanced accuracy | 72.42% | 81.55% | +9.13 pp | +| Brier score | 0.1058 | 0.0957 | -0.0101 | +| Safe early decisions | 20/37 | 25/37 | +5 | +| Valid full-trial cost reduction | 50.89% | 68.98% | +18.09 pp | +| Residual verification H20-hours | 0.5240 | 0.3310 | -36.84% | + +Both 0.95 policies have zero false accept and zero false reject on this +retrospective task. Only three 0.5-threshold classifications differ in favor +of instrumentation and none in favor of the strong baseline; McNemar's exact +two-sided p-value is 0.25. The cell-bootstrap accuracy-delta interval is +`[0.00,+18.18]` percentage points. The result is not robust to regularization: +at 0.1 the strong baseline is more accurate and the instrumentation policy +makes two unsafe decisions; at 10.0 the strong baseline is also more accurate. +Thus the stronger comparison still has enough point-estimate headroom for a +held-out test, but it materially weakens the evidence and makes a prospective +task-level result mandatory. + ## Interpretation There is enough headroom to run a held-out pilot, but not enough evidence to @@ -73,6 +102,9 @@ with three full repetitions. The registered protocol is - `runs/fidelity-headroom/prefix-metrics.json` - `runs/fidelity-headroom/test_analysis.py` - `runs/fidelity-headroom/test_prefix_analysis.py` +- `runs/fidelity-headroom/analyze_strong_baseline.py` +- `runs/fidelity-headroom/strong-baseline-metrics.json` +- `runs/fidelity-headroom/test_strong_baseline.py` ## Sanity block @@ -86,6 +118,8 @@ with three full repetitions. The registered protocol is | Outcome probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics | | Instrumentation probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics | | Layer-1 streams | 12 | 14,174 records | 58,725 records | 12 | Contiguous, zero drops | +| Matched frozen simulator anchors | 37 | pass rate 0.0688 | pass rate 1.0 | 12 pass-rate values | Every prefix matched exactly once | +| Frozen simulator anchor corpus | 92 | positive throughput | positive throughput | >1 | No duplicate cell/anchor run | Checked invariants: same folds/model family and cutoff; no full verdict in a feature; prefix-only Layer-1 slicing; non-negative costs/counters; bounded diff --git a/docs/fidelity-aware-harness-protocol-20260714.md b/docs/fidelity-aware-harness-protocol-20260714.md index 1bdb1b2..8bf05f0 100644 --- a/docs/fidelity-aware-harness-protocol-20260714.md +++ b/docs/fidelity-aware-harness-protocol-20260714.md @@ -52,6 +52,35 @@ difference is Z. The initial family is intentionally simple: a positive result then demonstrates value in the engine signal rather than capacity in a larger learner. A sequence model is admissible only as a later, paired ablation. +### Amendment A1: strengthen the calibration baseline before P2 + +Frozen 2026-07-14 13:08 Asia/Singapore, after P1 launch but before P1 +completion or analysis. A baseline audit found that the first frozen P1 +models use the simulator only to define candidate order; their feature vectors +do not contain the simulator's per-anchor prediction. This is insufficient +for the stronger term **outcome-only calibration**. P1 therefore remains a +prospective test of the originally frozen cross-workload predictor, but cannot +by itself open a contribution claim. + +For P2/P3, both nested models must additionally receive the identical frozen +simulator outputs available at that decision: predicted completed throughput +per GPU, predicted SLO pass rate, and predicted feasibility. The comparison +is consequently `sim + config + workload + real outcome prefix` versus that +exact vector plus real engine state. Simulator features, regularization, +cutoff, and thresholds are frozen before any P2 task. If telemetry does not +improve this stronger baseline, the harness has no independent contribution. + +The same audit also separates algorithm cost from benchmark-oracle cost. +Headline method cost includes every action the method would execute online: +simulator profiling/calibration, model onboarding, server startup, warm-up, +real prefix, continuation after abstention, method-requested confirmation, +logging overhead, failures, and cleanup. Exhaustive real-oracle runs and the +extra repetitions used only to construct 2-of-3 evaluation labels are common +benchmark annotation cost; they are reported separately and charged to no +method. A second, deliberately conservative table adds that common cost to +all methods. This prevents both hiding real method cost and making the +percentage gate mathematically depend on offline ground-truth annotation. + The frozen first policy uses a 5-second prefix, L2 regularization 1.0, and a two-sided abstaining threshold of 0.95: accept at `p(feasible)>=0.95`, reject at `p(feasible)<=0.05`, otherwise continue the exact same trial to completion. @@ -64,15 +93,15 @@ therefore not evidence; all claims come from subsequent held-out tasks. |---|---:|---:|---:|---:|---:| | Real-only oracle | no | no | full | optional diagnostic | every candidate/anchor | | Sim top-k + real final | yes | included in full run | full | no decision use | every shortlisted candidate/anchor | -| Outcome-only calibration | yes | yes | yes | no | only on abstention | -| Instrumentation-aware | yes | yes | yes | yes | only on abstention | +| Outcome-only calibration | yes, including its prediction features | yes | yes | no | only on abstention | +| Instrumentation-aware | same prediction features | yes | yes | yes | only on abstention | Tie buckets are expanded before top-k. `k` is selected on training tasks and is fixed on held-out tasks; an oracle per-task k is forbidden. Outcome-only -receives all information available outside the engine, including config and -workload features. Instrumentation cannot use any record submitted after the -cutoff. The full label, confirmation votes, simulator error, and later -requests are never model features. +receives all information available outside the engine, including config, +workload, and frozen simulator-prediction features. Instrumentation cannot use +any record submitted after the cutoff. The full label, confirmation votes, +realized simulator error, and later requests are never model features. ## Staged experiment diff --git a/runs/fidelity-headroom/analyze_strong_baseline.py b/runs/fidelity-headroom/analyze_strong_baseline.py new file mode 100644 index 0000000..4880afd --- /dev/null +++ b/runs/fidelity-headroom/analyze_strong_baseline.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Audit telemetry against a simulator-aware outcome calibration baseline. + +This is a retrospective headroom check. It strengthens the earlier +outcome-only baseline by giving both nested models the same per-anchor +Frontier throughput and SLO predictions. The only additional inputs to the +larger model are real engine Layer-1 features. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np + +from analyze_existing import ( + DEFAULT_REGULARIZATION, + REGULARIZATION_SENSITIVITY, + _classification_metrics, + _fit_logistic, + _group_bootstrap_delta, + _mcnemar_exact_p, + _sigmoid, +) +from analyze_prefixes import ( + INSTRUMENTATION_FEATURES, + OUTCOME_FEATURES, + PrefixExample, + build_examples, + numeric, + policy_metrics, + sha256_file, +) + + +SIMULATOR_FEATURES = ( + "log_sim_completed_throughput_per_gpu", + "sim_slo_pass_rate", + "sim_slo_feasible", +) + + +def load_simulator_features(raw_root: Path) -> tuple[dict[tuple[str, float], tuple[float, ...]], str]: + features: dict[tuple[str, float], tuple[float, ...]] = {} + digest = hashlib.sha256() + paths = sorted(raw_root.glob("*/trial-0001/run_manifest.json")) + for manifest_path in paths: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + run = manifest["run"] + if run["mode"] != "frozen-calibrated": + continue + scorer_path = manifest_path.parent / "scorer_output.json" + scorer = json.loads(scorer_path.read_text(encoding="utf-8")) + key = (str(run["cell_id"]), float(run["sampling_u"])) + if key in features: + raise ValueError(f"duplicate frozen simulator run: {key}") + throughput = float(scorer["throughput_requests_per_second_per_gpu"]) + pass_rate = float(scorer["slo"]["pass_rate"]) + if throughput <= 0 or not 0.0 <= pass_rate <= 1.0: + raise ValueError(f"invalid simulator output: {key}") + features[key] = ( + math.log(throughput), + pass_rate, + float(bool(scorer["slo"]["feasible"])), + ) + for path in (manifest_path, scorer_path): + digest.update(str(path.relative_to(raw_root)).encode()) + digest.update(path.read_bytes()) + return features, digest.hexdigest() + + +def simulator_row( + example: PrefixExample, + features: dict[tuple[str, float], tuple[float, ...]], +) -> tuple[float, ...]: + matches = [ + values + for (cell, anchor), values in features.items() + if cell == example.cell + and math.isclose(anchor, example.anchor, rel_tol=0.0, abs_tol=1e-12) + ] + if len(matches) != 1: + raise ValueError( + f"expected one simulator match for {example.cell}/{example.anchor}: {len(matches)}" + ) + return matches[0] + + +def grouped_predictions( + examples: list[PrefixExample], + simulator: dict[tuple[str, float], tuple[float, ...]], + *, + instrumentation_aware: bool, + regularization: float, +) -> tuple[np.ndarray, np.ndarray, list[str]]: + probabilities: list[float] = [] + labels: list[int] = [] + groups: list[str] = [] + for held_out in sorted({example.cell for example in examples}): + train = [example for example in examples if example.cell != held_out] + test = [example for example in examples if example.cell == held_out] + + def row(example: PrefixExample) -> np.ndarray: + values = example.outcome + simulator_row(example, simulator) + if instrumentation_aware: + values += example.instrumentation + return np.asarray((1.0, *values), dtype=np.float64) + + x_train = np.stack([row(example) for example in train]) + x_test = np.stack([row(example) for example in test]) + y_train = np.asarray([example.feasible for example in train], dtype=np.float64) + mean = x_train[:, 1:].mean(axis=0) + standard_deviation = x_train[:, 1:].std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + x_train[:, 1:] = (x_train[:, 1:] - mean) / standard_deviation + x_test[:, 1:] = (x_test[:, 1:] - mean) / standard_deviation + weights = _fit_logistic(x_train, y_train, regularization) + probabilities.extend(_sigmoid(x_test @ weights).tolist()) + labels.extend(example.feasible for example in test) + groups.extend(held_out for _ in test) + return ( + np.asarray(labels, dtype=np.int64), + np.asarray(probabilities, dtype=np.float64), + groups, + ) + + +def analyze( + phase6_path: Path, + phase6_raw_root: Path, + simulator_raw_root: Path, + simulator_metrics_path: Path, +) -> dict[str, Any]: + phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) + examples = build_examples(phase6, phase6_raw_root, 5.0) + simulator, simulator_raw_sha256 = load_simulator_features(simulator_raw_root) + red_flags = [] + try: + matched = [simulator_row(example, simulator) for example in examples] + except ValueError as error: + matched = [] + red_flags.append(str(error)) + + sensitivity = {} + if matched: + for regularization in REGULARIZATION_SENSITIVITY: + labels, baseline_probability, groups = grouped_predictions( + examples, + simulator, + instrumentation_aware=False, + regularization=regularization, + ) + instrument_labels, instrument_probability, instrument_groups = grouped_predictions( + examples, + simulator, + instrumentation_aware=True, + regularization=regularization, + ) + if not np.array_equal(labels, instrument_labels) or groups != instrument_groups: + raise AssertionError("nested baseline folds differ") + baseline_correct = (baseline_probability >= 0.5) == labels + instrument_correct = (instrument_probability >= 0.5) == labels + paired = { + "both_correct": int(np.sum(baseline_correct & instrument_correct)), + "sim_outcome_only_correct": int( + np.sum(baseline_correct & ~instrument_correct) + ), + "instrumentation_only_correct": int( + np.sum(~baseline_correct & instrument_correct) + ), + "both_wrong": int(np.sum(~baseline_correct & ~instrument_correct)), + } + paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p( + paired["sim_outcome_only_correct"], + paired["instrumentation_only_correct"], + ) + sensitivity[str(regularization)] = { + "sim_plus_outcome": { + "classification": _classification_metrics(labels, baseline_probability), + "policy_0p95": policy_metrics( + examples, labels, baseline_probability, 0.95 + ), + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": _classification_metrics(labels, instrument_probability), + "policy_0p95": policy_metrics( + examples, labels, instrument_probability, 0.95 + ), + }, + "paired_correctness": paired, + "group_bootstrap": _group_bootstrap_delta( + labels, + baseline_probability, + instrument_probability, + groups, + ), + } + + headline = sensitivity.get(str(DEFAULT_REGULARIZATION)) + simulator_pass_rates = [row[1] for row in matched] + labels = [example.feasible for example in examples] + if len(examples) != 37: + red_flags.append("examples_not_37") + if len(simulator) != 92: + red_flags.append("frozen_simulator_runs_not_92") + if len(set(labels)) != 2: + red_flags.append("single_label") + if matched and not all(0.0 <= value <= 1.0 for value in simulator_pass_rates): + red_flags.append("simulator_pass_rate_out_of_range") + + return { + "schema": "fidelity-strong-baseline-v1", + "status": "PASS" if not red_flags else "STOP", + "scope": "retrospective one-task headroom audit; not contribution evidence", + "comparison": ( + "same 5-second prefix, folds, logistic family, regularization, and frozen " + "Frontier outputs; the only nested difference is real Layer-1 engine state" + ), + "features": { + "shared_outcome": list(OUTCOME_FEATURES), + "shared_simulator": list(SIMULATOR_FEATURES), + "instrumentation_only": list(INSTRUMENTATION_FEATURES), + }, + "headline_regularization": DEFAULT_REGULARIZATION, + "headline": headline, + "regularization_sensitivity": sensitivity, + "provenance": { + "phase6_metrics": str(phase6_path.resolve()), + "phase6_metrics_sha256": sha256_file(phase6_path), + "phase6_raw_root": str(phase6_raw_root.resolve()), + "simulator_metrics": str(simulator_metrics_path.resolve()), + "simulator_metrics_sha256": sha256_file(simulator_metrics_path), + "simulator_raw_root": str(simulator_raw_root.resolve()), + "frozen_simulator_manifest_scorer_set_sha256": simulator_raw_sha256, + }, + "decision": { + "contribution_established": False, + "prospective_requirement": ( + "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks" + ), + }, + "sanity": { + "red_flags": red_flags, + "examples": numeric([1 for _ in examples]), + "labels": { + **numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + }, + "matched_simulator_pass_rate": numeric(simulator_pass_rates), + "frozen_simulator_runs": len(simulator), + "invariants": { + "all_examples_matched_once": len(matched) == len(examples), + "same_nested_folds": True, + "simulator_ratios_bounded": all( + 0.0 <= value <= 1.0 for value in simulator_pass_rates + ), + "labels_not_identical": len(set(labels)) == 2, + "per_config_results_not_all_identical": len(set(simulator_pass_rates)) > 1, + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--phase6-metrics", type=Path, required=True) + parser.add_argument("--phase6-raw-root", type=Path, required=True) + parser.add_argument("--simulator-raw-root", type=Path, required=True) + parser.add_argument("--simulator-metrics", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze( + args.phase6_metrics, + args.phase6_raw_root, + args.simulator_raw_root, + args.simulator_metrics, + ) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + "status": result["status"], + "output": str(args.output), + "red_flags": result["sanity"]["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/strong-baseline-metrics.json b/runs/fidelity-headroom/strong-baseline-metrics.json new file mode 100644 index 0000000..bf681ad --- /dev/null +++ b/runs/fidelity-headroom/strong-baseline-metrics.json @@ -0,0 +1,477 @@ +{ + "comparison": "same 5-second prefix, folds, logistic family, regularization, and frozen Frontier outputs; the only nested difference is real Layer-1 engine state", + "decision": { + "contribution_established": false, + "prospective_requirement": "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks" + }, + "features": { + "instrumentation_only": [ + "model_steps_per_second", + "waiting_mean", + "waiting_max", + "waiting_nonzero_share", + "running_mean", + "running_max", + "decode_batch_mean", + "decode_batch_max", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "preemptions" + ], + "shared_outcome": [ + "log_offered_rate_per_gpu", + "log2_tp", + "log2_max_num_seqs", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit" + ], + "shared_simulator": [ + "log_sim_completed_throughput_per_gpu", + "sim_slo_pass_rate", + "sim_slo_feasible" + ] + }, + "headline": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.18181818181818188 + ], + "point": 0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.04292727744470806, + 0.019924730979981074 + ], + "point": -0.010145365131402809 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "sim_outcome_only_correct": 0 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8108108108108109, + "balanced_accuracy": 0.7242063492063493, + "brier": 0.1058226346682949, + "confusion": { + "false_negative": 3, + "false_positive": 4, + "true_negative": 5, + "true_positive": 25 + }, + "log_loss": 0.3011048455679668 + }, + "policy_0p95": { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.5429431818208333, + "decision_coverage": 0.5405405405405406, + "early_accept": 16, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.5429431818208333, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5088695307144538, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.0956772695368921, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.288823031828762 + }, + "policy_0p95": { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7360063646722222, + "decision_coverage": 0.6756756756756757, + "early_accept": 20, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7360063646722222, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6898165884274738, + "valid_zero_error_policy": true + } + } + }, + "headline_regularization": 1.0, + "provenance": { + "frozen_simulator_manifest_scorer_set_sha256": "833842d96ecaa0b059ef99852621752f7989e63d100118b6025425fb119b7a55", + "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json", + "phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16", + "phase6_raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells", + "simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json", + "simulator_metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206", + "simulator_raw_root": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/raw" + }, + "regularization_sensitivity": { + "0.1": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.17500000000000004, + 0.0 + ], + "point": -0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.026383192545085435, + 0.0607951286646285 + ], + "point": 0.019228316404518567 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 0, + "mcnemar_exact_two_sided_p": 0.25, + "sim_outcome_only_correct": 3 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.10990776306815446, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.328357763455984 + }, + "policy_0p95": { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7402314096841667, + "decision_coverage": 0.6756756756756757, + "early_accept": 20, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7402314096841667, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6937764809990414, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8108108108108109, + "balanced_accuracy": 0.7619047619047619, + "brier": 0.12913607947267303, + "confusion": { + "false_negative": 4, + "false_positive": 3, + "true_negative": 6, + "true_positive": 24 + }, + "log_loss": 0.4373556318820343 + }, + "policy_0p95": { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.7469523484622221, + "decision_coverage": 0.7567567567567568, + "early_accept": 22, + "early_reject": 6, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9869795738005246, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns16", + "label_feasible": false, + "probability_feasible": 0.9855364057197005, + "remaining_h20_hours": 0.023106262014444445 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.03322417232055556, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7801765207827777, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + } + }, + "1.0": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.18181818181818188 + ], + "point": 0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.04292727744470806, + 0.019924730979981074 + ], + "point": -0.010145365131402809 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "sim_outcome_only_correct": 0 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8108108108108109, + "balanced_accuracy": 0.7242063492063493, + "brier": 0.1058226346682949, + "confusion": { + "false_negative": 3, + "false_positive": 4, + "true_negative": 5, + "true_positive": 25 + }, + "log_loss": 0.3011048455679668 + }, + "policy_0p95": { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.5429431818208333, + "decision_coverage": 0.5405405405405406, + "early_accept": 16, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.5429431818208333, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5088695307144538, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.0956772695368921, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.288823031828762 + }, + "policy_0p95": { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7360063646722222, + "decision_coverage": 0.6756756756756757, + "early_accept": 20, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7360063646722222, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6898165884274738, + "valid_zero_error_policy": true + } + } + }, + "10.0": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.13333333333333341, + 0.05555555555555558 + ], + "point": -0.027027027027027084 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.03091105649870874, + 0.01684192005239855 + ], + "point": -0.007318433328714388 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "sim_outcome_only_correct": 2 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8648648648648649, + "balanced_accuracy": 0.7222222222222222, + "brier": 0.10613344425735322, + "confusion": { + "false_negative": 0, + "false_positive": 5, + "true_negative": 4, + "true_positive": 28 + }, + "log_loss": 0.3404203142465075 + }, + "policy_0p95": { + "abstain_continue_full": 32, + "correctly_saved_h20_hours": 0.21727432337249997, + "decision_coverage": 0.13513513513513514, + "early_accept": 5, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.21727432337249997, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.20363877229302757, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7420634920634921, + "brier": 0.09881501092863883, + "confusion": { + "false_negative": 2, + "false_positive": 4, + "true_negative": 5, + "true_positive": 26 + }, + "log_loss": 0.312914193285738 + }, + "policy_0p95": { + "abstain_continue_full": 30, + "correctly_saved_h20_hours": 0.2384080185036111, + "decision_coverage": 0.1891891891891892, + "early_accept": 6, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.2384080185036111, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.22344617366339725, + "valid_zero_error_policy": true + } + } + } + }, + "sanity": { + "examples": { + "distinct_n": 1, + "max": 1.0, + "min": 1.0, + "n": 37 + }, + "frozen_simulator_runs": 92, + "invariants": { + "all_examples_matched_once": true, + "labels_not_identical": true, + "per_config_results_not_all_identical": true, + "same_nested_folds": true, + "simulator_ratios_bounded": true + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 37, + "negative": 9, + "positive": 28 + }, + "matched_simulator_pass_rate": { + "distinct_n": 12, + "max": 1.0, + "min": 0.06884057971014493, + "n": 37 + }, + "red_flags": [] + }, + "schema": "fidelity-strong-baseline-v1", + "scope": "retrospective one-task headroom audit; not contribution evidence", + "status": "PASS" +} diff --git a/runs/fidelity-headroom/test_strong_baseline.py b/runs/fidelity-headroom/test_strong_baseline.py new file mode 100644 index 0000000..5c1e9f1 --- /dev/null +++ b/runs/fidelity-headroom/test_strong_baseline.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +from pathlib import Path + +from analyze_strong_baseline import analyze + + +ROOT = Path(__file__).resolve().parents[2] +REPLAYSERVE = ROOT.parent / "replayserve" + + +def main() -> None: + result = analyze( + ROOT / "runs/opprof-phase6/phase6/metrics.json", + ROOT / "runs/opprof-phase6/phase6/solo-authoritative/cells", + REPLAYSERVE / "runs/simfid_s2rb/results/raw", + REPLAYSERVE / "runs/simfid_s2rb/results/metrics.json", + ) + assert result["status"] == "PASS", json.dumps(result["sanity"], indent=2) + assert result["sanity"]["frozen_simulator_runs"] == 92 + assert result["sanity"]["labels"]["n"] == 37 + headline = result["headline"] + assert headline["sim_plus_outcome"]["policy_0p95"]["false_accept"] == 0 + assert headline["sim_plus_outcome"]["policy_0p95"]["false_reject"] == 0 + assert ( + headline["sim_plus_outcome_plus_instrumentation"]["policy_0p95"][ + "false_accept" + ] + == 0 + ) + print("fidelity strong baseline: PASS") + + +if __name__ == "__main__": + main()