{
  "comparison": "same 5-second prefix, folds, logistic family, regularization, and frozen Frontier outputs; the only nested difference is real Layer-1 engine state",
  "decision": {
    "contribution_established": false,
    "prospective_requirement": "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks"
  },
  "features": {
    "instrumentation_only": [
      "model_steps_per_second",
      "waiting_mean",
      "waiting_max",
      "waiting_nonzero_share",
      "running_mean",
      "running_max",
      "decode_batch_mean",
      "decode_batch_max",
      "decode_batch_cv",
      "kv_usage_mean",
      "kv_usage_max",
      "kv_usage_end_minus_start",
      "graph_none_share",
      "graph_full_share",
      "padding_fraction",
      "prefill_token_fraction",
      "preemptions"
    ],
    "shared_outcome": [
      "log_offered_rate_per_gpu",
      "log2_tp",
      "log2_max_num_seqs",
      "admitted_fraction",
      "completed_over_admitted",
      "completed_pass_rate",
      "completed_fail_fraction_of_total",
      "outstanding_over_admitted",
      "ttft_max_over_slo_max",
      "ttft_mean_over_slo_max",
      "tpot_max_over_slo",
      "tpot_mean_over_slo",
      "admitted_input_tokens_mean_over_limit"
    ],
    "shared_simulator": [
      "log_sim_completed_throughput_per_gpu",
      "sim_slo_pass_rate",
      "sim_slo_feasible"
    ]
  },
  "headline": {
    "group_bootstrap": {
      "accuracy_delta_instrumentation_minus_outcome": {
        "ci95": [
          0.0,
          0.18181818181818188
        ],
        "point": 0.08108108108108103
      },
      "brier_delta_instrumentation_minus_outcome": {
        "ci95": [
          -0.04292727744470806,
          0.019924730979981074
        ],
        "point": -0.010145365131402809
      },
      "replicates": 10000,
      "seed": 20260714,
      "semantics": "group bootstrap over cells; diagnostic confidence interval"
    },
    "paired_correctness": {
      "both_correct": 30,
      "both_wrong": 4,
      "instrumentation_only_correct": 3,
      "mcnemar_exact_two_sided_p": 0.25,
      "sim_outcome_only_correct": 0
    },
    "sim_plus_outcome": {
      "classification": {
        "accuracy": 0.8108108108108109,
        "balanced_accuracy": 0.7242063492063493,
        "brier": 0.1058226346682949,
        "confusion": {
          "false_negative": 3,
          "false_positive": 4,
          "true_negative": 5,
          "true_positive": 25
        },
        "log_loss": 0.3011048455679668
      },
      "policy_0p95": {
        "abstain_continue_full": 17,
        "correctly_saved_h20_hours": 0.5429431818208333,
        "decision_coverage": 0.5405405405405406,
        "early_accept": 16,
        "early_reject": 4,
        "false_accept": 0,
        "false_accept_examples": [],
        "false_reject": 0,
        "false_reject_examples": [],
        "full_trial_h20_hours": 1.0669595034675,
        "invalidly_saved_h20_hours": 0.0,
        "remaining_h20_hours_at_cutoff": 0.957237281245278,
        "saved_h20_hours_if_decisions_used": 0.5429431818208333,
        "threshold": 0.95,
        "valid_cost_reduction_fraction": 0.5088695307144538,
        "valid_zero_error_policy": true
      }
    },
    "sim_plus_outcome_plus_instrumentation": {
      "classification": {
        "accuracy": 0.8918918918918919,
        "balanced_accuracy": 0.8154761904761905,
        "brier": 0.0956772695368921,
        "confusion": {
          "false_negative": 1,
          "false_positive": 3,
          "true_negative": 6,
          "true_positive": 27
        },
        "log_loss": 0.288823031828762
      },
      "policy_0p95": {
        "abstain_continue_full": 12,
        "correctly_saved_h20_hours": 0.7360063646722222,
        "decision_coverage": 0.6756756756756757,
        "early_accept": 20,
        "early_reject": 5,
        "false_accept": 0,
        "false_accept_examples": [],
        "false_reject": 0,
        "false_reject_examples": [],
        "full_trial_h20_hours": 1.0669595034675,
        "invalidly_saved_h20_hours": 0.0,
        "remaining_h20_hours_at_cutoff": 0.957237281245278,
        "saved_h20_hours_if_decisions_used": 0.7360063646722222,
        "threshold": 0.95,
        "valid_cost_reduction_fraction": 0.6898165884274738,
        "valid_zero_error_policy": true
      }
    }
  },
  "headline_regularization": 1.0,
  "provenance": {
    "frozen_simulator_manifest_scorer_set_sha256": "833842d96ecaa0b059ef99852621752f7989e63d100118b6025425fb119b7a55",
    "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json",
    "phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16",
    "phase6_raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells",
    "simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
    "simulator_metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206",
    "simulator_raw_root": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/raw"
  },
  "regularization_sensitivity": {
    "0.1": {
      "group_bootstrap": {
        "accuracy_delta_instrumentation_minus_outcome": {
          "ci95": [
            -0.17500000000000004,
            0.0
          ],
          "point": -0.08108108108108103
        },
        "brier_delta_instrumentation_minus_outcome": {
          "ci95": [
            -0.026383192545085435,
            0.0607951286646285
          ],
          "point": 0.019228316404518567
        },
        "replicates": 10000,
        "seed": 20260714,
        "semantics": "group bootstrap over cells; diagnostic confidence interval"
      },
      "paired_correctness": {
        "both_correct": 30,
        "both_wrong": 4,
        "instrumentation_only_correct": 0,
        "mcnemar_exact_two_sided_p": 0.25,
        "sim_outcome_only_correct": 3
      },
      "sim_plus_outcome": {
        "classification": {
          "accuracy": 0.8918918918918919,
          "balanced_accuracy": 0.8154761904761905,
          "brier": 0.10990776306815446,
          "confusion": {
            "false_negative": 1,
            "false_positive": 3,
            "true_negative": 6,
            "true_positive": 27
          },
          "log_loss": 0.328357763455984
        },
        "policy_0p95": {
          "abstain_continue_full": 12,
          "correctly_saved_h20_hours": 0.7402314096841667,
          "decision_coverage": 0.6756756756756757,
          "early_accept": 20,
          "early_reject": 5,
          "false_accept": 0,
          "false_accept_examples": [],
          "false_reject": 0,
          "false_reject_examples": [],
          "full_trial_h20_hours": 1.0669595034675,
          "invalidly_saved_h20_hours": 0.0,
          "remaining_h20_hours_at_cutoff": 0.957237281245278,
          "saved_h20_hours_if_decisions_used": 0.7402314096841667,
          "threshold": 0.95,
          "valid_cost_reduction_fraction": 0.6937764809990414,
          "valid_zero_error_policy": true
        }
      },
      "sim_plus_outcome_plus_instrumentation": {
        "classification": {
          "accuracy": 0.8108108108108109,
          "balanced_accuracy": 0.7619047619047619,
          "brier": 0.12913607947267303,
          "confusion": {
            "false_negative": 4,
            "false_positive": 3,
            "true_negative": 6,
            "true_positive": 24
          },
          "log_loss": 0.4373556318820343
        },
        "policy_0p95": {
          "abstain_continue_full": 9,
          "correctly_saved_h20_hours": 0.7469523484622221,
          "decision_coverage": 0.7567567567567568,
          "early_accept": 22,
          "early_reject": 6,
          "false_accept": 2,
          "false_accept_examples": [
            {
              "anchor": 0.49609375,
              "cell": "tp2_mns8",
              "label_feasible": false,
              "probability_feasible": 0.9869795738005246,
              "remaining_h20_hours": 0.010117910306111111
            },
            {
              "anchor": 0.033717411016,
              "cell": "tp4_mns16",
              "label_feasible": false,
              "probability_feasible": 0.9855364057197005,
              "remaining_h20_hours": 0.023106262014444445
            }
          ],
          "false_reject": 0,
          "false_reject_examples": [],
          "full_trial_h20_hours": 1.0669595034675,
          "invalidly_saved_h20_hours": 0.03322417232055556,
          "remaining_h20_hours_at_cutoff": 0.957237281245278,
          "saved_h20_hours_if_decisions_used": 0.7801765207827777,
          "threshold": 0.95,
          "valid_cost_reduction_fraction": null,
          "valid_zero_error_policy": false
        }
      }
    },
    "1.0": {
      "group_bootstrap": {
        "accuracy_delta_instrumentation_minus_outcome": {
          "ci95": [
            0.0,
            0.18181818181818188
          ],
          "point": 0.08108108108108103
        },
        "brier_delta_instrumentation_minus_outcome": {
          "ci95": [
            -0.04292727744470806,
            0.019924730979981074
          ],
          "point": -0.010145365131402809
        },
        "replicates": 10000,
        "seed": 20260714,
        "semantics": "group bootstrap over cells; diagnostic confidence interval"
      },
      "paired_correctness": {
        "both_correct": 30,
        "both_wrong": 4,
        "instrumentation_only_correct": 3,
        "mcnemar_exact_two_sided_p": 0.25,
        "sim_outcome_only_correct": 0
      },
      "sim_plus_outcome": {
        "classification": {
          "accuracy": 0.8108108108108109,
          "balanced_accuracy": 0.7242063492063493,
          "brier": 0.1058226346682949,
          "confusion": {
            "false_negative": 3,
            "false_positive": 4,
            "true_negative": 5,
            "true_positive": 25
          },
          "log_loss": 0.3011048455679668
        },
        "policy_0p95": {
          "abstain_continue_full": 17,
          "correctly_saved_h20_hours": 0.5429431818208333,
          "decision_coverage": 0.5405405405405406,
          "early_accept": 16,
          "early_reject": 4,
          "false_accept": 0,
          "false_accept_examples": [],
          "false_reject": 0,
          "false_reject_examples": [],
          "full_trial_h20_hours": 1.0669595034675,
          "invalidly_saved_h20_hours": 0.0,
          "remaining_h20_hours_at_cutoff": 0.957237281245278,
          "saved_h20_hours_if_decisions_used": 0.5429431818208333,
          "threshold": 0.95,
          "valid_cost_reduction_fraction": 0.5088695307144538,
          "valid_zero_error_policy": true
        }
      },
      "sim_plus_outcome_plus_instrumentation": {
        "classification": {
          "accuracy": 0.8918918918918919,
          "balanced_accuracy": 0.8154761904761905,
          "brier": 0.0956772695368921,
          "confusion": {
            "false_negative": 1,
            "false_positive": 3,
            "true_negative": 6,
            "true_positive": 27
          },
          "log_loss": 0.288823031828762
        },
        "policy_0p95": {
          "abstain_continue_full": 12,
          "correctly_saved_h20_hours": 0.7360063646722222,
          "decision_coverage": 0.6756756756756757,
          "early_accept": 20,
          "early_reject": 5,
          "false_accept": 0,
          "false_accept_examples": [],
          "false_reject": 0,
          "false_reject_examples": [],
          "full_trial_h20_hours": 1.0669595034675,
          "invalidly_saved_h20_hours": 0.0,
          "remaining_h20_hours_at_cutoff": 0.957237281245278,
          "saved_h20_hours_if_decisions_used": 0.7360063646722222,
          "threshold": 0.95,
          "valid_cost_reduction_fraction": 0.6898165884274738,
          "valid_zero_error_policy": true
        }
      }
    },
    "10.0": {
      "group_bootstrap": {
        "accuracy_delta_instrumentation_minus_outcome": {
          "ci95": [
            -0.13333333333333341,
            0.05555555555555558
          ],
          "point": -0.027027027027027084
        },
        "brier_delta_instrumentation_minus_outcome": {
          "ci95": [
            -0.03091105649870874,
            0.01684192005239855
          ],
          "point": -0.007318433328714388
        },
        "replicates": 10000,
        "seed": 20260714,
        "semantics": "group bootstrap over cells; diagnostic confidence interval"
      },
      "paired_correctness": {
        "both_correct": 30,
        "both_wrong": 4,
        "instrumentation_only_correct": 1,
        "mcnemar_exact_two_sided_p": 1.0,
        "sim_outcome_only_correct": 2
      },
      "sim_plus_outcome": {
        "classification": {
          "accuracy": 0.8648648648648649,
          "balanced_accuracy": 0.7222222222222222,
          "brier": 0.10613344425735322,
          "confusion": {
            "false_negative": 0,
            "false_positive": 5,
            "true_negative": 4,
            "true_positive": 28
          },
          "log_loss": 0.3404203142465075
        },
        "policy_0p95": {
          "abstain_continue_full": 32,
          "correctly_saved_h20_hours": 0.21727432337249997,
          "decision_coverage": 0.13513513513513514,
          "early_accept": 5,
          "early_reject": 0,
          "false_accept": 0,
          "false_accept_examples": [],
          "false_reject": 0,
          "false_reject_examples": [],
          "full_trial_h20_hours": 1.0669595034675,
          "invalidly_saved_h20_hours": 0.0,
          "remaining_h20_hours_at_cutoff": 0.957237281245278,
          "saved_h20_hours_if_decisions_used": 0.21727432337249997,
          "threshold": 0.95,
          "valid_cost_reduction_fraction": 0.20363877229302757,
          "valid_zero_error_policy": true
        }
      },
      "sim_plus_outcome_plus_instrumentation": {
        "classification": {
          "accuracy": 0.8378378378378378,
          "balanced_accuracy": 0.7420634920634921,
          "brier": 0.09881501092863883,
          "confusion": {
            "false_negative": 2,
            "false_positive": 4,
            "true_negative": 5,
            "true_positive": 26
          },
          "log_loss": 0.312914193285738
        },
        "policy_0p95": {
          "abstain_continue_full": 30,
          "correctly_saved_h20_hours": 0.2384080185036111,
          "decision_coverage": 0.1891891891891892,
          "early_accept": 6,
          "early_reject": 1,
          "false_accept": 0,
          "false_accept_examples": [],
          "false_reject": 0,
          "false_reject_examples": [],
          "full_trial_h20_hours": 1.0669595034675,
          "invalidly_saved_h20_hours": 0.0,
          "remaining_h20_hours_at_cutoff": 0.957237281245278,
          "saved_h20_hours_if_decisions_used": 0.2384080185036111,
          "threshold": 0.95,
          "valid_cost_reduction_fraction": 0.22344617366339725,
          "valid_zero_error_policy": true
        }
      }
    }
  },
  "sanity": {
    "examples": {
      "distinct_n": 1,
      "max": 1.0,
      "min": 1.0,
      "n": 37
    },
    "frozen_simulator_runs": 92,
    "invariants": {
      "all_examples_matched_once": true,
      "labels_not_identical": true,
      "per_config_results_not_all_identical": true,
      "same_nested_folds": true,
      "simulator_ratios_bounded": true
    },
    "labels": {
      "distinct_n": 2,
      "max": 1.0,
      "min": 0.0,
      "n": 37,
      "negative": 9,
      "positive": 28
    },
    "matched_simulator_pass_rate": {
      "distinct_n": 12,
      "max": 1.0,
      "min": 0.06884057971014493,
      "n": 37
    },
    "red_flags": []
  },
  "schema": "fidelity-strong-baseline-v1",
  "scope": "retrospective one-task headroom audit; not contribution evidence",
  "status": "PASS"
}