3697 lines
144 KiB
JSON
3697 lines
144 KiB
JSON
{
|
|
"cutoffs": {
|
|
"10": {
|
|
"cells": 12,
|
|
"completion_time_sources": {
|
|
"reconstructed_from_latency": 37
|
|
},
|
|
"examples": 37,
|
|
"headline": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.1333333333333333,
|
|
0.05555555555555547
|
|
],
|
|
"point": -0.027027027027027084
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.047874993358699436,
|
|
0.04753434529263568
|
|
],
|
|
"point": -0.0026830647286030934
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8531746031746033,
|
|
"brier": 0.07268572313801988,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 2,
|
|
"true_negative": 7,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.22136936001258795
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.7938369600363888,
|
|
"decision_coverage": 0.918918918918919,
|
|
"early_accept": 27,
|
|
"early_reject": 7,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8623652659223954,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.11022926282564288,
|
|
"remaining_h20_hours": 0.018465968481111112
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.025806101009444443,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.8196430610458332,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.7340601279138887,
|
|
"decision_coverage": 0.7837837837837838,
|
|
"early_accept": 24,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7340601279138887,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6879924922438714,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 15,
|
|
"correctly_saved_h20_hours": 0.620235621837222,
|
|
"decision_coverage": 0.5945945945945946,
|
|
"early_accept": 17,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.620235621837222,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5813113054633519,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.918918918918919,
|
|
"balanced_accuracy": 0.8333333333333333,
|
|
"brier": 0.07536878786662297,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.23513650573912553
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.7625537088033332,
|
|
"decision_coverage": 0.8648648648648649,
|
|
"early_accept": 27,
|
|
"early_reject": 5,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8486897200744762,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8216385310591849,
|
|
"remaining_h20_hours": 0.01680367415111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.024143806679444442,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7866975154827778,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 11,
|
|
"correctly_saved_h20_hours": 0.7042463331286111,
|
|
"decision_coverage": 0.7027027027027027,
|
|
"early_accept": 22,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7042463331286111,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6600497308847137,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 19,
|
|
"correctly_saved_h20_hours": 0.5876473660130556,
|
|
"decision_coverage": 0.4864864864864865,
|
|
"early_accept": 15,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.5876473660130556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5507682007641966,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 32,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 1,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 2
|
|
}
|
|
},
|
|
"headline_regularization": 1.0,
|
|
"label_sanity": {
|
|
"distinct_n": 2,
|
|
"max": 1.0,
|
|
"min": 0.0,
|
|
"n": 37,
|
|
"negative": 9,
|
|
"positive": 28,
|
|
"primary_adjudicated_disagreements": 3
|
|
},
|
|
"regularization_sensitivity": {
|
|
"0.1": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.08571428571428574,
|
|
0.07317073170731703
|
|
],
|
|
"point": 0.0
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.01721313553473963,
|
|
0.0846932121032576
|
|
],
|
|
"point": 0.019153359424323196
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.918918918918919,
|
|
"balanced_accuracy": 0.871031746031746,
|
|
"brier": 0.06720185291989068,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 2,
|
|
"true_negative": 7,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.22428253113260074
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 2,
|
|
"correctly_saved_h20_hours": 0.802005166048611,
|
|
"decision_coverage": 0.9459459459459459,
|
|
"early_accept": 27,
|
|
"early_reject": 8,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.905553117931728,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.02745974548325657,
|
|
"remaining_h20_hours": 0.018465968481111112
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.025806101009444443,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.8278112670580555,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.7653433791469442,
|
|
"decision_coverage": 0.8918918918918919,
|
|
"early_accept": 26,
|
|
"early_reject": 7,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.905553117931728,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.02745974548325657,
|
|
"remaining_h20_hours": 0.018465968481111112
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.025806101009444443,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7911494801563888,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 7,
|
|
"correctly_saved_h20_hours": 0.7226741330227777,
|
|
"decision_coverage": 0.8108108108108109,
|
|
"early_accept": 23,
|
|
"early_reject": 7,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.02745974548325657,
|
|
"remaining_h20_hours": 0.018465968481111112
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.018465968481111112,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7411401015038888,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.918918918918919,
|
|
"balanced_accuracy": 0.8333333333333333,
|
|
"brier": 0.04804849349556749,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.15081259765362642
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.7836433176388888,
|
|
"decision_coverage": 0.8918918918918919,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8878008796435344,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.007340132528333333,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7909834501672222,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.718791861695,
|
|
"decision_coverage": 0.7837837837837838,
|
|
"early_accept": 24,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.718791861695,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6736824212718535,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 10,
|
|
"correctly_saved_h20_hours": 0.7046020469899998,
|
|
"decision_coverage": 0.7297297297297297,
|
|
"early_accept": 23,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7046020469899998,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6603831211026486,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 33,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 1,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 1
|
|
}
|
|
},
|
|
"1.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.1333333333333333,
|
|
0.05555555555555547
|
|
],
|
|
"point": -0.027027027027027084
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.047874993358699436,
|
|
0.04753434529263568
|
|
],
|
|
"point": -0.0026830647286030934
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8531746031746033,
|
|
"brier": 0.07268572313801988,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 2,
|
|
"true_negative": 7,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.22136936001258795
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.7938369600363888,
|
|
"decision_coverage": 0.918918918918919,
|
|
"early_accept": 27,
|
|
"early_reject": 7,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8623652659223954,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.11022926282564288,
|
|
"remaining_h20_hours": 0.018465968481111112
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.025806101009444443,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.8196430610458332,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.7340601279138887,
|
|
"decision_coverage": 0.7837837837837838,
|
|
"early_accept": 24,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7340601279138887,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6879924922438714,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 15,
|
|
"correctly_saved_h20_hours": 0.620235621837222,
|
|
"decision_coverage": 0.5945945945945946,
|
|
"early_accept": 17,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.620235621837222,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5813113054633519,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.918918918918919,
|
|
"balanced_accuracy": 0.8333333333333333,
|
|
"brier": 0.07536878786662297,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.23513650573912553
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.7625537088033332,
|
|
"decision_coverage": 0.8648648648648649,
|
|
"early_accept": 27,
|
|
"early_reject": 5,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8486897200744762,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8216385310591849,
|
|
"remaining_h20_hours": 0.01680367415111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.024143806679444442,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7866975154827778,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 11,
|
|
"correctly_saved_h20_hours": 0.7042463331286111,
|
|
"decision_coverage": 0.7027027027027027,
|
|
"early_accept": 22,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7042463331286111,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6600497308847137,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 19,
|
|
"correctly_saved_h20_hours": 0.5876473660130556,
|
|
"decision_coverage": 0.4864864864864865,
|
|
"early_accept": 15,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.5876473660130556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5507682007641966,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 32,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 1,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 2
|
|
}
|
|
},
|
|
"10.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.0
|
|
],
|
|
"point": 0.0
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.026808476987744526,
|
|
0.02087777396654144
|
|
],
|
|
"point": -0.005783107568599871
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8378378378378378,
|
|
"balanced_accuracy": 0.7420634920634921,
|
|
"brier": 0.08901809268133021,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.2787387373669301
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.7770332858852778,
|
|
"decision_coverage": 0.8648648648648649,
|
|
"early_accept": 27,
|
|
"early_reject": 5,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8449371690787663,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.007340132528333333,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7843734184136111,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 17,
|
|
"correctly_saved_h20_hours": 0.6023881611255555,
|
|
"decision_coverage": 0.5405405405405406,
|
|
"early_accept": 16,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.6023881611255555,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.564583903295168,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 30,
|
|
"correctly_saved_h20_hours": 0.28353444813305556,
|
|
"decision_coverage": 0.1891891891891892,
|
|
"early_accept": 6,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.28353444813305556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.2657405901644815,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8378378378378378,
|
|
"balanced_accuracy": 0.7420634920634921,
|
|
"brier": 0.09480120024993008,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.31278388302658006
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.7566135917169443,
|
|
"decision_coverage": 0.7837837837837838,
|
|
"early_accept": 26,
|
|
"early_reject": 3,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8320431084673425,
|
|
"remaining_h20_hours": 0.007340132528333333
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.007340132528333333,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.7639537242452776,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 24,
|
|
"correctly_saved_h20_hours": 0.5226926016833333,
|
|
"decision_coverage": 0.35135135135135137,
|
|
"early_accept": 12,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.5226926016833333,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.48988982242028906,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 34,
|
|
"correctly_saved_h20_hours": 0.16934238506333332,
|
|
"decision_coverage": 0.08108108108108109,
|
|
"early_accept": 3,
|
|
"early_reject": 0,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.8475150590230554,
|
|
"saved_h20_hours_if_decisions_used": 0.16934238506333332,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.15871491327739184,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 31,
|
|
"both_wrong": 6,
|
|
"instrumentation_only_correct": 0,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 0
|
|
}
|
|
}
|
|
},
|
|
"remaining_h20_hours": {
|
|
"distinct_n": 37,
|
|
"max": 0.057199826214444444,
|
|
"min": 0.0012684118816666664,
|
|
"n": 37
|
|
}
|
|
},
|
|
"15": {
|
|
"cells": 12,
|
|
"completion_time_sources": {
|
|
"reconstructed_from_latency": 36
|
|
},
|
|
"examples": 36,
|
|
"headline": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.06896551724137923,
|
|
0.11428571428571421
|
|
],
|
|
"point": 0.02777777777777779
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.07972316141025643,
|
|
-0.004101962239149991
|
|
],
|
|
"point": -0.03744665409603411
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9166666666666666,
|
|
"balanced_accuracy": 0.8571428571428572,
|
|
"brier": 0.03946379437502599,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.12484297369457635
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.7164359347680556,
|
|
"decision_coverage": 0.9166666666666666,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.7164359347680556,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": 0.6740304458143536,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 6,
|
|
"correctly_saved_h20_hours": 0.6905889710727777,
|
|
"decision_coverage": 0.8333333333333334,
|
|
"early_accept": 25,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6905889710727777,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6497133511279798,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.5895315664361112,
|
|
"decision_coverage": 0.6666666666666666,
|
|
"early_accept": 20,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.5895315664361112,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5546374843344664,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8888888888888888,
|
|
"balanced_accuracy": 0.75,
|
|
"brier": 0.0769104484710601,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 4,
|
|
"true_negative": 4,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.23230947656220863
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 6,
|
|
"correctly_saved_h20_hours": 0.6709655458816667,
|
|
"decision_coverage": 0.8333333333333334,
|
|
"early_accept": 26,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8578028168458919,
|
|
"remaining_h20_hours": 0.0014445317586111114
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0014445317586111114,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6724100776402777,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 11,
|
|
"correctly_saved_h20_hours": 0.626660078601389,
|
|
"decision_coverage": 0.6944444444444444,
|
|
"early_accept": 22,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.626660078601389,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.5895683782116529,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 16,
|
|
"correctly_saved_h20_hours": 0.5494646261305556,
|
|
"decision_coverage": 0.5555555555555556,
|
|
"early_accept": 17,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.5494646261305556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5169420864266074,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 31,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 2,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 1
|
|
}
|
|
},
|
|
"headline_regularization": 1.0,
|
|
"label_sanity": {
|
|
"distinct_n": 2,
|
|
"max": 1.0,
|
|
"min": 0.0,
|
|
"n": 36,
|
|
"negative": 8,
|
|
"positive": 28,
|
|
"primary_adjudicated_disagreements": 3
|
|
},
|
|
"regularization_sensitivity": {
|
|
"0.1": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.13793103448275856
|
|
],
|
|
"point": 0.05555555555555558
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.08924754562701061,
|
|
0.007148232838160279
|
|
],
|
|
"point": -0.03187976039254431
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9444444444444444,
|
|
"balanced_accuracy": 0.875,
|
|
"brier": 0.03162164266499348,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.08770869035960555
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.7164359347680556,
|
|
"decision_coverage": 0.9166666666666666,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.7164359347680556,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": 0.6740304458143536,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.7164359347680556,
|
|
"decision_coverage": 0.9166666666666666,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.7164359347680556,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6740304458143536,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.6910240435036111,
|
|
"decision_coverage": 0.8888888888888888,
|
|
"early_accept": 26,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6910240435036111,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6501226718366222,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8888888888888888,
|
|
"balanced_accuracy": 0.7946428571428572,
|
|
"brier": 0.06350140305753779,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 3,
|
|
"true_negative": 5,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.2433253313244278
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.689579511745,
|
|
"decision_coverage": 0.8888888888888888,
|
|
"early_accept": 27,
|
|
"early_reject": 5,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9922655130114992,
|
|
"remaining_h20_hours": 0.0014445317586111114
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0014445317586111114,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6910240435036111,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.6783313931494445,
|
|
"decision_coverage": 0.8611111111111112,
|
|
"early_accept": 27,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9922655130114992,
|
|
"remaining_h20_hours": 0.0014445317586111114
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0014445317586111114,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6797759249080555,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.6449625425236111,
|
|
"decision_coverage": 0.7777777777777778,
|
|
"early_accept": 24,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9922655130114992,
|
|
"remaining_h20_hours": 0.0014445317586111114
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0014445317586111114,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6464070742822221,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 32,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 2,
|
|
"mcnemar_exact_two_sided_p": 0.5,
|
|
"outcome_only_correct": 0
|
|
}
|
|
},
|
|
"1.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.06896551724137923,
|
|
0.11428571428571421
|
|
],
|
|
"point": 0.02777777777777779
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.07972316141025643,
|
|
-0.004101962239149991
|
|
],
|
|
"point": -0.03744665409603411
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9166666666666666,
|
|
"balanced_accuracy": 0.8571428571428572,
|
|
"brier": 0.03946379437502599,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.12484297369457635
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.7164359347680556,
|
|
"decision_coverage": 0.9166666666666666,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.7164359347680556,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": 0.6740304458143536,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 6,
|
|
"correctly_saved_h20_hours": 0.6905889710727777,
|
|
"decision_coverage": 0.8333333333333334,
|
|
"early_accept": 25,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6905889710727777,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.6497133511279798,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.5895315664361112,
|
|
"decision_coverage": 0.6666666666666666,
|
|
"early_accept": 20,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.5895315664361112,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5546374843344664,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8888888888888888,
|
|
"balanced_accuracy": 0.75,
|
|
"brier": 0.0769104484710601,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 4,
|
|
"true_negative": 4,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.23230947656220863
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 6,
|
|
"correctly_saved_h20_hours": 0.6709655458816667,
|
|
"decision_coverage": 0.8333333333333334,
|
|
"early_accept": 26,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8578028168458919,
|
|
"remaining_h20_hours": 0.0014445317586111114
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0014445317586111114,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6724100776402777,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 11,
|
|
"correctly_saved_h20_hours": 0.626660078601389,
|
|
"decision_coverage": 0.6944444444444444,
|
|
"early_accept": 22,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.626660078601389,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.5895683782116529,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 16,
|
|
"correctly_saved_h20_hours": 0.5494646261305556,
|
|
"decision_coverage": 0.5555555555555556,
|
|
"early_accept": 17,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.5494646261305556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5169420864266074,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 31,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 2,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 1
|
|
}
|
|
},
|
|
"10.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.17142857142857137
|
|
],
|
|
"point": 0.08333333333333326
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.051136314558473485,
|
|
-0.009942733239401492
|
|
],
|
|
"point": -0.029809575955846893
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8888888888888888,
|
|
"balanced_accuracy": 0.8392857142857143,
|
|
"brier": 0.07023194254174596,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.23287532021698182
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.6924315406702778,
|
|
"decision_coverage": 0.8611111111111112,
|
|
"early_accept": 27,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.801686701254931,
|
|
"remaining_h20_hours": 0.004004611363888889
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.004004611363888889,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6964361520341666,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 14,
|
|
"correctly_saved_h20_hours": 0.5771830138255556,
|
|
"decision_coverage": 0.6111111111111112,
|
|
"early_accept": 19,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.5771830138255556,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.5430198364509199,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 26,
|
|
"correctly_saved_h20_hours": 0.30550869967055555,
|
|
"decision_coverage": 0.2777777777777778,
|
|
"early_accept": 8,
|
|
"early_reject": 2,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.30550869967055555,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.2874257907034979,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8055555555555556,
|
|
"balanced_accuracy": 0.6517857142857143,
|
|
"brier": 0.10004151849759285,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 5,
|
|
"true_negative": 3,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.32055333394171265
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 7,
|
|
"correctly_saved_h20_hours": 0.6877041925958333,
|
|
"decision_coverage": 0.8055555555555556,
|
|
"early_accept": 27,
|
|
"early_reject": 2,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8140722809073944,
|
|
"remaining_h20_hours": 0.004562354750555555
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.004562354750555555,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.6922665473463888,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 24,
|
|
"correctly_saved_h20_hours": 0.4570601465275,
|
|
"decision_coverage": 0.3333333333333333,
|
|
"early_accept": 11,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.4570601465275,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.43000698230978923,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 33,
|
|
"correctly_saved_h20_hours": 0.15267571839666666,
|
|
"decision_coverage": 0.08333333333333333,
|
|
"early_accept": 3,
|
|
"early_reject": 0,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.7379133138080555,
|
|
"saved_h20_hours_if_decisions_used": 0.15267571839666666,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.14363891806913368,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 29,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 3,
|
|
"mcnemar_exact_two_sided_p": 0.25,
|
|
"outcome_only_correct": 0
|
|
}
|
|
}
|
|
},
|
|
"remaining_h20_hours": {
|
|
"distinct_n": 36,
|
|
"max": 0.05164427065888889,
|
|
"min": 0.0014445317586111114,
|
|
"n": 36
|
|
}
|
|
},
|
|
"20": {
|
|
"cells": 12,
|
|
"completion_time_sources": {
|
|
"reconstructed_from_latency": 36
|
|
},
|
|
"examples": 36,
|
|
"headline": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.12195121951219512
|
|
],
|
|
"point": 0.05555555555555547
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.09816256823569453,
|
|
-0.009602741441109449
|
|
],
|
|
"point": -0.04807041919163683
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9166666666666666,
|
|
"balanced_accuracy": 0.8571428571428572,
|
|
"brier": 0.05902138516195827,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.24132948833387438
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 2,
|
|
"correctly_saved_h20_hours": 0.6191580696761111,
|
|
"decision_coverage": 0.9444444444444444,
|
|
"early_accept": 28,
|
|
"early_reject": 6,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8399807697230681,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.008919034978436878,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.009139434342777777,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6282975040188888,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.6101058238716667,
|
|
"decision_coverage": 0.8611111111111112,
|
|
"early_accept": 27,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.008919034978436878,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6174606812416666,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.5390392875763889,
|
|
"decision_coverage": 0.75,
|
|
"early_accept": 23,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.008919034978436878,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5463941449463888,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8611111111111112,
|
|
"balanced_accuracy": 0.7321428571428572,
|
|
"brier": 0.1070918043535951,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 4,
|
|
"true_negative": 4,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.437208492871466
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.5864084509805555,
|
|
"decision_coverage": 0.8888888888888888,
|
|
"early_accept": 28,
|
|
"early_reject": 4,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9419008835359353,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
},
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8529942983175343,
|
|
"remaining_h20_hours": 0.0012268335861111111
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8268198678404454,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0017338236710752416,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.01042191079861111,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5968303617791667,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.5682670563286111,
|
|
"decision_coverage": 0.7777777777777778,
|
|
"early_accept": 25,
|
|
"early_reject": 3,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9419008835359353,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0017338236710752416,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.007410500239722223,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5756775565683333,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 17,
|
|
"correctly_saved_h20_hours": 0.40106617994777777,
|
|
"decision_coverage": 0.5277777777777778,
|
|
"early_accept": 16,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0017338236710752416,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.4084210373177778,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 31,
|
|
"both_wrong": 3,
|
|
"instrumentation_only_correct": 2,
|
|
"mcnemar_exact_two_sided_p": 0.5,
|
|
"outcome_only_correct": 0
|
|
}
|
|
},
|
|
"headline_regularization": 1.0,
|
|
"label_sanity": {
|
|
"distinct_n": 2,
|
|
"max": 1.0,
|
|
"min": 0.0,
|
|
"n": 36,
|
|
"negative": 8,
|
|
"positive": 28,
|
|
"primary_adjudicated_disagreements": 3
|
|
},
|
|
"regularization_sensitivity": {
|
|
"0.1": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.1842105263157895
|
|
],
|
|
"point": 0.08333333333333326
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.1289106028344674,
|
|
-0.00039980805027801426
|
|
],
|
|
"point": -0.0571263639718715
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9444444444444444,
|
|
"balanced_accuracy": 0.9196428571428572,
|
|
"brier": 0.05738846072877349,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 1,
|
|
"true_negative": 7,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.3180853188579326
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 2,
|
|
"correctly_saved_h20_hours": 0.6191580696761111,
|
|
"decision_coverage": 0.9444444444444444,
|
|
"early_accept": 28,
|
|
"early_reject": 6,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8655614945720641,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.00028842593944516384,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.009139434342777777,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6282975040188888,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.6191580696761111,
|
|
"decision_coverage": 0.9166666666666666,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.00028842593944516384,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6265129270461112,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.6074562703719444,
|
|
"decision_coverage": 0.8888888888888888,
|
|
"early_accept": 26,
|
|
"early_reject": 6,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.00028842593944516384,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6148111277419444,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8611111111111112,
|
|
"balanced_accuracy": 0.7767857142857143,
|
|
"brier": 0.11451482470064499,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 3,
|
|
"true_negative": 5,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.7747412375287079
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.5848221568852777,
|
|
"decision_coverage": 0.9166666666666666,
|
|
"early_accept": 27,
|
|
"early_reject": 6,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9999405808371276,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
},
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9739951015698483,
|
|
"remaining_h20_hours": 0.0012268335861111111
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 1.6376668165938014e-05,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.008637333825833335,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5934594907111111,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.5791295938452777,
|
|
"decision_coverage": 0.8888888888888888,
|
|
"early_accept": 27,
|
|
"early_reject": 5,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9999405808371276,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
},
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9739951015698483,
|
|
"remaining_h20_hours": 0.0012268335861111111
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 1.6376668165938014e-05,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.008637333825833335,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5877669276711112,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.5676638636697222,
|
|
"decision_coverage": 0.8611111111111112,
|
|
"early_accept": 26,
|
|
"early_reject": 5,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9999405808371276,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
},
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9739951015698483,
|
|
"remaining_h20_hours": 0.0012268335861111111
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 1.6376668165938014e-05,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.008637333825833335,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5763011974955555,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 31,
|
|
"both_wrong": 2,
|
|
"instrumentation_only_correct": 3,
|
|
"mcnemar_exact_two_sided_p": 0.25,
|
|
"outcome_only_correct": 0
|
|
}
|
|
},
|
|
"1.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.12195121951219512
|
|
],
|
|
"point": 0.05555555555555547
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.09816256823569453,
|
|
-0.009602741441109449
|
|
],
|
|
"point": -0.04807041919163683
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9166666666666666,
|
|
"balanced_accuracy": 0.8571428571428572,
|
|
"brier": 0.05902138516195827,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.24132948833387438
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 2,
|
|
"correctly_saved_h20_hours": 0.6191580696761111,
|
|
"decision_coverage": 0.9444444444444444,
|
|
"early_accept": 28,
|
|
"early_reject": 6,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8399807697230681,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.008919034978436878,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.009139434342777777,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6282975040188888,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.6101058238716667,
|
|
"decision_coverage": 0.8611111111111112,
|
|
"early_accept": 27,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.008919034978436878,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6174606812416666,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.5390392875763889,
|
|
"decision_coverage": 0.75,
|
|
"early_accept": 23,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.008919034978436878,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5463941449463888,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8611111111111112,
|
|
"balanced_accuracy": 0.7321428571428572,
|
|
"brier": 0.1070918043535951,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 4,
|
|
"true_negative": 4,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.437208492871466
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.5864084509805555,
|
|
"decision_coverage": 0.8888888888888888,
|
|
"early_accept": 28,
|
|
"early_reject": 4,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9419008835359353,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
},
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8529942983175343,
|
|
"remaining_h20_hours": 0.0012268335861111111
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8268198678404454,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0017338236710752416,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.01042191079861111,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5968303617791667,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.5682670563286111,
|
|
"decision_coverage": 0.7777777777777778,
|
|
"early_accept": 25,
|
|
"early_reject": 3,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.5,
|
|
"cell": "tp1_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9419008835359353,
|
|
"remaining_h20_hours": 5.564286972222258e-05
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0017338236710752416,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.007410500239722223,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5756775565683333,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 17,
|
|
"correctly_saved_h20_hours": 0.40106617994777777,
|
|
"decision_coverage": 0.5277777777777778,
|
|
"early_accept": 16,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0017338236710752416,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.4084210373177778,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 31,
|
|
"both_wrong": 3,
|
|
"instrumentation_only_correct": 2,
|
|
"mcnemar_exact_two_sided_p": 0.5,
|
|
"outcome_only_correct": 0
|
|
}
|
|
},
|
|
"10.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.17142857142857137
|
|
],
|
|
"point": 0.08333333333333326
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.047457441513371,
|
|
-0.01422543161645023
|
|
],
|
|
"point": -0.029913894255621076
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.9166666666666666,
|
|
"balanced_accuracy": 0.8571428571428572,
|
|
"brier": 0.07346598213380073,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 2,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.24820863982463115
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 5,
|
|
"correctly_saved_h20_hours": 0.6026586119891666,
|
|
"decision_coverage": 0.8611111111111112,
|
|
"early_accept": 27,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8171600256181173,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.08736122237187642,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.009139434342777777,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6117980463319445,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.5909011698152777,
|
|
"decision_coverage": 0.7777777777777778,
|
|
"early_accept": 25,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.08736122237187642,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5982560271852778,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 27,
|
|
"correctly_saved_h20_hours": 0.27948134858055557,
|
|
"decision_coverage": 0.25,
|
|
"early_accept": 9,
|
|
"early_reject": 0,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.27948134858055557,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.26293898566315754,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8333333333333334,
|
|
"balanced_accuracy": 0.7142857142857143,
|
|
"brier": 0.10337987638942181,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 4,
|
|
"true_negative": 4,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.34599307331049317
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 7,
|
|
"correctly_saved_h20_hours": 0.6007090416925,
|
|
"decision_coverage": 0.8055555555555556,
|
|
"early_accept": 27,
|
|
"early_reject": 2,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8140697427804628,
|
|
"remaining_h20_hours": 0.0017845769727777771
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0444153615651932,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.009139434342777777,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.6098484760352777,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.5199227922302778,
|
|
"decision_coverage": 0.6666666666666666,
|
|
"early_accept": 23,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0444153615651932,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.5272776496002778,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 33,
|
|
"correctly_saved_h20_hours": 0.08992033662666665,
|
|
"decision_coverage": 0.08333333333333333,
|
|
"early_accept": 2,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns32",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.0444153615651932,
|
|
"remaining_h20_hours": 0.0073548573700000005
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0629133138080555,
|
|
"invalidly_saved_h20_hours": 0.0073548573700000005,
|
|
"remaining_h20_hours_at_cutoff": 0.6295799804747222,
|
|
"saved_h20_hours_if_decisions_used": 0.09727519399666666,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 30,
|
|
"both_wrong": 3,
|
|
"instrumentation_only_correct": 3,
|
|
"mcnemar_exact_two_sided_p": 0.25,
|
|
"outcome_only_correct": 0
|
|
}
|
|
}
|
|
},
|
|
"remaining_h20_hours": {
|
|
"distinct_n": 36,
|
|
"max": 0.04608871510333333,
|
|
"min": 5.564286972222258e-05,
|
|
"n": 36
|
|
}
|
|
},
|
|
"5": {
|
|
"cells": 12,
|
|
"completion_time_sources": {
|
|
"none_completed": 2,
|
|
"reconstructed_from_latency": 35
|
|
},
|
|
"examples": 37,
|
|
"headline": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.02777777777777779,
|
|
0.19999999999999996
|
|
],
|
|
"point": 0.10810810810810811
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.08739451923629489,
|
|
0.010803161364314168
|
|
],
|
|
"point": -0.03964626050589401
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8154761904761905,
|
|
"brier": 0.09006542265758688,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.2730810945072346
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.7828422774630556,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 23,
|
|
"early_reject": 5,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.918171067470718,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7929601877691667,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.7136471349655555,
|
|
"decision_coverage": 0.6756756756756757,
|
|
"early_accept": 21,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.918171067470718,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7237650452716666,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 15,
|
|
"correctly_saved_h20_hours": 0.6518691238655556,
|
|
"decision_coverage": 0.5945945945945946,
|
|
"early_accept": 18,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.6518691238655556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6109595741422738,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.7837837837837838,
|
|
"balanced_accuracy": 0.7063492063492063,
|
|
"brier": 0.1297116831634809,
|
|
"confusion": {
|
|
"false_negative": 4,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 24
|
|
},
|
|
"log_loss": 0.38484417997476916
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.745153891865,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 24,
|
|
"early_reject": 4,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8051358474368825,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9221804334720108,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8933807379877649,
|
|
"remaining_h20_hours": 0.022359229706666667
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.042037306932222226,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7871911987972221,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 14,
|
|
"correctly_saved_h20_hours": 0.6362291990002777,
|
|
"decision_coverage": 0.6216216216216216,
|
|
"early_accept": 19,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9221804334720108,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.6463471093063888,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 21,
|
|
"correctly_saved_h20_hours": 0.3878475556194444,
|
|
"decision_coverage": 0.43243243243243246,
|
|
"early_accept": 13,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.3878475556194444,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.3635072881013599,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 29,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 4,
|
|
"mcnemar_exact_two_sided_p": 0.125,
|
|
"outcome_only_correct": 0
|
|
}
|
|
},
|
|
"headline_regularization": 1.0,
|
|
"label_sanity": {
|
|
"distinct_n": 2,
|
|
"max": 1.0,
|
|
"min": 0.0,
|
|
"n": 37,
|
|
"negative": 9,
|
|
"positive": 28,
|
|
"primary_adjudicated_disagreements": 3
|
|
},
|
|
"regularization_sensitivity": {
|
|
"0.1": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.06980897009966774,
|
|
0.21875
|
|
],
|
|
"point": 0.08108108108108103
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.1649967051920624,
|
|
0.06447487686406284
|
|
],
|
|
"point": -0.05737762784013967
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8378378378378378,
|
|
"balanced_accuracy": 0.7797619047619048,
|
|
"brier": 0.13521691422103324,
|
|
"confusion": {
|
|
"false_negative": 3,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 25
|
|
},
|
|
"log_loss": 0.4511157566254769
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 3,
|
|
"correctly_saved_h20_hours": 0.8122652196319445,
|
|
"decision_coverage": 0.918918918918919,
|
|
"early_accept": 26,
|
|
"early_reject": 8,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9405657729239719,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9783420846880035,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns16",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9635702290378427,
|
|
"remaining_h20_hours": 0.023106262014444445
|
|
}
|
|
],
|
|
"false_reject": 2,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.21875,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.07387488532278061,
|
|
"remaining_h20_hours": 0.015631309493055554
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns16",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.17417966056624304,
|
|
"remaining_h20_hours": 0.03127135866722222
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.08968700740027778,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.9019522270322223,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 6,
|
|
"correctly_saved_h20_hours": 0.765429306841111,
|
|
"decision_coverage": 0.8378378378378378,
|
|
"early_accept": 24,
|
|
"early_reject": 7,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9405657729239719,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9783420846880035,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns16",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9635702290378427,
|
|
"remaining_h20_hours": 0.023106262014444445
|
|
}
|
|
],
|
|
"false_reject": 1,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.21875,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.07387488532278061,
|
|
"remaining_h20_hours": 0.015631309493055554
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.05841564873305555,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.8238449555741666,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.7469523484622221,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 22,
|
|
"early_reject": 6,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9783420846880035,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns16",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9635702290378427,
|
|
"remaining_h20_hours": 0.023106262014444445
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.03322417232055556,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7801765207827777,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.7567567567567568,
|
|
"balanced_accuracy": 0.7261904761904762,
|
|
"brier": 0.1925945420611729,
|
|
"confusion": {
|
|
"false_negative": 6,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 22
|
|
},
|
|
"log_loss": 0.7361068786964186
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 4,
|
|
"correctly_saved_h20_hours": 0.7745768340338888,
|
|
"decision_coverage": 0.8918918918918919,
|
|
"early_accept": 25,
|
|
"early_reject": 8,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.977871407650825,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9783309849310482,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9900156648758626,
|
|
"remaining_h20_hours": 0.022359229706666667
|
|
}
|
|
],
|
|
"false_reject": 3,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.21875,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.012882187111638552,
|
|
"remaining_h20_hours": 0.015631309493055554
|
|
},
|
|
{
|
|
"anchor": 0.2265625,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.016042537020255383,
|
|
"remaining_h20_hours": 0.01563306773861111
|
|
},
|
|
{
|
|
"anchor": 0.23046875,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.15321211351284938,
|
|
"remaining_h20_hours": 0.015631133334166665
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.08893281749805555,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.8635096515319445,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 8,
|
|
"correctly_saved_h20_hours": 0.6784465414575,
|
|
"decision_coverage": 0.7837837837837838,
|
|
"early_accept": 22,
|
|
"early_reject": 7,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.977871407650825,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9783309849310482,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9900156648758626,
|
|
"remaining_h20_hours": 0.022359229706666667
|
|
}
|
|
],
|
|
"false_reject": 2,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.21875,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.012882187111638552,
|
|
"remaining_h20_hours": 0.015631309493055554
|
|
},
|
|
{
|
|
"anchor": 0.2265625,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.016042537020255383,
|
|
"remaining_h20_hours": 0.01563306773861111
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0733016841638889,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7517482256213891,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.6471751827902779,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 21,
|
|
"early_reject": 7,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.977871407650825,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9783309849310482,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9900156648758626,
|
|
"remaining_h20_hours": 0.022359229706666667
|
|
}
|
|
],
|
|
"false_reject": 2,
|
|
"false_reject_examples": [
|
|
{
|
|
"anchor": 0.21875,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.012882187111638552,
|
|
"remaining_h20_hours": 0.015631309493055554
|
|
},
|
|
{
|
|
"anchor": 0.2265625,
|
|
"cell": "tp1_mns8",
|
|
"label_feasible": true,
|
|
"probability_feasible": 0.016042537020255383,
|
|
"remaining_h20_hours": 0.01563306773861111
|
|
}
|
|
],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0733016841638889,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7204768669541667,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 26,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 5,
|
|
"mcnemar_exact_two_sided_p": 0.453125,
|
|
"outcome_only_correct": 2
|
|
}
|
|
},
|
|
"1.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.02777777777777779,
|
|
0.19999999999999996
|
|
],
|
|
"point": 0.10810810810810811
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.08739451923629489,
|
|
0.010803161364314168
|
|
],
|
|
"point": -0.03964626050589401
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8154761904761905,
|
|
"brier": 0.09006542265758688,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.2730810945072346
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.7828422774630556,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 23,
|
|
"early_reject": 5,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.918171067470718,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7929601877691667,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.7136471349655555,
|
|
"decision_coverage": 0.6756756756756757,
|
|
"early_accept": 21,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.918171067470718,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7237650452716666,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 15,
|
|
"correctly_saved_h20_hours": 0.6518691238655556,
|
|
"decision_coverage": 0.5945945945945946,
|
|
"early_accept": 18,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.6518691238655556,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6109595741422738,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.7837837837837838,
|
|
"balanced_accuracy": 0.7063492063492063,
|
|
"brier": 0.1297116831634809,
|
|
"confusion": {
|
|
"false_negative": 4,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 24
|
|
},
|
|
"log_loss": 0.38484417997476916
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.745153891865,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 24,
|
|
"early_reject": 4,
|
|
"false_accept": 3,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.75,
|
|
"cell": "tp2_mns64",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8051358474368825,
|
|
"remaining_h20_hours": 0.009560166919444444
|
|
},
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9221804334720108,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.034252608017,
|
|
"cell": "tp4_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8933807379877649,
|
|
"remaining_h20_hours": 0.022359229706666667
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.042037306932222226,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7871911987972221,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 14,
|
|
"correctly_saved_h20_hours": 0.6362291990002777,
|
|
"decision_coverage": 0.6216216216216216,
|
|
"early_accept": 19,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9221804334720108,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.6463471093063888,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 21,
|
|
"correctly_saved_h20_hours": 0.3878475556194444,
|
|
"decision_coverage": 0.43243243243243246,
|
|
"early_accept": 13,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.3878475556194444,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.3635072881013599,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 29,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 4,
|
|
"mcnemar_exact_two_sided_p": 0.125,
|
|
"outcome_only_correct": 0
|
|
}
|
|
},
|
|
"10.0": {
|
|
"bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.08698849104859337,
|
|
0.07499999999999996
|
|
],
|
|
"point": 0.0
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.0419671863122155,
|
|
0.010410939632453053
|
|
],
|
|
"point": -0.015389088872407489
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"instrumentation_aware": {
|
|
"classification": {
|
|
"accuracy": 0.8378378378378378,
|
|
"balanced_accuracy": 0.7420634920634921,
|
|
"brier": 0.09695183324390907,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.31480154295615903
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.7917544064236111,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 24,
|
|
"early_reject": 4,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8905819069991598,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.8018723167297224,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 21,
|
|
"correctly_saved_h20_hours": 0.5056768371675,
|
|
"decision_coverage": 0.43243243243243246,
|
|
"early_accept": 13,
|
|
"early_reject": 3,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.5056768371675,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.4739419214357306,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 30,
|
|
"correctly_saved_h20_hours": 0.19128570447222223,
|
|
"decision_coverage": 0.1891891891891892,
|
|
"early_accept": 6,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.19128570447222223,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.17928112908743485,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"outcome_only": {
|
|
"classification": {
|
|
"accuracy": 0.8378378378378378,
|
|
"balanced_accuracy": 0.7043650793650793,
|
|
"brier": 0.11234092211631656,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 5,
|
|
"true_negative": 4,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.36773394130409404
|
|
},
|
|
"policies": [
|
|
{
|
|
"abstain_continue_full": 15,
|
|
"correctly_saved_h20_hours": 0.6981008277897223,
|
|
"decision_coverage": 0.5945945945945946,
|
|
"early_accept": 21,
|
|
"early_reject": 1,
|
|
"false_accept": 1,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.8520031544496152,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.010117910306111111,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7082187380958334,
|
|
"threshold": 0.8,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
},
|
|
{
|
|
"abstain_continue_full": 31,
|
|
"correctly_saved_h20_hours": 0.23290921672555553,
|
|
"decision_coverage": 0.16216216216216217,
|
|
"early_accept": 6,
|
|
"early_reject": 0,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.23290921672555553,
|
|
"threshold": 0.9,
|
|
"valid_cost_reduction_fraction": 0.21829246186816503,
|
|
"valid_zero_error_policy": true
|
|
},
|
|
{
|
|
"abstain_continue_full": 34,
|
|
"correctly_saved_h20_hours": 0.13888648644666668,
|
|
"decision_coverage": 0.08108108108108109,
|
|
"early_accept": 3,
|
|
"early_reject": 0,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.13888648644666668,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.13017034479312573,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
]
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 30,
|
|
"both_wrong": 5,
|
|
"instrumentation_only_correct": 1,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"outcome_only_correct": 1
|
|
}
|
|
}
|
|
},
|
|
"remaining_h20_hours": {
|
|
"distinct_n": 37,
|
|
"max": 0.06275538177,
|
|
"min": 0.0026573007705555556,
|
|
"n": 37
|
|
}
|
|
}
|
|
},
|
|
"cutoffs_s": [
|
|
5.0,
|
|
10.0,
|
|
15.0,
|
|
20.0
|
|
],
|
|
"decision": {
|
|
"contribution_established": false,
|
|
"reason": "This dataset contains one workload and reconstructed rather than exact request completion times. Three TP4 primary trials also disagree with their 2-of-3 labels. It can reject a missing-signal premise but cannot establish generalization or a paper-facing cost reduction."
|
|
},
|
|
"estimand": "2-of-3 adjudicated anchor feasibility from the first primary trial's identical short real prefix",
|
|
"headline_incremental_deltas": {
|
|
"10": {
|
|
"accuracy": -0.027027027027027084,
|
|
"brier": -0.0026830647286030934
|
|
},
|
|
"15": {
|
|
"accuracy": 0.02777777777777779,
|
|
"brier": -0.03744665409603411
|
|
},
|
|
"20": {
|
|
"accuracy": 0.05555555555555547,
|
|
"brier": -0.04807041919163683
|
|
},
|
|
"5": {
|
|
"accuracy": 0.10810810810810811,
|
|
"brier": -0.03964626050589401
|
|
}
|
|
},
|
|
"instrumentation_features": [
|
|
"model_steps_per_second",
|
|
"waiting_mean",
|
|
"waiting_max",
|
|
"waiting_nonzero_share",
|
|
"running_mean",
|
|
"running_max",
|
|
"decode_batch_mean",
|
|
"decode_batch_max",
|
|
"decode_batch_cv",
|
|
"kv_usage_mean",
|
|
"kv_usage_max",
|
|
"kv_usage_end_minus_start",
|
|
"graph_none_share",
|
|
"graph_full_share",
|
|
"padding_fraction",
|
|
"prefill_token_fraction",
|
|
"preemptions"
|
|
],
|
|
"model": "same L2 logistic model and folds; instrumentation model appends Layer-1 features",
|
|
"outcome_features": [
|
|
"log_offered_rate_per_gpu",
|
|
"log2_tp",
|
|
"log2_max_num_seqs",
|
|
"admitted_fraction",
|
|
"completed_over_admitted",
|
|
"completed_pass_rate",
|
|
"completed_fail_fraction_of_total",
|
|
"outstanding_over_admitted",
|
|
"ttft_max_over_slo_max",
|
|
"ttft_mean_over_slo_max",
|
|
"tpot_max_over_slo",
|
|
"tpot_mean_over_slo",
|
|
"admitted_input_tokens_mean_over_limit"
|
|
],
|
|
"provenance": {
|
|
"phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json",
|
|
"phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16",
|
|
"raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells"
|
|
},
|
|
"sanity": {
|
|
"cutoff_count": 4,
|
|
"invariants": {
|
|
"cutoffs_positive": true,
|
|
"full_trial_label_not_used_as_feature": true,
|
|
"paired_same_model_family": true,
|
|
"probabilities_checked_in_unit_interval": true,
|
|
"records_strictly_prefix_sliced": true
|
|
},
|
|
"red_flags": []
|
|
},
|
|
"schema": "fidelity-prefix-v1",
|
|
"scope": "retrospective single-workload prefix diagnostic; model selection, threshold choice, and contribution claims require held-out prospective tasks",
|
|
"split": "leave-one-configuration-cell-out",
|
|
"status": "PASS"
|
|
}
|