478 lines
16 KiB
JSON
478 lines
16 KiB
JSON
{
|
|
"comparison": "same 5-second prefix, folds, logistic family, regularization, and frozen Frontier outputs; the only nested difference is real Layer-1 engine state",
|
|
"decision": {
|
|
"contribution_established": false,
|
|
"prospective_requirement": "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks"
|
|
},
|
|
"features": {
|
|
"instrumentation_only": [
|
|
"model_steps_per_second",
|
|
"waiting_mean",
|
|
"waiting_max",
|
|
"waiting_nonzero_share",
|
|
"running_mean",
|
|
"running_max",
|
|
"decode_batch_mean",
|
|
"decode_batch_max",
|
|
"decode_batch_cv",
|
|
"kv_usage_mean",
|
|
"kv_usage_max",
|
|
"kv_usage_end_minus_start",
|
|
"graph_none_share",
|
|
"graph_full_share",
|
|
"padding_fraction",
|
|
"prefill_token_fraction",
|
|
"preemptions"
|
|
],
|
|
"shared_outcome": [
|
|
"log_offered_rate_per_gpu",
|
|
"log2_tp",
|
|
"log2_max_num_seqs",
|
|
"admitted_fraction",
|
|
"completed_over_admitted",
|
|
"completed_pass_rate",
|
|
"completed_fail_fraction_of_total",
|
|
"outstanding_over_admitted",
|
|
"ttft_max_over_slo_max",
|
|
"ttft_mean_over_slo_max",
|
|
"tpot_max_over_slo",
|
|
"tpot_mean_over_slo",
|
|
"admitted_input_tokens_mean_over_limit"
|
|
],
|
|
"shared_simulator": [
|
|
"log_sim_completed_throughput_per_gpu",
|
|
"sim_slo_pass_rate",
|
|
"sim_slo_feasible"
|
|
]
|
|
},
|
|
"headline": {
|
|
"group_bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.18181818181818188
|
|
],
|
|
"point": 0.08108108108108103
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.04292727744470806,
|
|
0.019924730979981074
|
|
],
|
|
"point": -0.010145365131402809
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 30,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 3,
|
|
"mcnemar_exact_two_sided_p": 0.25,
|
|
"sim_outcome_only_correct": 0
|
|
},
|
|
"sim_plus_outcome": {
|
|
"classification": {
|
|
"accuracy": 0.8108108108108109,
|
|
"balanced_accuracy": 0.7242063492063493,
|
|
"brier": 0.1058226346682949,
|
|
"confusion": {
|
|
"false_negative": 3,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 25
|
|
},
|
|
"log_loss": 0.3011048455679668
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 17,
|
|
"correctly_saved_h20_hours": 0.5429431818208333,
|
|
"decision_coverage": 0.5405405405405406,
|
|
"early_accept": 16,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.5429431818208333,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5088695307144538,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
},
|
|
"sim_plus_outcome_plus_instrumentation": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8154761904761905,
|
|
"brier": 0.0956772695368921,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.288823031828762
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.7360063646722222,
|
|
"decision_coverage": 0.6756756756756757,
|
|
"early_accept": 20,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7360063646722222,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6898165884274738,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
}
|
|
},
|
|
"headline_regularization": 1.0,
|
|
"provenance": {
|
|
"frozen_simulator_manifest_scorer_set_sha256": "833842d96ecaa0b059ef99852621752f7989e63d100118b6025425fb119b7a55",
|
|
"phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json",
|
|
"phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16",
|
|
"phase6_raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells",
|
|
"simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
|
|
"simulator_metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206",
|
|
"simulator_raw_root": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/raw"
|
|
},
|
|
"regularization_sensitivity": {
|
|
"0.1": {
|
|
"group_bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.17500000000000004,
|
|
0.0
|
|
],
|
|
"point": -0.08108108108108103
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.026383192545085435,
|
|
0.0607951286646285
|
|
],
|
|
"point": 0.019228316404518567
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 30,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 0,
|
|
"mcnemar_exact_two_sided_p": 0.25,
|
|
"sim_outcome_only_correct": 3
|
|
},
|
|
"sim_plus_outcome": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8154761904761905,
|
|
"brier": 0.10990776306815446,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.328357763455984
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.7402314096841667,
|
|
"decision_coverage": 0.6756756756756757,
|
|
"early_accept": 20,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7402314096841667,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6937764809990414,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
},
|
|
"sim_plus_outcome_plus_instrumentation": {
|
|
"classification": {
|
|
"accuracy": 0.8108108108108109,
|
|
"balanced_accuracy": 0.7619047619047619,
|
|
"brier": 0.12913607947267303,
|
|
"confusion": {
|
|
"false_negative": 4,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 24
|
|
},
|
|
"log_loss": 0.4373556318820343
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 9,
|
|
"correctly_saved_h20_hours": 0.7469523484622221,
|
|
"decision_coverage": 0.7567567567567568,
|
|
"early_accept": 22,
|
|
"early_reject": 6,
|
|
"false_accept": 2,
|
|
"false_accept_examples": [
|
|
{
|
|
"anchor": 0.49609375,
|
|
"cell": "tp2_mns8",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9869795738005246,
|
|
"remaining_h20_hours": 0.010117910306111111
|
|
},
|
|
{
|
|
"anchor": 0.033717411016,
|
|
"cell": "tp4_mns16",
|
|
"label_feasible": false,
|
|
"probability_feasible": 0.9855364057197005,
|
|
"remaining_h20_hours": 0.023106262014444445
|
|
}
|
|
],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.03322417232055556,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7801765207827777,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": null,
|
|
"valid_zero_error_policy": false
|
|
}
|
|
}
|
|
},
|
|
"1.0": {
|
|
"group_bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
0.0,
|
|
0.18181818181818188
|
|
],
|
|
"point": 0.08108108108108103
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.04292727744470806,
|
|
0.019924730979981074
|
|
],
|
|
"point": -0.010145365131402809
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 30,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 3,
|
|
"mcnemar_exact_two_sided_p": 0.25,
|
|
"sim_outcome_only_correct": 0
|
|
},
|
|
"sim_plus_outcome": {
|
|
"classification": {
|
|
"accuracy": 0.8108108108108109,
|
|
"balanced_accuracy": 0.7242063492063493,
|
|
"brier": 0.1058226346682949,
|
|
"confusion": {
|
|
"false_negative": 3,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 25
|
|
},
|
|
"log_loss": 0.3011048455679668
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 17,
|
|
"correctly_saved_h20_hours": 0.5429431818208333,
|
|
"decision_coverage": 0.5405405405405406,
|
|
"early_accept": 16,
|
|
"early_reject": 4,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.5429431818208333,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.5088695307144538,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
},
|
|
"sim_plus_outcome_plus_instrumentation": {
|
|
"classification": {
|
|
"accuracy": 0.8918918918918919,
|
|
"balanced_accuracy": 0.8154761904761905,
|
|
"brier": 0.0956772695368921,
|
|
"confusion": {
|
|
"false_negative": 1,
|
|
"false_positive": 3,
|
|
"true_negative": 6,
|
|
"true_positive": 27
|
|
},
|
|
"log_loss": 0.288823031828762
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 12,
|
|
"correctly_saved_h20_hours": 0.7360063646722222,
|
|
"decision_coverage": 0.6756756756756757,
|
|
"early_accept": 20,
|
|
"early_reject": 5,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.7360063646722222,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.6898165884274738,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
}
|
|
},
|
|
"10.0": {
|
|
"group_bootstrap": {
|
|
"accuracy_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.13333333333333341,
|
|
0.05555555555555558
|
|
],
|
|
"point": -0.027027027027027084
|
|
},
|
|
"brier_delta_instrumentation_minus_outcome": {
|
|
"ci95": [
|
|
-0.03091105649870874,
|
|
0.01684192005239855
|
|
],
|
|
"point": -0.007318433328714388
|
|
},
|
|
"replicates": 10000,
|
|
"seed": 20260714,
|
|
"semantics": "group bootstrap over cells; diagnostic confidence interval"
|
|
},
|
|
"paired_correctness": {
|
|
"both_correct": 30,
|
|
"both_wrong": 4,
|
|
"instrumentation_only_correct": 1,
|
|
"mcnemar_exact_two_sided_p": 1.0,
|
|
"sim_outcome_only_correct": 2
|
|
},
|
|
"sim_plus_outcome": {
|
|
"classification": {
|
|
"accuracy": 0.8648648648648649,
|
|
"balanced_accuracy": 0.7222222222222222,
|
|
"brier": 0.10613344425735322,
|
|
"confusion": {
|
|
"false_negative": 0,
|
|
"false_positive": 5,
|
|
"true_negative": 4,
|
|
"true_positive": 28
|
|
},
|
|
"log_loss": 0.3404203142465075
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 32,
|
|
"correctly_saved_h20_hours": 0.21727432337249997,
|
|
"decision_coverage": 0.13513513513513514,
|
|
"early_accept": 5,
|
|
"early_reject": 0,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.21727432337249997,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.20363877229302757,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
},
|
|
"sim_plus_outcome_plus_instrumentation": {
|
|
"classification": {
|
|
"accuracy": 0.8378378378378378,
|
|
"balanced_accuracy": 0.7420634920634921,
|
|
"brier": 0.09881501092863883,
|
|
"confusion": {
|
|
"false_negative": 2,
|
|
"false_positive": 4,
|
|
"true_negative": 5,
|
|
"true_positive": 26
|
|
},
|
|
"log_loss": 0.312914193285738
|
|
},
|
|
"policy_0p95": {
|
|
"abstain_continue_full": 30,
|
|
"correctly_saved_h20_hours": 0.2384080185036111,
|
|
"decision_coverage": 0.1891891891891892,
|
|
"early_accept": 6,
|
|
"early_reject": 1,
|
|
"false_accept": 0,
|
|
"false_accept_examples": [],
|
|
"false_reject": 0,
|
|
"false_reject_examples": [],
|
|
"full_trial_h20_hours": 1.0669595034675,
|
|
"invalidly_saved_h20_hours": 0.0,
|
|
"remaining_h20_hours_at_cutoff": 0.957237281245278,
|
|
"saved_h20_hours_if_decisions_used": 0.2384080185036111,
|
|
"threshold": 0.95,
|
|
"valid_cost_reduction_fraction": 0.22344617366339725,
|
|
"valid_zero_error_policy": true
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"sanity": {
|
|
"examples": {
|
|
"distinct_n": 1,
|
|
"max": 1.0,
|
|
"min": 1.0,
|
|
"n": 37
|
|
},
|
|
"frozen_simulator_runs": 92,
|
|
"invariants": {
|
|
"all_examples_matched_once": true,
|
|
"labels_not_identical": true,
|
|
"per_config_results_not_all_identical": true,
|
|
"same_nested_folds": true,
|
|
"simulator_ratios_bounded": true
|
|
},
|
|
"labels": {
|
|
"distinct_n": 2,
|
|
"max": 1.0,
|
|
"min": 0.0,
|
|
"n": 37,
|
|
"negative": 9,
|
|
"positive": 28
|
|
},
|
|
"matched_simulator_pass_rate": {
|
|
"distinct_n": 12,
|
|
"max": 1.0,
|
|
"min": 0.06884057971014493,
|
|
"n": 37
|
|
},
|
|
"red_flags": []
|
|
},
|
|
"schema": "fidelity-strong-baseline-v1",
|
|
"scope": "retrospective one-task headroom audit; not contribution evidence",
|
|
"status": "PASS"
|
|
}
|