7411 lines
241 KiB
JSON
7411 lines
241 KiB
JSON
{
|
|
"anomalies": [],
|
|
"cross_checks": [
|
|
{
|
|
"expected_percent": 58.0,
|
|
"name": "q30 high-pressure fixed-pd TTFT mean regret",
|
|
"observed_percent": 57.97674051508896,
|
|
"passed": true
|
|
},
|
|
{
|
|
"expected_percent": 33.0,
|
|
"name": "q235 A0 fixed-pd TPOT mean regret",
|
|
"observed_percent": 33.025510615656906,
|
|
"passed": true
|
|
},
|
|
{
|
|
"expected_percent": 33.0,
|
|
"name": "q235 A1 fixed-pd TPOT mean regret",
|
|
"observed_percent": 33.025510615656906,
|
|
"passed": true
|
|
},
|
|
{
|
|
"expected_percent": [
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0
|
|
],
|
|
"name": "q30 trace-pd all-objective regret",
|
|
"observed_percent": [
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0
|
|
],
|
|
"passed": true
|
|
},
|
|
{
|
|
"expected_percent": [
|
|
21.2,
|
|
0.3
|
|
],
|
|
"name": "q235 Trace-PO TTFT p90 A0 to A1 regret",
|
|
"observed_percent": [
|
|
21.17895828808376,
|
|
0.2607797240047338
|
|
],
|
|
"passed": true
|
|
},
|
|
{
|
|
"expected_percent": [
|
|
33.0,
|
|
37.2,
|
|
30.7,
|
|
34.6,
|
|
33.0,
|
|
37.2,
|
|
30.7,
|
|
34.6
|
|
],
|
|
"name": "q235 Fixed-PD TPOT/E2E regrets unchanged from A0 to A1",
|
|
"observed_percent": [
|
|
33.025510615656906,
|
|
37.228890038064755,
|
|
30.67524317507717,
|
|
34.56375801249614,
|
|
33.025510615656906,
|
|
37.228890038064755,
|
|
30.67524317507717,
|
|
34.56375801249614
|
|
],
|
|
"passed": true
|
|
}
|
|
],
|
|
"data_gaps": [],
|
|
"definitions": {
|
|
"config_uniform_scale": "geomean_c(sim_c / real_c) over paired configs",
|
|
"failure": "regret > 0.05",
|
|
"fragile_success": "winner match and real best-vs-second relative margin < 0.01",
|
|
"h_scale_verdict": "HOLDS iff failure equals (top-3 log-ratio spread > log1p(real relative margin))",
|
|
"log_ratio_spread_all": "max_c(log(sim_c / real_c)) - min_c(log(sim_c / real_c))",
|
|
"log_ratio_spread_real_top3": "log-ratio spread over the three configs with lowest real objective values",
|
|
"objective_direction": "all objectives are minimized",
|
|
"pair_inversion": "real_c > real_w and sim_c < sim_w, equivalently delta_c < -m_c",
|
|
"real_pair_margin_percent": "100 * log(real_c / real_w)",
|
|
"real_relative_margin_best_vs_second": "real_second / real_best - 1",
|
|
"regret": "real_sim_winner / real_best - 1, recomputed from per-config values",
|
|
"signed_pairwise_delta_percent": "100 * (log(sim_c / sim_w) - log(real_c / real_w)); negative favors c relative to real",
|
|
"winner_deciding_axis": "axis difference between real winner w and sim winner s; none for a winner match",
|
|
"winner_pair_tie": "sim_s == sim_w with real_s > real_w; a decision-boundary winner-label mismatch, not a strict inversion"
|
|
},
|
|
"directional_summary": {
|
|
"axis_inversions_by_case": [
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"n_inv_mixed": 6,
|
|
"n_inv_mns": 3,
|
|
"n_inv_tp": 7,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 4,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"n_inv_mixed": 24,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 8,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 4,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 6,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 2,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"n_inv_mixed": 5,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 5,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"n_inv_mixed": 4,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 4,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 2,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"n_inv_mixed": 4,
|
|
"n_inv_mns": 2,
|
|
"n_inv_tp": 4,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"n_inv_mixed": 4,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 4,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 2,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 2,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 2,
|
|
"n_tie_tp": 0
|
|
}
|
|
],
|
|
"axis_inversions_total": {
|
|
"mixed": 47,
|
|
"mns-axis": 17,
|
|
"tp-axis": 32
|
|
},
|
|
"axis_ties_total": {
|
|
"mixed": 0,
|
|
"mns-axis": 14,
|
|
"tp-axis": 0
|
|
},
|
|
"boundary_winner_ties": [
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "ttft_mean_ms",
|
|
"real_winner": "tp4_mns32",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "ttft_p90_ms",
|
|
"real_winner": "tp4_mns32",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "e2e_mean_ms",
|
|
"real_winner": "tp4_mns32",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "e2e_p90_ms",
|
|
"real_winner": "tp4_mns32",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "ttft_mean_ms",
|
|
"real_winner": "tp4_mns8",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "ttft_p90_ms",
|
|
"real_winner": "tp4_mns8",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "e2e_mean_ms",
|
|
"real_winner": "tp4_mns8",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "e2e_p90_ms",
|
|
"real_winner": "tp4_mns8",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"objective": "ttft_p90_ms",
|
|
"real_winner": "tp4_mns8",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"objective": "e2e_p90_ms",
|
|
"real_winner": "tp4_mns8",
|
|
"sim_winner": "tp4_mns16",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "ttft_p90_ms",
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "ttft_p90_ms",
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"objective": "ttft_p90_ms",
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_deciding_axis": "mns-axis"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"objective": "e2e_p90_ms",
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_deciding_axis": "mns-axis"
|
|
}
|
|
],
|
|
"fragile_success_count": 6,
|
|
"fragile_successes": [
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"objective": "e2e_p90_ms",
|
|
"real_relative_margin_percent": 0.07414165586019372
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "ttft_mean_ms",
|
|
"real_relative_margin_percent": 0.37425080108803854
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "ttft_mean_ms",
|
|
"real_relative_margin_percent": 0.20104594953702648
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "ttft_p90_ms",
|
|
"real_relative_margin_percent": 0.26166420965441084
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "e2e_mean_ms",
|
|
"real_relative_margin_percent": 0.20087777023818365
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "e2e_p90_ms",
|
|
"real_relative_margin_percent": 0.26228330848965253
|
|
}
|
|
],
|
|
"margin_robust_success_count": 17,
|
|
"material_failures": [
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"objective": "ttft_mean_ms",
|
|
"regret": 0.5797674051508896,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -523.7691675731886,
|
|
"winner_pair_m_s_percent": 45.7277624272319
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"objective": "ttft_p90_ms",
|
|
"regret": 0.5620663582204115,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -435.31434147282096,
|
|
"winner_pair_m_s_percent": 44.600953337070045
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.05493398001893279,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -101.70914835555358,
|
|
"winner_pair_m_s_percent": 5.3478186788827395
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"objective": "tpot_mean_ms",
|
|
"regret": 0.13962124906187534,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -52.50428401638987,
|
|
"winner_pair_m_s_percent": 13.069596953822204
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"objective": "tpot_p90_ms",
|
|
"regret": 0.1335710369665437,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -52.8208410178207,
|
|
"winner_pair_m_s_percent": 12.537285946185337
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"objective": "e2e_mean_ms",
|
|
"regret": 0.2692193946565338,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -59.60969731665489,
|
|
"winner_pair_m_s_percent": 23.840206161907986
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.2573048300400054,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -59.778209091748444,
|
|
"winner_pair_m_s_percent": 22.897040620669692
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "tpot_mean_ms",
|
|
"regret": 0.33025510615656906,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -63.96663007689711,
|
|
"winner_pair_m_s_percent": 28.537073298091865
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "tpot_p90_ms",
|
|
"regret": 0.37228890038064755,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -62.84476021464781,
|
|
"winner_pair_m_s_percent": 31.648007592775677
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "e2e_mean_ms",
|
|
"regret": 0.3067524317507717,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -61.50853261890787,
|
|
"winner_pair_m_s_percent": 26.754499952423572
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.3456375801249614,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -60.36185233190644,
|
|
"winner_pair_m_s_percent": 29.686793800126072
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"objective": "ttft_mean_ms",
|
|
"regret": 0.058746722254088946,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -12.020280158083734,
|
|
"winner_pair_m_s_percent": 5.708587111542836
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"objective": "e2e_mean_ms",
|
|
"regret": 0.05873813644442971,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -12.019469213946493,
|
|
"winner_pair_m_s_percent": 5.707776167405595
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.062164651568153984,
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_pair_delta_s_percent": -16.715978521917112,
|
|
"winner_pair_m_s_percent": 6.030894994380943
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"objective": "ttft_mean_ms",
|
|
"regret": 0.07039370625209918,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -36.29522004071061,
|
|
"winner_pair_m_s_percent": 6.802653056521642
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"objective": "ttft_p90_ms",
|
|
"regret": 0.2117895828808376,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -31.606834004650835,
|
|
"winner_pair_m_s_percent": 19.209826109044702
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"objective": "e2e_mean_ms",
|
|
"regret": 0.07037121566091509,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -36.29311886734452,
|
|
"winner_pair_m_s_percent": 6.800551883155552
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.21167227304868486,
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_pair_delta_s_percent": -31.59715282630199,
|
|
"winner_pair_m_s_percent": 19.20014493069586
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "tpot_mean_ms",
|
|
"regret": 0.33025510615656906,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -60.056263771346096,
|
|
"winner_pair_m_s_percent": 28.537073298091865
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "tpot_p90_ms",
|
|
"regret": 0.37228890038064755,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -61.12091996120764,
|
|
"winner_pair_m_s_percent": 31.648007592775677
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "e2e_mean_ms",
|
|
"regret": 0.3067524317507717,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -58.25805162281401,
|
|
"winner_pair_m_s_percent": 26.754499952423572
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.3456375801249614,
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_pair_delta_s_percent": -59.1984798883988,
|
|
"winner_pair_m_s_percent": 29.686793800126072
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"objective": "e2e_p90_ms",
|
|
"regret": 0.062164651568153984,
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_pair_delta_s_percent": -17.372678635438426,
|
|
"winner_pair_m_s_percent": 6.030894994380943
|
|
}
|
|
],
|
|
"sim_tie_winner_mismatch_count": 14,
|
|
"strict_winner_reversal_count": 33
|
|
},
|
|
"known_attribution": {
|
|
"q235_state": {
|
|
"components": [
|
|
{
|
|
"component": "total",
|
|
"exact_state_tp8_minus_tp4_ms": 10.902788634037428,
|
|
"own_composition_tp8_minus_tp4_ms": -20.06625468177714
|
|
},
|
|
{
|
|
"component": "moe_compute",
|
|
"exact_state_tp8_minus_tp4_ms": -1.9774463255033616,
|
|
"own_composition_tp8_minus_tp4_ms": -32.45444243169211
|
|
},
|
|
{
|
|
"component": "attention",
|
|
"exact_state_tp8_minus_tp4_ms": -2.130420408899373,
|
|
"own_composition_tp8_minus_tp4_ms": -7.461942625219683
|
|
},
|
|
{
|
|
"component": "tp_dp_communication",
|
|
"exact_state_tp8_minus_tp4_ms": -6.471757614086094,
|
|
"own_composition_tp8_minus_tp4_ms": -6.571107062807271
|
|
},
|
|
{
|
|
"component": "ep_communication",
|
|
"exact_state_tp8_minus_tp4_ms": 6.535508227248577,
|
|
"own_composition_tp8_minus_tp4_ms": 6.235402017662346
|
|
},
|
|
{
|
|
"component": "moe_routing",
|
|
"exact_state_tp8_minus_tp4_ms": 0.13840596821076812,
|
|
"own_composition_tp8_minus_tp4_ms": -0.2861742505642797
|
|
},
|
|
{
|
|
"component": "dense_mlp_compute",
|
|
"exact_state_tp8_minus_tp4_ms": -0.030752152939768296,
|
|
"own_composition_tp8_minus_tp4_ms": -0.12314137296116046
|
|
},
|
|
{
|
|
"component": "pipeline_communication",
|
|
"exact_state_tp8_minus_tp4_ms": 0.0,
|
|
"own_composition_tp8_minus_tp4_ms": 0.0
|
|
},
|
|
{
|
|
"component": "runtime_overhead",
|
|
"exact_state_tp8_minus_tp4_ms": 0.0,
|
|
"own_composition_tp8_minus_tp4_ms": 0.0
|
|
}
|
|
],
|
|
"configs": [
|
|
{
|
|
"config": "tp4_ep1_mns64",
|
|
"exact_state_coverage": 1.0,
|
|
"exact_state_matched_sim_ms": 34.25746674314369,
|
|
"real_exact_decode_batch_mean": 3.8896771904930825,
|
|
"real_token_weighted_iteration_ms": 18.47097461620307,
|
|
"sim_own_decode_batch_mean": 13.537037037037036
|
|
},
|
|
{
|
|
"config": "tp8_ep8_mns64",
|
|
"exact_state_coverage": 1.0,
|
|
"exact_state_matched_sim_ms": 45.160255377181116,
|
|
"real_exact_decode_batch_mean": 10.921314741035857,
|
|
"real_token_weighted_iteration_ms": 27.588699194406445,
|
|
"sim_own_decode_batch_mean": 18.93782383419689
|
|
}
|
|
],
|
|
"contrast_decomposition": {
|
|
"exact_state_matched_tp8_minus_tp4_ms": 10.902788634037428,
|
|
"residual_vs_exact_iteration_ms": 1.7850640558340523,
|
|
"residual_vs_frozen_real_tpot_ms": 3.9527886340374287,
|
|
"simulator_internal_tp8_minus_tp4_ms": -20.06625468177714,
|
|
"state_composition_shift_ms": 30.969043315814567
|
|
},
|
|
"reference": {
|
|
"exact_iteration_tp8_minus_tp4_ms": 9.117724578203376,
|
|
"observed_real_tp8_minus_tp4_ms": 6.949999999999999,
|
|
"real_tpot_ms": {
|
|
"tp4_ep1_mns64": 21.0442,
|
|
"tp8_ep8_mns64": 27.9942
|
|
}
|
|
},
|
|
"source": "q235-state-diag/state-diagnosis-exact.json",
|
|
"verdict": "Exact real state composition flips Frontier to the correct TP4 winner; the ranking error is caused by closed-loop state/composition mismatch, not a conditional stage reversal."
|
|
},
|
|
"q30_admission": {
|
|
"cells": [
|
|
{
|
|
"config": "tp2_mns64",
|
|
"mns": 64,
|
|
"real_required_slots_upper_bound": 7.548483722821714,
|
|
"real_ttft_ms": 194.56193759030603,
|
|
"real_waiting_max": 0,
|
|
"sim_first_scheduling_wait_ms": 38.22329108338011,
|
|
"sim_queue_free_ttft_ms": 191.02937373774816,
|
|
"sim_required_slots": 57.96430802744742,
|
|
"sim_tpot_over_real": 8.091316956395687,
|
|
"sim_ttft_ms": 229.2526648211283
|
|
},
|
|
{
|
|
"config": "tp4_mns16",
|
|
"mns": 16,
|
|
"real_required_slots_upper_bound": 14.637597063342735,
|
|
"real_ttft_ms": 124.49453797538061,
|
|
"real_waiting_max": 0,
|
|
"sim_first_scheduling_wait_ms": 64906.25631809643,
|
|
"sim_queue_free_ttft_ms": 134.74066995065758,
|
|
"sim_required_slots": 54.488824675063235,
|
|
"sim_tpot_over_real": 3.8276349505907024,
|
|
"sim_ttft_ms": 65040.99698804708
|
|
},
|
|
{
|
|
"config": "tp4_mns32",
|
|
"mns": 32,
|
|
"real_required_slots_upper_bound": 14.521569587596273,
|
|
"real_ttft_ms": 123.15859724408143,
|
|
"real_waiting_max": 0,
|
|
"sim_first_scheduling_wait_ms": 27180.676177927482,
|
|
"sim_queue_free_ttft_ms": 135.62147333612225,
|
|
"sim_required_slots": 65.89386162288272,
|
|
"sim_tpot_over_real": 4.6740581573385525,
|
|
"sim_ttft_ms": 27316.2976512636
|
|
},
|
|
{
|
|
"config": "tp4_mns64",
|
|
"mns": 64,
|
|
"real_required_slots_upper_bound": 14.661911968825338,
|
|
"real_ttft_ms": 123.17715882596677,
|
|
"real_waiting_max": 0,
|
|
"sim_first_scheduling_wait_ms": 6367.273779218055,
|
|
"sim_queue_free_ttft_ms": 136.7304324624105,
|
|
"sim_required_slots": 79.81630813400574,
|
|
"sim_tpot_over_real": 5.614117914776633,
|
|
"sim_ttft_ms": 6504.004211680465
|
|
}
|
|
],
|
|
"contrasts": {
|
|
"observed_sim_ttft_tp4_minus_tp2_ms": 27087.044986442474,
|
|
"queue_free_sim_ttft_tp4_minus_tp2_ms": -55.40790040162591,
|
|
"real_ttft_tp4_minus_tp2_ms": -71.4033403462246
|
|
},
|
|
"source": "q30-admission-diag/result.json",
|
|
"verdict": "Frontier overpredicts decode service time, so TP4's higher global arrival rate crosses the MNS admission cap only in simulation. The resulting first-scheduling queue reverses the TP topology ranking; without that queue Frontier itself predicts TP4 faster than TP2, matching real hardware."
|
|
}
|
|
},
|
|
"rows": [
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"config_uniform_scale": 45.899648513700804,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 4.140203056697399,
|
|
"log_ratio_spread_real_top3": 0.7350071901442927,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 425.7830890012937,
|
|
"signed_pairwise_delta_percent": -77.29228415027549
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 92.53155134784632,
|
|
"signed_pairwise_delta_percent": 154.92939260781299
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 92.42510316771971,
|
|
"signed_pairwise_delta_percent": 44.55096600091358
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 588.5391626639711,
|
|
"signed_pairwise_delta_percent": -174.96406393770422
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 494.19206953478573,
|
|
"signed_pairwise_delta_percent": -181.7812291449898
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 145.20093058332228,
|
|
"signed_pairwise_delta_percent": 67.76147877390028
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 42.379688777532195,
|
|
"signed_pairwise_delta_percent": -28.94975301351572
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 596.613115965754,
|
|
"signed_pairwise_delta_percent": -218.41901367549661
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 495.91805198396617,
|
|
"signed_pairwise_delta_percent": -218.17248485782983
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 324.25665835499575,
|
|
"signed_pairwise_delta_percent": -141.03952629372947
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 603.3510930798824,
|
|
"signed_pairwise_delta_percent": -259.0909130619269
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.5277512573119312,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp2_mns64",
|
|
"tp1_mns64"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-trace-pd/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"config_uniform_scale": 43.38413908996929,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 3.8627205241706646,
|
|
"log_ratio_spread_real_top3": 1.3676487927064054,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 413.27263799256224,
|
|
"signed_pairwise_delta_percent": -133.33363440024485
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 103.58793636206667,
|
|
"signed_pairwise_delta_percent": 92.65315392171725
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 104.88701154249979,
|
|
"signed_pairwise_delta_percent": 0.5820114595004977
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 550.2663402226464,
|
|
"signed_pairwise_delta_percent": -210.29242844968587
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 473.3308528278175,
|
|
"signed_pairwise_delta_percent": -229.66830605288865
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 130.25554778064452,
|
|
"signed_pairwise_delta_percent": 29.42287867963369
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.61341304162133,
|
|
"signed_pairwise_delta_percent": -44.11172534892322
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 559.1872645568208,
|
|
"signed_pairwise_delta_percent": -254.7846496300535
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 468.54545769902376,
|
|
"signed_pairwise_delta_percent": -259.82819430980805
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 333.04135988053577,
|
|
"signed_pairwise_delta_percent": -204.24777470078647
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 564.214798802028,
|
|
"signed_pairwise_delta_percent": -293.6188984953492
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.6755377028954586,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp2_mns64",
|
|
"tp1_mns32"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-trace-pd/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"config_uniform_scale": 4.798699439392796,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.34234204465206575,
|
|
"log_ratio_spread_real_top3": 0.11876167427448592,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 111.39211804152,
|
|
"signed_pairwise_delta_percent": -8.483533629441165
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 125.72442888367388,
|
|
"signed_pairwise_delta_percent": -4.073078922372719
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 127.21791091204592,
|
|
"signed_pairwise_delta_percent": 8.632369074762082
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 74.84126630438477,
|
|
"signed_pairwise_delta_percent": 1.447194103380467
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 73.532537849401,
|
|
"signed_pairwise_delta_percent": -13.742569389219161
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 97.95939493225517,
|
|
"signed_pairwise_delta_percent": -23.240697633755538
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 99.29270089893359,
|
|
"signed_pairwise_delta_percent": -14.63432527265468
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 31.337675062299986,
|
|
"signed_pairwise_delta_percent": 5.216266068044573
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 26.49069271125039,
|
|
"signed_pairwise_delta_percent": -6.6599013594039915
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 58.92488843099623,
|
|
"signed_pairwise_delta_percent": -25.60183539044448
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 66.79126824994552,
|
|
"signed_pairwise_delta_percent": -25.112476758256985
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.3033096673391522,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns16",
|
|
"tp2_mns8"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns8",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-trace-pd/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"config_uniform_scale": 4.291364819104713,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.349119999217536,
|
|
"log_ratio_spread_real_top3": 0.10928760145539185,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 125.15305260666022,
|
|
"signed_pairwise_delta_percent": -19.96596661376333
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 142.11708890779718,
|
|
"signed_pairwise_delta_percent": -17.29368054409164
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 143.30272831211315,
|
|
"signed_pairwise_delta_percent": -0.4539882281926477
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 81.3187358452952,
|
|
"signed_pairwise_delta_percent": -3.482599377639306
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 80.19578793426079,
|
|
"signed_pairwise_delta_percent": -18.566404630125845
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 107.78616263504968,
|
|
"signed_pairwise_delta_percent": -30.705392091898254
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 109.09052427237363,
|
|
"signed_pairwise_delta_percent": -19.929771139892004
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 34.15060711443027,
|
|
"signed_pairwise_delta_percent": 3.7395670641060086
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 28.455433619208037,
|
|
"signed_pairwise_delta_percent": -7.189193081433193
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 64.11332289157903,
|
|
"signed_pairwise_delta_percent": -28.87510274879303
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 75.37357913831967,
|
|
"signed_pairwise_delta_percent": -31.172432857647596
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.32916953334893906,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns16",
|
|
"tp2_mns8"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns8",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-trace-pd/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"config_uniform_scale": 8.262858317506733,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 1.0986813103518063,
|
|
"log_ratio_spread_real_top3": 0.5012037189370788,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 66.36499381882803,
|
|
"signed_pairwise_delta_percent": 108.79476716386034
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 58.8748864721832,
|
|
"signed_pairwise_delta_percent": 64.75911332399981
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 60.347657868883765,
|
|
"signed_pairwise_delta_percent": 35.940764769548515
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 112.0895297823733,
|
|
"signed_pairwise_delta_percent": 109.86813103518062
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 61.17455796538679,
|
|
"signed_pairwise_delta_percent": 76.12681299620277
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 32.51075059215534,
|
|
"signed_pairwise_delta_percent": 50.12037189370784
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 32.64897177653879,
|
|
"signed_pairwise_delta_percent": 6.771864363272045
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 104.80152491744161,
|
|
"signed_pairwise_delta_percent": 81.27184764853052
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.99697371409521,
|
|
"signed_pairwise_delta_percent": 63.386785575481056
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 6.3712519410935915,
|
|
"signed_pairwise_delta_percent": 40.82215218358013
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 103.31621796003591,
|
|
"signed_pairwise_delta_percent": 48.54179170585697
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.06578596194014197,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp4_mns32",
|
|
"tp2_mns32"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-trace-pd/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"config_uniform_scale": 8.066172032770895,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": true,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 1.2303978377958356,
|
|
"log_ratio_spread_real_top3": 0.7034476827520184,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 62.299232337248434,
|
|
"signed_pairwise_delta_percent": 108.56642733247624
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 66.51906134872392,
|
|
"signed_pairwise_delta_percent": 57.03741657339753
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 68.00975092247843,
|
|
"signed_pairwise_delta_percent": 28.839836578727095
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 97.52912455611215,
|
|
"signed_pairwise_delta_percent": 123.03978377958349
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 49.72701364992361,
|
|
"signed_pairwise_delta_percent": 83.73553771263259
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 33.62187114781614,
|
|
"signed_pairwise_delta_percent": 48.68626755803108
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 34.60052079875263,
|
|
"signed_pairwise_delta_percent": 8.505625400877559
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 94.65769118557269,
|
|
"signed_pairwise_delta_percent": 90.22793273806666
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 27.58788592694011,
|
|
"signed_pairwise_delta_percent": 70.3447682752018
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.07411418451216154,
|
|
"signed_pairwise_delta_percent": 45.007951726273106
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 91.4777538802895,
|
|
"signed_pairwise_delta_percent": 59.43122170260607
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0007414165586019372,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp4_mns32",
|
|
"tp4_mns16"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-trace-pd/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"config_uniform_scale": 80.27356282971952,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 6.647450003760477,
|
|
"log_ratio_spread_real_top3": 2.2919615656812473,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 2,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 95.49977993455686,
|
|
"signed_pairwise_delta_percent": 35.24651168874489
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 95.063546989325,
|
|
"signed_pairwise_delta_percent": -123.55932905047267
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 95.64389799119368,
|
|
"signed_pairwise_delta_percent": -534.2205177871797
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 94.7705561639161,
|
|
"signed_pairwise_delta_percent": 109.67959518505891
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 49.422396437751175,
|
|
"signed_pairwise_delta_percent": 60.17959107930238
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 49.333707226264345,
|
|
"signed_pairwise_delta_percent": -52.766300168579974
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 45.7277624272319,
|
|
"signed_pairwise_delta_percent": -523.7691675731886
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 46.01935636714559,
|
|
"signed_pairwise_delta_percent": 130.52448258886798
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.0788909971033689,
|
|
"signed_pairwise_delta_percent": 85.67453763656644
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 0.015070148334617827,
|
|
"signed_pairwise_delta_percent": -143.52161893155832
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 414.6352679329693,
|
|
"signed_pairwise_delta_percent": -263.0584045824042
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.00015071283938516977,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns64",
|
|
"tp4_mns16"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.5797674051508896,
|
|
"sim_winner": "tp2_mns64",
|
|
"sim_winner_real_relative_loss": 0.5797674051508896,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-pd/comparison.json",
|
|
"q30-fixed-hi/fixed-pd/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -523.7691675731886,
|
|
"winner_pair_m_s_percent": 45.7277624272319,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"config_uniform_scale": 130.66346534057038,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 7.092751831801099,
|
|
"log_ratio_spread_real_top3": 2.2268269641383185,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 93.72928763800273,
|
|
"signed_pairwise_delta_percent": 173.8072062575841
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 93.44021170742349,
|
|
"signed_pairwise_delta_percent": 21.330865292892188
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 93.88144073535756,
|
|
"signed_pairwise_delta_percent": -444.53250480136165
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 93.24983507093198,
|
|
"signed_pairwise_delta_percent": 244.41427013677844
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 49.20897259013651,
|
|
"signed_pairwise_delta_percent": 196.79655252285477
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 49.34236949428588,
|
|
"signed_pairwise_delta_percent": 90.24518667005819
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 44.600953337070045,
|
|
"signed_pairwise_delta_percent": -435.31434147282096
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 44.846370322374355,
|
|
"signed_pairwise_delta_percent": 264.74267837874817
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.23892035198680692,
|
|
"signed_pairwise_delta_percent": 222.6826964138318
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.06833719734037036,
|
|
"signed_pairwise_delta_percent": 142.72331929981317
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 470.42622660647976,
|
|
"signed_pairwise_delta_percent": -185.91602808451228
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0006836055252286144,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp4_mns32",
|
|
"tp4_mns16"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.5620663582204115,
|
|
"sim_winner": "tp2_mns64",
|
|
"sim_winner_real_relative_loss": 0.5620663582204115,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-pd/comparison.json",
|
|
"q30-fixed-hi/fixed-pd/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -435.31434147282096,
|
|
"winner_pair_m_s_percent": 44.600953337070045,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"config_uniform_scale": 6.313105858122109,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 1.1809570890904728,
|
|
"log_ratio_spread_real_top3": 0.9568100689791217,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 30.94382797908889,
|
|
"signed_pairwise_delta_percent": 73.27671364429281
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 29.57530277436313,
|
|
"signed_pairwise_delta_percent": 95.68100689791214
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 31.449801879589334,
|
|
"signed_pairwise_delta_percent": 102.52772025706145
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 31.054240651015142,
|
|
"signed_pairwise_delta_percent": 43.97849942435047
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 54.259465439535774,
|
|
"signed_pairwise_delta_percent": 6.807202298146797
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 56.81232707946078,
|
|
"signed_pairwise_delta_percent": 22.71082561174641
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 38.68791163842329,
|
|
"signed_pairwise_delta_percent": 59.286451340060744
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 36.9319004785833,
|
|
"signed_pairwise_delta_percent": -2.2053184202783296
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.66973031153608,
|
|
"signed_pairwise_delta_percent": -15.567988651985809
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 36.885145529252576,
|
|
"signed_pairwise_delta_percent": 4.4100686169577905
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.88435252075483,
|
|
"signed_pairwise_delta_percent": 22.735748881633384
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.34413815100360146,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp1_mns32",
|
|
"tp1_mns16"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns8",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-pd/comparison.json",
|
|
"q30-fixed-hi/fixed-pd/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"config_uniform_scale": 5.885039491382665,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 1.2232223312082269,
|
|
"log_ratio_spread_real_top3": 0.9511712499899991,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 32.76189094601604,
|
|
"signed_pairwise_delta_percent": 71.90461644842226
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 31.623627006928796,
|
|
"signed_pairwise_delta_percent": 95.11712499899991
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 33.72600206831461,
|
|
"signed_pairwise_delta_percent": 103.22068421989188
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 32.79701086217899,
|
|
"signed_pairwise_delta_percent": 42.366270249175436
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 70.38082644168276,
|
|
"signed_pairwise_delta_percent": -8.955970516126776
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 70.213016980584,
|
|
"signed_pairwise_delta_percent": 10.564547864752338
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.23602295916777,
|
|
"signed_pairwise_delta_percent": 60.90742373262441
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 40.39726148279639,
|
|
"signed_pairwise_delta_percent": -5.603938757169219
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.49621341698774,
|
|
"signed_pairwise_delta_percent": -19.101548900930826
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 40.815568423340856,
|
|
"signed_pairwise_delta_percent": 1.6982357807242388
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.453178330956014,
|
|
"signed_pairwise_delta_percent": 23.186909212748937
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.3719543690892244,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp1_mns32",
|
|
"tp1_mns16"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns8",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-pd/comparison.json",
|
|
"q30-fixed-hi/fixed-pd/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"config_uniform_scale": 18.86746837736289,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 2.2653893343616147,
|
|
"log_ratio_spread_real_top3": 1.1146207125801464,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 2,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.276181452945976,
|
|
"signed_pairwise_delta_percent": 84.93667289106678
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.7465468803864743,
|
|
"signed_pairwise_delta_percent": -39.82153729011294
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.3041295093334115,
|
|
"signed_pairwise_delta_percent": 143.94402186459692
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 19.530214414414008,
|
|
"signed_pairwise_delta_percent": 41.253386419289285
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 21.947794000268598,
|
|
"signed_pairwise_delta_percent": -34.6922795296003
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 4.610302806179068,
|
|
"signed_pairwise_delta_percent": -78.71974507782036
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.974254325999045,
|
|
"signed_pairwise_delta_percent": 113.33759030038964
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.5202495868550074,
|
|
"signed_pairwise_delta_percent": 33.73219440118163
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 0.7244237698333954,
|
|
"signed_pairwise_delta_percent": -26.525398366947872
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.6862244572033922,
|
|
"signed_pairwise_delta_percent": -82.59491157156452
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 113.12888745520367,
|
|
"signed_pairwise_delta_percent": -22.515920958436265
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.007270540664867875,
|
|
"real_top3_configs": [
|
|
"tp1_mns32",
|
|
"tp4_mns32",
|
|
"tp1_mns16"
|
|
],
|
|
"real_winner": "tp1_mns32",
|
|
"regret": 0.017005214686575565,
|
|
"sim_winner": "tp4_mns64",
|
|
"sim_winner_real_relative_loss": 0.017005214686575565,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-pd/comparison.json",
|
|
"q30-fixed-hi/fixed-pd/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -82.59491157156452,
|
|
"winner_pair_m_s_percent": 1.6862244572033922,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"config_uniform_scale": 24.706368987570023,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 2.74200205248491,
|
|
"log_ratio_spread_real_top3": 1.7249105689293742,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 2,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.058967777283412,
|
|
"signed_pairwise_delta_percent": 110.6461541912706
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.925759576793605,
|
|
"signed_pairwise_delta_percent": -63.428490447993504
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.0044647579152355,
|
|
"signed_pairwise_delta_percent": 172.49105689293737
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 33.27033924347824,
|
|
"signed_pairwise_delta_percent": 54.397757467940764
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 33.10740332058363,
|
|
"signed_pairwise_delta_percent": -32.26152641791503
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.3478186788827395,
|
|
"signed_pairwise_delta_percent": -101.70914835555358
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 4.530104156426464,
|
|
"signed_pairwise_delta_percent": 140.29978254043905
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 3.5336243843482795,
|
|
"signed_pairwise_delta_percent": 59.4958948992397
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 2.849041083001832,
|
|
"signed_pairwise_delta_percent": -8.454232788907941
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 3.5041851262986916,
|
|
"signed_pairwise_delta_percent": -92.1011626014233
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 156.6309394860069,
|
|
"signed_pairwise_delta_percent": -37.29992401503031
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.010095264385685976,
|
|
"real_top3_configs": [
|
|
"tp1_mns32",
|
|
"tp1_mns8",
|
|
"tp1_mns16"
|
|
],
|
|
"real_winner": "tp1_mns32",
|
|
"regret": 0.05493398001893279,
|
|
"sim_winner": "tp2_mns64",
|
|
"sim_winner_real_relative_loss": 0.05493398001893279,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-pd/comparison.json",
|
|
"q30-fixed-hi/fixed-pd/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -101.70914835555358,
|
|
"winner_pair_m_s_percent": 5.3478186788827395,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"config_uniform_scale": 0.964228039140724,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.2748488109291796,
|
|
"log_ratio_spread_real_top3": 0.0031834782899782255,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.05074090634112,
|
|
"signed_pairwise_delta_percent": -27.19082062892747
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.04075067896244,
|
|
"signed_pairwise_delta_percent": -27.180830401548782
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.23941070924307,
|
|
"signed_pairwise_delta_percent": -27.379490431829424
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.34480137033162,
|
|
"signed_pairwise_delta_percent": -27.484881092917966
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.753490275609735,
|
|
"signed_pairwise_delta_percent": -20.164334456356727
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.69449732837048,
|
|
"signed_pairwise_delta_percent": -20.10534150911747
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 50.79372465396658,
|
|
"signed_pairwise_delta_percent": -19.204568834713566
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 49.239894564408196,
|
|
"signed_pairwise_delta_percent": -17.650738745155188
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3457059423188482,
|
|
"signed_pairwise_delta_percent": -0.3457059423188482
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.11778366602921966,
|
|
"signed_pairwise_delta_percent": -0.11778366602921966
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3183478289978128,
|
|
"signed_pairwise_delta_percent": -0.3183478289978128
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0011785305823068803,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns64",
|
|
"tp4_mns8"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.0034630419451076033,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.0034630419451076033,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-po/comparison.json",
|
|
"q30-fixed-hi/fixed-po/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.3457059423188482,
|
|
"winner_pair_m_s_percent": 0.3457059423188482,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"config_uniform_scale": 0.9564015690834595,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.26887996664870883,
|
|
"log_ratio_spread_real_top3": 0.00304657671462559,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.62312189015677,
|
|
"signed_pairwise_delta_percent": -26.763201612741206
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.27398231390275,
|
|
"signed_pairwise_delta_percent": -26.414062036487184
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.66288725500065,
|
|
"signed_pairwise_delta_percent": -26.802966977585097
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.74791694228644,
|
|
"signed_pairwise_delta_percent": -26.887996664870883
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.29294809851041,
|
|
"signed_pairwise_delta_percent": -19.703792279257375
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.1818062042697,
|
|
"signed_pairwise_delta_percent": -19.59265038501667
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 50.60491466017445,
|
|
"signed_pairwise_delta_percent": -19.01575884092142
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 48.82452174557843,
|
|
"signed_pairwise_delta_percent": -17.235365926325404
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.5252213726458267,
|
|
"signed_pairwise_delta_percent": -0.5252213726458267
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3046576714625563,
|
|
"signed_pairwise_delta_percent": -0.3046576714625563
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0066591234348748005,
|
|
"signed_pairwise_delta_percent": -0.0066591234348748005
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 6.659345159421015e-05,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns8",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.005266030780421316,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.005266030780421316,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-po/comparison.json",
|
|
"q30-fixed-hi/fixed-po/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.5252213726458267,
|
|
"winner_pair_m_s_percent": 0.5252213726458267,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"config_uniform_scale": 0.9639779985207279,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.2746145677630782,
|
|
"log_ratio_spread_real_top3": 0.0031816282464492696,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.02634481547024,
|
|
"signed_pairwise_delta_percent": -27.16642453805659
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.01671464824831,
|
|
"signed_pairwise_delta_percent": -27.156794370834657
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.21675100585334,
|
|
"signed_pairwise_delta_percent": -27.356830728439697
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.32137705372146,
|
|
"signed_pairwise_delta_percent": -27.461456776307813
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.73931792837375,
|
|
"signed_pairwise_delta_percent": -20.15016210912074
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.679741891211606,
|
|
"signed_pairwise_delta_percent": -20.090586071958594
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 50.77867236420423,
|
|
"signed_pairwise_delta_percent": -19.189516544951225
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 49.225955262342325,
|
|
"signed_pairwise_delta_percent": -17.636799443089313
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3447193573234837,
|
|
"signed_pairwise_delta_percent": -0.3447193573234837
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.11768182780294648,
|
|
"signed_pairwise_delta_percent": -0.11768182780294648
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.31816282464494144,
|
|
"signed_pairwise_delta_percent": -0.31816282464494144
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0011775110003686695,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns64",
|
|
"tp4_mns8"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.0034531419781376194,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.0034531419781376194,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-po/comparison.json",
|
|
"q30-fixed-hi/fixed-po/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.3447193573234837,
|
|
"winner_pair_m_s_percent": 0.3447193573234837,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"config_uniform_scale": 0.9561532784377857,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.26866713457177865,
|
|
"log_ratio_spread_real_top3": 0.003023225018183584,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.59790659227713,
|
|
"signed_pairwise_delta_percent": -26.737986314861562
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.2500528083333,
|
|
"signed_pairwise_delta_percent": -26.390132530917743
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.63890275260015,
|
|
"signed_pairwise_delta_percent": -26.778982475184588
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 100.72663373459343,
|
|
"signed_pairwise_delta_percent": -26.86671345717787
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.27776494486162,
|
|
"signed_pairwise_delta_percent": -19.688609125608597
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.16674542969726,
|
|
"signed_pairwise_delta_percent": -19.577589610444228
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 50.5883392985462,
|
|
"signed_pairwise_delta_percent": -18.999183479293176
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 48.81412540227583,
|
|
"signed_pairwise_delta_percent": -17.224969583022798
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.5263180408757429,
|
|
"signed_pairwise_delta_percent": -0.5263180408757429
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3023225018183476,
|
|
"signed_pairwise_delta_percent": -0.3023225018183476
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.005990797056166493,
|
|
"signed_pairwise_delta_percent": -0.005990797056166493
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 5.990976507996848e-05,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns8",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.0052770552740579735,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.0052770552740579735,
|
|
"sources": [
|
|
"q30-fixed-hi/fixed-po/comparison.json",
|
|
"q30-fixed-hi/fixed-po/real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.5263180408757429,
|
|
"winner_pair_m_s_percent": 0.5263180408757429,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"config_uniform_scale": 1.368098672630302,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.4377687692283213,
|
|
"log_ratio_spread_real_top3": 0.015340372114717726,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 87.54418992315985,
|
|
"signed_pairwise_delta_percent": -43.361639510708514
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 87.65226356157126,
|
|
"signed_pairwise_delta_percent": -43.46971314911994
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 87.95942733528346,
|
|
"signed_pairwise_delta_percent": -43.77687692283213
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 87.90596709065582,
|
|
"signed_pairwise_delta_percent": -43.723416678204494
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 44.441336244863145,
|
|
"signed_pairwise_delta_percent": -22.421306800693213
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 42.65776119983087,
|
|
"signed_pairwise_delta_percent": -20.63773175566094
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.25101894269539,
|
|
"signed_pairwise_delta_percent": -19.230989498525457
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 40.74382542815109,
|
|
"signed_pairwise_delta_percent": -18.72379598398116
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.5340372114717864,
|
|
"signed_pairwise_delta_percent": -1.5340372114717864
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.3397123950439613,
|
|
"signed_pairwise_delta_percent": -1.3397123950439613
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.2598323976525228,
|
|
"signed_pairwise_delta_percent": -2.2598323976525228
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.013487267520547963,
|
|
"real_top3_configs": [
|
|
"tp4_mns16",
|
|
"tp4_mns64",
|
|
"tp4_mns32"
|
|
],
|
|
"real_winner": "tp4_mns16",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-pd-comparison.json",
|
|
"q30-expansion-lo/fixed-pd-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"config_uniform_scale": 1.4015522030046448,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.48740148963773433,
|
|
"log_ratio_spread_real_top3": 0.028938391079618264,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 83.24380892439706,
|
|
"signed_pairwise_delta_percent": -48.74014896377343
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 83.1240122706535,
|
|
"signed_pairwise_delta_percent": -48.620352310029865
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 83.23797700238447,
|
|
"signed_pairwise_delta_percent": -48.73431704176084
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 83.20129545481906,
|
|
"signed_pairwise_delta_percent": -48.69763549419543
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.904473317043724,
|
|
"signed_pairwise_delta_percent": -23.31237555807283
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 39.314739780466134,
|
|
"signed_pairwise_delta_percent": -20.722642021495243
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 38.33192154778196,
|
|
"signed_pairwise_delta_percent": -19.73982378881107
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 36.994448671903356,
|
|
"signed_pairwise_delta_percent": -18.402350912932466
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 3.2108167899433555,
|
|
"signed_pairwise_delta_percent": -3.2108167899433555
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.7450653265328064,
|
|
"signed_pairwise_delta_percent": -1.7450653265328064
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.8938391079618246,
|
|
"signed_pairwise_delta_percent": -2.8938391079618246
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.01760380548676732,
|
|
"real_top3_configs": [
|
|
"tp4_mns16",
|
|
"tp4_mns64",
|
|
"tp4_mns8"
|
|
],
|
|
"real_winner": "tp4_mns16",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-pd-comparison.json",
|
|
"q30-expansion-lo/fixed-pd-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"config_uniform_scale": 4.951803445364717,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.5265893738689182,
|
|
"log_ratio_spread_real_top3": 0.0023698681934412047,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 6,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 13.069596953822204,
|
|
"signed_pairwise_delta_percent": -52.50428401638987
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 13.062412988745601,
|
|
"signed_pairwise_delta_percent": -52.49710005131326
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.912704892995365,
|
|
"signed_pairwise_delta_percent": -52.347391955563026
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 13.224250324324174,
|
|
"signed_pairwise_delta_percent": -52.658937386891836
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 7.931511304661838,
|
|
"signed_pairwise_delta_percent": -38.89275046110161
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.285385999289323,
|
|
"signed_pairwise_delta_percent": -40.2466251557291
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.543865055930773,
|
|
"signed_pairwise_delta_percent": -40.50510421237055
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.455313972221232,
|
|
"signed_pairwise_delta_percent": -40.416553128661
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.7798181655007458,
|
|
"signed_pairwise_delta_percent": -0.7798181655007458
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.23698681934411808,
|
|
"signed_pairwise_delta_percent": -0.23698681934411808
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.039348303682490725,
|
|
"signed_pairwise_delta_percent": -0.039348303682490725
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0003935604614297983,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns8",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.13962124906187534,
|
|
"sim_winner": "tp1_mns16",
|
|
"sim_winner_real_relative_loss": 0.13962124906187534,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-pd-comparison.json",
|
|
"q30-expansion-lo/fixed-pd-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -52.50428401638987,
|
|
"winner_pair_m_s_percent": 13.069596953822204,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"config_uniform_scale": 4.949511590379519,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.5320917895477677,
|
|
"log_ratio_spread_real_top3": 0.0027149664654271266,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 6,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.537285946185337,
|
|
"signed_pairwise_delta_percent": -52.8208410178207
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.710829179768771,
|
|
"signed_pairwise_delta_percent": -52.994384251404135
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.566252787665844,
|
|
"signed_pairwise_delta_percent": -52.8498078593012
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.925623883141398,
|
|
"signed_pairwise_delta_percent": -53.20917895477676
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.207508741892353,
|
|
"signed_pairwise_delta_percent": -40.867096818654424
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.12717383854323,
|
|
"signed_pairwise_delta_percent": -40.7867619153053
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.376587887539571,
|
|
"signed_pairwise_delta_percent": -41.03617596430164
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 9.357480041752579,
|
|
"signed_pairwise_delta_percent": -41.01706811851465
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.0112043752863755,
|
|
"signed_pairwise_delta_percent": -2.0112043752863755
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2245770721418584,
|
|
"signed_pairwise_delta_percent": -0.2245770721418584
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.27149664654272265,
|
|
"signed_pairwise_delta_percent": -0.27149664654272265
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002248294353297764,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.1335710369665437,
|
|
"sim_winner": "tp1_mns16",
|
|
"sim_winner_real_relative_loss": 0.1335710369665437,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-pd-comparison.json",
|
|
"q30-expansion-lo/fixed-pd-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -52.8208410178207,
|
|
"winner_pair_m_s_percent": 12.537285946185337,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"config_uniform_scale": 4.4225746146806655,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.5980564765480212,
|
|
"log_ratio_spread_real_top3": 0.0019192061388726422,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 6,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 23.840206161907986,
|
|
"signed_pairwise_delta_percent": -59.60969731665489
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 23.856001111110555,
|
|
"signed_pairwise_delta_percent": -59.625492265857446
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 23.79706751418482,
|
|
"signed_pairwise_delta_percent": -59.56655866893173
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 24.036156500055224,
|
|
"signed_pairwise_delta_percent": -59.805647654802115
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.27084643218256,
|
|
"signed_pairwise_delta_percent": -41.25705263893593
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 13.179118203297904,
|
|
"signed_pairwise_delta_percent": -42.16532441005128
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 13.206003717586201,
|
|
"signed_pairwise_delta_percent": -42.19220992433957
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 13.059567610922084,
|
|
"signed_pairwise_delta_percent": -42.04577381767545
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.5379386855583932,
|
|
"signed_pairwise_delta_percent": -0.5379386855583932
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.19192061388725232,
|
|
"signed_pairwise_delta_percent": -0.19192061388725232
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.11294449835385692,
|
|
"signed_pairwise_delta_percent": -0.11294449835385692
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0011300830467204648,
|
|
"real_top3_configs": [
|
|
"tp4_mns32",
|
|
"tp4_mns8",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns32",
|
|
"regret": 0.2692193946565338,
|
|
"sim_winner": "tp1_mns16",
|
|
"sim_winner_real_relative_loss": 0.2692193946565338,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-pd-comparison.json",
|
|
"q30-expansion-lo/fixed-pd-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -59.60969731665489,
|
|
"winner_pair_m_s_percent": 23.840206161907986,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"config_uniform_scale": 4.421436305428993,
|
|
"expected_config_count": 12,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.5995516935141478,
|
|
"log_ratio_spread_real_top3": 0.0016899142191908467,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 6,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 2,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 22.897040620669692,
|
|
"signed_pairwise_delta_percent": -59.778209091748444
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 23.046064773358367,
|
|
"signed_pairwise_delta_percent": -59.92723324443712
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 22.841076346337463,
|
|
"signed_pairwise_delta_percent": -59.72224481741621
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 23.074000880336015,
|
|
"signed_pairwise_delta_percent": -59.95516935141476
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.569832422090965,
|
|
"signed_pairwise_delta_percent": -42.327734313472305
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.566770407129498,
|
|
"signed_pairwise_delta_percent": -42.32467229851083
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.592012573206555,
|
|
"signed_pairwise_delta_percent": -42.34991446458789
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 12.476154151010087,
|
|
"signed_pairwise_delta_percent": -42.23405604239143
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.7798712357940356,
|
|
"signed_pairwise_delta_percent": -0.7798712357940356
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.16899142191907698,
|
|
"signed_pairwise_delta_percent": -0.16899142191907698
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.11976959438291016,
|
|
"signed_pairwise_delta_percent": -0.11976959438291016
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0011984134680460556,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns64",
|
|
"tp4_mns32"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.2573048300400054,
|
|
"sim_winner": "tp1_mns16",
|
|
"sim_winner_real_relative_loss": 0.2573048300400054,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-pd-comparison.json",
|
|
"q30-expansion-lo/fixed-pd-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -59.778209091748444,
|
|
"winner_pair_m_s_percent": 22.897040620669692,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"config_uniform_scale": 1.2178864428064151,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.20880946208450285,
|
|
"log_ratio_spread_real_top3": 0.01086052419322281,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 85.73029954680493,
|
|
"signed_pairwise_delta_percent": -20.172637981772546
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 85.68328241018153,
|
|
"signed_pairwise_delta_percent": -20.12562084514915
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 86.43860777348267,
|
|
"signed_pairwise_delta_percent": -20.88094620845029
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 86.17619129616834,
|
|
"signed_pairwise_delta_percent": -20.618529731135947
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.62103939690888,
|
|
"signed_pairwise_delta_percent": -10.126432818195235
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.189828478080884,
|
|
"signed_pairwise_delta_percent": -9.69522189936724
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 38.78141976101907,
|
|
"signed_pairwise_delta_percent": -7.286813182305435
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 39.70095967038979,
|
|
"signed_pairwise_delta_percent": -8.20635309167615
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.4830480235035413,
|
|
"signed_pairwise_delta_percent": -1.4830480235035413
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3264759887775993,
|
|
"signed_pairwise_delta_percent": -0.3264759887775993
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.0860524193222814,
|
|
"signed_pairwise_delta_percent": -1.0860524193222814
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.003270095020734498,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.014940997472395745,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.014940997472395745,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-po-comparison.json",
|
|
"q30-expansion-lo/fixed-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -1.4830480235035413,
|
|
"winner_pair_m_s_percent": 1.4830480235035413,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"config_uniform_scale": 1.1837027909165883,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.16051303629507518,
|
|
"log_ratio_spread_real_top3": 0.02273410040317836,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 80.70853515712606,
|
|
"signed_pairwise_delta_percent": -15.150873592066905
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 80.8156741203624,
|
|
"signed_pairwise_delta_percent": -15.258012555303235
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 81.60896519456668,
|
|
"signed_pairwise_delta_percent": -16.05130362950752
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 81.12861014030412,
|
|
"signed_pairwise_delta_percent": -15.570948575244959
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.49241217671676,
|
|
"signed_pairwise_delta_percent": -5.997805597996803
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.08952422311525,
|
|
"signed_pairwise_delta_percent": -5.5949176443952915
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 34.46421799160749,
|
|
"signed_pairwise_delta_percent": -2.9696114128875353
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 36.11912088646263,
|
|
"signed_pairwise_delta_percent": -4.624514307742672
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 3.527877494419683,
|
|
"signed_pairwise_delta_percent": -3.527877494419683
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.273410040317854,
|
|
"signed_pairwise_delta_percent": -2.273410040317854
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.6470456517583268,
|
|
"signed_pairwise_delta_percent": -0.6470456517583268
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0064914351440450435,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns64",
|
|
"tp4_mns32"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.035908453871728696,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.035908453871728696,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-po-comparison.json",
|
|
"q30-expansion-lo/fixed-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -3.527877494419683,
|
|
"winner_pair_m_s_percent": 3.527877494419683,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"config_uniform_scale": 1.217293846920945,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.20839845449892702,
|
|
"log_ratio_spread_real_top3": 0.010876152767897662,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 85.68903876109678,
|
|
"signed_pairwise_delta_percent": -20.131377196064392
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 85.64280282131507,
|
|
"signed_pairwise_delta_percent": -20.08514125628268
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 86.3975070149251,
|
|
"signed_pairwise_delta_percent": -20.839845449892714
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 86.1361690595838,
|
|
"signed_pairwise_delta_percent": -20.578507494551413
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.599166472131735,
|
|
"signed_pairwise_delta_percent": -10.104559893418097
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 41.167571965772595,
|
|
"signed_pairwise_delta_percent": -9.672965387058952
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 38.758357485895,
|
|
"signed_pairwise_delta_percent": -7.263750907181354
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 39.67737178487652,
|
|
"signed_pairwise_delta_percent": -8.182765206162879
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.4785789280404857,
|
|
"signed_pairwise_delta_percent": -1.4785789280404857
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3281651404981352,
|
|
"signed_pairwise_delta_percent": -0.3281651404981352
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.0876152767897598,
|
|
"signed_pairwise_delta_percent": -1.0876152767897598
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.003287041917935163,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.014895639803871319,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.014895639803871319,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-po-comparison.json",
|
|
"q30-expansion-lo/fixed-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -1.4785789280404857,
|
|
"winner_pair_m_s_percent": 1.4785789280404857,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"config_uniform_scale": 1.1830544739129631,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.1602197871525023,
|
|
"log_ratio_spread_real_top3": 0.023698379402236264,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 80.67769169969341,
|
|
"signed_pairwise_delta_percent": -15.120030134634243
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 80.7830378068094,
|
|
"signed_pairwise_delta_percent": -15.225376241750244
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 81.5796402803094,
|
|
"signed_pairwise_delta_percent": -16.021978715250242
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 81.09692218428181,
|
|
"signed_pairwise_delta_percent": -15.539260619222638
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.48755808141785,
|
|
"signed_pairwise_delta_percent": -5.992951502697896
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 37.073974944994774,
|
|
"signed_pairwise_delta_percent": -5.579368366274817
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 34.45716604214575,
|
|
"signed_pairwise_delta_percent": -2.96255946342579
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 36.112698620380655,
|
|
"signed_pairwise_delta_percent": -4.618092041660699
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 3.527334206322636,
|
|
"signed_pairwise_delta_percent": -3.527334206322636
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.369837940223627,
|
|
"signed_pairwise_delta_percent": -2.369837940223627
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.6581754137676994,
|
|
"signed_pairwise_delta_percent": -0.6581754137676994
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.006603461479437911,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns64",
|
|
"tp4_mns32"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.03590282591969052,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.03590282591969052,
|
|
"sources": [
|
|
"q30-expansion-lo/fixed-po-comparison.json",
|
|
"q30-expansion-lo/fixed-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -3.527334206322636,
|
|
"winner_pair_m_s_percent": 3.527334206322636,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"config_uniform_scale": 0.6570740275710648,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.18223262538030371,
|
|
"log_ratio_spread_real_top3": 0.02603063729306998,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 3,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.72747001936281,
|
|
"signed_pairwise_delta_percent": -17.953572454869516
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.92536482192115,
|
|
"signed_pairwise_delta_percent": -18.151244664387878
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.84293846190077,
|
|
"signed_pairwise_delta_percent": -18.06852174527367
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.99716010252367,
|
|
"signed_pairwise_delta_percent": -18.223262538030372
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 46.24106247791962,
|
|
"signed_pairwise_delta_percent": -15.983259499872476
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 46.02969238739204,
|
|
"signed_pairwise_delta_percent": -15.772028622022393
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 43.35154374968354,
|
|
"signed_pairwise_delta_percent": -13.093873141924501
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 44.35514908862859,
|
|
"signed_pairwise_delta_percent": -14.09734611058145
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 3.137475730979236,
|
|
"signed_pairwise_delta_percent": -3.138149298412063
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.700370444058652,
|
|
"signed_pairwise_delta_percent": -1.7008263369350451
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 2.6023901618741747,
|
|
"signed_pairwise_delta_percent": -2.6030637293070016
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.01714909028663425,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp4_mns32",
|
|
"tp4_mns8"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.03187213306057868,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.03187213306057868,
|
|
"sources": [
|
|
"q30-expansion-lo/trace-po-comparison.json",
|
|
"q30-expansion-lo/trace-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -3.138149298412063,
|
|
"winner_pair_m_s_percent": 3.137475730979236,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"config_uniform_scale": 0.4780578821676308,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.17717267764722577,
|
|
"log_ratio_spread_real_top3": 0.0019733480856019003,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 102.02546049460537,
|
|
"signed_pairwise_delta_percent": -17.02774649052323
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 102.51013137549631,
|
|
"signed_pairwise_delta_percent": -17.51241737141418
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 102.71498176880472,
|
|
"signed_pairwise_delta_percent": -17.71726776472258
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 102.02298163614279,
|
|
"signed_pairwise_delta_percent": -17.025267632060647
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 53.38866226862059,
|
|
"signed_pairwise_delta_percent": -16.77030936999181
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 53.072230658401054,
|
|
"signed_pairwise_delta_percent": -16.453877759772272
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.17810724030649,
|
|
"signed_pairwise_delta_percent": -14.55975434167771
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.19432721693139,
|
|
"signed_pairwise_delta_percent": -14.57597431830261
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.37707949175865374,
|
|
"signed_pairwise_delta_percent": -0.37707949175865374
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.1973348085601952,
|
|
"signed_pairwise_delta_percent": -0.1973348085601952
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.19364778265184052,
|
|
"signed_pairwise_delta_percent": -0.19364778265184052
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0019383540105721853,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns64",
|
|
"tp4_mns32"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.0037779133092612494,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.0037779133092612494,
|
|
"sources": [
|
|
"q30-expansion-lo/trace-po-comparison.json",
|
|
"q30-expansion-lo/trace-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.37707949175865374,
|
|
"winner_pair_m_s_percent": 0.37707949175865374,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"config_uniform_scale": 0.6569411821071235,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.18204547796659498,
|
|
"log_ratio_spread_real_top3": 0.0260064332624374,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 3,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.70924994261264,
|
|
"signed_pairwise_delta_percent": -17.93535237811935
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.90683362884803,
|
|
"signed_pairwise_delta_percent": -18.132713471314766
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.82486083494068,
|
|
"signed_pairwise_delta_percent": -18.050444118313578
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 90.97844536115277,
|
|
"signed_pairwise_delta_percent": -18.204547796659476
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 46.2297703769945,
|
|
"signed_pairwise_delta_percent": -15.971967398947356
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 46.01838840420611,
|
|
"signed_pairwise_delta_percent": -15.760724638836459
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 43.33991907299428,
|
|
"signed_pairwise_delta_percent": -13.08224846523523
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 44.344292942369236,
|
|
"signed_pairwise_delta_percent": -14.08648996432209
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 3.1380840852501946,
|
|
"signed_pairwise_delta_percent": -3.138757652683021
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.6981488657060708,
|
|
"signed_pairwise_delta_percent": -1.698604758582464
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns8",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 2.599969758810901,
|
|
"signed_pairwise_delta_percent": -2.600643326243728
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.017126493773631513,
|
|
"real_top3_configs": [
|
|
"tp4_mns64",
|
|
"tp4_mns32",
|
|
"tp4_mns8"
|
|
],
|
|
"real_winner": "tp4_mns64",
|
|
"regret": 0.03187841051786555,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.03187841051786555,
|
|
"sources": [
|
|
"q30-expansion-lo/trace-po-comparison.json",
|
|
"q30-expansion-lo/trace-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -3.138757652683021,
|
|
"winner_pair_m_s_percent": 3.1380840852501946,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"config_uniform_scale": 0.4780257567661343,
|
|
"expected_config_count": 12,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.17668938090174957,
|
|
"log_ratio_spread_real_top3": 0.001510603131868793,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 12,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.97733324187712,
|
|
"signed_pairwise_delta_percent": -16.979619237794985
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 102.4618944001636,
|
|
"signed_pairwise_delta_percent": -17.464180396081463
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 102.66665209425709,
|
|
"signed_pairwise_delta_percent": -17.668938090174958
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp1_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 101.97521651914867,
|
|
"signed_pairwise_delta_percent": -16.977502515066533
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 53.34185487949904,
|
|
"signed_pairwise_delta_percent": -16.72350198087026
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 53.025317262729466,
|
|
"signed_pairwise_delta_percent": -16.406964364100684
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp2_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.13076854333851,
|
|
"signed_pairwise_delta_percent": -14.51241564470973
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp2_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 51.14883628784737,
|
|
"signed_pairwise_delta_percent": -14.530483389218585
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns16",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.33103325097428193,
|
|
"signed_pairwise_delta_percent": -0.33103325097428193
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns32",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.1510603131868839,
|
|
"signed_pairwise_delta_percent": -0.1510603131868839
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.1471039901586335,
|
|
"signed_pairwise_delta_percent": -0.1471039901586335
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_mns8",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0014721224115223919,
|
|
"real_top3_configs": [
|
|
"tp4_mns8",
|
|
"tp4_mns64",
|
|
"tp4_mns32"
|
|
],
|
|
"real_winner": "tp4_mns8",
|
|
"regret": 0.003315817711349034,
|
|
"sim_winner": "tp4_mns16",
|
|
"sim_winner_real_relative_loss": 0.003315817711349034,
|
|
"sources": [
|
|
"q30-expansion-lo/trace-po-comparison.json",
|
|
"q30-expansion-lo/trace-po-real-audit.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.33103325097428193,
|
|
"winner_pair_m_s_percent": 0.33103325097428193,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"config_uniform_scale": 0.9905469229359986,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.060652013241479515,
|
|
"log_ratio_spread_real_top3": 0.05042015844107084,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 4.149707946615582,
|
|
"signed_pairwise_delta_percent": -5.042015844107099
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.172893426656431,
|
|
"signed_pairwise_delta_percent": -6.065201324147949
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3735522251855693,
|
|
"signed_pairwise_delta_percent": -0.3735522251855693
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0037425080108803854,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.04237011756982145,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.04237011756982145,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -5.042015844107099,
|
|
"winner_pair_m_s_percent": 4.149707946615582,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"config_uniform_scale": 1.0202314731471502,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.02402007999917405,
|
|
"log_ratio_spread_real_top3": 0.02402007999917405,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.0837798588092087,
|
|
"signed_pairwise_delta_percent": 2.1933999669826525
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.325627595184895,
|
|
"signed_pairwise_delta_percent": 1.9515522306069657
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.20860803293475833,
|
|
"signed_pairwise_delta_percent": -0.20860803293475833
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002088257708717345,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.002088257708717345,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.002088257708717345,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.20860803293475833,
|
|
"winner_pair_m_s_percent": 0.20860803293475833,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"config_uniform_scale": 3.0245083955601917,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.6396663007689712,
|
|
"log_ratio_spread_real_top3": 0.6391084715731847,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.6769338761256622,
|
|
"signed_pairwise_delta_percent": -0.6769338761256622
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 28.537073298091865,
|
|
"signed_pairwise_delta_percent": -63.96663007689711
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 28.481290378513215,
|
|
"signed_pairwise_delta_percent": -63.910847157318464
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.006792302522137694,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.33025510615656906,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.33025510615656906,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -63.96663007689711,
|
|
"winner_pair_m_s_percent": 28.537073298091865,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"config_uniform_scale": 3.06180896406935,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.6289288207236795,
|
|
"log_ratio_spread_real_top3": 0.628447602146478,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2660356852052706,
|
|
"signed_pairwise_delta_percent": -0.2660356852052706
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 31.648007592775677,
|
|
"signed_pairwise_delta_percent": -62.84476021464781
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 31.696129450495825,
|
|
"signed_pairwise_delta_percent": -62.89288207236796
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002663898741543047,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.37228890038064755,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.37228890038064755,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -62.84476021464781,
|
|
"winner_pair_m_s_percent": 31.648007592775677,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"config_uniform_scale": 2.912666802729827,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.6150853261890787,
|
|
"log_ratio_spread_real_top3": 0.6147276719238838,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.6988689401937037,
|
|
"signed_pairwise_delta_percent": -0.6988689401937037
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 26.754499952423572,
|
|
"signed_pairwise_delta_percent": -61.50853261890787
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 26.718734525904075,
|
|
"signed_pairwise_delta_percent": -61.47276719238837
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.007013167281255672,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.3067524317507717,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.3067524317507717,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -61.50853261890787,
|
|
"winner_pair_m_s_percent": 26.754499952423572,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"config_uniform_scale": 2.9547444028283523,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.6045691264386757,
|
|
"log_ratio_spread_real_top3": 0.6036185233190644,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2938181405538578,
|
|
"signed_pairwise_delta_percent": -0.2938181405538578
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 29.686793800126072,
|
|
"signed_pairwise_delta_percent": -60.36185233190644
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 29.781854112087196,
|
|
"signed_pairwise_delta_percent": -60.45691264386757
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002942502091140886,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.3456375801249614,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.3456375801249614,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -60.36185233190644,
|
|
"winner_pair_m_s_percent": 29.686793800126072,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"config_uniform_scale": 0.9643054515322584,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.1213620172981578,
|
|
"log_ratio_spread_real_top3": 0.12020280158083751,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.708587111542836,
|
|
"signed_pairwise_delta_percent": -12.020280158083734
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.824508683274874,
|
|
"signed_pairwise_delta_percent": -12.13620172981577
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.20084412263250578,
|
|
"signed_pairwise_delta_percent": -0.20084412263250578
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002010459495370265,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.058746722254088946,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.058746722254088946,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -12.020280158083734,
|
|
"winner_pair_m_s_percent": 5.708587111542836,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"config_uniform_scale": 0.9406625738485695,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.0831623213185602,
|
|
"log_ratio_spread_real_top3": 0.07972272936918397,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.6605798903418276,
|
|
"signed_pairwise_delta_percent": -7.972272936918415
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 2.0045390852794442,
|
|
"signed_pairwise_delta_percent": -8.316232131856031
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.26132246488070326,
|
|
"signed_pairwise_delta_percent": -0.26132246488070326
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0026166420965441084,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.016744441542810407,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.016744441542810407,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -7.972272936918415,
|
|
"winner_pair_m_s_percent": 1.6605798903418276,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"config_uniform_scale": 0.9641676390687596,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.12135084408558959,
|
|
"log_ratio_spread_real_top3": 0.12019469213946486,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.707776167405595,
|
|
"signed_pairwise_delta_percent": -12.019469213946493
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.823391362018052,
|
|
"signed_pairwise_delta_percent": -12.13508440855895
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.20067628063206866,
|
|
"signed_pairwise_delta_percent": -0.20067628063206866
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0020087777023818365,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.05873813644442971,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.05873813644442971,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -12.019469213946493,
|
|
"winner_pair_m_s_percent": 5.707776167405595,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"config_uniform_scale": 0.9405407803694738,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.08316619889204555,
|
|
"log_ratio_spread_real_top3": 0.07973880510831316,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.6621874642547243,
|
|
"signed_pairwise_delta_percent": -7.973880510831312
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 2.004926842627968,
|
|
"signed_pairwise_delta_percent": -8.316619889204555
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2619399460772448,
|
|
"signed_pairwise_delta_percent": -0.2619399460772448
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0026228330848965253,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.016760786592592503,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.016760786592592503,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -7.973880510831312,
|
|
"winner_pair_m_s_percent": 1.6621874642547243,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"config_uniform_scale": 1.4928001864415554,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 1.016421300063101,
|
|
"log_ratio_spread_real_top3": 1.016421300063101,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 314.42970560873886,
|
|
"signed_pairwise_delta_percent": 98.99891938297203
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 331.9449017039404,
|
|
"signed_pairwise_delta_percent": 101.64213000631013
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 348.43412520640095,
|
|
"signed_pairwise_delta_percent": 34.81502191302237
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 22.203359079184416,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"config_uniform_scale": 1.5105914717935105,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.9648739788503272,
|
|
"log_ratio_spread_real_top3": 0.9648739788503272,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 357.1169800531145,
|
|
"signed_pairwise_delta_percent": 87.72634904159466
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 376.2523321691806,
|
|
"signed_pairwise_delta_percent": 96.48739788503273
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 392.34130165358084,
|
|
"signed_pairwise_delta_percent": 24.022421630469815
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 34.55816479168436,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"config_uniform_scale": 1.340235629202675,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.6151851510539637,
|
|
"log_ratio_spread_real_top3": 0.43053455008809527,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 23.577777935433357,
|
|
"signed_pairwise_delta_percent": 42.06672378709799
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 15.538065881069413,
|
|
"signed_pairwise_delta_percent": 43.05345500880952
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 29.89518418107898,
|
|
"signed_pairwise_delta_percent": -18.465060096586843
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.16810252500411127,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"config_uniform_scale": 1.2809934431185706,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.4940487275442631,
|
|
"log_ratio_spread_real_top3": 0.2975515038921607,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 38.93167860985212,
|
|
"signed_pairwise_delta_percent": 24.34566988904276
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 23.332498967838475,
|
|
"signed_pairwise_delta_percent": 29.755150389216045
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 42.55321100556,
|
|
"signed_pairwise_delta_percent": -19.649722365210227
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.26279180695073645,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"config_uniform_scale": 1.414672147250406,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.6925792799032979,
|
|
"log_ratio_spread_real_top3": 0.6925792799032979,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 11.295399365368032,
|
|
"signed_pairwise_delta_percent": 50.146824333458675
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 9.087763981481082,
|
|
"signed_pairwise_delta_percent": 51.16511829066085
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 0.6135354983447174,
|
|
"signed_pairwise_delta_percent": -18.092809699668955
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.006154214824712234,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.006154214824712234,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.006154214824712234,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -18.092809699668955,
|
|
"winner_pair_m_s_percent": 0.6135354983447174,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"config_uniform_scale": 1.4729688476415597,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.7152768252374071,
|
|
"log_ratio_spread_real_top3": 0.7152768252374071,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 13.943039926940672,
|
|
"signed_pairwise_delta_percent": 50.65884104036849
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 10.455222177040667,
|
|
"signed_pairwise_delta_percent": 54.81170400182358
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 6.030894994380943,
|
|
"signed_pairwise_delta_percent": -16.715978521917112
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.062164651568153984,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.062164651568153984,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.062164651568153984,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -16.715978521917112,
|
|
"winner_pair_m_s_percent": 6.030894994380943,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"config_uniform_scale": 0.753106573816306,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.36295220040710596,
|
|
"log_ratio_spread_real_top3": 0.3458080890155827,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.119345640134235,
|
|
"signed_pairwise_delta_percent": -34.58080890155828
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 6.802653056521642,
|
|
"signed_pairwise_delta_percent": -36.29522004071061
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.3657297298272386,
|
|
"signed_pairwise_delta_percent": -1.3745139177357077
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.013750984200462835,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.07039370625209918,
|
|
"sim_winner": "tp4_ep1_mns64",
|
|
"sim_winner_real_relative_loss": 0.07039370625209918,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -36.29522004071061,
|
|
"winner_pair_m_s_percent": 6.802653056521642,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"config_uniform_scale": 0.7665080250950821,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.3179742793323852,
|
|
"log_ratio_spread_real_top3": 0.3160683400465084,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 19.209826109044702,
|
|
"signed_pairwise_delta_percent": -31.606834004650835
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 19.400420037632387,
|
|
"signed_pairwise_delta_percent": -31.79742793323852
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.26044028368209854,
|
|
"signed_pairwise_delta_percent": -0.26044028368209854
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002607797240047338,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.2117895828808376,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.2117895828808376,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -31.606834004650835,
|
|
"winner_pair_m_s_percent": 19.209826109044702,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"config_uniform_scale": 0.7530456732974741,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.3629311886734452,
|
|
"log_ratio_spread_real_top3": 0.34578727672972004,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 5.1172644115479695,
|
|
"signed_pairwise_delta_percent": -34.57872767297201
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 6.800551883155552,
|
|
"signed_pairwise_delta_percent": -36.29311886734452
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.3654615828579657,
|
|
"signed_pairwise_delta_percent": -1.3742457707664346
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0137482658615673,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.07037121566091509,
|
|
"sim_winner": "tp4_ep1_mns64",
|
|
"sim_winner_real_relative_loss": 0.07037121566091509,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -36.29311886734452,
|
|
"winner_pair_m_s_percent": 6.800551883155552,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-po",
|
|
"config_uniform_scale": 0.7664791140961692,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.3178774778616069,
|
|
"log_ratio_spread_real_top3": 0.3159715282630199,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 19.20014493069586,
|
|
"signed_pairwise_delta_percent": -31.59715282630199
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 19.390739890554553,
|
|
"signed_pairwise_delta_percent": -31.787747786160686
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2516272531701875,
|
|
"signed_pairwise_delta_percent": -0.2516272531701875
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0025194410024500513,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.21167227304868486,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"sim_winner_real_relative_loss": 0.21167227304868486,
|
|
"sources": [
|
|
"q235-fourcase-a0/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mixed",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -31.59715282630199,
|
|
"winner_pair_m_s_percent": 19.20014493069586,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"config_uniform_scale": 0.8052174107907835,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": true,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.2684736063166279,
|
|
"log_ratio_spread_real_top3": 0.2684736063166279,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 4.149707946615582,
|
|
"signed_pairwise_delta_percent": 26.47380840647722
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.172893426656431,
|
|
"signed_pairwise_delta_percent": 25.450622926436374
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.3735522251855693,
|
|
"signed_pairwise_delta_percent": -0.3735522251855693
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0037425080108803854,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"config_uniform_scale": 0.8369995512232968,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.30454826263257295,
|
|
"log_ratio_spread_real_top3": 0.30454826263257295,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.0837798588092087,
|
|
"signed_pairwise_delta_percent": 30.246218230322537
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.325627595184895,
|
|
"signed_pairwise_delta_percent": 30.004370493946848
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.20860803293475833,
|
|
"signed_pairwise_delta_percent": -0.20860803293475833
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002088257708717345,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.002088257708717345,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.002088257708717345,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.20860803293475833,
|
|
"winner_pair_m_s_percent": 0.20860803293475833,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"config_uniform_scale": 2.608078839444179,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.6005626377134611,
|
|
"log_ratio_spread_real_top3": 0.6000048085176745,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.6769338761256622,
|
|
"signed_pairwise_delta_percent": -0.6769338761256622
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 28.537073298091865,
|
|
"signed_pairwise_delta_percent": -60.056263771346096
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 28.481290378513215,
|
|
"signed_pairwise_delta_percent": -60.00048085176745
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.006792302522137694,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.33025510615656906,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.33025510615656906,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -60.056263771346096,
|
|
"winner_pair_m_s_percent": 28.537073298091865,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"config_uniform_scale": 2.586955736602833,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.6116904181892778,
|
|
"log_ratio_spread_real_top3": 0.6112091996120763,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2660356852052706,
|
|
"signed_pairwise_delta_percent": -0.2660356852052706
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 31.648007592775677,
|
|
"signed_pairwise_delta_percent": -61.12091996120764
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 31.696129450495825,
|
|
"signed_pairwise_delta_percent": -61.169041818927795
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002663898741543047,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.37228890038064755,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.37228890038064755,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -61.12091996120764,
|
|
"winner_pair_m_s_percent": 31.648007592775677,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"config_uniform_scale": 2.50822061424693,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.5825805162281401,
|
|
"log_ratio_spread_real_top3": 0.5822228619629451,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.6988689401937037,
|
|
"signed_pairwise_delta_percent": -0.6988689401937037
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 26.754499952423572,
|
|
"signed_pairwise_delta_percent": -58.25805162281401
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 26.718734525904075,
|
|
"signed_pairwise_delta_percent": -58.222286196294526
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.007013167281255672,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.3067524317507717,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.3067524317507717,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -58.25805162281401,
|
|
"winner_pair_m_s_percent": 26.754499952423572,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"config_uniform_scale": 2.494675463839052,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.5929354020035993,
|
|
"log_ratio_spread_real_top3": 0.591984798883988,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 1,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 1,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2938181405538578,
|
|
"signed_pairwise_delta_percent": -0.2938181405538578
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 29.686793800126072,
|
|
"signed_pairwise_delta_percent": -59.1984798883988
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 29.781854112087196,
|
|
"signed_pairwise_delta_percent": -59.29354020035993
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002942502091140886,
|
|
"real_top3_configs": [
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"regret": 0.3456375801249614,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.3456375801249614,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "tp-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -59.1984798883988,
|
|
"winner_pair_m_s_percent": 29.686793800126072,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"config_uniform_scale": 0.7759575081370588,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": true,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.24794491019938592,
|
|
"log_ratio_spread_real_top3": 0.24794491019938592,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.708587111542836,
|
|
"signed_pairwise_delta_percent": 24.593646897306066
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.824508683274874,
|
|
"signed_pairwise_delta_percent": 24.477725325574028
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.20084412263250578,
|
|
"signed_pairwise_delta_percent": -0.20084412263250578
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002010459495370265,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"config_uniform_scale": 0.7569325524828211,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": true,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.2890297658333315,
|
|
"log_ratio_spread_real_top3": 0.2890297658333315,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.6605798903418276,
|
|
"signed_pairwise_delta_percent": 28.641654118452443
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.0045390852794442,
|
|
"signed_pairwise_delta_percent": 28.297694923514825
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.26132246488070326,
|
|
"signed_pairwise_delta_percent": -0.26132246488070326
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0026166420965441084,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"config_uniform_scale": 0.7758466131757196,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": true,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.24795134122075396,
|
|
"log_ratio_spread_real_top3": 0.24795134122075396,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.707776167405595,
|
|
"signed_pairwise_delta_percent": 24.594457841443305
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.823391362018052,
|
|
"signed_pairwise_delta_percent": 24.47884264683085
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.20067628063206866,
|
|
"signed_pairwise_delta_percent": -0.20067628063206866
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0020087777023818365,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"config_uniform_scale": 0.7568345476811307,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": true,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.28901986490616804,
|
|
"log_ratio_spread_real_top3": 0.28901986490616804,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 1.6621874642547243,
|
|
"signed_pairwise_delta_percent": 28.640046544539548
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 2.004926842627968,
|
|
"signed_pairwise_delta_percent": 28.297307166166302
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2619399460772448,
|
|
"signed_pairwise_delta_percent": -0.2619399460772448
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0026228330848965253,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"config_uniform_scale": 1.2167509036113942,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 1.5440051366971033,
|
|
"log_ratio_spread_real_top3": 1.5440051366971033,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 314.42970560873886,
|
|
"signed_pairwise_delta_percent": 150.7153663041929
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 331.9449017039404,
|
|
"signed_pairwise_delta_percent": 154.40051366971034
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 348.43412520640095,
|
|
"signed_pairwise_delta_percent": 106.41277848941169
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 22.203359079184416,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"config_uniform_scale": 1.218869500962875,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 1.511462353051269,
|
|
"log_ratio_spread_real_top3": 1.511462353051269,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 357.1169800531145,
|
|
"signed_pairwise_delta_percent": 142.91720964148476
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 376.2523321691806,
|
|
"signed_pairwise_delta_percent": 151.14623530512694
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 392.34130165358084,
|
|
"signed_pairwise_delta_percent": 101.89467468744536
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 34.55816479168436,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"config_uniform_scale": 1.3379683648861502,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.46724919425743383,
|
|
"log_ratio_spread_real_top3": 0.28229702985939353,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 23.577777935433357,
|
|
"signed_pairwise_delta_percent": 26.996972943866705
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 15.538065881069413,
|
|
"signed_pairwise_delta_percent": 28.229702985939365
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 29.89518418107898,
|
|
"signed_pairwise_delta_percent": -18.495216439804036
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.16810252500411127,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"config_uniform_scale": 1.220753300748686,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": false,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.4532917510074562,
|
|
"log_ratio_spread_real_top3": 0.2226509791024174,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "tpot_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 38.93167860985212,
|
|
"signed_pairwise_delta_percent": 15.892799069061242
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 23.332498967838475,
|
|
"signed_pairwise_delta_percent": 22.265097910241728
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 42.55321100556,
|
|
"signed_pairwise_delta_percent": -23.064077190503873
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.26279180695073645,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"sim_winner_real_relative_loss": 0.0,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "none",
|
|
"winner_match": true,
|
|
"winner_pair_delta_s_percent": 0.0,
|
|
"winner_pair_m_s_percent": 0.0,
|
|
"winner_pair_relation": "match",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"config_uniform_scale": 1.430237446084041,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.487848464399216,
|
|
"log_ratio_spread_real_top3": 0.487848464399216,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 11.295399365368032,
|
|
"signed_pairwise_delta_percent": 30.727731611159037
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 9.087763981481082,
|
|
"signed_pairwise_delta_percent": 32.85538085186
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 0.6135354983447174,
|
|
"signed_pairwise_delta_percent": -15.929465588061584
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.006154214824712234,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.006154214824712234,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.006154214824712234,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -15.929465588061584,
|
|
"winner_pair_m_s_percent": 0.6135354983447174,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"config_uniform_scale": 1.4917712567044108,
|
|
"expected_config_count": 4,
|
|
"failure": true,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "HOLDS",
|
|
"log_ratio_spread_all": 0.49679676904370246,
|
|
"log_ratio_spread_real_top3": 0.49679676904370246,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 13.943039926940672,
|
|
"signed_pairwise_delta_percent": 29.838662224128264
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 10.455222177040667,
|
|
"signed_pairwise_delta_percent": 32.30699826893181
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 6.030894994380943,
|
|
"signed_pairwise_delta_percent": -17.372678635438426
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.062164651568153984,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns64"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.062164651568153984,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.062164651568153984,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -17.372678635438426,
|
|
"winner_pair_m_s_percent": 6.030894994380943,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"config_uniform_scale": 0.5481032351618105,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.3220111524478986,
|
|
"log_ratio_spread_real_top3": 0.3220111524478986,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.119345640134235,
|
|
"signed_pairwise_delta_percent": 30.823724522814285
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 6.802653056521642,
|
|
"signed_pairwise_delta_percent": 29.107141724120687
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.3657297298272386,
|
|
"signed_pairwise_delta_percent": -1.3773907219755823
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.013750984200462835,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.013750984200462835,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"sim_winner_real_relative_loss": 0.013750984200462835,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -1.3773907219755823,
|
|
"winner_pair_m_s_percent": 1.3657297298272386,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"config_uniform_scale": 0.5678289728094312,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.31919899251598766,
|
|
"log_ratio_spread_real_top3": 0.31919899251598766,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "ttft_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 19.209826109044702,
|
|
"signed_pairwise_delta_percent": 31.659458967916663
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 19.400420037632387,
|
|
"signed_pairwise_delta_percent": 31.468865039328982
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.26044028368209854,
|
|
"signed_pairwise_delta_percent": -0.26044028368209854
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.002607797240047338,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.002607797240047338,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.002607797240047338,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.26044028368209854,
|
|
"winner_pair_m_s_percent": 0.26044028368209854,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"config_uniform_scale": 0.548058912389237,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.32202928326406854,
|
|
"log_ratio_spread_real_top3": 0.32202928326406854,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 1,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 0,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_mean_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 5.1172644115479695,
|
|
"signed_pairwise_delta_percent": 30.825805751400548
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 6.800551883155552,
|
|
"signed_pairwise_delta_percent": 29.109242897486777
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": true,
|
|
"real_pair_margin_percent": 1.3654615828579657,
|
|
"signed_pairwise_delta_percent": -1.3771225750063092
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0137482658615673,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns128",
|
|
"tp8_ep8_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"regret": 0.0137482658615673,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"sim_winner_real_relative_loss": 0.0137482658615673,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -1.3771225750063092,
|
|
"winner_pair_m_s_percent": 1.3654615828579657,
|
|
"winner_pair_relation": "reversal",
|
|
"winner_pair_strict_reversal": true
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"config_uniform_scale": 0.5678075555479308,
|
|
"expected_config_count": 4,
|
|
"failure": false,
|
|
"fragile_success": false,
|
|
"h_scale_condition_spread_gt_log_margin": true,
|
|
"h_scale_verdict": "COUNTEREXAMPLE",
|
|
"log_ratio_spread_all": 0.3192076739943569,
|
|
"log_ratio_spread_real_top3": 0.3192076739943569,
|
|
"missing_configs": [],
|
|
"n_inv_mixed": 0,
|
|
"n_inv_mns": 0,
|
|
"n_inv_tp": 0,
|
|
"n_tie_mixed": 0,
|
|
"n_tie_mns": 1,
|
|
"n_tie_tp": 0,
|
|
"objective": "e2e_p90_ms",
|
|
"paired_config_count": 4,
|
|
"pairwise_vs_real_winner": [
|
|
{
|
|
"axis_vs_real_winner": "mixed",
|
|
"config": "tp4_ep1_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 19.20014493069586,
|
|
"signed_pairwise_delta_percent": 31.669140146265505
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "tp-axis",
|
|
"config": "tp4_ep1_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 19.390739890554553,
|
|
"signed_pairwise_delta_percent": 31.478545186406816
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "mns-axis",
|
|
"config": "tp8_ep8_mns128",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.2516272531701875,
|
|
"signed_pairwise_delta_percent": -0.2516272531701875
|
|
},
|
|
{
|
|
"axis_vs_real_winner": "none",
|
|
"config": "tp8_ep8_mns64",
|
|
"inversion": false,
|
|
"real_pair_margin_percent": 0.0,
|
|
"signed_pairwise_delta_percent": 0.0
|
|
}
|
|
],
|
|
"real_relative_margin_best_vs_second": 0.0025194410024500513,
|
|
"real_top3_configs": [
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"regret": 0.0025194410024500513,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"sim_winner_real_relative_loss": 0.0025194410024500513,
|
|
"sources": [
|
|
"q235-ablation-a1/comparison.json"
|
|
],
|
|
"winner_deciding_axis": "mns-axis",
|
|
"winner_match": false,
|
|
"winner_pair_delta_s_percent": -0.2516272531701875,
|
|
"winner_pair_m_s_percent": 0.2516272531701875,
|
|
"winner_pair_relation": "sim-tie",
|
|
"winner_pair_strict_reversal": false
|
|
}
|
|
],
|
|
"schema": "frontier-split-rootcause-decomposition-v2",
|
|
"summary": {
|
|
"case_count": 14,
|
|
"data_gap_count": 0,
|
|
"failure_count": 23,
|
|
"h_scale_counterexample_count": 40,
|
|
"h_scale_counterexamples": [
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-pd",
|
|
"objective": "e2e_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"objective": "tpot_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"objective": "tpot_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd high",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po high",
|
|
"objective": "e2e_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-pd low",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 fixed-po low",
|
|
"objective": "e2e_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q30 trace-po low",
|
|
"objective": "e2e_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-pd",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 fixed-po",
|
|
"objective": "e2e_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"objective": "tpot_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"objective": "tpot_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A0 trace-pd",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-pd",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 fixed-po",
|
|
"objective": "e2e_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"objective": "tpot_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-pd",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"objective": "ttft_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"objective": "ttft_p90_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"objective": "e2e_mean_ms"
|
|
},
|
|
{
|
|
"case": "q235 A1 trace-po",
|
|
"objective": "e2e_p90_ms"
|
|
}
|
|
],
|
|
"row_count": 70,
|
|
"winner_mismatch_count": 47
|
|
}
|
|
}
|