5223 lines
183 KiB
JSON
5223 lines
183 KiB
JSON
{
|
|
"historical_modes": {
|
|
"historical-per-tp-calibration": {
|
|
"false_feasibility": {
|
|
"overall": {
|
|
"agreement": 64,
|
|
"false_feasible": 21,
|
|
"false_infeasible": 7
|
|
},
|
|
"per_cell": {
|
|
"tp1_mns16": {
|
|
"agreement": 7,
|
|
"false_feasible": 1,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp1_mns32": {
|
|
"agreement": 6,
|
|
"false_feasible": 2,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp1_mns64": {
|
|
"agreement": 6,
|
|
"false_feasible": 2,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp1_mns8": {
|
|
"agreement": 6,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 2
|
|
},
|
|
"tp2_mns16": {
|
|
"agreement": 7,
|
|
"false_feasible": 1,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp2_mns32": {
|
|
"agreement": 3,
|
|
"false_feasible": 5,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp2_mns64": {
|
|
"agreement": 2,
|
|
"false_feasible": 6,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp2_mns8": {
|
|
"agreement": 3,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
},
|
|
"tp4_mns16": {
|
|
"agreement": 6,
|
|
"false_feasible": 1,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp4_mns32": {
|
|
"agreement": 6,
|
|
"false_feasible": 1,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp4_mns64": {
|
|
"agreement": 6,
|
|
"false_feasible": 1,
|
|
"false_infeasible": 0
|
|
},
|
|
"tp4_mns8": {
|
|
"agreement": 6,
|
|
"false_feasible": 1,
|
|
"false_infeasible": 0
|
|
}
|
|
}
|
|
},
|
|
"metrics": {
|
|
"kendall_tau_b": {
|
|
"both_tied": 4,
|
|
"concordant": 58,
|
|
"discordant": 0,
|
|
"pair_count": 66,
|
|
"real_only_tied": 1,
|
|
"sim_only_tied": 3,
|
|
"tau_b": 0.9668009539030813
|
|
},
|
|
"named_interactions": {
|
|
"relations": {
|
|
"tp2_mns32_gt_tp4_mns16": true,
|
|
"tp4_mns16_eq_tp4_mns32": true,
|
|
"tp4_mns16_eq_tp4_mns64": true,
|
|
"tp4_mns16_gt_tp1_mns16": true,
|
|
"tp4_mns16_gt_tp2_mns16": true,
|
|
"tp4_mns16_gt_tp4_mns8": true
|
|
},
|
|
"simulator_top_bucket_cells": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
],
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": true
|
|
},
|
|
"pairwise_direction": {
|
|
"exact_sign_accuracy": 0.9393939393939394,
|
|
"exact_sign_agreement": 62,
|
|
"non_tied_concordance": 1.0,
|
|
"non_tied_concordant": 58,
|
|
"non_tied_pair_count": 58,
|
|
"pair_count": 66
|
|
},
|
|
"per_mns_tp_direction": {
|
|
"accuracy": 1.0,
|
|
"comparisons": 8,
|
|
"exact_sign_agreement": 8,
|
|
"lines": [
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 2,
|
|
"line_class_agrees": true,
|
|
"mns": 8,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 2,
|
|
"line_class_agrees": true,
|
|
"mns": 16,
|
|
"real_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 2,
|
|
"line_class_agrees": true,
|
|
"mns": 32,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 2,
|
|
"line_class_agrees": true,
|
|
"mns": 64,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
}
|
|
]
|
|
},
|
|
"per_tp_mns_direction": {
|
|
"accuracy": 0.6666666666666666,
|
|
"comparisons": 9,
|
|
"exact_sign_agreement": 6,
|
|
"lines": [
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 2,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1,
|
|
0
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 1
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
0,
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
1,
|
|
0
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 2
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 3,
|
|
"line_class_agrees": true,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
"real_line_class": "increasing",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 4
|
|
}
|
|
]
|
|
},
|
|
"tie_buckets": {
|
|
"real": {
|
|
"buckets": {
|
|
"tp1_mns16": 715736,
|
|
"tp1_mns32": 695431,
|
|
"tp1_mns64": 695431,
|
|
"tp1_mns8": 639593,
|
|
"tp2_mns16": 692893,
|
|
"tp2_mns32": 1000000,
|
|
"tp2_mns64": 992385,
|
|
"tp2_mns8": 692893,
|
|
"tp4_mns16": 743654,
|
|
"tp4_mns32": 743654,
|
|
"tp4_mns64": 743654,
|
|
"tp4_mns8": 390862
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp2_mns32"
|
|
],
|
|
"interval": [
|
|
3.2833333333333328,
|
|
3.2833366166666664
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 992385,
|
|
"cells": [
|
|
"tp2_mns64"
|
|
],
|
|
"interval": [
|
|
3.25833075,
|
|
3.258334033333333
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 743654,
|
|
"cells": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
2.4416639666666664,
|
|
2.4416672499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 715736,
|
|
"cells": [
|
|
"tp1_mns16"
|
|
],
|
|
"interval": [
|
|
2.3499998666666664,
|
|
2.3500031499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 695431,
|
|
"cells": [
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
2.283331783333333,
|
|
2.2833350666666665
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 692893,
|
|
"cells": [
|
|
"tp2_mns8",
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
2.274998683333333,
|
|
2.2750019666666663
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 639593,
|
|
"cells": [
|
|
"tp1_mns8"
|
|
],
|
|
"interval": [
|
|
2.0999970166666664,
|
|
2.1000002999999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 390862,
|
|
"cells": [
|
|
"tp4_mns8"
|
|
],
|
|
"interval": [
|
|
1.2833302333333332,
|
|
1.2833335166666666
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 3.283333333333333e-06
|
|
},
|
|
"simulator": {
|
|
"buckets": {
|
|
"tp1_mns16": 635555,
|
|
"tp1_mns32": 635555,
|
|
"tp1_mns64": 635555,
|
|
"tp1_mns8": 457777,
|
|
"tp2_mns16": 613333,
|
|
"tp2_mns32": 1000000,
|
|
"tp2_mns64": 1000000,
|
|
"tp2_mns8": 464444,
|
|
"tp4_mns16": 666666,
|
|
"tp4_mns32": 666666,
|
|
"tp4_mns64": 666666,
|
|
"tp4_mns8": 352222
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
],
|
|
"interval": [
|
|
3.7499999999999996,
|
|
3.75000375
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 666666,
|
|
"cells": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
2.4999974999999997,
|
|
2.50000125
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 635555,
|
|
"cells": [
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
2.38333125,
|
|
2.3833349999999998
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 613333,
|
|
"cells": [
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
2.29999875,
|
|
2.3000024999999997
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 464444,
|
|
"cells": [
|
|
"tp2_mns8"
|
|
],
|
|
"interval": [
|
|
1.7416649999999998,
|
|
1.7416687499999999
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 457777,
|
|
"cells": [
|
|
"tp1_mns8"
|
|
],
|
|
"interval": [
|
|
1.71666375,
|
|
1.7166674999999998
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 352222,
|
|
"cells": [
|
|
"tp4_mns8"
|
|
],
|
|
"interval": [
|
|
1.3208324999999999,
|
|
1.32083625
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 3.7499999999999997e-06
|
|
}
|
|
},
|
|
"top1": {
|
|
"candidate_cells": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
],
|
|
"effective_k": 2,
|
|
"optimistic_regret": 0.0,
|
|
"worst_case_regret": 0.0076142131979695165
|
|
},
|
|
"top5": {
|
|
"maximum_exact_five_overlap": 5,
|
|
"minimum_exact_five_overlap": 5,
|
|
"optimistic_regret": 0.0,
|
|
"real_cutoff": {
|
|
"boundary": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"choose_from_boundary": 3,
|
|
"expanded_effective_k": 5,
|
|
"higher": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
]
|
|
},
|
|
"simulator_cutoff": {
|
|
"boundary": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"choose_from_boundary": 3,
|
|
"expanded_effective_k": 5,
|
|
"higher": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
]
|
|
},
|
|
"worst_case_regret": 0.0
|
|
}
|
|
},
|
|
"ranking": [
|
|
{
|
|
"cell": "tp2_mns32",
|
|
"rank": 1,
|
|
"score": 3.75
|
|
},
|
|
{
|
|
"cell": "tp2_mns64",
|
|
"rank": 2,
|
|
"score": 3.75
|
|
},
|
|
{
|
|
"cell": "tp4_mns16",
|
|
"rank": 3,
|
|
"score": 2.5
|
|
},
|
|
{
|
|
"cell": "tp4_mns32",
|
|
"rank": 4,
|
|
"score": 2.5
|
|
},
|
|
{
|
|
"cell": "tp4_mns64",
|
|
"rank": 5,
|
|
"score": 2.5
|
|
},
|
|
{
|
|
"cell": "tp1_mns16",
|
|
"rank": 6,
|
|
"score": 2.3833333333333333
|
|
},
|
|
{
|
|
"cell": "tp1_mns32",
|
|
"rank": 7,
|
|
"score": 2.3833333333333333
|
|
},
|
|
{
|
|
"cell": "tp1_mns64",
|
|
"rank": 8,
|
|
"score": 2.3833333333333333
|
|
},
|
|
{
|
|
"cell": "tp2_mns16",
|
|
"rank": 9,
|
|
"score": 2.3
|
|
},
|
|
{
|
|
"cell": "tp2_mns8",
|
|
"rank": 10,
|
|
"score": 1.7416666666666667
|
|
},
|
|
{
|
|
"cell": "tp1_mns8",
|
|
"rank": 11,
|
|
"score": 1.7166666666666666
|
|
},
|
|
{
|
|
"cell": "tp4_mns8",
|
|
"rank": 12,
|
|
"score": 1.3208333333333333
|
|
}
|
|
],
|
|
"simulated_scores": {
|
|
"tp1_mns16": 2.3833333333333333,
|
|
"tp1_mns32": 2.3833333333333333,
|
|
"tp1_mns64": 2.3833333333333333,
|
|
"tp1_mns8": 1.7166666666666666,
|
|
"tp2_mns16": 2.3,
|
|
"tp2_mns32": 3.75,
|
|
"tp2_mns64": 3.75,
|
|
"tp2_mns8": 1.7416666666666667,
|
|
"tp4_mns16": 2.5,
|
|
"tp4_mns32": 2.5,
|
|
"tp4_mns64": 2.5,
|
|
"tp4_mns8": 1.3208333333333333
|
|
}
|
|
},
|
|
"historical-profile-only": {
|
|
"false_feasibility": {
|
|
"overall": {
|
|
"agreement": 37,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 55
|
|
},
|
|
"per_cell": {
|
|
"tp1_mns16": {
|
|
"agreement": 3,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
},
|
|
"tp1_mns32": {
|
|
"agreement": 4,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 4
|
|
},
|
|
"tp1_mns64": {
|
|
"agreement": 4,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 4
|
|
},
|
|
"tp1_mns8": {
|
|
"agreement": 5,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 3
|
|
},
|
|
"tp2_mns16": {
|
|
"agreement": 2,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 6
|
|
},
|
|
"tp2_mns32": {
|
|
"agreement": 5,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 3
|
|
},
|
|
"tp2_mns64": {
|
|
"agreement": 6,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 2
|
|
},
|
|
"tp2_mns8": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 7
|
|
},
|
|
"tp4_mns16": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 6
|
|
},
|
|
"tp4_mns32": {
|
|
"agreement": 2,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
},
|
|
"tp4_mns64": {
|
|
"agreement": 2,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
},
|
|
"tp4_mns8": {
|
|
"agreement": 2,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
}
|
|
}
|
|
},
|
|
"metrics": {
|
|
"kendall_tau_b": {
|
|
"both_tied": 2,
|
|
"concordant": 23,
|
|
"discordant": 23,
|
|
"pair_count": 66,
|
|
"real_only_tied": 3,
|
|
"sim_only_tied": 15,
|
|
"tau_b": 0.0
|
|
},
|
|
"named_interactions": {
|
|
"relations": {
|
|
"tp2_mns32_gt_tp4_mns16": false,
|
|
"tp4_mns16_eq_tp4_mns32": false,
|
|
"tp4_mns16_eq_tp4_mns64": false,
|
|
"tp4_mns16_gt_tp1_mns16": false,
|
|
"tp4_mns16_gt_tp2_mns16": false,
|
|
"tp4_mns16_gt_tp4_mns8": false
|
|
},
|
|
"simulator_top_bucket_cells": [
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
"pairwise_direction": {
|
|
"exact_sign_accuracy": 0.3787878787878788,
|
|
"exact_sign_agreement": 25,
|
|
"non_tied_concordance": 0.5,
|
|
"non_tied_concordant": 23,
|
|
"non_tied_pair_count": 46,
|
|
"pair_count": 66
|
|
},
|
|
"per_mns_tp_direction": {
|
|
"accuracy": 0.0,
|
|
"comparisons": 8,
|
|
"exact_sign_agreement": 0,
|
|
"lines": [
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": false,
|
|
"mns": 8,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
0
|
|
],
|
|
"sim_line_class": "decreasing"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": true,
|
|
"mns": 16,
|
|
"real_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": true,
|
|
"mns": 32,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": true,
|
|
"mns": 64,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"sim_line_class": "non-monotonic"
|
|
}
|
|
]
|
|
},
|
|
"per_tp_mns_direction": {
|
|
"accuracy": 0.2222222222222222,
|
|
"comparisons": 9,
|
|
"exact_sign_agreement": 2,
|
|
"lines": [
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1,
|
|
0
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat",
|
|
"tp": 1
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": true,
|
|
"real_adjacent_signs": [
|
|
0,
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
-1,
|
|
0
|
|
],
|
|
"sim_line_class": "non-monotonic",
|
|
"tp": 2
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": true,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
"real_line_class": "increasing",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
1,
|
|
0
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 4
|
|
}
|
|
]
|
|
},
|
|
"tie_buckets": {
|
|
"real": {
|
|
"buckets": {
|
|
"tp1_mns16": 715736,
|
|
"tp1_mns32": 695431,
|
|
"tp1_mns64": 695431,
|
|
"tp1_mns8": 639593,
|
|
"tp2_mns16": 692893,
|
|
"tp2_mns32": 1000000,
|
|
"tp2_mns64": 992385,
|
|
"tp2_mns8": 692893,
|
|
"tp4_mns16": 743654,
|
|
"tp4_mns32": 743654,
|
|
"tp4_mns64": 743654,
|
|
"tp4_mns8": 390862
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp2_mns32"
|
|
],
|
|
"interval": [
|
|
3.2833333333333328,
|
|
3.2833366166666664
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 992385,
|
|
"cells": [
|
|
"tp2_mns64"
|
|
],
|
|
"interval": [
|
|
3.25833075,
|
|
3.258334033333333
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 743654,
|
|
"cells": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
2.4416639666666664,
|
|
2.4416672499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 715736,
|
|
"cells": [
|
|
"tp1_mns16"
|
|
],
|
|
"interval": [
|
|
2.3499998666666664,
|
|
2.3500031499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 695431,
|
|
"cells": [
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
2.283331783333333,
|
|
2.2833350666666665
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 692893,
|
|
"cells": [
|
|
"tp2_mns8",
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
2.274998683333333,
|
|
2.2750019666666663
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 639593,
|
|
"cells": [
|
|
"tp1_mns8"
|
|
],
|
|
"interval": [
|
|
2.0999970166666664,
|
|
2.1000002999999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 390862,
|
|
"cells": [
|
|
"tp4_mns8"
|
|
],
|
|
"interval": [
|
|
1.2833302333333332,
|
|
1.2833335166666666
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 3.283333333333333e-06
|
|
},
|
|
"simulator": {
|
|
"buckets": {
|
|
"tp1_mns16": 832807,
|
|
"tp1_mns32": 832807,
|
|
"tp1_mns64": 832807,
|
|
"tp1_mns8": 832807,
|
|
"tp2_mns16": 902208,
|
|
"tp2_mns32": 0,
|
|
"tp2_mns64": 0,
|
|
"tp2_mns8": 0,
|
|
"tp4_mns16": 0,
|
|
"tp4_mns32": 1000000,
|
|
"tp4_mns64": 1000000,
|
|
"tp4_mns8": 0
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
1.3208333333333333,
|
|
1.3208346541666667
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 902208,
|
|
"cells": [
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
1.1916664,
|
|
1.1916677208333333
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 832807,
|
|
"cells": [
|
|
"tp1_mns8",
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
1.0999992458333334,
|
|
1.1000005666666666
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 0,
|
|
"cells": [
|
|
"tp2_mns8",
|
|
"tp2_mns32",
|
|
"tp2_mns64",
|
|
"tp4_mns8",
|
|
"tp4_mns16"
|
|
],
|
|
"interval": [
|
|
0.0,
|
|
1.3208333333333333e-06
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 1.3208333333333333e-06
|
|
}
|
|
},
|
|
"top1": {
|
|
"candidate_cells": [
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"effective_k": 2,
|
|
"optimistic_regret": 0.25634517766497456,
|
|
"worst_case_regret": 0.25634517766497456
|
|
},
|
|
"top5": {
|
|
"maximum_exact_five_overlap": 2,
|
|
"minimum_exact_five_overlap": 2,
|
|
"optimistic_regret": 0.25634517766497456,
|
|
"real_cutoff": {
|
|
"boundary": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"choose_from_boundary": 3,
|
|
"expanded_effective_k": 5,
|
|
"higher": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
]
|
|
},
|
|
"simulator_cutoff": {
|
|
"boundary": [
|
|
"tp1_mns8",
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"choose_from_boundary": 2,
|
|
"expanded_effective_k": 7,
|
|
"higher": [
|
|
"tp4_mns32",
|
|
"tp4_mns64",
|
|
"tp2_mns16"
|
|
]
|
|
},
|
|
"worst_case_regret": 0.25634517766497456
|
|
}
|
|
},
|
|
"ranking": [
|
|
{
|
|
"cell": "tp4_mns32",
|
|
"rank": 1,
|
|
"score": 1.3208333333333333
|
|
},
|
|
{
|
|
"cell": "tp4_mns64",
|
|
"rank": 2,
|
|
"score": 1.3208333333333333
|
|
},
|
|
{
|
|
"cell": "tp2_mns16",
|
|
"rank": 3,
|
|
"score": 1.1916666666666667
|
|
},
|
|
{
|
|
"cell": "tp1_mns16",
|
|
"rank": 4,
|
|
"score": 1.1
|
|
},
|
|
{
|
|
"cell": "tp1_mns32",
|
|
"rank": 5,
|
|
"score": 1.1
|
|
},
|
|
{
|
|
"cell": "tp1_mns64",
|
|
"rank": 6,
|
|
"score": 1.1
|
|
},
|
|
{
|
|
"cell": "tp1_mns8",
|
|
"rank": 7,
|
|
"score": 1.1
|
|
},
|
|
{
|
|
"cell": "tp2_mns32",
|
|
"rank": 8,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp2_mns64",
|
|
"rank": 9,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp2_mns8",
|
|
"rank": 10,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp4_mns16",
|
|
"rank": 11,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp4_mns8",
|
|
"rank": 12,
|
|
"score": 0.0
|
|
}
|
|
],
|
|
"simulated_scores": {
|
|
"tp1_mns16": 1.1,
|
|
"tp1_mns32": 1.1,
|
|
"tp1_mns64": 1.1,
|
|
"tp1_mns8": 1.1,
|
|
"tp2_mns16": 1.1916666666666667,
|
|
"tp2_mns32": 0.0,
|
|
"tp2_mns64": 0.0,
|
|
"tp2_mns8": 0.0,
|
|
"tp4_mns16": 0.0,
|
|
"tp4_mns32": 1.3208333333333333,
|
|
"tp4_mns64": 1.3208333333333333,
|
|
"tp4_mns8": 0.0
|
|
}
|
|
}
|
|
},
|
|
"real_scores": {
|
|
"tp1_mns16": 2.35,
|
|
"tp1_mns32": 2.283333333333333,
|
|
"tp1_mns64": 2.283333333333333,
|
|
"tp1_mns8": 2.1,
|
|
"tp2_mns16": 2.275,
|
|
"tp2_mns32": 3.283333333333333,
|
|
"tp2_mns64": 3.2583333333333333,
|
|
"tp2_mns8": 2.275,
|
|
"tp4_mns16": 2.441666666666667,
|
|
"tp4_mns32": 2.441666666666667,
|
|
"tp4_mns64": 2.441666666666667,
|
|
"tp4_mns8": 1.2833333333333334
|
|
},
|
|
"schema": "frontier-qwen30-s2-profile-ablation.v1",
|
|
"scope": {
|
|
"calibration_a_tp": 1.0,
|
|
"cells": 12,
|
|
"probes": 92,
|
|
"trace_horizon_seconds": 60
|
|
},
|
|
"sources": {
|
|
"ground_truth": "/home/gahow/phd/replayserve/docs/assets/simfid_s2r/ground_truth.json",
|
|
"historical_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
|
|
"replacement_profile_shards": [
|
|
{
|
|
"path": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2-s2/results-shard-00/metrics.json",
|
|
"runs": 46
|
|
},
|
|
{
|
|
"path": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2-s2/results-shard-01/metrics.json",
|
|
"runs": 46
|
|
}
|
|
]
|
|
},
|
|
"vllm020_profile_only": {
|
|
"SLO-gated": {
|
|
"cell_score_details": {
|
|
"tp1_mns16": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp1_mns32": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp1_mns64": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp1_mns8": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp2_mns16": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp2_mns32": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp2_mns64": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp2_mns8": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp4_mns16": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp4_mns32": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp4_mns64": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
},
|
|
"tp4_mns8": {
|
|
"feasible": false,
|
|
"feasible_anchor_count": 0,
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.0,
|
|
"selected_probe_index": null,
|
|
"selected_sampling_u": null
|
|
}
|
|
},
|
|
"false_feasibility": {
|
|
"overall": {
|
|
"agreement": 30,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 62
|
|
},
|
|
"per_cell": {
|
|
"tp1_mns16": {
|
|
"agreement": 2,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 6
|
|
},
|
|
"tp1_mns32": {
|
|
"agreement": 3,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
},
|
|
"tp1_mns64": {
|
|
"agreement": 3,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
},
|
|
"tp1_mns8": {
|
|
"agreement": 4,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 4
|
|
},
|
|
"tp2_mns16": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 7
|
|
},
|
|
"tp2_mns32": {
|
|
"agreement": 5,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 3
|
|
},
|
|
"tp2_mns64": {
|
|
"agreement": 6,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 2
|
|
},
|
|
"tp2_mns8": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 7
|
|
},
|
|
"tp4_mns16": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 6
|
|
},
|
|
"tp4_mns32": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 6
|
|
},
|
|
"tp4_mns64": {
|
|
"agreement": 1,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 6
|
|
},
|
|
"tp4_mns8": {
|
|
"agreement": 2,
|
|
"false_feasible": 0,
|
|
"false_infeasible": 5
|
|
}
|
|
}
|
|
},
|
|
"loao": {
|
|
"defined_replicates": 92,
|
|
"range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals",
|
|
"ranges": {
|
|
"pairwise_exact_sign_accuracy": {
|
|
"max": 0.07575757575757576,
|
|
"min": 0.07575757575757576
|
|
},
|
|
"pairwise_non_tied_concordance": {
|
|
"max": null,
|
|
"min": null
|
|
},
|
|
"tau_b": {
|
|
"max": 0.0,
|
|
"min": 0.0
|
|
},
|
|
"top1_optimistic_regret": {
|
|
"max": 0.0,
|
|
"min": 0.0
|
|
},
|
|
"top1_worst_case_regret": {
|
|
"max": 0.6091370558375634,
|
|
"min": 0.6091370558375634
|
|
},
|
|
"top5_maximum_exact_five_overlap": {
|
|
"max": 5,
|
|
"min": 5
|
|
},
|
|
"top5_minimum_exact_five_overlap": {
|
|
"max": 0,
|
|
"min": 0
|
|
},
|
|
"top5_optimistic_regret": {
|
|
"max": 0.0,
|
|
"min": 0.0
|
|
},
|
|
"top5_worst_case_regret": {
|
|
"max": 0.30456852791878175,
|
|
"min": 0.30456852791878175
|
|
}
|
|
},
|
|
"replicate_count": 92,
|
|
"replicates": [
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.07575757575757576,
|
|
"pairwise_non_tied_concordance": null,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.0,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.6091370558375634,
|
|
"top5_maximum_exact_five_overlap": 5,
|
|
"top5_minimum_exact_five_overlap": 0,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.30456852791878175,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
}
|
|
],
|
|
"tp2_mns32_unique_global_best_count": 0,
|
|
"trap_reproduced_count": 0,
|
|
"undefined": [],
|
|
"undefined_replicates": 0
|
|
},
|
|
"metrics": {
|
|
"kendall_tau_b": {
|
|
"both_tied": 5,
|
|
"concordant": 0,
|
|
"discordant": 0,
|
|
"pair_count": 66,
|
|
"real_only_tied": 0,
|
|
"sim_only_tied": 61,
|
|
"tau_b": 0.0
|
|
},
|
|
"named_interactions": {
|
|
"relations": {
|
|
"tp2_mns32_gt_tp4_mns16": false,
|
|
"tp4_mns16_eq_tp4_mns32": true,
|
|
"tp4_mns16_eq_tp4_mns64": true,
|
|
"tp4_mns16_gt_tp1_mns16": false,
|
|
"tp4_mns16_gt_tp2_mns16": false,
|
|
"tp4_mns16_gt_tp4_mns8": false
|
|
},
|
|
"simulator_top_bucket_cells": [
|
|
"tp1_mns8",
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64",
|
|
"tp2_mns8",
|
|
"tp2_mns16",
|
|
"tp2_mns32",
|
|
"tp2_mns64",
|
|
"tp4_mns8",
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
"pairwise_direction": {
|
|
"exact_sign_accuracy": 0.07575757575757576,
|
|
"exact_sign_agreement": 5,
|
|
"non_tied_concordance": null,
|
|
"non_tied_concordant": 0,
|
|
"non_tied_pair_count": 0,
|
|
"pair_count": 66
|
|
},
|
|
"per_mns_tp_direction": {
|
|
"accuracy": 0.0,
|
|
"comparisons": 8,
|
|
"exact_sign_agreement": 0,
|
|
"lines": [
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": false,
|
|
"mns": 8,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": false,
|
|
"mns": 16,
|
|
"real_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": false,
|
|
"mns": 32,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 0,
|
|
"line_class_agrees": false,
|
|
"mns": 64,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat"
|
|
}
|
|
]
|
|
},
|
|
"per_tp_mns_direction": {
|
|
"accuracy": 0.4444444444444444,
|
|
"comparisons": 9,
|
|
"exact_sign_agreement": 4,
|
|
"lines": [
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1,
|
|
0
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat",
|
|
"tp": 1
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
0,
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat",
|
|
"tp": 2
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 2,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
"real_line_class": "increasing",
|
|
"sim_adjacent_signs": [
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
"sim_line_class": "flat",
|
|
"tp": 4
|
|
}
|
|
]
|
|
},
|
|
"tie_buckets": {
|
|
"real": {
|
|
"buckets": {
|
|
"tp1_mns16": 715736,
|
|
"tp1_mns32": 695431,
|
|
"tp1_mns64": 695431,
|
|
"tp1_mns8": 639593,
|
|
"tp2_mns16": 692893,
|
|
"tp2_mns32": 1000000,
|
|
"tp2_mns64": 992385,
|
|
"tp2_mns8": 692893,
|
|
"tp4_mns16": 743654,
|
|
"tp4_mns32": 743654,
|
|
"tp4_mns64": 743654,
|
|
"tp4_mns8": 390862
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp2_mns32"
|
|
],
|
|
"interval": [
|
|
3.2833333333333328,
|
|
3.2833366166666664
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 992385,
|
|
"cells": [
|
|
"tp2_mns64"
|
|
],
|
|
"interval": [
|
|
3.25833075,
|
|
3.258334033333333
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 743654,
|
|
"cells": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
2.4416639666666664,
|
|
2.4416672499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 715736,
|
|
"cells": [
|
|
"tp1_mns16"
|
|
],
|
|
"interval": [
|
|
2.3499998666666664,
|
|
2.3500031499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 695431,
|
|
"cells": [
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
2.283331783333333,
|
|
2.2833350666666665
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 692893,
|
|
"cells": [
|
|
"tp2_mns8",
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
2.274998683333333,
|
|
2.2750019666666663
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 639593,
|
|
"cells": [
|
|
"tp1_mns8"
|
|
],
|
|
"interval": [
|
|
2.0999970166666664,
|
|
2.1000002999999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 390862,
|
|
"cells": [
|
|
"tp4_mns8"
|
|
],
|
|
"interval": [
|
|
1.2833302333333332,
|
|
1.2833335166666666
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 3.283333333333333e-06
|
|
},
|
|
"simulator": {
|
|
"buckets": {
|
|
"tp1_mns16": 0,
|
|
"tp1_mns32": 0,
|
|
"tp1_mns64": 0,
|
|
"tp1_mns8": 0,
|
|
"tp2_mns16": 0,
|
|
"tp2_mns32": 0,
|
|
"tp2_mns64": 0,
|
|
"tp2_mns8": 0,
|
|
"tp4_mns16": 0,
|
|
"tp4_mns32": 0,
|
|
"tp4_mns64": 0,
|
|
"tp4_mns8": 0
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 0,
|
|
"cells": [
|
|
"tp1_mns8",
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64",
|
|
"tp2_mns8",
|
|
"tp2_mns16",
|
|
"tp2_mns32",
|
|
"tp2_mns64",
|
|
"tp4_mns8",
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
0.0,
|
|
1e-09
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 1e-09
|
|
}
|
|
},
|
|
"top1": {
|
|
"candidate_cells": [
|
|
"tp1_mns8",
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64",
|
|
"tp2_mns8",
|
|
"tp2_mns16",
|
|
"tp2_mns32",
|
|
"tp2_mns64",
|
|
"tp4_mns8",
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"effective_k": 12,
|
|
"optimistic_regret": 0.0,
|
|
"worst_case_regret": 0.6091370558375634
|
|
},
|
|
"top5": {
|
|
"maximum_exact_five_overlap": 5,
|
|
"minimum_exact_five_overlap": 0,
|
|
"optimistic_regret": 0.0,
|
|
"real_cutoff": {
|
|
"boundary": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"choose_from_boundary": 3,
|
|
"expanded_effective_k": 5,
|
|
"higher": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
]
|
|
},
|
|
"simulator_cutoff": {
|
|
"boundary": [
|
|
"tp1_mns8",
|
|
"tp1_mns16",
|
|
"tp1_mns32",
|
|
"tp1_mns64",
|
|
"tp2_mns8",
|
|
"tp2_mns16",
|
|
"tp2_mns32",
|
|
"tp2_mns64",
|
|
"tp4_mns8",
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"choose_from_boundary": 5,
|
|
"expanded_effective_k": 12,
|
|
"higher": []
|
|
},
|
|
"worst_case_regret": 0.30456852791878175
|
|
}
|
|
},
|
|
"ranking": [
|
|
{
|
|
"cell": "tp1_mns16",
|
|
"rank": 1,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp1_mns32",
|
|
"rank": 2,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp1_mns64",
|
|
"rank": 3,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp1_mns8",
|
|
"rank": 4,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp2_mns16",
|
|
"rank": 5,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp2_mns32",
|
|
"rank": 6,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp2_mns64",
|
|
"rank": 7,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp2_mns8",
|
|
"rank": 8,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp4_mns16",
|
|
"rank": 9,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp4_mns32",
|
|
"rank": 10,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp4_mns64",
|
|
"rank": 11,
|
|
"score": 0.0
|
|
},
|
|
{
|
|
"cell": "tp4_mns8",
|
|
"rank": 12,
|
|
"score": 0.0
|
|
}
|
|
],
|
|
"reading": "SLO-gated",
|
|
"simulated_scores": {
|
|
"tp1_mns16": 0.0,
|
|
"tp1_mns32": 0.0,
|
|
"tp1_mns64": 0.0,
|
|
"tp1_mns8": 0.0,
|
|
"tp2_mns16": 0.0,
|
|
"tp2_mns32": 0.0,
|
|
"tp2_mns64": 0.0,
|
|
"tp2_mns8": 0.0,
|
|
"tp4_mns16": 0.0,
|
|
"tp4_mns32": 0.0,
|
|
"tp4_mns64": 0.0,
|
|
"tp4_mns8": 0.0
|
|
}
|
|
},
|
|
"throughput-proxy": {
|
|
"cell_score_details": {
|
|
"tp1_mns16": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.8453202347375465,
|
|
"selected_probe_index": 1,
|
|
"selected_sampling_u": 0.25,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp1_mns32": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 1.4213857973581445,
|
|
"selected_probe_index": 0,
|
|
"selected_sampling_u": 0.5,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp1_mns64": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 2.1437448608949605,
|
|
"selected_probe_index": 0,
|
|
"selected_sampling_u": 0.5,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp1_mns8": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.5319246206192596,
|
|
"selected_probe_index": 1,
|
|
"selected_sampling_u": 0.25,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp2_mns16": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.6441840113363596,
|
|
"selected_probe_index": 4,
|
|
"selected_sampling_u": 0.46875,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp2_mns32": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 1.093439153724593,
|
|
"selected_probe_index": 4,
|
|
"selected_sampling_u": 0.78125,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp2_mns64": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 1.7227463064785613,
|
|
"selected_probe_index": 2,
|
|
"selected_sampling_u": 0.875,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp2_mns8": {
|
|
"recorded_anchor_count_after_removal": 8,
|
|
"score": 0.3900292387118045,
|
|
"selected_probe_index": 4,
|
|
"selected_sampling_u": 0.46875,
|
|
"valid_anchor_count": 8
|
|
},
|
|
"tp4_mns16": {
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.4355571852372527,
|
|
"selected_probe_index": 5,
|
|
"selected_sampling_u": 0.033182214016,
|
|
"valid_anchor_count": 7
|
|
},
|
|
"tp4_mns32": {
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.760215065879823,
|
|
"selected_probe_index": 5,
|
|
"selected_sampling_u": 0.033182214016,
|
|
"valid_anchor_count": 7
|
|
},
|
|
"tp4_mns64": {
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 1.2313111686958826,
|
|
"selected_probe_index": 5,
|
|
"selected_sampling_u": 0.033182214016,
|
|
"valid_anchor_count": 7
|
|
},
|
|
"tp4_mns8": {
|
|
"recorded_anchor_count_after_removal": 7,
|
|
"score": 0.25160540206585935,
|
|
"selected_probe_index": 0,
|
|
"selected_sampling_u": 0.034252608017,
|
|
"valid_anchor_count": 7
|
|
}
|
|
},
|
|
"loao": {
|
|
"defined_replicates": 92,
|
|
"range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals",
|
|
"ranges": {
|
|
"pairwise_exact_sign_accuracy": {
|
|
"max": 0.6515151515151515,
|
|
"min": 0.6515151515151515
|
|
},
|
|
"pairwise_non_tied_concordance": {
|
|
"max": 0.7049180327868853,
|
|
"min": 0.7049180327868853
|
|
},
|
|
"tau_b": {
|
|
"max": 0.39400626128204935,
|
|
"min": 0.39400626128204935
|
|
},
|
|
"top1_optimistic_regret": {
|
|
"max": 0.30456852791878175,
|
|
"min": 0.30456852791878175
|
|
},
|
|
"top1_worst_case_regret": {
|
|
"max": 0.30456852791878175,
|
|
"min": 0.30456852791878175
|
|
},
|
|
"top5_maximum_exact_five_overlap": {
|
|
"max": 3,
|
|
"min": 3
|
|
},
|
|
"top5_minimum_exact_five_overlap": {
|
|
"max": 3,
|
|
"min": 3
|
|
},
|
|
"top5_optimistic_regret": {
|
|
"max": 0.0,
|
|
"min": 0.0
|
|
},
|
|
"top5_worst_case_regret": {
|
|
"max": 0.0,
|
|
"min": 0.0
|
|
}
|
|
},
|
|
"replicate_count": 92,
|
|
"replicates": [
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns8",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns16",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns32",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp1_mns64",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns8",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns16",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns32",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp2_mns64",
|
|
"removed_probe_index": 7,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns8",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns16",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns32",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 0,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 1,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 2,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 3,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 4,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 5,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
{
|
|
"pairwise_exact_sign_accuracy": 0.6515151515151515,
|
|
"pairwise_non_tied_concordance": 0.7049180327868853,
|
|
"removed_cell_id": "tp4_mns64",
|
|
"removed_probe_index": 6,
|
|
"tau_b": 0.39400626128204935,
|
|
"top1_optimistic_regret": 0.30456852791878175,
|
|
"top1_worst_case_regret": 0.30456852791878175,
|
|
"top5_maximum_exact_five_overlap": 3,
|
|
"top5_minimum_exact_five_overlap": 3,
|
|
"top5_optimistic_regret": 0.0,
|
|
"top5_worst_case_regret": 0.0,
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
}
|
|
],
|
|
"tp2_mns32_unique_global_best_count": 0,
|
|
"trap_reproduced_count": 0,
|
|
"undefined": [],
|
|
"undefined_replicates": 0
|
|
},
|
|
"metrics": {
|
|
"kendall_tau_b": {
|
|
"both_tied": 0,
|
|
"concordant": 43,
|
|
"discordant": 18,
|
|
"pair_count": 66,
|
|
"real_only_tied": 5,
|
|
"sim_only_tied": 0,
|
|
"tau_b": 0.39400626128204935
|
|
},
|
|
"named_interactions": {
|
|
"relations": {
|
|
"tp2_mns32_gt_tp4_mns16": true,
|
|
"tp4_mns16_eq_tp4_mns32": false,
|
|
"tp4_mns16_eq_tp4_mns64": false,
|
|
"tp4_mns16_gt_tp1_mns16": false,
|
|
"tp4_mns16_gt_tp2_mns16": false,
|
|
"tp4_mns16_gt_tp4_mns8": true
|
|
},
|
|
"simulator_top_bucket_cells": [
|
|
"tp1_mns64"
|
|
],
|
|
"tp2_mns32_unique_global_best": false,
|
|
"trap_reproduced": false
|
|
},
|
|
"pairwise_direction": {
|
|
"exact_sign_accuracy": 0.6515151515151515,
|
|
"exact_sign_agreement": 43,
|
|
"non_tied_concordance": 0.7049180327868853,
|
|
"non_tied_concordant": 43,
|
|
"non_tied_pair_count": 61,
|
|
"pair_count": 66
|
|
},
|
|
"per_mns_tp_direction": {
|
|
"accuracy": 0.5,
|
|
"comparisons": 8,
|
|
"exact_sign_agreement": 4,
|
|
"lines": [
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"mns": 8,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
-1
|
|
],
|
|
"sim_line_class": "decreasing"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"mns": 16,
|
|
"real_adjacent_signs": [
|
|
-1,
|
|
1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
-1
|
|
],
|
|
"sim_line_class": "decreasing"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"mns": 32,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
-1
|
|
],
|
|
"sim_line_class": "decreasing"
|
|
},
|
|
{
|
|
"comparisons": 2,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"mns": 64,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
-1,
|
|
-1
|
|
],
|
|
"sim_line_class": "decreasing"
|
|
}
|
|
]
|
|
},
|
|
"per_tp_mns_direction": {
|
|
"accuracy": 0.3333333333333333,
|
|
"comparisons": 9,
|
|
"exact_sign_agreement": 3,
|
|
"lines": [
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
-1,
|
|
0
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
1,
|
|
1
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 1
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": false,
|
|
"real_adjacent_signs": [
|
|
0,
|
|
1,
|
|
-1
|
|
],
|
|
"real_line_class": "non-monotonic",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
1,
|
|
1
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 2
|
|
},
|
|
{
|
|
"comparisons": 3,
|
|
"exact_sign_agreement": 1,
|
|
"line_class_agrees": true,
|
|
"real_adjacent_signs": [
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
"real_line_class": "increasing",
|
|
"sim_adjacent_signs": [
|
|
1,
|
|
1,
|
|
1
|
|
],
|
|
"sim_line_class": "increasing",
|
|
"tp": 4
|
|
}
|
|
]
|
|
},
|
|
"tie_buckets": {
|
|
"real": {
|
|
"buckets": {
|
|
"tp1_mns16": 715736,
|
|
"tp1_mns32": 695431,
|
|
"tp1_mns64": 695431,
|
|
"tp1_mns8": 639593,
|
|
"tp2_mns16": 692893,
|
|
"tp2_mns32": 1000000,
|
|
"tp2_mns64": 992385,
|
|
"tp2_mns8": 692893,
|
|
"tp4_mns16": 743654,
|
|
"tp4_mns32": 743654,
|
|
"tp4_mns64": 743654,
|
|
"tp4_mns8": 390862
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp2_mns32"
|
|
],
|
|
"interval": [
|
|
3.2833333333333328,
|
|
3.2833366166666664
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 992385,
|
|
"cells": [
|
|
"tp2_mns64"
|
|
],
|
|
"interval": [
|
|
3.25833075,
|
|
3.258334033333333
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 743654,
|
|
"cells": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
2.4416639666666664,
|
|
2.4416672499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 715736,
|
|
"cells": [
|
|
"tp1_mns16"
|
|
],
|
|
"interval": [
|
|
2.3499998666666664,
|
|
2.3500031499999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 695431,
|
|
"cells": [
|
|
"tp1_mns32",
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
2.283331783333333,
|
|
2.2833350666666665
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 692893,
|
|
"cells": [
|
|
"tp2_mns8",
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
2.274998683333333,
|
|
2.2750019666666663
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 639593,
|
|
"cells": [
|
|
"tp1_mns8"
|
|
],
|
|
"interval": [
|
|
2.0999970166666664,
|
|
2.1000002999999996
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 390862,
|
|
"cells": [
|
|
"tp4_mns8"
|
|
],
|
|
"interval": [
|
|
1.2833302333333332,
|
|
1.2833335166666666
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 3.283333333333333e-06
|
|
},
|
|
"simulator": {
|
|
"buckets": {
|
|
"tp1_mns16": 394319,
|
|
"tp1_mns32": 663038,
|
|
"tp1_mns64": 1000000,
|
|
"tp1_mns8": 248128,
|
|
"tp2_mns16": 300494,
|
|
"tp2_mns32": 510060,
|
|
"tp2_mns64": 803615,
|
|
"tp2_mns8": 181938,
|
|
"tp4_mns16": 203175,
|
|
"tp4_mns32": 354620,
|
|
"tp4_mns64": 574373,
|
|
"tp4_mns8": 117367
|
|
},
|
|
"occupied_intervals": [
|
|
{
|
|
"bucket_id": 1000000,
|
|
"cells": [
|
|
"tp1_mns64"
|
|
],
|
|
"interval": [
|
|
2.1437448608949605,
|
|
2.1437470046398217
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 803615,
|
|
"cells": [
|
|
"tp2_mns64"
|
|
],
|
|
"interval": [
|
|
1.7227455263881037,
|
|
1.7227476701329647
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 663038,
|
|
"cells": [
|
|
"tp1_mns32"
|
|
],
|
|
"interval": [
|
|
1.421384305078073,
|
|
1.4213864488229337
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 574373,
|
|
"cells": [
|
|
"tp4_mns64"
|
|
],
|
|
"interval": [
|
|
1.2313091669868212,
|
|
1.2313113107316822
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 510060,
|
|
"cells": [
|
|
"tp2_mns32"
|
|
],
|
|
"interval": [
|
|
1.0934385037480836,
|
|
1.0934406474929446
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 394319,
|
|
"cells": [
|
|
"tp1_mns16"
|
|
],
|
|
"interval": [
|
|
0.84531932980324,
|
|
0.8453214735481008
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 354620,
|
|
"cells": [
|
|
"tp4_mns32"
|
|
],
|
|
"interval": [
|
|
0.760214802570571,
|
|
0.7602169463154318
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 300494,
|
|
"cells": [
|
|
"tp2_mns16"
|
|
],
|
|
"interval": [
|
|
0.6441824682297703,
|
|
0.6441846119746312
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 248128,
|
|
"cells": [
|
|
"tp1_mns8"
|
|
],
|
|
"interval": [
|
|
0.5319231248441448,
|
|
0.5319252685890057
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 203175,
|
|
"cells": [
|
|
"tp4_mns16"
|
|
],
|
|
"interval": [
|
|
0.43555536211233364,
|
|
0.4355575058571945
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 181938,
|
|
"cells": [
|
|
"tp2_mns8"
|
|
],
|
|
"interval": [
|
|
0.39002865250150737,
|
|
0.3900307962463682
|
|
]
|
|
},
|
|
{
|
|
"bucket_id": 117367,
|
|
"cells": [
|
|
"tp4_mns8"
|
|
],
|
|
"interval": [
|
|
0.25160490308865885,
|
|
0.25160704683351975
|
|
]
|
|
}
|
|
],
|
|
"tolerance": 2.1437448608949606e-06
|
|
}
|
|
},
|
|
"top1": {
|
|
"candidate_cells": [
|
|
"tp1_mns64"
|
|
],
|
|
"effective_k": 1,
|
|
"optimistic_regret": 0.30456852791878175,
|
|
"worst_case_regret": 0.30456852791878175
|
|
},
|
|
"top5": {
|
|
"maximum_exact_five_overlap": 3,
|
|
"minimum_exact_five_overlap": 3,
|
|
"optimistic_regret": 0.0,
|
|
"real_cutoff": {
|
|
"boundary": [
|
|
"tp4_mns16",
|
|
"tp4_mns32",
|
|
"tp4_mns64"
|
|
],
|
|
"choose_from_boundary": 3,
|
|
"expanded_effective_k": 5,
|
|
"higher": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
]
|
|
},
|
|
"simulator_cutoff": {
|
|
"boundary": [
|
|
"tp2_mns32"
|
|
],
|
|
"choose_from_boundary": 1,
|
|
"expanded_effective_k": 5,
|
|
"higher": [
|
|
"tp1_mns64",
|
|
"tp2_mns64",
|
|
"tp1_mns32",
|
|
"tp4_mns64"
|
|
]
|
|
},
|
|
"worst_case_regret": 0.0
|
|
}
|
|
},
|
|
"ranking": [
|
|
{
|
|
"cell": "tp1_mns64",
|
|
"rank": 1,
|
|
"score": 2.1437448608949605
|
|
},
|
|
{
|
|
"cell": "tp2_mns64",
|
|
"rank": 2,
|
|
"score": 1.7227463064785613
|
|
},
|
|
{
|
|
"cell": "tp1_mns32",
|
|
"rank": 3,
|
|
"score": 1.4213857973581445
|
|
},
|
|
{
|
|
"cell": "tp4_mns64",
|
|
"rank": 4,
|
|
"score": 1.2313111686958826
|
|
},
|
|
{
|
|
"cell": "tp2_mns32",
|
|
"rank": 5,
|
|
"score": 1.093439153724593
|
|
},
|
|
{
|
|
"cell": "tp1_mns16",
|
|
"rank": 6,
|
|
"score": 0.8453202347375465
|
|
},
|
|
{
|
|
"cell": "tp4_mns32",
|
|
"rank": 7,
|
|
"score": 0.760215065879823
|
|
},
|
|
{
|
|
"cell": "tp2_mns16",
|
|
"rank": 8,
|
|
"score": 0.6441840113363596
|
|
},
|
|
{
|
|
"cell": "tp1_mns8",
|
|
"rank": 9,
|
|
"score": 0.5319246206192596
|
|
},
|
|
{
|
|
"cell": "tp4_mns16",
|
|
"rank": 10,
|
|
"score": 0.4355571852372527
|
|
},
|
|
{
|
|
"cell": "tp2_mns8",
|
|
"rank": 11,
|
|
"score": 0.3900292387118045
|
|
},
|
|
{
|
|
"cell": "tp4_mns8",
|
|
"rank": 12,
|
|
"score": 0.25160540206585935
|
|
}
|
|
],
|
|
"reading": "throughput-proxy",
|
|
"simulated_scores": {
|
|
"tp1_mns16": 0.8453202347375465,
|
|
"tp1_mns32": 1.4213857973581445,
|
|
"tp1_mns64": 2.1437448608949605,
|
|
"tp1_mns8": 0.5319246206192596,
|
|
"tp2_mns16": 0.6441840113363596,
|
|
"tp2_mns32": 1.093439153724593,
|
|
"tp2_mns64": 1.7227463064785613,
|
|
"tp2_mns8": 0.3900292387118045,
|
|
"tp4_mns16": 0.4355571852372527,
|
|
"tp4_mns32": 0.760215065879823,
|
|
"tp4_mns64": 1.2313111686958826,
|
|
"tp4_mns8": 0.25160540206585935
|
|
}
|
|
}
|
|
}
|
|
}
|