953 lines
33 KiB
JSON
953 lines
33 KiB
JSON
{
|
|
"cases": {
|
|
"fixed-pd": {
|
|
"comparison": {
|
|
"e2e_mean_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.3067524317507717,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
},
|
|
"e2e_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.3456375801249614,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
},
|
|
"tpot_mean_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.33025510615656906,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
},
|
|
"tpot_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp4_ep1_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.37228890038064755,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
},
|
|
"ttft_mean_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.04237011756982145,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
},
|
|
"ttft_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 1.0,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [],
|
|
"selected_real_regret": 0.002088257708717345,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
}
|
|
},
|
|
"real": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 5729.31677040774,
|
|
"e2e_p90_ms": 5871.668232022785,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 21.044200828709027,
|
|
"tpot_p90_ms": 21.59612646244247,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp4_ep1_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp4_ep1_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp4_ep1_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 363.00995954954925,
|
|
"ttft_p90_ms": 369.09779999405146
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 5769.497427325912,
|
|
"e2e_p90_ms": 5888.945628073998,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 21.187139407074238,
|
|
"tpot_p90_ms": 21.653656356547977,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp4_ep1_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp4_ep1_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp4_ep1_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 366.7432916322386,
|
|
"ttft_p90_ms": 369.99153497163206
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 7486.7986220007915,
|
|
"e2e_p90_ms": 7901.13743103575,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 27.994155607374484,
|
|
"tpot_p90_ms": 29.63612463562658,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp8_ep8_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp8_ep8_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp8_ep8_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 348.25438050341427,
|
|
"ttft_p90_ms": 365.8816539682448
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 7484.121415326708,
|
|
"e2e_p90_ms": 7908.651847974397,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 27.978544004773987,
|
|
"tpot_p90_ms": 29.650389521346227,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp8_ep8_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp8_ep8_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-pd/real/tp8_ep8_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 349.5577253122725,
|
|
"ttft_p90_ms": 365.11919100303203
|
|
}
|
|
},
|
|
"sim": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 22734.095381397616,
|
|
"e2e_p50_ms": 23441.595611495926,
|
|
"e2e_p90_ms": 23484.306701188103,
|
|
"e2e_p95_ms": 23489.511452611594,
|
|
"tpot_mean_ms": 87.77349881741111,
|
|
"tpot_p50_ms": 90.6067822139126,
|
|
"tpot_p90_ms": 90.60678221391389,
|
|
"tpot_p95_ms": 90.60678221391389,
|
|
"ttft_mean_ms": 351.8531829577845,
|
|
"ttft_p50_ms": 351.44748294051453,
|
|
"ttft_p90_ms": 381.14669406792245,
|
|
"ttft_p95_ms": 385.3185272085611
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 22734.095381397616,
|
|
"e2e_p50_ms": 23441.595611495926,
|
|
"e2e_p90_ms": 23484.306701188103,
|
|
"e2e_p95_ms": 23489.511452611594,
|
|
"tpot_mean_ms": 87.77349881741111,
|
|
"tpot_p50_ms": 90.6067822139126,
|
|
"tpot_p90_ms": 90.60678221391389,
|
|
"tpot_p95_ms": 90.60678221391389,
|
|
"ttft_mean_ms": 351.8531829577845,
|
|
"ttft_p50_ms": 351.44748294051453,
|
|
"ttft_p90_ms": 381.14669406792245,
|
|
"ttft_p95_ms": 385.3185272085611
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 16059.899811799207,
|
|
"e2e_p50_ms": 17256.45308650381,
|
|
"e2e_p90_ms": 17280.553823012277,
|
|
"e2e_p95_ms": 17282.746891876577,
|
|
"tpot_mean_ms": 61.58781555229474,
|
|
"tpot_p50_ms": 66.32324900760649,
|
|
"tpot_p90_ms": 66.3246446898248,
|
|
"tpot_p95_ms": 66.3246446898248,
|
|
"ttft_mean_ms": 355.00684596404665,
|
|
"ttft_p50_ms": 355.16695480338,
|
|
"ttft_p90_ms": 368.8582878708644,
|
|
"ttft_p95_ms": 370.503089520021
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 16059.899811799207,
|
|
"e2e_p50_ms": 17256.45308650381,
|
|
"e2e_p90_ms": 17280.553823012277,
|
|
"e2e_p95_ms": 17282.746891876577,
|
|
"tpot_mean_ms": 61.58781555229474,
|
|
"tpot_p50_ms": 66.32324900760649,
|
|
"tpot_p90_ms": 66.3246446898248,
|
|
"tpot_p95_ms": 66.3246446898248,
|
|
"ttft_mean_ms": 355.00684596404665,
|
|
"ttft_p50_ms": 355.16695480338,
|
|
"ttft_p90_ms": 368.8582878708644,
|
|
"ttft_p95_ms": 370.503089520021
|
|
}
|
|
}
|
|
},
|
|
"fixed-po": {
|
|
"comparison": {
|
|
"e2e_mean_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.05873813644442971,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
},
|
|
"e2e_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.016760786592592503,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
},
|
|
"ttft_mean_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.058746722254088946,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
},
|
|
"ttft_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.016744441542810407,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
}
|
|
},
|
|
"real": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 345.40159726161653,
|
|
"e2e_p90_ms": 346.7378600034863,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp4_ep1_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp4_ep1_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp4_ep1_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 345.35322551680514,
|
|
"ttft_p90_ms": 346.6896569589153
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 345.8011649264437,
|
|
"e2e_p90_ms": 347.9283060878515,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp4_ep1_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp4_ep1_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp4_ep1_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 345.75379653310387,
|
|
"ttft_p90_ms": 347.88418107200414
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 326.2389304512842,
|
|
"e2e_p90_ms": 341.0220619989559,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp8_ep8_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp8_ep8_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp8_ep8_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 326.19059710668336,
|
|
"ttft_p90_ms": 340.98013502079993
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 326.89427194042366,
|
|
"e2e_p90_ms": 341.91650594584644,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp8_ep8_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp8_ep8_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/fixed-po/real/tp8_ep8_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 326.846390089937,
|
|
"ttft_p90_ms": 341.87235799618065
|
|
}
|
|
},
|
|
"sim": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 313.84871569626137,
|
|
"e2e_p50_ms": 313.848715696261,
|
|
"e2e_p90_ms": 313.8487156962668,
|
|
"e2e_p95_ms": 313.8487156962668,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 313.84871569626137,
|
|
"ttft_p50_ms": 313.848715696261,
|
|
"ttft_p90_ms": 313.8487156962668,
|
|
"ttft_p95_ms": 313.8487156962668
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 313.84871569626137,
|
|
"e2e_p50_ms": 313.848715696261,
|
|
"e2e_p90_ms": 313.8487156962668,
|
|
"e2e_p95_ms": 313.8487156962668,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 313.84871569626137,
|
|
"ttft_p50_ms": 313.848715696261,
|
|
"ttft_p90_ms": 313.8487156962668,
|
|
"ttft_p95_ms": 313.8487156962668
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 334.2963928223074,
|
|
"e2e_p50_ms": 334.2963928222744,
|
|
"e2e_p90_ms": 334.2963928224325,
|
|
"e2e_p95_ms": 334.2963928224325,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 334.2963928223074,
|
|
"ttft_p50_ms": 334.2963928222744,
|
|
"ttft_p90_ms": 334.2963928224325,
|
|
"ttft_p95_ms": 334.2963928224325
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 334.2963928223074,
|
|
"e2e_p50_ms": 334.2963928222744,
|
|
"e2e_p90_ms": 334.2963928224325,
|
|
"e2e_p95_ms": 334.2963928224325,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 334.2963928223074,
|
|
"ttft_p50_ms": 334.2963928222744,
|
|
"ttft_p90_ms": 334.2963928224325,
|
|
"ttft_p95_ms": 334.2963928224325
|
|
}
|
|
}
|
|
},
|
|
"trace-pd": {
|
|
"comparison": {
|
|
"e2e_mean_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.8333333333333334,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [
|
|
[
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.006154214824712234,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
},
|
|
"e2e_p90_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.6666666666666666,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
[
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.062164651568153984,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": false
|
|
},
|
|
"tpot_mean_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.6666666666666666,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"winner_match": true
|
|
},
|
|
"tpot_p90_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.6666666666666666,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns64",
|
|
"winner_match": true
|
|
},
|
|
"ttft_mean_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.6666666666666666,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": true
|
|
},
|
|
"ttft_p90_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.6666666666666666,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.0,
|
|
"sim_winner": "tp8_ep8_mns128",
|
|
"winner_match": true
|
|
}
|
|
},
|
|
"real": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 203026.22048668287,
|
|
"e2e_p90_ms": 344183.7428959552,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 58.09653520572589,
|
|
"tpot_p90_ms": 75.82137218925735,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp4_ep1_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp4_ep1_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp4_ep1_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 26487.547554662306,
|
|
"ttft_p90_ms": 93044.83354603872
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 198593.25371425177,
|
|
"e2e_p90_ms": 332386.17597706616,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 53.60856802543236,
|
|
"tpot_p90_ms": 64.87020396877685,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp4_ep1_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp4_ep1_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp4_ep1_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 31557.98602907994,
|
|
"ttft_p90_ms": 112666.82129900437
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 182457.35910624202,
|
|
"e2e_p90_ms": 318000.89644105174,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 61.88514020253078,
|
|
"tpot_p90_ms": 78.6175951982538,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp8_ep8_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp8_ep8_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp8_ep8_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 1141.539354895564,
|
|
"ttft_p90_ms": 2616.6939180111513
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 181341.34550937495,
|
|
"e2e_p90_ms": 299389.4552709535,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": 45.89371812653489,
|
|
"tpot_p90_ms": 51.37046630467056,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp8_ep8_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp8_ep8_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-pd/real/tp8_ep8_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 37215.26084531421,
|
|
"ttft_p90_ms": 132333.42359995004
|
|
}
|
|
},
|
|
"sim": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 385157.8895230749,
|
|
"e2e_p50_ms": 367024.8243163692,
|
|
"e2e_p90_ms": 673951.1520207045,
|
|
"e2e_p95_ms": 785718.9717121953,
|
|
"tpot_mean_ms": 100.38179934319204,
|
|
"tpot_p50_ms": 100.42433178975269,
|
|
"tpot_p90_ms": 113.67517808832463,
|
|
"tpot_p95_ms": 121.63558651348922,
|
|
"ttft_mean_ms": 59067.296381594686,
|
|
"ttft_p50_ms": 908.4160127742393,
|
|
"ttft_p90_ms": 200789.2407497713,
|
|
"ttft_p95_ms": 289881.2920798696
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 380604.1804371099,
|
|
"e2e_p50_ms": 378158.20614125655,
|
|
"e2e_p90_ms": 678448.1689084907,
|
|
"e2e_p95_ms": 763969.1713963022,
|
|
"tpot_mean_ms": 93.54579486627527,
|
|
"tpot_p50_ms": 92.86650455014843,
|
|
"tpot_p90_ms": 102.66261969117096,
|
|
"tpot_p95_ms": 112.87824516729198,
|
|
"ttft_mean_ms": 72259.33174260205,
|
|
"ttft_p50_ms": 2897.238841988653,
|
|
"ttft_p90_ms": 265395.14114254655,
|
|
"ttft_p95_ms": 273163.7753733101
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 174939.1805624549,
|
|
"e2e_p50_ms": 166723.75338099696,
|
|
"e2e_p90_ms": 317439.7347771481,
|
|
"e2e_p95_ms": 371471.8509246631,
|
|
"tpot_mean_ms": 58.37204729277776,
|
|
"tpot_p50_ms": 54.29394782230334,
|
|
"tpot_p90_ms": 75.91440592454555,
|
|
"tpot_p95_ms": 84.58610544605006,
|
|
"ttft_mean_ms": 945.9090659083846,
|
|
"ttft_p50_ms": 669.9533543429972,
|
|
"ttft_p90_ms": 2348.607298432925,
|
|
"ttft_p95_ms": 2781.46390848989
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 208352.4497977196,
|
|
"e2e_p50_ms": 207573.22184026084,
|
|
"e2e_p90_ms": 353236.8601573669,
|
|
"e2e_p95_ms": 402705.2991150841,
|
|
"tpot_mean_ms": 52.06723645713691,
|
|
"tpot_p50_ms": 51.64653531173072,
|
|
"tpot_p90_ms": 60.374786839964116,
|
|
"tpot_p95_ms": 62.94158800335912,
|
|
"ttft_mean_ms": 43679.663681651284,
|
|
"ttft_p50_ms": 2772.0384805371,
|
|
"ttft_p90_ms": 151027.16878398336,
|
|
"ttft_p95_ms": 165014.86227423933
|
|
}
|
|
}
|
|
},
|
|
"trace-po": {
|
|
"comparison": {
|
|
"e2e_mean_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.07037121566091509,
|
|
"sim_winner": "tp4_ep1_mns64",
|
|
"winner_match": false
|
|
},
|
|
"e2e_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.21167227304868486,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
},
|
|
"ttft_mean_ms": {
|
|
"informative_pairs": 6,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns128",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp4_ep1_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp8_ep8_mns64",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.07039370625209918,
|
|
"sim_winner": "tp4_ep1_mns64",
|
|
"winner_match": false
|
|
},
|
|
"ttft_p90_ms": {
|
|
"informative_pairs": 4,
|
|
"pair_direction_agreement": 0.0,
|
|
"real_winner": "tp8_ep8_mns64",
|
|
"reversals": [
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns64",
|
|
"tp8_ep8_mns128"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns64"
|
|
],
|
|
[
|
|
"tp4_ep1_mns128",
|
|
"tp8_ep8_mns128"
|
|
]
|
|
],
|
|
"selected_real_regret": 0.2117895828808376,
|
|
"sim_winner": "tp4_ep1_mns128",
|
|
"winner_match": false
|
|
}
|
|
},
|
|
"real": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 1083.6389664896267,
|
|
"e2e_p90_ms": 3156.966591021046,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp4_ep1_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp4_ep1_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp4_ep1_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 1083.561832460368,
|
|
"ttft_p90_ms": 3156.9307829486206
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 1102.0341126990652,
|
|
"e2e_p90_ms": 3162.989347940311,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp4_ep1_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp4_ep1_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp4_ep1_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 1101.9558890756384,
|
|
"ttft_p90_ms": 3162.9534389358014
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 1029.581229927413,
|
|
"e2e_p90_ms": 2612.0267439400777,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp8_ep8_mns128/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp8_ep8_mns128/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp8_ep8_mns128/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 1029.4865175674956,
|
|
"ttft_p90_ms": 2611.9744409807026
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 1043.7361864025345,
|
|
"e2e_p90_ms": 2605.4624350508675,
|
|
"request_samples": 387,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"trials": [
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp8_ep8_mns64/trial1/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp8_ep8_mns64/trial2/results/result.json",
|
|
"/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1/real/trace-po/real/tp8_ep8_mns64/trial3/results/result.json"
|
|
],
|
|
"ttft_mean_ms": 1043.6429704051557,
|
|
"ttft_p90_ms": 2605.180658050813
|
|
}
|
|
},
|
|
"sim": {
|
|
"tp4_ep1_mns128": {
|
|
"e2e_mean_ms": 691.7873605232029,
|
|
"e2e_p50_ms": 211.81272326305088,
|
|
"e2e_p90_ms": 2068.417017674378,
|
|
"e2e_p95_ms": 2843.8890088954663,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 691.7873605232029,
|
|
"ttft_p50_ms": 211.81272326305088,
|
|
"ttft_p90_ms": 2068.417017674378,
|
|
"ttft_p95_ms": 2843.8890088954663
|
|
},
|
|
"tp4_ep1_mns64": {
|
|
"e2e_mean_ms": 691.5722223601863,
|
|
"e2e_p50_ms": 210.89579132517144,
|
|
"e2e_p90_ms": 2068.417017674378,
|
|
"e2e_p95_ms": 2843.8890088954663,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 691.5722223601863,
|
|
"ttft_p50_ms": 210.89579132517144,
|
|
"ttft_p90_ms": 2068.417017674378,
|
|
"ttft_p95_ms": 2843.8890088954663
|
|
},
|
|
"tp8_ep8_mns128": {
|
|
"e2e_mean_ms": 928.7998403823643,
|
|
"e2e_p50_ms": 593.0899264387222,
|
|
"e2e_p90_ms": 2341.4108333476474,
|
|
"e2e_p95_ms": 2898.493085937801,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 928.7998403823643,
|
|
"ttft_p50_ms": 593.0899264387222,
|
|
"ttft_p90_ms": 2341.4108333476474,
|
|
"ttft_p95_ms": 2898.493085937801
|
|
},
|
|
"tp8_ep8_mns64": {
|
|
"e2e_mean_ms": 928.7182564423873,
|
|
"e2e_p50_ms": 593.0899264387222,
|
|
"e2e_p90_ms": 2341.4108333476474,
|
|
"e2e_p95_ms": 2898.493085937801,
|
|
"tpot_mean_ms": null,
|
|
"tpot_p50_ms": null,
|
|
"tpot_p90_ms": null,
|
|
"tpot_p95_ms": null,
|
|
"ttft_mean_ms": 928.7182564423873,
|
|
"ttft_p50_ms": 593.0899264387222,
|
|
"ttft_p90_ms": 2341.4108333476474,
|
|
"ttft_p95_ms": 2898.493085937801
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"schema": "qwen235-v020-simulator-real-comparison-v1"
|
|
}
|