223 lines
5.9 KiB
JSON
223 lines
5.9 KiB
JSON
{
|
|
"all_corrected_batches_within_measured_support": true,
|
|
"decision": "STATIC_WHOLE_CURVE_FIXES_BC8",
|
|
"mns16_rankings_fast_to_slow": {
|
|
"original": [
|
|
"tp2_mns16",
|
|
"tp1_mns16",
|
|
"tp4_mns16"
|
|
],
|
|
"real": [
|
|
"tp2_mns16",
|
|
"tp4_mns16",
|
|
"tp1_mns16"
|
|
],
|
|
"whole_curve": [
|
|
"tp2_mns16",
|
|
"tp4_mns16",
|
|
"tp1_mns16"
|
|
]
|
|
},
|
|
"points": [
|
|
{
|
|
"config": "tp1_mns16",
|
|
"mns": 16,
|
|
"original": {
|
|
"decode_b_gt_1_fraction": 0.0,
|
|
"decode_batch_histogram": {
|
|
"1": 16383
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
5.493362394
|
|
]
|
|
},
|
|
"e2e_mean_ms": 864.3525078423435,
|
|
"tpot_mean_ms": 5.493362394020072,
|
|
"ttft_mean_ms": 166.6954838017941
|
|
},
|
|
"original_tpot_residual_ms": 0.616915450202109,
|
|
"real_tpot_mean_ms": 4.876446943817963,
|
|
"tp": 1,
|
|
"usage": {
|
|
"collective:moe:tp1-b1:structural-zero": 16383,
|
|
"moe:tp1-b1": 16383
|
|
},
|
|
"whole_curve": {
|
|
"decode_b_gt_1_fraction": 0.0,
|
|
"decode_batch_histogram": {
|
|
"1": 16383
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
5.493362394
|
|
]
|
|
},
|
|
"e2e_mean_ms": 864.3525078423435,
|
|
"tpot_mean_ms": 5.493362394020072,
|
|
"ttft_mean_ms": 166.6954838017941
|
|
},
|
|
"whole_curve_tpot_residual_ms": 0.616915450202109
|
|
},
|
|
{
|
|
"config": "tp2_mns16",
|
|
"mns": 16,
|
|
"original": {
|
|
"decode_b_gt_1_fraction": 0.0,
|
|
"decode_batch_histogram": {
|
|
"1": 16383
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
4.928319445
|
|
]
|
|
},
|
|
"e2e_mean_ms": 744.4521695355745,
|
|
"tpot_mean_ms": 4.9283194445308744,
|
|
"ttft_mean_ms": 118.55560008015348
|
|
},
|
|
"original_tpot_residual_ms": 0.6235698385840074,
|
|
"real_tpot_mean_ms": 4.304749605946867,
|
|
"tp": 2,
|
|
"usage": {
|
|
"collective:attention:tp2-b1": 16383,
|
|
"collective:moe:tp2-b1": 16383,
|
|
"fused_norm_deduction:attn:tp2-b1": 16383,
|
|
"fused_norm_deduction:mlp:tp2-b1": 16383,
|
|
"moe:tp2-b1": 16383
|
|
},
|
|
"whole_curve": {
|
|
"decode_b_gt_1_fraction": 0.0,
|
|
"decode_batch_histogram": {
|
|
"1": 16383
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
4.928319445
|
|
]
|
|
},
|
|
"e2e_mean_ms": 744.4521695355745,
|
|
"tpot_mean_ms": 4.9283194445308744,
|
|
"ttft_mean_ms": 118.55560008015348
|
|
},
|
|
"whole_curve_tpot_residual_ms": 0.6235698385840074
|
|
},
|
|
{
|
|
"config": "tp4_mns16",
|
|
"mns": 16,
|
|
"original": {
|
|
"decode_b_gt_1_fraction": 0.4814983594604448,
|
|
"decode_batch_histogram": {
|
|
"1": 5689,
|
|
"2": 5283
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
4.738406304
|
|
],
|
|
"2": [
|
|
5.411344412
|
|
]
|
|
},
|
|
"e2e_mean_ms": 840.3052136178922,
|
|
"tpot_mean_ms": 5.863803518571768,
|
|
"ttft_mean_ms": 95.60216675927755
|
|
},
|
|
"original_tpot_residual_ms": 1.4143723087101598,
|
|
"real_tpot_mean_ms": 4.449431209861608,
|
|
"tp": 4,
|
|
"usage": {
|
|
"collective:attention:tp4-b1": 7393,
|
|
"collective:attention:tp4-b2": 4431,
|
|
"collective:moe:tp4-b1": 7393,
|
|
"collective:moe:tp4-b2": 4431,
|
|
"fused_norm_deduction:attn:tp4-b1": 7393,
|
|
"fused_norm_deduction:attn:tp4-b2": 4431,
|
|
"fused_norm_deduction:mlp:tp4-b1": 7393,
|
|
"fused_norm_deduction:mlp:tp4-b2": 4431,
|
|
"moe:tp4-b1": 7393,
|
|
"moe:tp4-b2": 4431,
|
|
"whole_decode:tp4-b2:target_ms=4.623176437": 4431
|
|
},
|
|
"whole_curve": {
|
|
"decode_b_gt_1_fraction": 0.37474627875507444,
|
|
"decode_batch_histogram": {
|
|
"1": 7393,
|
|
"2": 4431
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
4.738406304
|
|
],
|
|
"2": [
|
|
4.623176437
|
|
]
|
|
},
|
|
"e2e_mean_ms": 777.2050553866409,
|
|
"tpot_mean_ms": 5.367470055757525,
|
|
"ttft_mean_ms": 95.5363583054353
|
|
},
|
|
"whole_curve_tpot_residual_ms": 0.918038845895917
|
|
},
|
|
{
|
|
"config": "tp4_mns32",
|
|
"mns": 32,
|
|
"original": {
|
|
"decode_b_gt_1_fraction": 0.4814983594604448,
|
|
"decode_batch_histogram": {
|
|
"1": 5689,
|
|
"2": 5283
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
4.738406304
|
|
],
|
|
"2": [
|
|
5.411344412
|
|
]
|
|
},
|
|
"e2e_mean_ms": 840.3052136178922,
|
|
"tpot_mean_ms": 5.863803518571768,
|
|
"ttft_mean_ms": 95.60216675927755
|
|
},
|
|
"original_tpot_residual_ms": 1.4977765516959103,
|
|
"real_tpot_mean_ms": 4.366026966875857,
|
|
"tp": 4,
|
|
"usage": {
|
|
"collective:attention:tp4-b1": 7393,
|
|
"collective:attention:tp4-b2": 4431,
|
|
"collective:moe:tp4-b1": 7393,
|
|
"collective:moe:tp4-b2": 4431,
|
|
"fused_norm_deduction:attn:tp4-b1": 7393,
|
|
"fused_norm_deduction:attn:tp4-b2": 4431,
|
|
"fused_norm_deduction:mlp:tp4-b1": 7393,
|
|
"fused_norm_deduction:mlp:tp4-b2": 4431,
|
|
"moe:tp4-b1": 7393,
|
|
"moe:tp4-b2": 4431,
|
|
"whole_decode:tp4-b2:target_ms=4.623176437": 4431
|
|
},
|
|
"whole_curve": {
|
|
"decode_b_gt_1_fraction": 0.37474627875507444,
|
|
"decode_batch_histogram": {
|
|
"1": 7393,
|
|
"2": 4431
|
|
},
|
|
"decode_service_ms": {
|
|
"1": [
|
|
4.738406304
|
|
],
|
|
"2": [
|
|
4.623176437
|
|
]
|
|
},
|
|
"e2e_mean_ms": 777.2050553866409,
|
|
"tpot_mean_ms": 5.367470055757525,
|
|
"ttft_mean_ms": 95.5363583054353
|
|
},
|
|
"whole_curve_tpot_residual_ms": 1.0014430888816674
|
|
}
|
|
],
|
|
"schema": "frontier-decode-grid-bc8-verdict.v1",
|
|
"whole_curve_restores_real_mns16_ranking": true
|
|
}
|