358 lines
12 KiB
JSON
358 lines
12 KiB
JSON
{
|
|
"contract": {
|
|
"component_time": "sum of CUDA kernel durations inside execute range",
|
|
"timing": "CUDA graph-on GPU execute annotations",
|
|
"tp_aggregation": "per-rank; slowest rank mean approximates critical path"
|
|
},
|
|
"label": "tp4-b6-r1",
|
|
"rank_summary": {
|
|
"component_rank_mean_ms": {
|
|
"attention": 0.926535269230769,
|
|
"collective": 0.5587644230769231,
|
|
"linear_norm_rope": 0.8382221538461538,
|
|
"moe": 2.378592788461537,
|
|
"other": 0.02582126923076923,
|
|
"output_head": 0.048266576923076926,
|
|
"router": 0.1886060961538461
|
|
},
|
|
"ranks": 4,
|
|
"slowest_rank_execute_mean_ms": 5.43262976923077,
|
|
"slowest_rank_kernel_busy_mean_ms": 4.971441538461537
|
|
},
|
|
"ranks": [
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9201209999999999,
|
|
"n": 13,
|
|
"p50_ms": 0.920411,
|
|
"p95_ms": 0.9225939999999999,
|
|
"population_std_ms": 0.0019136943731553082
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.5660879230769231,
|
|
"n": 13,
|
|
"p50_ms": 0.5659190000000001,
|
|
"p95_ms": 0.5763190000000002,
|
|
"population_std_ms": 0.005072051101743609
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8359634615384616,
|
|
"n": 13,
|
|
"p50_ms": 0.8359020000000003,
|
|
"p95_ms": 0.8375410000000001,
|
|
"population_std_ms": 0.0010538181579662785
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.3794084615384605,
|
|
"n": 13,
|
|
"p50_ms": 2.3725099999999975,
|
|
"p95_ms": 2.4436449999999983,
|
|
"population_std_ms": 0.038306336006573204
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025853076923076923,
|
|
"n": 13,
|
|
"p50_ms": 0.025792,
|
|
"p95_ms": 0.026593,
|
|
"population_std_ms": 0.0003076432653223938
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.04835438461538462,
|
|
"n": 13,
|
|
"p50_ms": 0.048193,
|
|
"p95_ms": 0.048991999999999994,
|
|
"population_std_ms": 0.0003663435836776379
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.18904192307692302,
|
|
"n": 13,
|
|
"p50_ms": 0.1891829999999999,
|
|
"p95_ms": 0.1920339999999999,
|
|
"population_std_ms": 0.0016883016710719592
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_1(1)": 1,
|
|
"execute_context_0(0)_generation_6(6)": 13,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 5.431982538461539,
|
|
"n": 13,
|
|
"p50_ms": 5.424736,
|
|
"p95_ms": 5.495328000000001,
|
|
"population_std_ms": 0.03904621042109629
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 4.96483023076923,
|
|
"n": 13,
|
|
"p50_ms": 4.958829999999999,
|
|
"p95_ms": 5.028786999999998,
|
|
"population_std_ms": 0.0381875548699665
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.46715230769230887,
|
|
"n": 13,
|
|
"p50_ms": 0.46785100000000135,
|
|
"p95_ms": 0.4726150000000029,
|
|
"population_std_ms": 0.0034291495555826275
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r1-20260723-20260723T091102857920Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798079116726333.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9270714615384612,
|
|
"n": 13,
|
|
"p50_ms": 0.9273969999999994,
|
|
"p95_ms": 0.9299869999999996,
|
|
"population_std_ms": 0.0017639532793125299
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.5647607692307692,
|
|
"n": 13,
|
|
"p50_ms": 0.5641320000000001,
|
|
"p95_ms": 0.5718079999999998,
|
|
"population_std_ms": 0.004140332932615274
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8313579230769229,
|
|
"n": 13,
|
|
"p50_ms": 0.8313590000000001,
|
|
"p95_ms": 0.8333470000000002,
|
|
"population_std_ms": 0.0011534636442897947
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.3681258461538452,
|
|
"n": 13,
|
|
"p50_ms": 2.3586600000000004,
|
|
"p95_ms": 2.4394819999999995,
|
|
"population_std_ms": 0.04033740793290489
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.02615892307692308,
|
|
"n": 13,
|
|
"p50_ms": 0.026175999999999998,
|
|
"p95_ms": 0.026528000000000003,
|
|
"population_std_ms": 0.00026144170155812974
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.048209615384615385,
|
|
"n": 13,
|
|
"p50_ms": 0.048096,
|
|
"p95_ms": 0.049089,
|
|
"population_std_ms": 0.0004454711487612834
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.1910713846153846,
|
|
"n": 13,
|
|
"p50_ms": 0.19120299999999996,
|
|
"p95_ms": 0.1940459999999999,
|
|
"population_std_ms": 0.0014778622574756317
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_1(1)": 1,
|
|
"execute_context_0(0)_generation_6(6)": 13,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 5.430220692307693,
|
|
"n": 13,
|
|
"p50_ms": 5.419602,
|
|
"p95_ms": 5.493810000000001,
|
|
"population_std_ms": 0.03883185853707986
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 4.956755923076922,
|
|
"n": 13,
|
|
"p50_ms": 4.948898,
|
|
"p95_ms": 5.020686999999999,
|
|
"population_std_ms": 0.039974607321805765
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.4734647692307705,
|
|
"n": 13,
|
|
"p50_ms": 0.47312300000000196,
|
|
"p95_ms": 0.47953300000000176,
|
|
"population_std_ms": 0.0027594904533504113
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r1-20260723-20260723T091102857920Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798079120793210.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9307573846153845,
|
|
"n": 13,
|
|
"p50_ms": 0.930942,
|
|
"p95_ms": 0.9329269999999996,
|
|
"population_std_ms": 0.0011322629851124393
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.5509683846153847,
|
|
"n": 13,
|
|
"p50_ms": 0.5507769999999997,
|
|
"p95_ms": 0.559011,
|
|
"population_std_ms": 0.0047502838868763225
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8407734615384612,
|
|
"n": 13,
|
|
"p50_ms": 0.8405719999999995,
|
|
"p95_ms": 0.8421149999999998,
|
|
"population_std_ms": 0.0010200678420412281
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.380922769230769,
|
|
"n": 13,
|
|
"p50_ms": 2.3736940000000004,
|
|
"p95_ms": 2.450815999999999,
|
|
"population_std_ms": 0.04052726901758431
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025844076923076924,
|
|
"n": 13,
|
|
"p50_ms": 0.025858,
|
|
"p95_ms": 0.026433,
|
|
"population_std_ms": 0.0004114569725216555
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.04823130769230769,
|
|
"n": 13,
|
|
"p50_ms": 0.048128,
|
|
"p95_ms": 0.048799999999999996,
|
|
"population_std_ms": 0.0003355393785477529
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.18870923076923074,
|
|
"n": 13,
|
|
"p50_ms": 0.18834999999999993,
|
|
"p95_ms": 0.19122899999999987,
|
|
"population_std_ms": 0.001473603806155092
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_1(1)": 1,
|
|
"execute_context_0(0)_generation_6(6)": 13,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 5.43262976923077,
|
|
"n": 13,
|
|
"p50_ms": 5.423109,
|
|
"p95_ms": 5.49578,
|
|
"population_std_ms": 0.039102580987318745
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 4.966206615384615,
|
|
"n": 13,
|
|
"p50_ms": 4.956755999999999,
|
|
"p95_ms": 5.031173999999998,
|
|
"population_std_ms": 0.039860749997842636
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.46642315384615496,
|
|
"n": 13,
|
|
"p50_ms": 0.4660140000000004,
|
|
"p95_ms": 0.47152800000000017,
|
|
"population_std_ms": 0.002432888149003149
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r1-20260723-20260723T091102857920Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r1/traces/profile/dp0_pp0_tp2_dcp0_ep2_rank2.1784798079116114132.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9281912307692306,
|
|
"n": 13,
|
|
"p50_ms": 0.9282559999999997,
|
|
"p95_ms": 0.9303809999999997,
|
|
"population_std_ms": 0.0012537519785301848
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.5532406153846153,
|
|
"n": 13,
|
|
"p50_ms": 0.5540510000000001,
|
|
"p95_ms": 0.560127,
|
|
"population_std_ms": 0.004131487002529642
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8447937692307695,
|
|
"n": 13,
|
|
"p50_ms": 0.8444480000000002,
|
|
"p95_ms": 0.8477320000000004,
|
|
"population_std_ms": 0.0012637563149865787
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.385914076923075,
|
|
"n": 13,
|
|
"p50_ms": 2.3754019999999985,
|
|
"p95_ms": 2.4513039999999986,
|
|
"population_std_ms": 0.03863391885270188
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025429,
|
|
"n": 13,
|
|
"p50_ms": 0.025372999999999996,
|
|
"p95_ms": 0.02592,
|
|
"population_std_ms": 0.00022032458572960754
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.048271,
|
|
"n": 13,
|
|
"p50_ms": 0.048223999999999996,
|
|
"p95_ms": 0.049121000000000005,
|
|
"population_std_ms": 0.0004103602544856638
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.18560184615384612,
|
|
"n": 13,
|
|
"p50_ms": 0.18550399999999995,
|
|
"p95_ms": 0.18627299999999997,
|
|
"population_std_ms": 0.00039899813883387426
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_1(1)": 1,
|
|
"execute_context_0(0)_generation_6(6)": 13,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 5.429620846153846,
|
|
"n": 13,
|
|
"p50_ms": 5.421929,
|
|
"p95_ms": 5.4902809999999995,
|
|
"population_std_ms": 0.0385510445131288
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 4.971441538461537,
|
|
"n": 13,
|
|
"p50_ms": 4.965375999999999,
|
|
"p95_ms": 5.033802999999999,
|
|
"population_std_ms": 0.03898058259148536
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.4581793076923092,
|
|
"n": 13,
|
|
"p50_ms": 0.45836400000000044,
|
|
"p95_ms": 0.46242600000000156,
|
|
"population_std_ms": 0.002535451754152047
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r1-20260723-20260723T091102857920Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r1/traces/profile/dp0_pp0_tp3_dcp0_ep3_rank3.1784798079118504722.pt.trace.json.gz"
|
|
}
|
|
],
|
|
"schema": "frontier-decode-batch-trace.v1"
|
|
}
|