354 lines
12 KiB
JSON
354 lines
12 KiB
JSON
{
|
|
"contract": {
|
|
"component_time": "sum of CUDA kernel durations inside execute range",
|
|
"timing": "CUDA graph-on GPU execute annotations",
|
|
"tp_aggregation": "per-rank; slowest rank mean approximates critical path"
|
|
},
|
|
"label": "tp4-b6-r3",
|
|
"rank_summary": {
|
|
"component_rank_mean_ms": {
|
|
"attention": 0.9331604285714286,
|
|
"collective": 3.192824375,
|
|
"linear_norm_rope": 0.8404031785714285,
|
|
"moe": 2.399789857142857,
|
|
"other": 0.025532517857142854,
|
|
"output_head": 0.048368125,
|
|
"router": 0.19489794642857136
|
|
},
|
|
"ranks": 4,
|
|
"slowest_rank_execute_mean_ms": 8.633801928571428,
|
|
"slowest_rank_kernel_busy_mean_ms": 8.521951642857141
|
|
},
|
|
"ranks": [
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9356066428571428,
|
|
"n": 14,
|
|
"p50_ms": 0.9354089999999998,
|
|
"p95_ms": 0.9385260000000003,
|
|
"population_std_ms": 0.001676932599682409
|
|
},
|
|
"collective": {
|
|
"mean_ms": 4.066270071428571,
|
|
"n": 14,
|
|
"p50_ms": 0.55129,
|
|
"p95_ms": 16.453366,
|
|
"population_std_ms": 5.619590739357225
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8387703571428572,
|
|
"n": 14,
|
|
"p50_ms": 0.8386610000000003,
|
|
"p95_ms": 0.8405769999999998,
|
|
"population_std_ms": 0.001042224242880996
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.403109357142857,
|
|
"n": 14,
|
|
"p50_ms": 2.3945385000000012,
|
|
"p95_ms": 2.4938269999999996,
|
|
"population_std_ms": 0.04863547144920812
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025629785714285713,
|
|
"n": 14,
|
|
"p50_ms": 0.025696499999999997,
|
|
"p95_ms": 0.02624,
|
|
"population_std_ms": 0.00037226741129221103
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.04842978571428571,
|
|
"n": 14,
|
|
"p50_ms": 0.048384,
|
|
"p95_ms": 0.049248,
|
|
"population_std_ms": 0.00035716626351871235
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.19324292857142852,
|
|
"n": 14,
|
|
"p50_ms": 0.19292599999999996,
|
|
"p95_ms": 0.1981709999999999,
|
|
"population_std_ms": 0.0022171815591706862
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_6(6)": 14,
|
|
"execute_context_1(2048)_generation_2(2)": 1,
|
|
"execute_context_3(6144)_generation_3(3)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 8.630969642857142,
|
|
"n": 14,
|
|
"p50_ms": 5.180047500000001,
|
|
"p95_ms": 20.987705000000002,
|
|
"population_std_ms": 5.5894495457541336
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 8.511058928571428,
|
|
"n": 14,
|
|
"p50_ms": 5.061531499999999,
|
|
"p95_ms": 20.864821999999997,
|
|
"population_std_ms": 5.589475958722368
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.11991071428571518,
|
|
"n": 14,
|
|
"p50_ms": 0.12010500000000235,
|
|
"p95_ms": 0.12316100000000052,
|
|
"population_std_ms": 0.002551737571274248
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 14,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784799692262262652.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9304471428571427,
|
|
"n": 14,
|
|
"p50_ms": 0.930654,
|
|
"p95_ms": 0.9319719999999995,
|
|
"population_std_ms": 0.0009635016389000793
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.5773663571428572,
|
|
"n": 14,
|
|
"p50_ms": 0.5793035000000002,
|
|
"p95_ms": 0.59031,
|
|
"population_std_ms": 0.008760785031745086
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8337531428571427,
|
|
"n": 14,
|
|
"p50_ms": 0.8337829999999997,
|
|
"p95_ms": 0.8351629999999995,
|
|
"population_std_ms": 0.0008970194182593586
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.3872937857142857,
|
|
"n": 14,
|
|
"p50_ms": 2.3779580000000005,
|
|
"p95_ms": 2.4758159999999987,
|
|
"population_std_ms": 0.04801953596369253
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.02552642857142857,
|
|
"n": 14,
|
|
"p50_ms": 0.025569500000000002,
|
|
"p95_ms": 0.025981999999999998,
|
|
"population_std_ms": 0.00035502126984283196
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.048183142857142855,
|
|
"n": 14,
|
|
"p50_ms": 0.048112,
|
|
"p95_ms": 0.049249,
|
|
"population_std_ms": 0.00048348686747460613
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.19290549999999992,
|
|
"n": 14,
|
|
"p50_ms": 0.19270699999999996,
|
|
"p95_ms": 0.1958099999999999,
|
|
"population_std_ms": 0.0013893215584799379
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_6(6)": 14,
|
|
"execute_context_1(2048)_generation_2(2)": 1,
|
|
"execute_context_3(6144)_generation_3(3)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 8.221614142857144,
|
|
"n": 14,
|
|
"p50_ms": 5.1788165,
|
|
"p95_ms": 21.941601,
|
|
"population_std_ms": 5.439025316289738
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 4.9954754999999995,
|
|
"n": 14,
|
|
"p50_ms": 4.988138500000001,
|
|
"p95_ms": 5.080260999999998,
|
|
"population_std_ms": 0.04987686944831562
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 3.2261386428571432,
|
|
"n": 14,
|
|
"p50_ms": 0.12194000000000038,
|
|
"p95_ms": 16.981787,
|
|
"population_std_ms": 5.465271976091736
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 14,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784799692310997248.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9339937142857143,
|
|
"n": 14,
|
|
"p50_ms": 0.9335875,
|
|
"p95_ms": 0.9370900000000002,
|
|
"population_std_ms": 0.001917785904502957
|
|
},
|
|
"collective": {
|
|
"mean_ms": 4.071160642857143,
|
|
"n": 14,
|
|
"p50_ms": 0.5595504999999998,
|
|
"p95_ms": 16.441921000000004,
|
|
"population_std_ms": 5.617545177819675
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8418829285714284,
|
|
"n": 14,
|
|
"p50_ms": 0.8420469999999998,
|
|
"p95_ms": 0.8443809999999999,
|
|
"population_std_ms": 0.0015434930914874625
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.3997612142857148,
|
|
"n": 14,
|
|
"p50_ms": 2.3921425000000003,
|
|
"p95_ms": 2.4903699999999995,
|
|
"population_std_ms": 0.04851648560788174
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025574571428571426,
|
|
"n": 14,
|
|
"p50_ms": 0.025535999999999996,
|
|
"p95_ms": 0.026114000000000002,
|
|
"population_std_ms": 0.0002516030644503687
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.04837264285714286,
|
|
"n": 14,
|
|
"p50_ms": 0.048512,
|
|
"p95_ms": 0.048960000000000004,
|
|
"population_std_ms": 0.00043459432761120634
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.1906739285714285,
|
|
"n": 14,
|
|
"p50_ms": 0.19033549999999994,
|
|
"p95_ms": 0.19376099999999996,
|
|
"population_std_ms": 0.0011997104093599232
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_6(6)": 14,
|
|
"execute_context_1(2048)_generation_2(2)": 1,
|
|
"execute_context_3(6144)_generation_3(3)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 8.630474571428572,
|
|
"n": 14,
|
|
"p50_ms": 5.184212499999999,
|
|
"p95_ms": 20.985098999999998,
|
|
"population_std_ms": 5.5897660056633045
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 8.511419642857144,
|
|
"n": 14,
|
|
"p50_ms": 5.0612965,
|
|
"p95_ms": 20.866541000000005,
|
|
"population_std_ms": 5.589603699080663
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.11905492857142821,
|
|
"n": 14,
|
|
"p50_ms": 0.1185754999999995,
|
|
"p95_ms": 0.1234339999999996,
|
|
"population_std_ms": 0.0022576749881340562
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 14,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp2_dcp0_ep2_rank2.1784799692265838678.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 0.9325942142857142,
|
|
"n": 14,
|
|
"p50_ms": 0.9325534999999999,
|
|
"p95_ms": 0.9368339999999997,
|
|
"population_std_ms": 0.0022950169367122264
|
|
},
|
|
"collective": {
|
|
"mean_ms": 4.056500428571428,
|
|
"n": 14,
|
|
"p50_ms": 0.5449905,
|
|
"p95_ms": 16.437329999999996,
|
|
"population_std_ms": 5.618098886085954
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.8472062857142859,
|
|
"n": 14,
|
|
"p50_ms": 0.8469400000000002,
|
|
"p95_ms": 0.8496459999999999,
|
|
"population_std_ms": 0.001113636605808041
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.4089950714285697,
|
|
"n": 14,
|
|
"p50_ms": 2.400836999999998,
|
|
"p95_ms": 2.491555999999999,
|
|
"population_std_ms": 0.04692533473274991
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025399285714285715,
|
|
"n": 14,
|
|
"p50_ms": 0.025391999999999998,
|
|
"p95_ms": 0.025988,
|
|
"population_std_ms": 0.00023926925316514007
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.04848692857142857,
|
|
"n": 14,
|
|
"p50_ms": 0.048528,
|
|
"p95_ms": 0.04912,
|
|
"population_std_ms": 0.0004430105552250606
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.20276942857142852,
|
|
"n": 14,
|
|
"p50_ms": 0.2021774999999999,
|
|
"p95_ms": 0.20793499999999998,
|
|
"population_std_ms": 0.0026834677382150296
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_6(6)": 14,
|
|
"execute_context_1(2048)_generation_2(2)": 1,
|
|
"execute_context_3(6144)_generation_3(3)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 8.633801928571428,
|
|
"n": 14,
|
|
"p50_ms": 5.179864499999999,
|
|
"p95_ms": 20.988474,
|
|
"population_std_ms": 5.589972337133016
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 8.521951642857141,
|
|
"n": 14,
|
|
"p50_ms": 5.071154499999998,
|
|
"p95_ms": 20.875197999999994,
|
|
"population_std_ms": 5.589281480492545
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.11185028571428772,
|
|
"n": 14,
|
|
"p50_ms": 0.11232300000000128,
|
|
"p95_ms": 0.11633899999999997,
|
|
"population_std_ms": 0.0023228722192460186
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 14,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp4-b6-r3-20260723-20260723T093751542606Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp4-b6-r3/traces/profile/dp0_pp0_tp3_dcp0_ep3_rank3.1784799692262484339.pt.trace.json.gz"
|
|
}
|
|
],
|
|
"schema": "frontier-decode-batch-trace.v1"
|
|
}
|