190 lines
6.2 KiB
JSON
190 lines
6.2 KiB
JSON
{
|
|
"contract": {
|
|
"component_time": "sum of CUDA kernel durations inside execute range",
|
|
"timing": "CUDA graph-on GPU execute annotations",
|
|
"tp_aggregation": "per-rank; slowest rank mean approximates critical path"
|
|
},
|
|
"label": "tp2-b6-r1",
|
|
"rank_summary": {
|
|
"component_rank_mean_ms": {
|
|
"attention": 1.0590365714285717,
|
|
"collective": 0.48200882142857143,
|
|
"linear_norm_rope": 0.9312401785714285,
|
|
"moe": 3.678020964285713,
|
|
"other": 0.02528782142857143,
|
|
"output_head": 0.08801067857142857,
|
|
"router": 0.1985437857142856
|
|
},
|
|
"ranks": 2,
|
|
"slowest_rank_execute_mean_ms": 6.570553642857143,
|
|
"slowest_rank_kernel_busy_mean_ms": 6.462748499999998
|
|
},
|
|
"ranks": [
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 1.056612428571429,
|
|
"n": 14,
|
|
"p50_ms": 1.0567985000000006,
|
|
"p95_ms": 1.0597780000000006,
|
|
"population_std_ms": 0.0016962762879018302
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.4931447857142857,
|
|
"n": 14,
|
|
"p50_ms": 0.4922715000000001,
|
|
"p95_ms": 0.5031650000000001,
|
|
"population_std_ms": 0.0056488543235214805
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.9297670714285715,
|
|
"n": 14,
|
|
"p50_ms": 0.9300385,
|
|
"p95_ms": 0.9327710000000003,
|
|
"population_std_ms": 0.0016833948889195887
|
|
},
|
|
"moe": {
|
|
"mean_ms": 3.6738179999999985,
|
|
"n": 14,
|
|
"p50_ms": 3.667117499999998,
|
|
"p95_ms": 3.9196679999999993,
|
|
"population_std_ms": 0.1767424054840424
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025222642857142857,
|
|
"n": 14,
|
|
"p50_ms": 0.025185,
|
|
"p95_ms": 0.025983,
|
|
"population_std_ms": 0.00032333101286073835
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.08808714285714285,
|
|
"n": 14,
|
|
"p50_ms": 0.08807999999999999,
|
|
"p95_ms": 0.08870399999999999,
|
|
"population_std_ms": 0.0003292721100381563
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.19489707142857132,
|
|
"n": 14,
|
|
"p50_ms": 0.19094449999999988,
|
|
"p95_ms": 0.23763299999999998,
|
|
"population_std_ms": 0.012033189735443558
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_6(6)": 14,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 6.569652285714286,
|
|
"n": 14,
|
|
"p50_ms": 6.5619125,
|
|
"p95_ms": 6.809498,
|
|
"population_std_ms": 0.16639578247764245
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 6.461549142857142,
|
|
"n": 14,
|
|
"p50_ms": 6.452550999999998,
|
|
"p95_ms": 6.705629999999999,
|
|
"population_std_ms": 0.166627622430915
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.10810314285714392,
|
|
"n": 14,
|
|
"p50_ms": 0.10853050000000275,
|
|
"p95_ms": 0.1127690000000019,
|
|
"population_std_ms": 0.0025825165428091904
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 14,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b6-r1-20260723-20260723T091712228992Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798447891492657.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 1.0614607142857144,
|
|
"n": 14,
|
|
"p50_ms": 1.0611505000000003,
|
|
"p95_ms": 1.0646370000000003,
|
|
"population_std_ms": 0.0017739766639056317
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.47087285714285715,
|
|
"n": 14,
|
|
"p50_ms": 0.46880499999999997,
|
|
"p95_ms": 0.48576,
|
|
"population_std_ms": 0.005842167135296126
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.9327132857142855,
|
|
"n": 14,
|
|
"p50_ms": 0.9321509999999997,
|
|
"p95_ms": 0.9383449999999995,
|
|
"population_std_ms": 0.002275944997721476
|
|
},
|
|
"moe": {
|
|
"mean_ms": 3.6822239285714278,
|
|
"n": 14,
|
|
"p50_ms": 3.6781699999999997,
|
|
"p95_ms": 3.9217039999999996,
|
|
"population_std_ms": 0.17642357594982436
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.025353000000000004,
|
|
"n": 14,
|
|
"p50_ms": 0.02536,
|
|
"p95_ms": 0.026433,
|
|
"population_std_ms": 0.0005135631829260575
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.08793421428571428,
|
|
"n": 14,
|
|
"p50_ms": 0.0880325,
|
|
"p95_ms": 0.088833,
|
|
"population_std_ms": 0.0005008674159569053
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.2021904999999999,
|
|
"n": 14,
|
|
"p50_ms": 0.1997179999999999,
|
|
"p95_ms": 0.237536,
|
|
"population_std_ms": 0.009894411602732434
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_6(6)": 14,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 6.570553642857143,
|
|
"n": 14,
|
|
"p50_ms": 6.5663730000000005,
|
|
"p95_ms": 6.809703000000001,
|
|
"population_std_ms": 0.16604228180350736
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 6.462748499999998,
|
|
"n": 14,
|
|
"p50_ms": 6.456941499999999,
|
|
"p95_ms": 6.703455999999998,
|
|
"population_std_ms": 0.16541565407168776
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.10780514285714433,
|
|
"n": 14,
|
|
"p50_ms": 0.10772100000000107,
|
|
"p95_ms": 0.1121450000000026,
|
|
"population_std_ms": 0.0019970330441916122
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 14,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b6-r1-20260723-20260723T091712228992Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798447895471197.pt.trace.json.gz"
|
|
}
|
|
],
|
|
"schema": "frontier-decode-batch-trace.v1"
|
|
}
|