192 lines
6.3 KiB
JSON
192 lines
6.3 KiB
JSON
{
|
|
"contract": {
|
|
"component_time": "sum of CUDA kernel durations inside execute range",
|
|
"timing": "CUDA graph-on GPU execute annotations",
|
|
"tp_aggregation": "per-rank; slowest rank mean approximates critical path"
|
|
},
|
|
"label": "tp2-b6-r2",
|
|
"rank_summary": {
|
|
"component_rank_mean_ms": {
|
|
"attention": 1.0463109615384618,
|
|
"collective": 0.48337815384615385,
|
|
"linear_norm_rope": 0.929004346153846,
|
|
"moe": 3.7078099230769217,
|
|
"other": 0.025587923076923078,
|
|
"output_head": 0.08800392307692309,
|
|
"router": 0.19478769230769236
|
|
},
|
|
"ranks": 2,
|
|
"slowest_rank_execute_mean_ms": 6.9378406923076925,
|
|
"slowest_rank_kernel_busy_mean_ms": 6.477923153846153
|
|
},
|
|
"ranks": [
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 1.0448520000000003,
|
|
"n": 13,
|
|
"p50_ms": 1.045254000000001,
|
|
"p95_ms": 1.0465970000000002,
|
|
"population_std_ms": 0.0012648827856913572
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.4906066923076923,
|
|
"n": 13,
|
|
"p50_ms": 0.48854500000000006,
|
|
"p95_ms": 0.5128349999999999,
|
|
"population_std_ms": 0.007319711768018179
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.9276054615384616,
|
|
"n": 13,
|
|
"p50_ms": 0.927922,
|
|
"p95_ms": 0.9300229999999996,
|
|
"population_std_ms": 0.0012124997456231853
|
|
},
|
|
"moe": {
|
|
"mean_ms": 3.7032171538461522,
|
|
"n": 13,
|
|
"p50_ms": 3.6941509999999984,
|
|
"p95_ms": 3.9345319999999977,
|
|
"population_std_ms": 0.18506379688215108
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.02531215384615385,
|
|
"n": 13,
|
|
"p50_ms": 0.025345,
|
|
"p95_ms": 0.025984,
|
|
"population_std_ms": 0.0003609239950148982
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.08801007692307693,
|
|
"n": 13,
|
|
"p50_ms": 0.087968,
|
|
"p95_ms": 0.08851200000000001,
|
|
"population_std_ms": 0.0003446964018777317
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.19831961538461543,
|
|
"n": 13,
|
|
"p50_ms": 0.1956510000000001,
|
|
"p95_ms": 0.23433300000000007,
|
|
"population_std_ms": 0.010532589860037432
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_1(1)": 1,
|
|
"execute_context_0(0)_generation_6(6)": 13,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 6.937815615384616,
|
|
"n": 13,
|
|
"p50_ms": 6.920955,
|
|
"p95_ms": 7.177563,
|
|
"population_std_ms": 0.17465452547487034
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 6.477923153846153,
|
|
"n": 13,
|
|
"p50_ms": 6.464021999999998,
|
|
"p95_ms": 6.714968999999998,
|
|
"population_std_ms": 0.17390358211193135
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.4598924615384625,
|
|
"n": 13,
|
|
"p50_ms": 0.46010100000000165,
|
|
"p95_ms": 0.4628480000000019,
|
|
"population_std_ms": 0.0022957863280165477
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b6-r2-20260723-20260723T093020293408Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r2/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784799236964172371.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 1.0477699230769233,
|
|
"n": 13,
|
|
"p50_ms": 1.0473580000000002,
|
|
"p95_ms": 1.0502500000000003,
|
|
"population_std_ms": 0.0016845846994043283
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.47614961538461537,
|
|
"n": 13,
|
|
"p50_ms": 0.47488799999999987,
|
|
"p95_ms": 0.4852860000000001,
|
|
"population_std_ms": 0.006279926721013708
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.9304032307692304,
|
|
"n": 13,
|
|
"p50_ms": 0.9300610000000001,
|
|
"p95_ms": 0.9343409999999996,
|
|
"population_std_ms": 0.0015794439213190966
|
|
},
|
|
"moe": {
|
|
"mean_ms": 3.712402692307691,
|
|
"n": 13,
|
|
"p50_ms": 3.7038869999999973,
|
|
"p95_ms": 3.951868000000003,
|
|
"population_std_ms": 0.18554981438267218
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.02586369230769231,
|
|
"n": 13,
|
|
"p50_ms": 0.025888,
|
|
"p95_ms": 0.026400000000000003,
|
|
"population_std_ms": 0.0002951522122606689
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.08799776923076924,
|
|
"n": 13,
|
|
"p50_ms": 0.087937,
|
|
"p95_ms": 0.088992,
|
|
"population_std_ms": 0.00042456876834413246
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.19125576923076928,
|
|
"n": 13,
|
|
"p50_ms": 0.18793900000000005,
|
|
"p95_ms": 0.23312300000000005,
|
|
"population_std_ms": 0.012158267147240543
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_1(1)": 1,
|
|
"execute_context_0(0)_generation_6(6)": 13,
|
|
"execute_context_2(2049)_generation_4(4)": 1,
|
|
"execute_context_4(8191)_generation_1(1)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 6.9378406923076925,
|
|
"n": 13,
|
|
"p50_ms": 6.920646,
|
|
"p95_ms": 7.1769359999999995,
|
|
"population_std_ms": 0.17404824161814114
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 6.471842692307691,
|
|
"n": 13,
|
|
"p50_ms": 6.4540469999999965,
|
|
"p95_ms": 6.708258000000003,
|
|
"population_std_ms": 0.17518788350506015
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.465998000000001,
|
|
"n": 13,
|
|
"p50_ms": 0.4667860000000026,
|
|
"p95_ms": 0.4705280000000025,
|
|
"population_std_ms": 0.0037250578518997
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_6(6)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b6-r2-20260723-20260723T093020293408Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r2/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784799236965538368.pt.trace.json.gz"
|
|
}
|
|
],
|
|
"schema": "frontier-decode-batch-trace.v1"
|
|
}
|