192 lines
6.4 KiB
JSON
192 lines
6.4 KiB
JSON
{
|
|
"contract": {
|
|
"component_time": "sum of CUDA kernel durations inside execute range",
|
|
"timing": "CUDA graph-on GPU execute annotations",
|
|
"tp_aggregation": "per-rank; slowest rank mean approximates critical path"
|
|
},
|
|
"label": "tp2-b8-r1",
|
|
"rank_summary": {
|
|
"component_rank_mean_ms": {
|
|
"attention": 1.4162915000000003,
|
|
"collective": 0.5046254615384616,
|
|
"linear_norm_rope": 0.933206192307692,
|
|
"moe": 2.681186538461537,
|
|
"other": 0.023443884615384616,
|
|
"output_head": 0.08826711538461537,
|
|
"router": 0.19981553846153838
|
|
},
|
|
"ranks": 2,
|
|
"slowest_rank_execute_mean_ms": 5.967337846153845,
|
|
"slowest_rank_kernel_busy_mean_ms": 5.849403615384613
|
|
},
|
|
"ranks": [
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 1.4268944615384618,
|
|
"n": 13,
|
|
"p50_ms": 1.4273899999999997,
|
|
"p95_ms": 1.4289299999999996,
|
|
"population_std_ms": 0.0013227596802130133
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.4871666153846154,
|
|
"n": 13,
|
|
"p50_ms": 0.4847079999999998,
|
|
"p95_ms": 0.5161030000000001,
|
|
"population_std_ms": 0.009071831835526453
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.9357503846153842,
|
|
"n": 13,
|
|
"p50_ms": 0.9351129999999993,
|
|
"p95_ms": 0.9375759999999999,
|
|
"population_std_ms": 0.0012191190727774577
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.6808982307692295,
|
|
"n": 13,
|
|
"p50_ms": 2.729251,
|
|
"p95_ms": 2.8241379999999983,
|
|
"population_std_ms": 0.11820978926868417
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.023988000000000002,
|
|
"n": 13,
|
|
"p50_ms": 0.023935,
|
|
"p95_ms": 0.024639999999999995,
|
|
"population_std_ms": 0.0002731643630321658
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.08829546153846153,
|
|
"n": 13,
|
|
"p50_ms": 0.088256,
|
|
"p95_ms": 0.08921599999999999,
|
|
"population_std_ms": 0.00048060836693389233
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.2012756923076922,
|
|
"n": 13,
|
|
"p50_ms": 0.20156699999999994,
|
|
"p95_ms": 0.20582599999999993,
|
|
"population_std_ms": 0.0022949889352713113
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_8(8)": 13,
|
|
"execute_context_1(2048)_generation_1(1)": 1,
|
|
"execute_context_3(4098)_generation_5(5)": 1,
|
|
"execute_context_4(8190)_generation_2(2)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 5.966980846153846,
|
|
"n": 13,
|
|
"p50_ms": 6.004943,
|
|
"p95_ms": 6.112047,
|
|
"population_std_ms": 0.11913819132925879
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 5.844268846153844,
|
|
"n": 13,
|
|
"p50_ms": 5.885293999999999,
|
|
"p95_ms": 5.987602999999998,
|
|
"population_std_ms": 0.11849233575983938
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.12271200000000157,
|
|
"n": 13,
|
|
"p50_ms": 0.12307100000000126,
|
|
"p95_ms": 0.12728500000000142,
|
|
"population_std_ms": 0.0027851573357023283
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_8(8)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b8-r1-20260723-20260723T091715999468Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b8-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798455024962606.pt.trace.json.gz"
|
|
},
|
|
{
|
|
"components": {
|
|
"attention": {
|
|
"mean_ms": 1.405688538461539,
|
|
"n": 13,
|
|
"p50_ms": 1.4057520000000006,
|
|
"p95_ms": 1.4114940000000005,
|
|
"population_std_ms": 0.0022431369386294517
|
|
},
|
|
"collective": {
|
|
"mean_ms": 0.5220843076923077,
|
|
"n": 13,
|
|
"p50_ms": 0.5193970000000001,
|
|
"p95_ms": 0.5542680000000001,
|
|
"population_std_ms": 0.010336781774023671
|
|
},
|
|
"linear_norm_rope": {
|
|
"mean_ms": 0.9306619999999997,
|
|
"n": 13,
|
|
"p50_ms": 0.9303679999999996,
|
|
"p95_ms": 0.9333509999999994,
|
|
"population_std_ms": 0.0014067406955851874
|
|
},
|
|
"moe": {
|
|
"mean_ms": 2.6814748461538445,
|
|
"n": 13,
|
|
"p50_ms": 2.7311729999999965,
|
|
"p95_ms": 2.8229829999999962,
|
|
"population_std_ms": 0.11917174241136594
|
|
},
|
|
"other": {
|
|
"mean_ms": 0.02289976923076923,
|
|
"n": 13,
|
|
"p50_ms": 0.022943999999999996,
|
|
"p95_ms": 0.02352,
|
|
"population_std_ms": 0.0002890651121343674
|
|
},
|
|
"output_head": {
|
|
"mean_ms": 0.08823876923076922,
|
|
"n": 13,
|
|
"p50_ms": 0.088193,
|
|
"p95_ms": 0.088768,
|
|
"population_std_ms": 0.0003492319383240336
|
|
},
|
|
"router": {
|
|
"mean_ms": 0.19835538461538452,
|
|
"n": 13,
|
|
"p50_ms": 0.1982099999999999,
|
|
"p95_ms": 0.2010899999999999,
|
|
"population_std_ms": 0.0022037652233764197
|
|
}
|
|
},
|
|
"execute_annotation_histogram": {
|
|
"execute_context_0(0)_generation_8(8)": 13,
|
|
"execute_context_1(2048)_generation_1(1)": 1,
|
|
"execute_context_3(4098)_generation_5(5)": 1,
|
|
"execute_context_4(8190)_generation_2(2)": 1
|
|
},
|
|
"execute_wall": {
|
|
"mean_ms": 5.967337846153845,
|
|
"n": 13,
|
|
"p50_ms": 6.0104049999999996,
|
|
"p95_ms": 6.1097969999999995,
|
|
"population_std_ms": 0.11987574925913026
|
|
},
|
|
"gpu_kernel_busy": {
|
|
"mean_ms": 5.849403615384613,
|
|
"n": 13,
|
|
"p50_ms": 5.889734999999996,
|
|
"p95_ms": 5.993167999999996,
|
|
"population_std_ms": 0.11980136802451119
|
|
},
|
|
"non_kernel_gap": {
|
|
"mean_ms": 0.11793423076923243,
|
|
"n": 13,
|
|
"p50_ms": 0.11715500000000123,
|
|
"p95_ms": 0.12245800000000262,
|
|
"population_std_ms": 0.002861606035183301
|
|
},
|
|
"selected_execute_annotation": "execute_context_0(0)_generation_8(8)",
|
|
"steps": 13,
|
|
"trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b8-r1-20260723-20260723T091715999468Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b8-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798455030360286.pt.trace.json.gz"
|
|
}
|
|
],
|
|
"schema": "frontier-decode-batch-trace.v1"
|
|
}
|