{ "contract": { "component_time": "sum of CUDA kernel durations inside execute range", "timing": "CUDA graph-on GPU execute annotations", "tp_aggregation": "per-rank; slowest rank mean approximates critical path" }, "label": "tp2-b6-r1", "rank_summary": { "component_rank_mean_ms": { "attention": 1.0590365714285717, "collective": 0.48200882142857143, "linear_norm_rope": 0.9312401785714285, "moe": 3.678020964285713, "other": 0.02528782142857143, "output_head": 0.08801067857142857, "router": 0.1985437857142856 }, "ranks": 2, "slowest_rank_execute_mean_ms": 6.570553642857143, "slowest_rank_kernel_busy_mean_ms": 6.462748499999998 }, "ranks": [ { "components": { "attention": { "mean_ms": 1.056612428571429, "n": 14, "p50_ms": 1.0567985000000006, "p95_ms": 1.0597780000000006, "population_std_ms": 0.0016962762879018302 }, "collective": { "mean_ms": 0.4931447857142857, "n": 14, "p50_ms": 0.4922715000000001, "p95_ms": 0.5031650000000001, "population_std_ms": 0.0056488543235214805 }, "linear_norm_rope": { "mean_ms": 0.9297670714285715, "n": 14, "p50_ms": 0.9300385, "p95_ms": 0.9327710000000003, "population_std_ms": 0.0016833948889195887 }, "moe": { "mean_ms": 3.6738179999999985, "n": 14, "p50_ms": 3.667117499999998, "p95_ms": 3.9196679999999993, "population_std_ms": 0.1767424054840424 }, "other": { "mean_ms": 0.025222642857142857, "n": 14, "p50_ms": 0.025185, "p95_ms": 0.025983, "population_std_ms": 0.00032333101286073835 }, "output_head": { "mean_ms": 0.08808714285714285, "n": 14, "p50_ms": 0.08807999999999999, "p95_ms": 0.08870399999999999, "population_std_ms": 0.0003292721100381563 }, "router": { "mean_ms": 0.19489707142857132, "n": 14, "p50_ms": 0.19094449999999988, "p95_ms": 0.23763299999999998, "population_std_ms": 0.012033189735443558 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_6(6)": 14, "execute_context_2(2049)_generation_4(4)": 1, "execute_context_4(8191)_generation_1(1)": 1 }, "execute_wall": { "mean_ms": 6.569652285714286, "n": 14, "p50_ms": 6.5619125, "p95_ms": 6.809498, "population_std_ms": 0.16639578247764245 }, "gpu_kernel_busy": { "mean_ms": 6.461549142857142, "n": 14, "p50_ms": 6.452550999999998, "p95_ms": 6.705629999999999, "population_std_ms": 0.166627622430915 }, "non_kernel_gap": { "mean_ms": 0.10810314285714392, "n": 14, "p50_ms": 0.10853050000000275, "p95_ms": 0.1127690000000019, "population_std_ms": 0.0025825165428091904 }, "selected_execute_annotation": "execute_context_0(0)_generation_6(6)", "steps": 14, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b6-r1-20260723-20260723T091712228992Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798447891492657.pt.trace.json.gz" }, { "components": { "attention": { "mean_ms": 1.0614607142857144, "n": 14, "p50_ms": 1.0611505000000003, "p95_ms": 1.0646370000000003, "population_std_ms": 0.0017739766639056317 }, "collective": { "mean_ms": 0.47087285714285715, "n": 14, "p50_ms": 0.46880499999999997, "p95_ms": 0.48576, "population_std_ms": 0.005842167135296126 }, "linear_norm_rope": { "mean_ms": 0.9327132857142855, "n": 14, "p50_ms": 0.9321509999999997, "p95_ms": 0.9383449999999995, "population_std_ms": 0.002275944997721476 }, "moe": { "mean_ms": 3.6822239285714278, "n": 14, "p50_ms": 3.6781699999999997, "p95_ms": 3.9217039999999996, "population_std_ms": 0.17642357594982436 }, "other": { "mean_ms": 0.025353000000000004, "n": 14, "p50_ms": 0.02536, "p95_ms": 0.026433, "population_std_ms": 0.0005135631829260575 }, "output_head": { "mean_ms": 0.08793421428571428, "n": 14, "p50_ms": 0.0880325, "p95_ms": 0.088833, "population_std_ms": 0.0005008674159569053 }, "router": { "mean_ms": 0.2021904999999999, "n": 14, "p50_ms": 0.1997179999999999, "p95_ms": 0.237536, "population_std_ms": 0.009894411602732434 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_6(6)": 14, "execute_context_2(2049)_generation_4(4)": 1, "execute_context_4(8191)_generation_1(1)": 1 }, "execute_wall": { "mean_ms": 6.570553642857143, "n": 14, "p50_ms": 6.5663730000000005, "p95_ms": 6.809703000000001, "population_std_ms": 0.16604228180350736 }, "gpu_kernel_busy": { "mean_ms": 6.462748499999998, "n": 14, "p50_ms": 6.456941499999999, "p95_ms": 6.703455999999998, "population_std_ms": 0.16541565407168776 }, "non_kernel_gap": { "mean_ms": 0.10780514285714433, "n": 14, "p50_ms": 0.10772100000000107, "p95_ms": 0.1121450000000026, "population_std_ms": 0.0019970330441916122 }, "selected_execute_annotation": "execute_context_0(0)_generation_6(6)", "steps": 14, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b6-r1-20260723-20260723T091712228992Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b6-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798447895471197.pt.trace.json.gz" } ], "schema": "frontier-decode-batch-trace.v1" }