{ "contract": { "component_time": "sum of CUDA kernel durations inside execute range", "timing": "CUDA graph-on GPU execute annotations", "tp_aggregation": "per-rank; slowest rank mean approximates critical path" }, "label": "tp2-b2-r2", "rank_summary": { "component_rank_mean_ms": { "attention": 0.7975021874999999, "collective": 0.44700175, "linear_norm_rope": 0.9334021562499997, "moe": 2.1485421874999986, "other": 0.02076096875, "output_head": 0.08701734375, "router": 0.19057953124999993 }, "ranks": 2, "slowest_rank_execute_mean_ms": 4.7298366875, "slowest_rank_kernel_busy_mean_ms": 4.625673687499998 }, "ranks": [ { "components": { "attention": { "mean_ms": 0.7957946874999998, "n": 16, "p50_ms": 0.7977499999999997, "p95_ms": 0.8024729999999999, "population_std_ms": 0.0050081016328389115 }, "collective": { "mean_ms": 0.4559536875, "n": 16, "p50_ms": 0.44857899999999995, "p95_ms": 0.5749780000000001, "population_std_ms": 0.03100173565044134 }, "linear_norm_rope": { "mean_ms": 0.9314522499999998, "n": 16, "p50_ms": 0.9312219999999996, "p95_ms": 0.9362239999999998, "population_std_ms": 0.0018965892906741145 }, "moe": { "mean_ms": 2.144734312499999, "n": 16, "p50_ms": 2.2638159999999994, "p95_ms": 2.2729309999999985, "population_std_ms": 0.16050482071137254 }, "other": { "mean_ms": 0.0205803125, "n": 16, "p50_ms": 0.020513, "p95_ms": 0.021342, "population_std_ms": 0.00034404573074483917 }, "output_head": { "mean_ms": 0.0869003125, "n": 16, "p50_ms": 0.086897, "p95_ms": 0.087585, "population_std_ms": 0.0003536547961554459 }, "router": { "mean_ms": 0.19025812499999992, "n": 16, "p50_ms": 0.19022449999999994, "p95_ms": 0.19071799999999992, "population_std_ms": 0.00028991245984778943 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_2(2)": 16 }, "execute_wall": { "mean_ms": 4.72958625, "n": 16, "p50_ms": 4.841987, "p95_ms": 4.974675, "population_std_ms": 0.1744621378768544 }, "gpu_kernel_busy": { "mean_ms": 4.625673687499998, "n": 16, "p50_ms": 4.7410514999999975, "p95_ms": 4.871186999999999, "population_std_ms": 0.1763796075070467 }, "non_kernel_gap": { "mean_ms": 0.10391256250000158, "n": 16, "p50_ms": 0.10354800000000175, "p95_ms": 0.10959400000000219, "population_std_ms": 0.0028733005187579386 }, "selected_execute_annotation": "execute_context_0(0)_generation_2(2)", "steps": 16, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b2-r2-20260723-20260723T092320882080Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r2/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798812071346091.pt.trace.json.gz" }, { "components": { "attention": { "mean_ms": 0.7992096874999999, "n": 16, "p50_ms": 0.8017949999999999, "p95_ms": 0.8056339999999999, "population_std_ms": 0.00523807464292403 }, "collective": { "mean_ms": 0.4380498125, "n": 16, "p50_ms": 0.4294910000000001, "p95_ms": 0.56768, "population_std_ms": 0.033749115542519664 }, "linear_norm_rope": { "mean_ms": 0.9353520624999997, "n": 16, "p50_ms": 0.9359214999999996, "p95_ms": 0.9375480000000005, "population_std_ms": 0.0017075802861340613 }, "moe": { "mean_ms": 2.1523500624999987, "n": 16, "p50_ms": 2.2702614999999993, "p95_ms": 2.284628, "population_std_ms": 0.16187233535855888 }, "other": { "mean_ms": 0.020941625000000002, "n": 16, "p50_ms": 0.0209265, "p95_ms": 0.021598, "population_std_ms": 0.0003365376789826063 }, "output_head": { "mean_ms": 0.087134375, "n": 16, "p50_ms": 0.087281, "p95_ms": 0.087585, "population_std_ms": 0.0004719001317810776 }, "router": { "mean_ms": 0.19090093749999995, "n": 16, "p50_ms": 0.19088099999999994, "p95_ms": 0.19164899999999993, "population_std_ms": 0.0003892426089134459 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_2(2)": 16 }, "execute_wall": { "mean_ms": 4.7298366875, "n": 16, "p50_ms": 4.8450835, "p95_ms": 4.970972, "population_std_ms": 0.1741141631820623 }, "gpu_kernel_busy": { "mean_ms": 4.6239385624999985, "n": 16, "p50_ms": 4.7412614999999985, "p95_ms": 4.869705999999997, "population_std_ms": 0.17688744584409086 }, "non_kernel_gap": { "mean_ms": 0.10589812500000201, "n": 16, "p50_ms": 0.10663300000000042, "p95_ms": 0.11212500000000247, "population_std_ms": 0.003380934446920671 }, "selected_execute_annotation": "execute_context_0(0)_generation_2(2)", "steps": 16, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b2-r2-20260723-20260723T092320882080Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r2/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798812076480600.pt.trace.json.gz" } ], "schema": "frontier-decode-batch-trace.v1" }