{ "contract": { "component_time": "sum of CUDA kernel durations inside execute range", "timing": "CUDA graph-on GPU execute annotations", "tp_aggregation": "per-rank; slowest rank mean approximates critical path" }, "label": "tp2-b2-r1", "rank_summary": { "component_rank_mean_ms": { "attention": 0.7962288124999999, "collective": 0.470928375, "linear_norm_rope": 0.9330912187499996, "moe": 2.136206281249999, "other": 0.02065321875, "output_head": 0.08696040625000001, "router": 0.19041484374999995 }, "ranks": 2, "slowest_rank_execute_mean_ms": 4.7613618125, "slowest_rank_kernel_busy_mean_ms": 4.657866374999998 }, "ranks": [ { "components": { "attention": { "mean_ms": 0.7951932499999999, "n": 16, "p50_ms": 0.7978159999999996, "p95_ms": 0.8009929999999998, "population_std_ms": 0.004837302056673742 }, "collective": { "mean_ms": 0.4532200625, "n": 16, "p50_ms": 0.44667250000000003, "p95_ms": 0.579522, "population_std_ms": 0.032947570582800075 }, "linear_norm_rope": { "mean_ms": 0.9318721874999996, "n": 16, "p50_ms": 0.9318319999999995, "p95_ms": 0.9354359999999993, "population_std_ms": 0.001684765236566671 }, "moe": { "mean_ms": 2.133037374999999, "n": 16, "p50_ms": 2.260420499999999, "p95_ms": 2.2688760000000006, "population_std_ms": 0.1538413522138454 }, "other": { "mean_ms": 0.0205200625, "n": 16, "p50_ms": 0.0205595, "p95_ms": 0.020864, "population_std_ms": 0.0002014638518289322 }, "output_head": { "mean_ms": 0.086960375, "n": 16, "p50_ms": 0.087073, "p95_ms": 0.087424, "population_std_ms": 0.0003634009278675549 }, "router": { "mean_ms": 0.19029662499999994, "n": 16, "p50_ms": 0.19019399999999992, "p95_ms": 0.19120399999999993, "population_std_ms": 0.00036799488362612264 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_2(2)": 16 }, "execute_wall": { "mean_ms": 4.7613618125, "n": 16, "p50_ms": 4.842258, "p95_ms": 5.225267, "population_std_ms": 0.20309347556335333 }, "gpu_kernel_busy": { "mean_ms": 4.611099937499998, "n": 16, "p50_ms": 4.738485999999997, "p95_ms": 4.880249, "population_std_ms": 0.17253325791549196 }, "non_kernel_gap": { "mean_ms": 0.15026187500000182, "n": 16, "p50_ms": 0.10683750000000192, "p95_ms": 0.7480410000000006, "population_std_ms": 0.15527935203322837 }, "selected_execute_annotation": "execute_context_0(0)_generation_2(2)", "steps": 16, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b2-r1-20260723-20260723T091051245390Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798066564146891.pt.trace.json.gz" }, { "components": { "attention": { "mean_ms": 0.7972643749999999, "n": 16, "p50_ms": 0.8003265, "p95_ms": 0.8037429999999999, "population_std_ms": 0.005557197617448423 }, "collective": { "mean_ms": 0.4886366875, "n": 16, "p50_ms": 0.4386005, "p95_ms": 1.0788600000000004, "population_std_ms": 0.15686321799051514 }, "linear_norm_rope": { "mean_ms": 0.9343102499999997, "n": 16, "p50_ms": 0.9339574999999996, "p95_ms": 0.9376900000000002, "population_std_ms": 0.0018319578836589897 }, "moe": { "mean_ms": 2.1393751874999984, "n": 16, "p50_ms": 2.2672624999999984, "p95_ms": 2.276405999999998, "population_std_ms": 0.15445752086424064 }, "other": { "mean_ms": 0.020786375, "n": 16, "p50_ms": 0.020785, "p95_ms": 0.021567999999999997, "population_std_ms": 0.0003711985444139028 }, "output_head": { "mean_ms": 0.0869604375, "n": 16, "p50_ms": 0.087008, "p95_ms": 0.087937, "population_std_ms": 0.0004370792217593405 }, "router": { "mean_ms": 0.19053306249999993, "n": 16, "p50_ms": 0.19049349999999995, "p95_ms": 0.19151799999999988, "population_std_ms": 0.0003664594910678971 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_2(2)": 16 }, "execute_wall": { "mean_ms": 4.7613106875, "n": 16, "p50_ms": 4.8443950000000005, "p95_ms": 5.2236769999999995, "population_std_ms": 0.20362942305009554 }, "gpu_kernel_busy": { "mean_ms": 4.657866374999998, "n": 16, "p50_ms": 4.739760999999998, "p95_ms": 5.121143999999998, "population_std_ms": 0.20551660892883905 }, "non_kernel_gap": { "mean_ms": 0.10344431250000213, "n": 16, "p50_ms": 0.10326550000000179, "p95_ms": 0.10955400000000282, "population_std_ms": 0.002999292535722995 }, "selected_execute_annotation": "execute_context_0(0)_generation_2(2)", "steps": 16, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b2-r1-20260723-20260723T091051245390Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b2-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798066565640462.pt.trace.json.gz" } ], "schema": "frontier-decode-batch-trace.v1" }