{ "contract": { "component_time": "sum of CUDA kernel durations inside execute range", "timing": "CUDA graph-on GPU execute annotations", "tp_aggregation": "per-rank; slowest rank mean approximates critical path" }, "label": "tp2-b4-r1", "rank_summary": { "component_rank_mean_ms": { "attention": 0.9284945714285713, "collective": 0.4770398571428571, "linear_norm_rope": 0.9251340357142854, "moe": 2.5554174642857146, "other": 0.021663214285714286, "output_head": 0.08753164285714285, "router": 0.1898966428571428 }, "ranks": 2, "slowest_rank_execute_mean_ms": 5.294425214285715, "slowest_rank_kernel_busy_mean_ms": 5.187257214285713 }, "ranks": [ { "components": { "attention": { "mean_ms": 0.9336339285714287, "n": 14, "p50_ms": 0.933751, "p95_ms": 0.9427530000000001, "population_std_ms": 0.0036295537007084476 }, "collective": { "mean_ms": 0.47076142857142855, "n": 14, "p50_ms": 0.4640835000000001, "p95_ms": 0.5701469999999997, "population_std_ms": 0.0277061237241214 }, "linear_norm_rope": { "mean_ms": 0.9244245714285712, "n": 14, "p50_ms": 0.9243979999999998, "p95_ms": 0.9272680000000004, "population_std_ms": 0.0014581096232689887 }, "moe": { "mean_ms": 2.554062214285715, "n": 14, "p50_ms": 2.594499500000001, "p95_ms": 2.694474, "population_std_ms": 0.11681852632925924 }, "other": { "mean_ms": 0.02227242857142857, "n": 14, "p50_ms": 0.022274000000000002, "p95_ms": 0.022625, "population_std_ms": 0.00020562612198068582 }, "output_head": { "mean_ms": 0.087598, "n": 14, "p50_ms": 0.08764849999999999, "p95_ms": 0.088257, "population_std_ms": 0.00042077038190171175 }, "router": { "mean_ms": 0.1903450714285714, "n": 14, "p50_ms": 0.19036999999999993, "p95_ms": 0.19091199999999997, "population_std_ms": 0.00033161088735480956 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_1(1)": 1, "execute_context_0(0)_generation_4(4)": 14, "execute_context_3(6144)_generation_1(1)": 1 }, "execute_wall": { "mean_ms": 5.292617, "n": 14, "p50_ms": 5.351565000000001, "p95_ms": 5.420654000000001, "population_std_ms": 0.12026193009071014 }, "gpu_kernel_busy": { "mean_ms": 5.183097642857143, "n": 14, "p50_ms": 5.2450280000000005, "p95_ms": 5.312519, "population_std_ms": 0.12140493871020902 }, "non_kernel_gap": { "mean_ms": 0.10951935714285696, "n": 14, "p50_ms": 0.1093335000000013, "p95_ms": 0.11392299999999889, "population_std_ms": 0.002305499592809895 }, "selected_execute_annotation": "execute_context_0(0)_generation_4(4)", "steps": 14, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b4-r1-20260723-20260723T091055280404Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b4-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784798075195956333.pt.trace.json.gz" }, { "components": { "attention": { "mean_ms": 0.9233552142857139, "n": 14, "p50_ms": 0.9231514999999997, "p95_ms": 0.92672, "population_std_ms": 0.002063045328030955 }, "collective": { "mean_ms": 0.4833182857142857, "n": 14, "p50_ms": 0.47641600000000006, "p95_ms": 0.5747199999999999, "population_std_ms": 0.025529410631741578 }, "linear_norm_rope": { "mean_ms": 0.9258434999999997, "n": 14, "p50_ms": 0.9259644999999999, "p95_ms": 0.9298849999999995, "population_std_ms": 0.0017809597633218826 }, "moe": { "mean_ms": 2.556772714285714, "n": 14, "p50_ms": 2.5972645000000014, "p95_ms": 2.697822, "population_std_ms": 0.11678101165455461 }, "other": { "mean_ms": 0.021054, "n": 14, "p50_ms": 0.021136000000000002, "p95_ms": 0.021632000000000002, "population_std_ms": 0.0003493892630617853 }, "output_head": { "mean_ms": 0.08746528571428572, "n": 14, "p50_ms": 0.087424, "p95_ms": 0.087936, "population_std_ms": 0.00027621250322672924 }, "router": { "mean_ms": 0.18944821428571418, "n": 14, "p50_ms": 0.1893899999999999, "p95_ms": 0.18998899999999994, "population_std_ms": 0.00036123541563669255 } }, "execute_annotation_histogram": { "execute_context_0(0)_generation_1(1)": 1, "execute_context_0(0)_generation_4(4)": 14, "execute_context_3(6144)_generation_1(1)": 1 }, "execute_wall": { "mean_ms": 5.294425214285715, "n": 14, "p50_ms": 5.3559085, "p95_ms": 5.420421, "population_std_ms": 0.12112366521521874 }, "gpu_kernel_busy": { "mean_ms": 5.187257214285713, "n": 14, "p50_ms": 5.247959, "p95_ms": 5.314969999999999, "population_std_ms": 0.12129808972773216 }, "non_kernel_gap": { "mean_ms": 0.10716800000000092, "n": 14, "p50_ms": 0.10673850000000096, "p95_ms": 0.11011199999999732, "population_std_ms": 0.0018639973942341857 }, "selected_execute_annotation": "execute_context_0(0)_generation_4(4)", "steps": 14, "trace": "runs/frontier-decode-batch-grid-v0/fleet-artifacts/decode-grid-tp2-b4-r1-20260723-20260723T091055280404Z/artifacts/runs/frontier-decode-batch-grid-v0/remote-outputs/tp2-b4-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784798075201036386.pt.trace.json.gz" } ], "schema": "frontier-decode-batch-trace.v1" }