chore: vendor sglang v0.5.10 snapshot

This commit is contained in:
2026-04-24 12:29:36 +00:00
parent 78f0d15221
commit bded08301f
4308 changed files with 1200894 additions and 2 deletions

View File

@@ -0,0 +1,330 @@
"""
Benchmark SGLang vs Aiter custom all-reduce across message sizes.
Usage:
torchrun --nproc_per_node=2 benchmark_aiter.py
torchrun --nproc_per_node=4 benchmark_aiter.py
torchrun --nproc_per_node=8 benchmark_aiter.py
"""
import argparse
import os
import sys
import time
from typing import List, Optional, Tuple
import torch
import torch.distributed as dist
def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark SGLang vs Aiter custom all-reduce across message sizes."
)
parser.add_argument(
"--backend",
type=str,
default="gloo",
help="Process group backend for the custom-AR control path (must NOT be nccl).",
)
parser.add_argument(
"--warmup",
type=int,
default=5,
help="Warmup iterations per size per implementation.",
)
parser.add_argument(
"--iters-small",
type=int,
default=50,
help="Benchmark iterations for sizes <= 1MB.",
)
parser.add_argument(
"--iters-large",
type=int,
default=20,
help="Benchmark iterations for sizes > 1MB.",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Print per-iteration timings on rank 0 for debugging.",
)
return parser.parse_args()
def get_env_rank_world() -> Tuple[int, int, int]:
rank = int(os.environ.get("RANK", "0"))
world_size = int(os.environ.get("WORLD_SIZE", "1"))
local_rank = int(os.environ.get("LOCAL_RANK", str(rank)))
return rank, world_size, local_rank
def init_dist(backend: str):
rank, world_size, _ = get_env_rank_world()
if not dist.is_initialized():
dist.init_process_group(
backend=backend,
init_method="env://",
rank=rank,
world_size=world_size,
)
def get_device(local_rank: int) -> torch.device:
torch.cuda.set_device(local_rank)
return torch.device(f"cuda:{local_rank}")
def human_size(num_bytes: int) -> str:
units = [("B", 1), ("K", 1024), ("M", 1024 * 1024), ("G", 1024 * 1024 * 1024)]
for suf, base in reversed(units):
if num_bytes % base == 0 and num_bytes >= base:
val = num_bytes // base
return f"{val}{suf}"
return f"{num_bytes}B"
def get_message_sizes() -> List[int]:
return [
32 * 1024,
64 * 1024,
128 * 1024,
256 * 1024,
512 * 1024,
1 * 1024 * 1024,
2 * 1024 * 1024,
4 * 1024 * 1024,
8 * 1024 * 1024,
16 * 1024 * 1024,
32 * 1024 * 1024,
64 * 1024 * 1024,
]
@torch.inference_mode()
def run_once(comm, inp: torch.Tensor) -> Optional[torch.Tensor]:
if hasattr(comm, "all_reduce_unreg"):
return comm.all_reduce_unreg(inp)
if hasattr(comm, "custom_all_reduce"):
return comm.custom_all_reduce(inp)
raise RuntimeError("No known all-reduce method found on the communicator.")
@torch.inference_mode()
def bench_impl(
name: str,
comm,
sizes: List[int],
device: torch.device,
warmup: int,
iters_small: int,
iters_large: int,
verbose: bool,
pg: Optional[dist.ProcessGroup] = None,
) -> List[Tuple[int, Optional[float]]]:
rank = dist.get_rank()
world_size = dist.get_world_size()
results: List[Tuple[int, Optional[float]]] = []
for size_bytes in sizes:
elems = size_bytes // 2 # float16: 2 bytes per element
inp = torch.empty(elems, dtype=torch.float16, device=device)
inp.uniform_(0, 1)
disabled = False
dist.barrier(group=pg)
for _ in range(warmup):
torch.cuda.synchronize()
out = run_once(comm, inp)
torch.cuda.synchronize()
if out is None:
disabled = True
break
dist.barrier(group=pg)
if disabled:
if rank == 0:
print(
f"[{name}] {human_size(size_bytes)}: custom AR disabled (skipped)"
)
results.append((size_bytes, None))
continue
num_iters = iters_small if size_bytes <= (1 * 1024 * 1024) else iters_large
times_ms: List[float] = []
for it in range(num_iters):
dist.barrier(group=pg)
torch.cuda.synchronize()
t0 = time.perf_counter()
out = run_once(comm, inp)
torch.cuda.synchronize()
t1 = time.perf_counter()
dist.barrier(group=pg)
if out is None:
disabled = True
break
dt_ms = (t1 - t0) * 1000.0
times_ms.append(dt_ms)
if verbose and rank == 0:
print(
f"[{name}] size={human_size(size_bytes)} iter={it} time={dt_ms:.3f} ms"
)
if disabled or not times_ms:
if rank == 0:
print(
f"[{name}] {human_size(size_bytes)}: custom AR disabled (no timings)"
)
results.append((size_bytes, None))
continue
avg_ms_local = sum(times_ms) / len(times_ms)
avg_tensor = torch.tensor([avg_ms_local], dtype=torch.float64, device=device)
gather_list = [torch.zeros_like(avg_tensor) for _ in range(world_size)]
dist.all_gather(gather_list, avg_tensor, group=pg)
if rank == 0:
avg_ms = float(torch.stack(gather_list).mean().item())
print(
f"[{name}] {human_size(size_bytes)}: {avg_ms:.3f} ms (avg across ranks)"
)
results.append((size_bytes, avg_ms))
else:
results.append((size_bytes, None))
return results
def main():
args = parse_args()
rank, world_size, local_rank = get_env_rank_world()
if world_size not in (2, 4, 6, 8):
print(
f"[rank {rank}] WARNING: world_size={world_size} not in supported set (2,4,6,8). "
"Custom AR may disable itself.",
file=sys.stderr,
)
init_dist(args.backend)
device = get_device(local_rank)
# Import after dist init; some libs query torch dist state on import
sgl_comm = None
aiter_comm = None
HAVE_SGLANG = False
HAVE_AITER = False
try:
from sglang.srt.distributed.device_communicators.custom_all_reduce import (
CustomAllreduce as SGLCustomAllreduce,
)
HAVE_SGLANG = True
except Exception as e:
if rank == 0:
print(f"SGLang CustomAllreduce import failed: {e}", file=sys.stderr)
try:
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as AiterCustomAllreduce,
)
HAVE_AITER = True
except Exception as e:
if rank == 0:
print(f"Aiter CustomAllreduce import failed: {e}", file=sys.stderr)
if rank == 0:
print(f"Initialized PG backend={args.backend} world_size={world_size}")
print(f"Device: {device.type}:{device.index}")
print(f"SGLang available: {HAVE_SGLANG}, Aiter available: {HAVE_AITER}")
pg = dist.group.WORLD
sizes = get_message_sizes()
max_size = max(sizes) if sizes else (64 * 1024 * 1024)
if HAVE_SGLANG:
try:
sgl_comm = SGLCustomAllreduce(group=pg, device=device, max_size=max_size)
except Exception as e:
if rank == 0:
print(
f"Failed to construct SGLang CustomAllreduce: {e}", file=sys.stderr
)
sgl_comm = None
if HAVE_AITER:
try:
aiter_comm = AiterCustomAllreduce(
group=pg, device=device, max_size=max_size
)
except Exception as e:
if rank == 0:
print(
f"Failed to construct Aiter CustomAllreduce: {e}", file=sys.stderr
)
aiter_comm = None
sgl_results: List[Tuple[int, Optional[float]]] = []
aiter_results: List[Tuple[int, Optional[float]]] = []
if sgl_comm is not None:
sgl_results = bench_impl(
name="SGLang",
comm=sgl_comm,
sizes=sizes,
device=device,
warmup=args.warmup,
iters_small=args.iters_small,
iters_large=args.iters_large,
verbose=args.verbose,
pg=pg,
)
if aiter_comm is not None:
aiter_results = bench_impl(
name="Aiter",
comm=aiter_comm,
sizes=sizes,
device=device,
warmup=args.warmup,
iters_small=args.iters_small,
iters_large=args.iters_large,
verbose=args.verbose,
pg=pg,
)
for comm in (sgl_comm, aiter_comm):
if comm is not None and hasattr(comm, "close"):
try:
comm.close()
except Exception:
pass
if dist.get_rank() == 0:
print("\nResults (avg ms across ranks; None = disabled/unavailable):")
header = f"{'Size':>8} {'SGLang(ms)':>12} {'Aiter(ms)':>11}"
print(header)
print("-" * len(header))
sgl_map = {s: v for s, v in sgl_results if v is not None}
aiter_map = {s: v for s, v in aiter_results if v is not None}
for s in sizes:
sgl_ms = sgl_map.get(s, None)
aiter_ms = aiter_map.get(s, None)
print(
f"{human_size(s):>8} {('%.3f' % sgl_ms) if sgl_ms is not None else 'None':>12} "
f"{('%.3f' % aiter_ms) if aiter_ms is not None else 'None':>11}"
)
dist.barrier()
dist.destroy_process_group()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,351 @@
"""
Benchmark SGLang custom all-reduce vs Torch symm-mem all-reduce across message sizes.
Usage:
torchrun --nproc_per_node=2 benchmark_all_reduce.py
torchrun --nproc_per_node=4 benchmark_all_reduce.py
torchrun --nproc_per_node=8 benchmark_all_reduce.py
"""
import argparse
import os
import sys
import time
from typing import List, Optional, Tuple
import torch
import torch.distributed as dist
from sglang.srt.distributed.parallel_state import (
destroy_distributed_environment,
destroy_model_parallel,
init_distributed_environment,
initialize_model_parallel,
)
def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark SGLang custom all-reduce vs Torch symm-mem all-reduce across message sizes."
)
parser.add_argument(
"--backend",
type=str,
default="gloo",
help="Process group backend for the custom-AR control path (must NOT be nccl).",
)
parser.add_argument(
"--warmup",
type=int,
default=5,
help="Warmup iterations per size per implementation.",
)
parser.add_argument(
"--iters-small",
type=int,
default=50,
help="Benchmark iterations for sizes <= 1MB.",
)
parser.add_argument(
"--iters-large",
type=int,
default=20,
help="Benchmark iterations for sizes > 1MB.",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Print per-iteration timings on rank 0 for debugging.",
)
return parser.parse_args()
def get_env_rank_world() -> Tuple[int, int, int]:
rank = int(os.environ.get("RANK", "0"))
world_size = int(os.environ.get("WORLD_SIZE", "1"))
local_rank = int(os.environ.get("LOCAL_RANK", str(rank)))
return rank, world_size, local_rank
def init_dist(backend: str):
rank, world_size, _ = get_env_rank_world()
if not dist.is_initialized():
dist.init_process_group(
backend=backend,
init_method="env://",
rank=rank,
world_size=world_size,
)
device = torch.device(f"cuda:{rank}")
torch.cuda.set_device(device)
distributed_init_method = f"tcp://localhost:23456"
init_distributed_environment(
world_size=world_size,
rank=rank,
distributed_init_method=distributed_init_method,
local_rank=rank,
)
initialize_model_parallel(tensor_model_parallel_size=world_size)
return dist.group.WORLD
def get_device(local_rank: int) -> torch.device:
torch.cuda.set_device(local_rank)
return torch.device(f"cuda:{local_rank}")
def human_size(num_bytes: int) -> str:
units = [("B", 1), ("K", 1024), ("M", 1024 * 1024), ("G", 1024 * 1024 * 1024)]
for suf, base in reversed(units):
if num_bytes % base == 0 and num_bytes >= base:
val = num_bytes // base
return f"{val}{suf}"
return f"{num_bytes}B"
def get_message_sizes() -> List[int]:
return [
32 * 1024,
64 * 1024,
128 * 1024,
256 * 1024,
512 * 1024,
1 * 1024 * 1024,
2 * 1024 * 1024,
4 * 1024 * 1024,
8 * 1024 * 1024,
16 * 1024 * 1024,
32 * 1024 * 1024,
64 * 1024 * 1024,
]
@torch.inference_mode()
def run_once(comm, inp: torch.Tensor) -> Optional[torch.Tensor]:
if hasattr(comm, "custom_all_reduce"):
return comm.custom_all_reduce(inp)
if hasattr(comm, "all_reduce"):
return comm.all_reduce(inp)
raise RuntimeError("No known all-reduce method found on the communicator.")
@torch.inference_mode()
def bench_impl(
name: str,
comm,
sizes: List[int],
device: torch.device,
warmup: int,
iters_small: int,
iters_large: int,
verbose: bool,
pg: Optional[dist.ProcessGroup] = None,
) -> List[Tuple[int, Optional[float]]]:
rank = dist.get_rank()
world_size = dist.get_world_size()
results: List[Tuple[int, Optional[float]]] = []
for size_bytes in sizes:
elems = size_bytes // 2 # float16: 2 bytes per element
inp = torch.empty(elems, dtype=torch.float16, device=device)
inp.uniform_(0, 1)
disabled = False
dist.barrier(group=pg)
for _ in range(warmup):
torch.cuda.synchronize()
out = run_once(comm, inp)
torch.cuda.synchronize()
if out is None:
disabled = True
break
dist.barrier(group=pg)
if disabled:
if rank == 0:
print(
f"[{name}] {human_size(size_bytes)}: custom AR disabled (skipped)"
)
results.append((size_bytes, None))
continue
num_iters = iters_small if size_bytes <= (1 * 1024 * 1024) else iters_large
times_ms: List[float] = []
for it in range(num_iters):
dist.barrier(group=pg)
torch.cuda.synchronize()
t0 = time.perf_counter()
out = run_once(comm, inp)
torch.cuda.synchronize()
t1 = time.perf_counter()
dist.barrier(group=pg)
if out is None:
disabled = True
break
dt_ms = (t1 - t0) * 1000.0
times_ms.append(dt_ms)
if verbose and rank == 0:
print(
f"[{name}] size={human_size(size_bytes)} iter={it} time={dt_ms:.3f} ms"
)
if disabled or not times_ms:
if rank == 0:
print(
f"[{name}] {human_size(size_bytes)}: custom AR disabled (no timings)"
)
results.append((size_bytes, None))
continue
avg_ms_local = sum(times_ms) / len(times_ms)
avg_tensor = torch.tensor([avg_ms_local], dtype=torch.float64, device=device)
gather_list = [torch.zeros_like(avg_tensor) for _ in range(world_size)]
dist.all_gather(gather_list, avg_tensor, group=pg)
if rank == 0:
avg_ms = float(torch.stack(gather_list).mean().item())
print(
f"[{name}] {human_size(size_bytes)}: {avg_ms:.3f} ms (avg across ranks)"
)
results.append((size_bytes, avg_ms))
else:
results.append((size_bytes, None))
return results
def main():
args = parse_args()
rank, world_size, local_rank = get_env_rank_world()
if world_size not in (2, 4, 6, 8):
print(
f"[rank {rank}] WARNING: world_size={world_size} not in supported set (2,4,6,8). "
"Custom AR may disable itself.",
file=sys.stderr,
)
group = init_dist(args.backend)
device = get_device(local_rank)
# Import after dist init; some libs query torch dist state on import
torch_symm_mem_comm = None
HAVE_SGLANG_CUSTOM = False
HAVE_TORCH_SYMM_MEM = False
try:
from sglang.srt.distributed.device_communicators.custom_all_reduce import (
CustomAllreduce as SGLCustomAllreduce,
)
HAVE_SGLANG_CUSTOM = True
except Exception as e:
if rank == 0:
print(f"SGLang CustomAllreduce import failed: {e}", file=sys.stderr)
try:
from sglang.srt.distributed.device_communicators.torch_symm_mem import (
TorchSymmMemCommunicator as TorchSymmMemAllreduce,
)
HAVE_TORCH_SYMM_MEM = True
except Exception as e:
if rank == 0:
print(f"TorchSymmMemAllreduce import failed: {e}", file=sys.stderr)
if rank == 0:
print(f"Initialized PG backend={args.backend} world_size={world_size}")
print(f"Device: {device.type}:{device.index}")
print(
f"SGLang Custom available: {HAVE_SGLANG_CUSTOM}, Torch Symm-Mem available: {HAVE_TORCH_SYMM_MEM}"
)
sizes = get_message_sizes()
max_size = max(sizes) if sizes else (128 * 1024 * 1024)
if HAVE_SGLANG_CUSTOM:
try:
sgl_custom_comm = SGLCustomAllreduce(
group=group, device=device, max_size=max_size
)
except Exception as e:
if rank == 0:
print(
f"Failed to construct SGLangCustomAllreduce: {e}", file=sys.stderr
)
sgl_custom_comm = None
if HAVE_TORCH_SYMM_MEM:
try:
torch_symm_mem_comm = TorchSymmMemAllreduce(group=group, device=device)
except Exception as e:
if rank == 0:
print(
f"Failed to construct TorchSymmMemAllreduce: {e}", file=sys.stderr
)
torch_symm_mem_comm = None
sgl_custom_results: List[Tuple[int, Optional[float]]] = []
symm_mem_results: List[Tuple[int, Optional[float]]] = []
if sgl_custom_comm is not None:
sgl_custom_results = bench_impl(
name="SGLangCustom",
comm=sgl_custom_comm,
sizes=sizes,
device=device,
warmup=args.warmup,
iters_small=args.iters_small,
iters_large=args.iters_large,
verbose=args.verbose,
pg=group,
)
if torch_symm_mem_comm is not None:
symm_mem_results = bench_impl(
name="TorchSymmMem",
comm=torch_symm_mem_comm,
sizes=sizes,
device=device,
warmup=args.warmup,
iters_small=args.iters_small,
iters_large=args.iters_large,
verbose=args.verbose,
pg=group,
)
for comm in (sgl_custom_comm, torch_symm_mem_comm):
if comm is not None and hasattr(comm, "close"):
try:
comm.close()
except Exception:
pass
if dist.get_rank() == 0:
print(
f"\nResults (avg ms across {world_size} ranks; None = disabled/unavailable):"
)
header = f"{'Size':>8} {'CustomAR(ms)':>12} {'TorchSymmMem(ms)':>11}"
print(header)
print("-" * len(header))
sgl_custom_map = {s: v for s, v in sgl_custom_results if v is not None}
symm_mem_map = {s: v for s, v in symm_mem_results if v is not None}
for s in sizes:
sgl_ms = sgl_custom_map.get(s, None)
symm_mem_ms = symm_mem_map.get(s, None)
print(
f"{human_size(s):>8} {('%.3f' % sgl_ms) if sgl_ms is not None else 'None':>12} "
f"{('%.3f' % symm_mem_ms) if symm_mem_ms is not None else 'None':>11}"
)
torch.distributed.barrier(group=group)
destroy_model_parallel()
destroy_distributed_environment()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,536 @@
"""
Benchmark fused allreduce+rmsnorm on AMD with correctness checks.
This script targets the same fused op used by SGLang:
`tensor_model_parallel_fused_allreduce_rmsnorm`.
It reports:
- eager mode latency (prefill-like)
- graph mode latency (decode-like)
- fused availability (whether fused path returns non-None)
- correctness (fused output matches split allreduce + rmsnorm reference)
Usage example:
torchrun --nproc_per_node=8 \
benchmark/kernels/all_reduce/benchmark_fused_ar_rms_amd.py \
--dtype bfloat16 \
--prefill-shapes 2048x8192,8192x8192 \
--decode-shapes 1x8192,4x8192,16x8192 \
--warmup 10 --iters 30 --repeats 5
"""
import argparse
import csv
import os
import statistics
from typing import Dict, List, Optional, Sequence, Tuple
import torch
import torch.distributed as dist
import torch.nn.functional as F
from sglang.srt.distributed.communication_op import (
tensor_model_parallel_all_reduce,
tensor_model_parallel_fused_allreduce_rmsnorm,
)
from sglang.srt.distributed.parallel_state import (
destroy_distributed_environment,
destroy_model_parallel,
graph_capture,
init_distributed_environment,
initialize_model_parallel,
set_custom_all_reduce,
)
Shape = Tuple[int, int]
def parse_shapes(raw: str) -> List[Shape]:
shapes: List[Shape] = []
for item in [x.strip() for x in raw.split(",") if x.strip()]:
if "x" not in item:
raise ValueError(f"Invalid shape '{item}', expected MxN format.")
m_str, n_str = item.split("x", 1)
m = int(m_str)
n = int(n_str)
if m <= 0 or n <= 0:
raise ValueError(f"Invalid shape '{item}', both dims must be positive.")
shapes.append((m, n))
if not shapes:
raise ValueError("Empty shape list is not allowed.")
return shapes
def dtype_from_name(name: str) -> torch.dtype:
mapping = {
"float16": torch.float16,
"fp16": torch.float16,
"bfloat16": torch.bfloat16,
"bf16": torch.bfloat16,
}
if name not in mapping:
raise ValueError(f"Unsupported dtype: {name}")
return mapping[name]
def check_close(
a: torch.Tensor, b: torch.Tensor, dtype: torch.dtype
) -> Tuple[bool, str]:
if dtype == torch.bfloat16:
rtol, atol = 2e-2, 1.25e-1
else:
rtol, atol = 1e-2, 2e-2
try:
torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
return True, "PASS"
except AssertionError:
max_diff = torch.max(torch.abs(a - b)).item()
mean_diff = torch.mean(torch.abs(a - b)).item()
return False, f"FAIL(max={max_diff:.6f},mean={mean_diff:.6f})"
def _measure_us(
fn,
warmup: int,
iters: int,
repeats: int,
device: torch.device,
) -> Tuple[float, Dict[str, float]]:
for _ in range(warmup):
fn()
torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
samples_us: List[float] = []
for _ in range(max(1, repeats)):
_barrier(device)
torch.cuda.synchronize()
start_event.record()
for _ in range(iters):
fn()
end_event.record()
end_event.synchronize()
samples_us.append(start_event.elapsed_time(end_event) * 1000.0 / iters)
sorted_samples = sorted(samples_us)
p50 = float(statistics.median(sorted_samples))
p95 = float(sorted_samples[int((len(sorted_samples) - 1) * 0.95)])
return p50, {
"p50_us": p50,
"p95_us": p95,
"min_us": float(sorted_samples[0]),
"max_us": float(sorted_samples[-1]),
}
def _barrier(device: torch.device):
try:
dist.barrier(device_ids=[device.index])
except TypeError:
dist.barrier()
def _mean_across_ranks(value: float, device: torch.device) -> float:
t = torch.tensor([value], dtype=torch.float64, device=device)
dist.all_reduce(t, op=dist.ReduceOp.SUM)
t /= dist.get_world_size()
return float(t.item())
def _all_true_across_ranks(value: bool, device: torch.device) -> bool:
t = torch.tensor([1 if value else 0], dtype=torch.int32, device=device)
dist.all_reduce(t, op=dist.ReduceOp.MIN)
return bool(int(t.item()))
def _make_inputs(
shape: Shape,
dtype: torch.dtype,
seed: int,
residual_mode: str,
rank: int,
device: torch.device,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
m, n = shape
torch.manual_seed(seed + rank * 17)
x = torch.randn((m, n), dtype=torch.float32, device=device).to(dtype)
if residual_mode == "self":
residual = x.clone()
elif residual_mode == "random":
residual = torch.randn((m, n), dtype=torch.float32, device=device).to(dtype)
elif residual_mode == "zero":
residual = torch.zeros((m, n), dtype=dtype, device=device)
else:
raise ValueError(f"Unknown residual_mode: {residual_mode}")
weight = torch.randn((n,), dtype=torch.float32, device=device).to(dtype)
return x, residual, weight
def _split_reference(
x: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor, eps: float
) -> Tuple[torch.Tensor, torch.Tensor]:
ar_out = tensor_model_parallel_all_reduce(x.clone())
residual_out = ar_out + residual
out = F.rms_norm(
input=residual_out,
normalized_shape=(residual_out.shape[-1],),
weight=weight,
eps=eps,
)
return out, residual_out
def bench_eager(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
eps: float,
warmup: int,
iters: int,
repeats: int,
) -> Dict[str, object]:
split_fn = lambda: _split_reference(x, residual, weight, eps)
split_us, split_stats = _measure_us(split_fn, warmup, iters, repeats, x.device)
fused_probe = tensor_model_parallel_fused_allreduce_rmsnorm(
x.clone(), residual.clone(), weight, eps
)
fused_available = fused_probe is not None
fused_us: Optional[float] = None
fused_stats: Optional[Dict[str, float]] = None
if fused_available:
fused_fn = lambda: tensor_model_parallel_fused_allreduce_rmsnorm(
x, residual, weight, eps
)
fused_us, fused_stats = _measure_us(fused_fn, warmup, iters, repeats, x.device)
ref_out, ref_residual = _split_reference(x, residual, weight, eps)
if fused_available:
fused_out, fused_residual = tensor_model_parallel_fused_allreduce_rmsnorm(
x.clone(), residual.clone(), weight, eps
)
out_ok, out_detail = check_close(fused_out, ref_out, x.dtype)
res_ok, res_detail = check_close(fused_residual, ref_residual, x.dtype)
correctness_ok = out_ok and res_ok
correctness_detail = f"out={out_detail}, residual={res_detail}"
else:
correctness_ok = True
correctness_detail = "SKIP(fused_unavailable)"
return {
"split_us": split_us,
"split_stats": split_stats,
"fused_available": fused_available,
"fused_us": fused_us,
"fused_stats": fused_stats,
"correctness_ok": correctness_ok,
"correctness_detail": correctness_detail,
}
def bench_graph(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
eps: float,
warmup: int,
iters: int,
repeats: int,
) -> Dict[str, object]:
split_x = x.clone()
split_res = residual.clone()
split_graph_out: Optional[torch.Tensor] = None
with graph_capture() as gc:
split_graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(split_graph, stream=gc.stream):
split_graph_out, _ = _split_reference(split_x, split_res, weight, eps)
def split_replay():
split_graph.replay()
split_us, split_stats = _measure_us(split_replay, warmup, iters, repeats, x.device)
fused_probe = tensor_model_parallel_fused_allreduce_rmsnorm(
x.clone(), residual.clone(), weight, eps
)
fused_available = fused_probe is not None
fused_us: Optional[float] = None
fused_stats: Optional[Dict[str, float]] = None
fused_graph_out: Optional[torch.Tensor] = None
fused_graph_residual: Optional[torch.Tensor] = None
if fused_available:
fused_x = x.clone()
fused_res = residual.clone()
with graph_capture() as gc:
fused_graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(fused_graph, stream=gc.stream):
fused_graph_out, fused_graph_residual = (
tensor_model_parallel_fused_allreduce_rmsnorm(
fused_x, fused_res, weight, eps
)
)
def fused_replay():
fused_graph.replay()
fused_us, fused_stats = _measure_us(
fused_replay, warmup, iters, repeats, x.device
)
ref_out, ref_residual = _split_reference(x, residual, weight, eps)
if (
fused_available
and fused_graph_out is not None
and fused_graph_residual is not None
):
fused_graph.replay()
torch.cuda.synchronize()
out_ok, out_detail = check_close(fused_graph_out, ref_out, x.dtype)
res_ok, res_detail = check_close(fused_graph_residual, ref_residual, x.dtype)
correctness_ok = out_ok and res_ok
correctness_detail = f"out={out_detail}, residual={res_detail}"
else:
correctness_ok = True
correctness_detail = "SKIP(fused_unavailable)"
return {
"split_us": split_us,
"split_stats": split_stats,
"fused_available": fused_available,
"fused_us": fused_us,
"fused_stats": fused_stats,
"correctness_ok": correctness_ok,
"correctness_detail": correctness_detail,
}
def _shape_bytes(shape: Shape, dtype: torch.dtype) -> int:
m, n = shape
return m * n * torch.tensor([], dtype=dtype).element_size()
def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark fused allreduce+rmsnorm (prefill eager + decode graph)."
)
parser.add_argument(
"--dtype",
type=str,
default="bf16",
choices=["fp16", "bf16", "float16", "bfloat16"],
)
parser.add_argument("--eps", type=float, default=1e-6)
parser.add_argument("--seed", type=int, default=1234)
parser.add_argument(
"--residual-mode",
type=str,
default="self",
choices=["self", "random", "zero"],
help="Use residual=x (self) to match aiter test behavior by default.",
)
parser.add_argument(
"--prefill-shapes",
type=str,
default="2048x8192,8192x8192,16384x8192",
help="Comma-separated MxN shapes for eager mode.",
)
parser.add_argument(
"--decode-shapes",
type=str,
default="1x8192,2x8192,4x8192,8x8192,16x8192",
help="Comma-separated MxN shapes for graph mode.",
)
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--iters", type=int, default=30)
parser.add_argument("--repeats", type=int, default=5)
parser.add_argument(
"--mode",
type=str,
default="both",
choices=["eager", "graph", "both"],
)
parser.add_argument(
"--csv-out",
type=str,
default=None,
help="Optional output CSV path (written on rank 0 only).",
)
return parser.parse_args()
def main():
args = parse_args()
dtype = dtype_from_name(args.dtype)
rank = int(os.environ.get("RANK", "0"))
world_size = int(os.environ.get("WORLD_SIZE", "1"))
local_rank = int(os.environ.get("LOCAL_RANK", str(rank)))
torch.cuda.set_device(local_rank % torch.cuda.device_count())
device = torch.device(f"cuda:{local_rank % torch.cuda.device_count()}")
set_custom_all_reduce(True)
init_distributed_environment(
world_size=world_size,
rank=rank,
local_rank=local_rank,
distributed_init_method="env://",
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=world_size)
prefill_shapes = parse_shapes(args.prefill_shapes)
decode_shapes = parse_shapes(args.decode_shapes)
if rank == 0:
print(
"Config: "
f"world_size={world_size}, dtype={dtype}, residual_mode={args.residual_mode}, "
f"warmup={args.warmup}, iters={args.iters}, repeats={args.repeats}"
)
run_modes: Sequence[str]
if args.mode == "both":
run_modes = ("eager", "graph")
else:
run_modes = (args.mode,)
csv_rows: List[Dict[str, object]] = []
for mode in run_modes:
shapes = prefill_shapes if mode == "eager" else decode_shapes
if rank == 0:
phase_name = "prefill(eager)" if mode == "eager" else "decode(graph)"
print("\n" + "=" * 120)
print(f"Mode: {phase_name}")
print(
"| Shape | Input bytes/rank | Split p50 (us) | Fused p50 (us) | Speedup | Fused available | Correctness |"
)
print(
"|:------|-----------------:|---------------:|---------------:|--------:|:----------------|:------------|"
)
for shape in shapes:
x, residual, weight = _make_inputs(
shape=shape,
dtype=dtype,
seed=args.seed,
residual_mode=args.residual_mode,
rank=rank,
device=device,
)
if mode == "eager":
metrics = bench_eager(
x=x,
residual=residual,
weight=weight,
eps=args.eps,
warmup=args.warmup,
iters=args.iters,
repeats=args.repeats,
)
else:
metrics = bench_graph(
x=x,
residual=residual,
weight=weight,
eps=args.eps,
warmup=args.warmup,
iters=args.iters,
repeats=args.repeats,
)
split_us = _mean_across_ranks(float(metrics["split_us"]), device)
fused_available = _all_true_across_ranks(
bool(metrics["fused_available"]), device
)
correctness_ok = _all_true_across_ranks(
bool(metrics["correctness_ok"]), device
)
fused_us: Optional[float] = None
if fused_available and metrics["fused_us"] is not None:
fused_us = _mean_across_ranks(float(metrics["fused_us"]), device)
if rank == 0:
m, n = shape
shape_str = f"{m}x{n}"
bytes_per_rank = _shape_bytes(shape, dtype)
if fused_us is not None and fused_us > 0:
speedup = split_us / fused_us
speedup_str = f"{speedup:.3f}x"
fused_str = f"{fused_us:.1f}"
else:
speedup_str = "N/A"
fused_str = "N/A"
correctness_text = (
"PASS" if correctness_ok else str(metrics["correctness_detail"])
)
print(
f"| {shape_str} | {bytes_per_rank} | {split_us:.1f} | {fused_str} | "
f"{speedup_str} | {str(fused_available)} | {correctness_text} |"
)
csv_rows.append(
{
"mode": mode,
"shape": shape_str,
"m": m,
"n": n,
"bytes_per_rank": bytes_per_rank,
"split_p50_us": split_us,
"fused_p50_us": fused_us if fused_us is not None else "",
"speedup_split_over_fused": (
split_us / fused_us
if fused_us is not None and fused_us > 0
else ""
),
"fused_available": fused_available,
"correctness_ok": correctness_ok,
"correctness_detail": correctness_text,
"dtype": str(dtype),
"world_size": world_size,
"residual_mode": args.residual_mode,
"warmup": args.warmup,
"iters": args.iters,
"repeats": args.repeats,
}
)
if rank == 0 and args.csv_out:
os.makedirs(os.path.dirname(args.csv_out) or ".", exist_ok=True)
fieldnames = [
"mode",
"shape",
"m",
"n",
"bytes_per_rank",
"split_p50_us",
"fused_p50_us",
"speedup_split_over_fused",
"fused_available",
"correctness_ok",
"correctness_detail",
"dtype",
"world_size",
"residual_mode",
"warmup",
"iters",
"repeats",
]
with open(args.csv_out, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(csv_rows)
print(f"\nSaved CSV to: {args.csv_out}")
_barrier(device)
destroy_model_parallel()
destroy_distributed_environment()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,224 @@
"""For Now, MSCCL is only supported on TP16 and TP8 case
export WORLD_SIZE=1
export RANK=0
export MASTER_ADDR=127.0.0.1
export MASTER_PORT=12345
torchrun --nproc_per_node gpu \
--nnodes $WORLD_SIZE \
--node_rank $RANK \
--master_addr $MASTER_ADDR \
--master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py
"""
import os
from contextlib import nullcontext
from typing import List
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
from sglang.srt.distributed import init_distributed_environment
from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
from sglang.srt.distributed.parallel_state import (
get_tensor_model_parallel_group,
graph_capture,
initialize_model_parallel,
set_mscclpp_all_reduce,
)
def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor:
dist.all_reduce(torch_input, group=group)
return torch_input
def msccl_allreduce(
msccl_input: torch.Tensor, msccl_comm: PyMscclppCommunicator
) -> torch.Tensor:
return msccl_comm.all_reduce(msccl_input)
def pynccl_allreduce(
msccl_input: torch.Tensor, pynccl_comm: PyNcclCommunicator
) -> torch.Tensor:
pynccl_comm.all_reduce(msccl_input)
return msccl_input
def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10):
graph_input = inp_randn.clone()
with graph_capture() as graph_capture_context:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for _ in range(graph_loop):
graph_out = func(graph_input)
graph.replay()
func_output = graph_out.clone()
for _ in range(warmup_loop):
graph.replay()
torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
latencies: List[float] = []
for _ in range(test_loop):
torch.cuda.synchronize()
dist.barrier()
start_event.record()
graph.replay()
end_event.record()
end_event.synchronize()
latencies.append(start_event.elapsed_time(end_event))
func_cost_us = sum(latencies) / len(latencies) / graph_loop * 1000
graph.reset()
return func_output, func_cost_us
def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10):
eager_input = inp_randn.clone()
eager_output = func(eager_input)
func_output = eager_output.clone()
for _ in range(warmup_loop):
func(eager_input)
torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
torch.cuda.synchronize()
start_event.record()
for _ in range(test_loop):
func(eager_input)
end_event.record()
torch.cuda.synchronize()
func_cost_us = start_event.elapsed_time(end_event) / test_loop * 1000
return func_output, func_cost_us
def get_torch_prof_ctx(do_prof: bool):
ctx = (
torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
record_shapes=True,
with_stack=True,
)
if do_prof
else nullcontext()
)
return ctx
def human_readable_size(size, decimal_places=1):
for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
if size < 1024.0 or unit == "PiB":
break
size /= 1024.0
return f"{size:.{decimal_places}f} {unit}"
try:
from tabulate import tabulate
except ImportError:
print("tabulate not installed, skipping table printing")
tabulate = None
def print_markdown_table(data):
if tabulate is not None:
print(tabulate(data, headers="keys", tablefmt="github"))
return
headers = data[0].keys()
header_row = "| " + " | ".join(headers) + " |"
separator = "| " + " | ".join(["---"] * len(headers)) + " |"
rows = []
for item in data:
row = "| " + " | ".join(str(item[key]) for key in headers) + " |"
rows.append(row)
markdown_table = "\n".join([header_row, separator] + rows)
print(markdown_table)
if __name__ == "__main__":
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
if not dist.is_initialized():
dist.init_process_group(backend="nccl")
world, world_size = dist.group.WORLD, dist.get_world_size()
rank = dist.get_rank()
torch.cuda.set_device(rank % 8)
device = torch.cuda.current_device()
set_mscclpp_all_reduce(True)
init_distributed_environment(
world_size=world_size,
rank=rank,
local_rank=rank % 8,
)
initialize_model_parallel(tensor_model_parallel_size=world_size)
group = get_tensor_model_parallel_group().device_group
cpu_group = get_tensor_model_parallel_group().cpu_group
pynccl_comm = get_tensor_model_parallel_group().pynccl_comm
pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm
dist.barrier()
profile = False
dtype = torch.bfloat16
ctx = get_torch_prof_ctx(profile)
result = []
with ctx:
for i in range(10, 20):
sz = 2**i
if sz * dtype.itemsize > 2**20:
break
inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device)
memory = torch.empty_like(inp_randn)
memory_out = torch.empty_like(memory)
torch_eager_output, torch_eager_time = _bench_eager_time(
lambda inp: torch_allreduce(inp, group), inp_randn
)
msccl_eager_output, msccl_eager_time = _bench_eager_time(
lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn
)
msccl_graph_output, msccl_graph_time = _bench_graph_time(
lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn
)
# since pynccl is inplace op, this return result is not correct if graph loop > 1
_, pynccl_graph_time = _bench_graph_time(
lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn
)
torch.testing.assert_close(torch_eager_output, msccl_graph_output)
torch.testing.assert_close(torch_eager_output, msccl_eager_output)
result.append(
{
"msg_size": human_readable_size(inp_randn.nbytes),
"torch eager time": torch_eager_time,
"msccl eager time": msccl_eager_time,
"msccl graph time": msccl_graph_time,
"pynccl graph time": pynccl_graph_time,
}
)
if rank == 0:
print(f"sz={sz}, dtype={dtype}: correctness check PASS!")
if rank == 0:
print_markdown_table(result)
if profile:
prof_dir = f"prof/msccl"
os.makedirs(prof_dir, exist_ok=True)
ctx.export_chrome_trace(f"{prof_dir}/trace_rank{dist.get_rank()}.json.gz")

View File

@@ -0,0 +1,248 @@
"""For Now, TORCH_SYMM_MEM is only supported on following limited tp case
SM90: {
2: 64 * MiB, # 64 MB
4: 64 * MiB, # 64 MB
6: 128 * MiB, # 128 MB
8: 128 * MiB, # 128 MB
},
SM100: {
2: 64 * MiB, # 64 MB
4: 64 * MiB, # 64 MB
6: 128 * MiB, # 128 MB
8: 128 * MiB, # 128 MB
}
export WORLD_SIZE=8
export RANK=0
export MASTER_ADDR=127.0.0.1
export MASTER_PORT=12345
torchrun --nproc_per_node gpu \
--nnodes $WORLD_SIZE \
--node_rank $RANK \
--master_addr $MASTER_ADDR \
--master_port $MASTER_PORT ./benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py
"""
import os
from contextlib import nullcontext
from typing import List
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
from sglang.srt.distributed import init_distributed_environment
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
from sglang.srt.distributed.device_communicators.torch_symm_mem import (
TorchSymmMemCommunicator,
)
from sglang.srt.distributed.parallel_state import (
get_tensor_model_parallel_group,
graph_capture,
initialize_model_parallel,
set_torch_symm_mem_all_reduce,
)
from sglang.utils import is_in_ci
IS_CI = is_in_ci()
def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor:
dist.all_reduce(torch_input, group=group)
return torch_input
def torch_symm_mem_allreduce(
torch_symm_mem_input: torch.Tensor, torch_symm_mem_comm: TorchSymmMemCommunicator
) -> torch.Tensor:
return torch_symm_mem_comm.all_reduce(torch_symm_mem_input)
def pynccl_allreduce(
pynccl_input: torch.Tensor, pynccl_comm: PyNcclCommunicator
) -> torch.Tensor:
pynccl_comm.all_reduce(pynccl_input)
return pynccl_input
def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10):
graph_input = inp_randn.clone()
with graph_capture() as graph_capture_context:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for _ in range(graph_loop):
graph_out = func(graph_input)
graph.replay()
func_output = graph_out.clone()
for _ in range(warmup_loop):
graph.replay()
torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
latencies: List[float] = []
for _ in range(test_loop):
torch.cuda.synchronize()
dist.barrier()
start_event.record()
graph.replay()
end_event.record()
end_event.synchronize()
latencies.append(start_event.elapsed_time(end_event))
func_cost_us = sum(latencies) / len(latencies) / graph_loop * 1000
graph.reset()
return func_output, func_cost_us
def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10):
eager_input = inp_randn.clone()
eager_output = func(eager_input)
func_output = eager_output.clone()
for _ in range(warmup_loop):
func(eager_input)
torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
torch.cuda.synchronize()
start_event.record()
for _ in range(test_loop):
func(eager_input)
end_event.record()
torch.cuda.synchronize()
func_cost_us = start_event.elapsed_time(end_event) / test_loop * 1000
return func_output, func_cost_us
def get_torch_prof_ctx(do_prof: bool):
ctx = (
torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
record_shapes=True,
with_stack=True,
)
if do_prof
else nullcontext()
)
return ctx
def human_readable_size(size, decimal_places=1):
for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
if size < 1024.0 or unit == "PiB":
break
size /= 1024.0
return f"{size:.{decimal_places}f} {unit}"
try:
from tabulate import tabulate
except ImportError:
print("tabulate not installed, skipping table printing")
tabulate = None
def print_markdown_table(data):
if tabulate is not None:
print(tabulate(data, headers="keys", tablefmt="github"))
return
headers = data[0].keys()
header_row = "| " + " | ".join(headers) + " |"
separator = "| " + " | ".join(["---"] * len(headers)) + " |"
rows = []
for item in data:
row = "| " + " | ".join(str(item[key]) for key in headers) + " |"
rows.append(row)
markdown_table = "\n".join([header_row, separator] + rows)
print(markdown_table)
if __name__ == "__main__":
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
if not dist.is_initialized():
dist.init_process_group(backend="nccl")
world, world_size = dist.group.WORLD, dist.get_world_size()
rank = dist.get_rank()
torch.cuda.set_device(rank % 8)
device = torch.cuda.current_device()
set_torch_symm_mem_all_reduce(True)
init_distributed_environment(
world_size=world_size,
rank=rank,
local_rank=rank % 8,
)
initialize_model_parallel(tensor_model_parallel_size=world_size)
group = get_tensor_model_parallel_group().device_group
cpu_group = get_tensor_model_parallel_group().cpu_group
pynccl_comm = get_tensor_model_parallel_group().pynccl_comm
torch_symm_mem_comm = get_tensor_model_parallel_group().torch_symm_mem_comm
dist.barrier()
profile = False
dtype = torch.bfloat16
ctx = get_torch_prof_ctx(profile)
result = []
with ctx:
if IS_CI:
i_range = range(10, 11)
else:
i_range = range(10, 20)
for i in i_range:
sz = 2**i
if sz * dtype.itemsize > 2**24:
break
inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device)
memory = torch.empty_like(inp_randn)
memory_out = torch.empty_like(memory)
torch_eager_output, torch_eager_time = _bench_eager_time(
lambda inp: torch_allreduce(inp, group), inp_randn
)
symm_mem_eager_output, symm_mem_eager_time = _bench_eager_time(
lambda inp: torch_symm_mem_allreduce(inp, torch_symm_mem_comm),
inp_randn,
)
symm_mem_graph_output, symm_mem_graph_time = _bench_graph_time(
lambda inp: torch_symm_mem_allreduce(inp, torch_symm_mem_comm),
inp_randn,
)
# since pynccl is inplace op, this return result is not correct if graph loop > 1
_, pynccl_graph_time = _bench_graph_time(
lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn
)
torch.testing.assert_close(torch_eager_output, symm_mem_graph_output)
torch.testing.assert_close(torch_eager_output, symm_mem_eager_output)
result.append(
{
"msg_size": human_readable_size(inp_randn.nbytes),
"torch eager time": torch_eager_time,
"symm mem eager time": symm_mem_eager_time,
"symm mem graph time": symm_mem_graph_time,
"pynccl graph time": pynccl_graph_time,
}
)
if rank == 0:
print(f"sz={sz}, dtype={dtype}: correctness check PASS!")
if rank == 0:
print_markdown_table(result)
if profile:
prof_dir = f"prof/torch_symm_mem"
os.makedirs(prof_dir, exist_ok=True)
ctx.export_chrome_trace(f"{prof_dir}/trace_rank{dist.get_rank()}.json.gz")