50 lines
1.3 KiB
Python
50 lines
1.3 KiB
Python
from __future__ import annotations
|
|
|
|
import statistics
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
import torch
|
|
|
|
from kernels.triton.row_softmax import triton_row_softmax
|
|
from reference.torch_row_softmax import torch_row_softmax
|
|
|
|
|
|
def benchmark(fn, *args, warmup: int = 5, reps: int = 25) -> float:
|
|
for _ in range(warmup):
|
|
fn(*args)
|
|
if args[0].is_cuda:
|
|
torch.cuda.synchronize()
|
|
times_ms = []
|
|
for _ in range(reps):
|
|
if args[0].is_cuda:
|
|
torch.cuda.synchronize()
|
|
start = time.perf_counter()
|
|
fn(*args)
|
|
if args[0].is_cuda:
|
|
torch.cuda.synchronize()
|
|
times_ms.append((time.perf_counter() - start) * 1e3)
|
|
return statistics.median(times_ms)
|
|
|
|
|
|
def main() -> None:
|
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
x = torch.randn(4096, 1024, device=device)
|
|
ref_ms = benchmark(torch_row_softmax, x)
|
|
print(f"torch_row_softmax: {ref_ms:.3f} ms")
|
|
if device == "cuda":
|
|
try:
|
|
triton_ms = benchmark(triton_row_softmax, x)
|
|
print(f"triton_row_softmax: {triton_ms:.3f} ms")
|
|
except (NotImplementedError, RuntimeError) as exc:
|
|
print(f"triton_row_softmax: skipped ({exc})")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|