Implement all 5 Triton kernel labs
- vector_add: basic masked load/store with block indexing - row_softmax: single-pass numerically stable softmax per row - tiled_matmul: K-dimension tile loop with edge masking (IEEE precision) - online_softmax: two-pass running max/sum recurrence across blocks - flash_attention_fwd: blockwise Q/K/V with online softmax, causal support All 26 tests pass on RTX 5090 (CUDA 12.8, Triton 3.6).
This commit is contained in:
@@ -35,11 +35,24 @@ if TRITON_AVAILABLE:
|
||||
):
|
||||
pid_m = tl.program_id(axis=0)
|
||||
pid_n = tl.program_id(axis=1)
|
||||
# TODO(student): compute the tile owned by this program instance.
|
||||
# TODO(student): loop over K tiles and accumulate partial products.
|
||||
# TODO(student): use masking on edge tiles.
|
||||
# TODO(student): store the output tile.
|
||||
pass
|
||||
offs_m = pid_m * block_m + tl.arange(0, block_m)
|
||||
offs_n = pid_n * block_n + tl.arange(0, block_n)
|
||||
offs_k = tl.arange(0, block_k)
|
||||
a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak
|
||||
b_ptrs = b_ptr + offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn
|
||||
acc = tl.zeros((block_m, block_n), dtype=tl.float32)
|
||||
for ki in range(0, tl.cdiv(k, block_k)):
|
||||
k_offset = ki * block_k
|
||||
a_mask = (offs_m[:, None] < m) & ((k_offset + offs_k[None, :]) < k)
|
||||
b_mask = ((k_offset + offs_k[:, None]) < k) & (offs_n[None, :] < n)
|
||||
a_tile = tl.load(a_ptrs, mask=a_mask, other=0.0)
|
||||
b_tile = tl.load(b_ptrs, mask=b_mask, other=0.0)
|
||||
acc += tl.dot(a_tile, b_tile, input_precision="ieee")
|
||||
a_ptrs += block_k * stride_ak
|
||||
b_ptrs += block_k * stride_bk
|
||||
c_mask = (offs_m[:, None] < m) & (offs_n[None, :] < n)
|
||||
c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
|
||||
tl.store(c_ptrs, acc, mask=c_mask)
|
||||
|
||||
|
||||
def triton_tiled_matmul(
|
||||
@@ -57,5 +70,17 @@ def triton_tiled_matmul(
|
||||
raise ValueError(f"incompatible shapes: {a.shape} and {b.shape}")
|
||||
if not a.is_cuda or not b.is_cuda:
|
||||
raise ValueError("Triton kernels in this lab expect CUDA tensors.")
|
||||
raise NotImplementedError("TODO(student): implement the tiled Triton matmul path.")
|
||||
m, k = a.shape
|
||||
_, n = b.shape
|
||||
c = torch.empty((m, n), device=a.device, dtype=a.dtype)
|
||||
grid = (triton.cdiv(m, block_m), triton.cdiv(n, block_n))
|
||||
tiled_matmul_kernel[grid](
|
||||
a, b, c,
|
||||
m, n, k,
|
||||
a.stride(0), a.stride(1),
|
||||
b.stride(0), b.stride(1),
|
||||
c.stride(0), c.stride(1),
|
||||
block_m=block_m, block_n=block_n, block_k=block_k,
|
||||
)
|
||||
return c
|
||||
|
||||
|
||||
Reference in New Issue
Block a user