Gahow Wang
c8e8153702
CUDA kernels (csrc/): - common.cuh: shared warp_reduce_sum/max, block_reduce_sum/max - normalization/rmsnorm.cu: RMSNorm (F32 + BF16) - normalization/layernorm.cu: LayerNorm with Welford (F32 + BF16) - activation/activations.cu: GELU tanh-approx + SiLU (F32 + BF16) - reduce/softmax.cu: safe softmax, 3-pass (F32 + BF16) - embedding/embedding.cu: gather lookup (F32 + BF16) - embedding/rope.cu: RoPE in-place + precomputed cos/sin cache (F32 + BF16) Rust wrappers (xserv-kernels/src/): - rmsnorm.rs, layernorm.rs, activation.rs, softmax.rs, embedding.rs, rope.rs - RopeCache struct with GPU-side precomputation Tests: 12 new tests (ops_test.rs), all passing with good precision: - F32: max_err 1e-6 ~ 1e-9 - BF16: max_err 2e-3 ~ 7e-3 Total: 29 kernel tests + 27 prior = 56 tests passing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
56 lines
1.8 KiB
Plaintext
56 lines
1.8 KiB
Plaintext
#include <cuda_bf16.h>
|
|
|
|
// Embedding lookup: out[seq_idx] = table[token_ids[seq_idx]]
|
|
// Grid: num_tokens, Block: handles hidden_size elements per token.
|
|
|
|
__global__ void embedding_f32(
|
|
const float* __restrict__ table, // [vocab_size, hidden_size]
|
|
const int* __restrict__ token_ids, // [num_tokens]
|
|
float* __restrict__ out, // [num_tokens, hidden_size]
|
|
int hidden_size
|
|
) {
|
|
int token_idx = blockIdx.x;
|
|
int tid = token_ids[token_idx];
|
|
const float* row = table + tid * hidden_size;
|
|
float* dst = out + token_idx * hidden_size;
|
|
|
|
for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
|
|
dst[i] = row[i];
|
|
}
|
|
}
|
|
|
|
__global__ void embedding_bf16(
|
|
const __nv_bfloat16* __restrict__ table,
|
|
const int* __restrict__ token_ids,
|
|
__nv_bfloat16* __restrict__ out,
|
|
int hidden_size
|
|
) {
|
|
int token_idx = blockIdx.x;
|
|
int tid = token_ids[token_idx];
|
|
const __nv_bfloat16* row = table + tid * hidden_size;
|
|
__nv_bfloat16* dst = out + token_idx * hidden_size;
|
|
|
|
for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
|
|
dst[i] = row[i];
|
|
}
|
|
}
|
|
|
|
extern "C" {
|
|
|
|
void launch_embedding_f32(const void* table, const void* token_ids, void* out,
|
|
int num_tokens, int hidden_size, void* stream) {
|
|
int block = (hidden_size < 256) ? hidden_size : 256;
|
|
embedding_f32<<<num_tokens, block, 0, (cudaStream_t)stream>>>(
|
|
(const float*)table, (const int*)token_ids, (float*)out, hidden_size);
|
|
}
|
|
|
|
void launch_embedding_bf16(const void* table, const void* token_ids, void* out,
|
|
int num_tokens, int hidden_size, void* stream) {
|
|
int block = (hidden_size < 256) ? hidden_size : 256;
|
|
embedding_bf16<<<num_tokens, block, 0, (cudaStream_t)stream>>>(
|
|
(const __nv_bfloat16*)table, (const int*)token_ids,
|
|
(__nv_bfloat16*)out, hidden_size);
|
|
}
|
|
|
|
}
|