cuda: bf16 cuBLAS GemmEx (16BF in/out, fp32 accum) + cast kernels
Add the bf16 compute primitives for T12 mixed precision: - DType::BF16 (half::bf16 as TensorDType), 2 bytes. - cublasGemmEx / cublasGemmStridedBatchedEx FFI + CUDA_R_16BF / CUBLAS_COMPUTE_32F constants (values per xserv gemm.rs). - cublas::gemm_ex / gemm_ex_strided_batched: same row-major⟺col-major transpose algebra as sgemm, bf16 in/out, fp32 accumulation. - csrc/ops/cast.cu: f32<->bf16 cast + bf16 elementwise (add/mul/scale/ silu(+dx)/add_bias/sum_rows), each load->fp32->compute->store bf16. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,12 +1,16 @@
|
||||
//! Tensor data types.
|
||||
//!
|
||||
//! T2 only needs `F32`, but the enum + `TensorDType` trait are structured so
|
||||
//! half-precision types (F16/BF16) can be added later (T7 mixed precision)
|
||||
//! without touching call sites.
|
||||
//! T2 only needs `F32`; `BF16` was added in T12 for mixed-precision training
|
||||
//! (bf16 linears / activations, fp32 master weights — see
|
||||
//! `docs/11-bf16-mixed-precision.md`). The enum + `TensorDType` trait keep call
|
||||
//! sites dtype-polymorphic.
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum DType {
|
||||
F32,
|
||||
/// bfloat16: 1 sign / 8 exponent / 7 mantissa. Same exponent range as f32
|
||||
/// (so no loss scaling needed), ~2-3 decimal digits. The T12 AMP compute type.
|
||||
BF16,
|
||||
/// 32-bit signed integers. Used for cross-entropy targets (token ids).
|
||||
I32,
|
||||
}
|
||||
@@ -15,6 +19,7 @@ impl DType {
|
||||
pub fn size_bytes(self) -> usize {
|
||||
match self {
|
||||
DType::F32 => 4,
|
||||
DType::BF16 => 2,
|
||||
DType::I32 => 4,
|
||||
}
|
||||
}
|
||||
@@ -22,6 +27,7 @@ impl DType {
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
DType::F32 => "f32",
|
||||
DType::BF16 => "bf16",
|
||||
DType::I32 => "i32",
|
||||
}
|
||||
}
|
||||
@@ -50,6 +56,16 @@ impl TensorDType for f32 {
|
||||
}
|
||||
}
|
||||
|
||||
impl TensorDType for half::bf16 {
|
||||
const DTYPE: DType = DType::BF16;
|
||||
fn to_f64(self) -> f64 {
|
||||
self.to_f64()
|
||||
}
|
||||
fn from_f64(v: f64) -> Self {
|
||||
half::bf16::from_f64(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl TensorDType for i32 {
|
||||
const DTYPE: DType = DType::I32;
|
||||
fn to_f64(self) -> f64 {
|
||||
|
||||
Reference in New Issue
Block a user