Merge t18-dropout into main
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> # Conflicts: # README.md # crates/xtrain-autodiff/tests/autograd.rs # crates/xtrain-model/src/model.rs # crates/xtrain-train/src/bin/train.rs # crates/xtrain-train/src/train_loop.rs # docs/evolution.md
This commit is contained in:
@@ -500,3 +500,48 @@ unsafe extern "C" {
|
||||
s: CudaStream,
|
||||
);
|
||||
}
|
||||
|
||||
// Dropout (Phase T18, csrc/ops/dropout.cu). A counter-based (stateless) RNG: the
|
||||
// keep/drop decision for element `i` is `hash(seed, i)` — no global state, so a
|
||||
// re-run with the same `seed` reproduces the same mask (compatible with T13
|
||||
// activation recomputation). Forward writes `out = x ⊙ mask` and the fp32 `mask`
|
||||
// buffer (mask[i] = (1/(1-p)) if kept else 0, the inverted-dropout scale);
|
||||
// backward applies the SAME mask: dx = d ⊙ mask. fp32 + bf16 activation variants
|
||||
// (mask is fp32 in both; the uniform is computed in fp32, dtype-independent).
|
||||
#[cfg(not(no_cuda))]
|
||||
unsafe extern "C" {
|
||||
pub fn launch_dropout_fwd_f32(
|
||||
x: *const f32,
|
||||
out: *mut f32,
|
||||
mask: *mut f32,
|
||||
p: f32,
|
||||
scale: f32,
|
||||
seed: u64,
|
||||
n: i32,
|
||||
s: CudaStream,
|
||||
);
|
||||
pub fn launch_dropout_bwd_f32(
|
||||
d: *const f32,
|
||||
mask: *const f32,
|
||||
dx: *mut f32,
|
||||
n: i32,
|
||||
s: CudaStream,
|
||||
);
|
||||
pub fn launch_dropout_fwd_bf16(
|
||||
x: *const c_void,
|
||||
out: *mut c_void,
|
||||
mask: *mut f32,
|
||||
p: f32,
|
||||
scale: f32,
|
||||
seed: u64,
|
||||
n: i32,
|
||||
s: CudaStream,
|
||||
);
|
||||
pub fn launch_dropout_bwd_bf16(
|
||||
d: *const c_void,
|
||||
mask: *const f32,
|
||||
dx: *mut c_void,
|
||||
n: i32,
|
||||
s: CudaStream,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user