Merge t18-dropout into main
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> # Conflicts: # README.md # crates/xtrain-autodiff/tests/autograd.rs # crates/xtrain-model/src/model.rs # crates/xtrain-train/src/bin/train.rs # crates/xtrain-train/src/train_loop.rs # docs/evolution.md
This commit is contained in:
@@ -140,6 +140,31 @@ pub fn swiglu(gate: &Var, up: &Var) -> Var {
|
||||
mul(&silu(gate), up)
|
||||
}
|
||||
|
||||
/// Dropout (Phase T18). With probability `p` zero each element, scale the kept
|
||||
/// ones by `1/(1-p)` (inverted dropout — `E[out] == x`). The keep/drop mask is
|
||||
/// drawn by a counter-based RNG from `(seed, element index)`, so it is fully
|
||||
/// determined by `seed` (same `seed` ⇒ same mask: stable across the T13 recompute
|
||||
/// re-run, and held fixed across the ± perturbation of a finite-diff grad-check).
|
||||
/// Forward caches the per-element scale `mask`; **backward applies the same mask**
|
||||
/// (`dx = d ⊙ mask`), making dropout a fixed elementwise linear map of `x`.
|
||||
///
|
||||
/// `p == 0` is a no-op: returns `x.clone()` (no node added) so the default graph
|
||||
/// is bit-identical to the no-dropout path. eval-time identity is handled by the
|
||||
/// caller simply not invoking dropout (the model's train/eval switch).
|
||||
pub fn dropout(x: &Var, p: f32, seed: u64) -> Var {
|
||||
if p == 0.0 {
|
||||
return x.clone();
|
||||
}
|
||||
let (out, mask) = x.value().dropout(p, seed);
|
||||
Var::from_op(
|
||||
out,
|
||||
vec![x.clone()],
|
||||
Box::new(move |d, parents| {
|
||||
Var::push_grad(&parents[0], Tensor::dropout_backward(d, &mask));
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
/// RoPE (rotate_half) over `x:[tokens,heads,head_dim]` with per-sequence position
|
||||
/// `row % period` (`period` = sequence length; `period == tokens` for a single
|
||||
/// sequence). Orthogonal map, so the backward is the inverse rotation of `dy` — no
|
||||
|
||||
Reference in New Issue
Block a user