test+bins: flash grad-check, flash==composed, PyTorch parity, --flash flag
autograd: flash_attention_batched_bwd (dQ/dK/dV finite-diff, seq>tile) + flash_matches_composed_fwd. model/tests/flash.rs: flash==composed on-vs-off (logits/loss/every param grad), fp32 + bf16. parity_dump: XTRAIN_PARITY_FLASH dumps the flash path for the same parity.py oracle (PyTorch SDPA parity at B>1). train + train_ddp get the --flash flag. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -67,7 +67,7 @@ fn dump_for_parity() {
|
||||
|
||||
// Same deterministic init as the overfit test.
|
||||
let mut seed = 1u64;
|
||||
let model = TinyTransformer::new(cfg, device, |shape| {
|
||||
let mut model = TinyTransformer::new(cfg, device, |shape| {
|
||||
seed = seed.wrapping_add(1);
|
||||
let n: usize = shape.iter().product();
|
||||
if shape.len() == 1 {
|
||||
@@ -76,6 +76,14 @@ fn dump_for_parity() {
|
||||
fill(n, seed, 0.08)
|
||||
}
|
||||
});
|
||||
// T14: with XTRAIN_PARITY_FLASH set, dump from the fused flash-attention path.
|
||||
// flash is the SAME SDPA math, so the SAME parity.py PyTorch oracle is the
|
||||
// reference for both paths — running this once per path checks flash against
|
||||
// PyTorch at B>1 (forward logits + every parameter grad).
|
||||
if std::env::var("XTRAIN_PARITY_FLASH").is_ok() {
|
||||
model = model.with_flash(true);
|
||||
println!("parity: FLASH attention path");
|
||||
}
|
||||
|
||||
// config + ids
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user