test+bins: flash grad-check, flash==composed, PyTorch parity, --flash flag
autograd: flash_attention_batched_bwd (dQ/dK/dV finite-diff, seq>tile) + flash_matches_composed_fwd. model/tests/flash.rs: flash==composed on-vs-off (logits/loss/every param grad), fp32 + bf16. parity_dump: XTRAIN_PARITY_FLASH dumps the flash path for the same parity.py oracle (PyTorch SDPA parity at B>1). train + train_ddp get the --flash flag. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -89,6 +89,9 @@ fn main() {
|
||||
// rank checkpoints its own forward/backward; exact grads, lower peak activation
|
||||
// memory (lets dim1024 batch32 fit). Opt-in; default off.
|
||||
let recompute = args.iter().any(|a| a == "--recompute");
|
||||
// Fused flash-attention (Phase T14): single fused SDPA kernel, online softmax,
|
||||
// no materialized [bh,S,S] scores. Opt-in; default off keeps the composed path.
|
||||
let flash = args.iter().any(|a| a == "--flash");
|
||||
let ckpt: Option<PathBuf> = args
|
||||
.iter()
|
||||
.position(|a| a == "--ckpt")
|
||||
@@ -174,6 +177,9 @@ fn main() {
|
||||
if recompute {
|
||||
println!("activation recompute: ON (per-block gradient checkpointing)");
|
||||
}
|
||||
if flash {
|
||||
println!("flash-attention: ON (fused SDPA kernel, no materialized scores)");
|
||||
}
|
||||
let results = launch(
|
||||
&devices,
|
||||
&train_corpus,
|
||||
@@ -187,6 +193,9 @@ fn main() {
|
||||
if recompute {
|
||||
m = m.with_recompute(true);
|
||||
}
|
||||
if flash {
|
||||
m = m.with_flash(true);
|
||||
}
|
||||
m
|
||||
},
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user