train: --bf16 flag (fp32-master AMP) + bf16 correctness test

- TinyTransformer::with_compute_dtype(BF16): embedding stays fp32
  master then casts to bf16; each linear casts its fp32 weight to bf16
  on the fly; logits cast back to fp32 for cross-entropy. Default F32
  reproduces the v0-v4 forward graph bit-for-bit.
- --bf16 flag on bin/train and bin/train_ddp (off by default).
- tests/bf16.rs: same fp32 master weights run fp32 vs bf16; assert
  loss/logits/grads within a loose bf16 tol, no NaN, and grads are
  fp32 (master untouched).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 14:14:55 +08:00
parent b0086b5214
commit 0a2a4dcaa8
4 changed files with 241 additions and 16 deletions

View File

@@ -82,6 +82,9 @@ fn main() {
let val_tokens: usize = flag(&args, "--val-tokens", 0);
let eval_every: usize = flag(&args, "--eval-every", 0);
let eval_batches: usize = flag(&args, "--eval-batches", 64);
// bf16 mixed precision (Phase T12): fp32 master weights, bf16 linears +
// activations. Opt-in; default fp32 reproduces v0v4 numerics.
let bf16 = args.iter().any(|a| a == "--bf16");
let ckpt: Option<PathBuf> = args
.iter()
.position(|a| a == "--ckpt")
@@ -161,12 +164,22 @@ fn main() {
eval every {eval_every}"
);
if bf16 {
println!("bf16 mixed precision: ON (fp32 master weights)");
}
let results = launch(
&devices,
&train_corpus,
valid.as_ref(),
&dcfg,
move |device| build_model(cfg, device),
move |device| {
let m = build_model(cfg, device);
if bf16 {
m.with_compute_dtype(xtrain_tensor::DType::BF16)
} else {
m
}
},
);
let r0 = &results[0];
let start = r0.losses.first().copied().unwrap_or(0.0);