train: --bf16 flag (fp32-master AMP) + bf16 correctness test
- TinyTransformer::with_compute_dtype(BF16): embedding stays fp32 master then casts to bf16; each linear casts its fp32 weight to bf16 on the fly; logits cast back to fp32 for cross-entropy. Default F32 reproduces the v0-v4 forward graph bit-for-bit. - --bf16 flag on bin/train and bin/train_ddp (off by default). - tests/bf16.rs: same fp32 master weights run fp32 vs bf16; assert loss/logits/grads within a loose bf16 tol, no NaN, and grads are fp32 (master untouched). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -82,6 +82,9 @@ fn main() {
|
||||
let val_tokens: usize = flag(&args, "--val-tokens", 0);
|
||||
let eval_every: usize = flag(&args, "--eval-every", 0);
|
||||
let eval_batches: usize = flag(&args, "--eval-batches", 64);
|
||||
// bf16 mixed precision (Phase T12): fp32 master weights, bf16 linears +
|
||||
// activations. Opt-in; default fp32 reproduces v0–v4 numerics.
|
||||
let bf16 = args.iter().any(|a| a == "--bf16");
|
||||
let ckpt: Option<PathBuf> = args
|
||||
.iter()
|
||||
.position(|a| a == "--ckpt")
|
||||
@@ -161,12 +164,22 @@ fn main() {
|
||||
eval every {eval_every}"
|
||||
);
|
||||
|
||||
if bf16 {
|
||||
println!("bf16 mixed precision: ON (fp32 master weights)");
|
||||
}
|
||||
let results = launch(
|
||||
&devices,
|
||||
&train_corpus,
|
||||
valid.as_ref(),
|
||||
&dcfg,
|
||||
move |device| build_model(cfg, device),
|
||||
move |device| {
|
||||
let m = build_model(cfg, device);
|
||||
if bf16 {
|
||||
m.with_compute_dtype(xtrain_tensor::DType::BF16)
|
||||
} else {
|
||||
m
|
||||
}
|
||||
},
|
||||
);
|
||||
let r0 = &results[0];
|
||||
let start = r0.losses.first().copied().unwrap_or(0.0);
|
||||
|
||||
Reference in New Issue
Block a user