Merge t16-grad-accum into main
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> # Conflicts: # README.md # docs/evolution.md
This commit is contained in:
@@ -74,6 +74,10 @@ fn main() {
|
||||
// Optimization knobs (mirror bin/train).
|
||||
let steps: usize = flag(&args, "--steps", 100);
|
||||
let batch: usize = flag(&args, "--batch", 16);
|
||||
// Micro-batch gradient accumulation (Phase T16): effective global batch =
|
||||
// accum_steps × batch, all-reducing only at the accumulation boundary. Default
|
||||
// 1 = no accumulation (bit-identical to the pre-T16 DDP path).
|
||||
let accum_steps: usize = flag(&args, "--accum-steps", 1).max(1);
|
||||
let seq_len: usize = flag(&args, "--seq", 64);
|
||||
let max_lr: f32 = flag(&args, "--max-lr", 3e-3);
|
||||
let min_lr: f32 = flag(&args, "--min-lr", max_lr * 0.1);
|
||||
@@ -150,6 +154,7 @@ fn main() {
|
||||
let dcfg = DdpConfig {
|
||||
seq_len,
|
||||
batch_size: batch,
|
||||
accum_steps,
|
||||
steps,
|
||||
schedule: LrSchedule {
|
||||
max_lr,
|
||||
@@ -167,8 +172,9 @@ fn main() {
|
||||
};
|
||||
|
||||
println!(
|
||||
"training: {steps} steps, seq {seq_len}, global batch {batch}, lr {max_lr:.1e}→{min_lr:.1e}, \
|
||||
eval every {eval_every}"
|
||||
"training: {steps} steps, seq {seq_len}, global batch {batch} × accum {accum_steps} = \
|
||||
effective global batch {}, lr {max_lr:.1e}→{min_lr:.1e}, eval every {eval_every}",
|
||||
batch * accum_steps
|
||||
);
|
||||
|
||||
if bf16 {
|
||||
|
||||
Reference in New Issue
Block a user