Merge t16-grad-accum into main

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

# Conflicts:
#	README.md
#	docs/evolution.md
This commit is contained in:
2026-06-18 00:37:11 +08:00
10 changed files with 697 additions and 48 deletions

View File

@@ -74,6 +74,10 @@ fn main() {
// Optimization knobs (mirror bin/train).
let steps: usize = flag(&args, "--steps", 100);
let batch: usize = flag(&args, "--batch", 16);
// Micro-batch gradient accumulation (Phase T16): effective global batch =
// accum_steps × batch, all-reducing only at the accumulation boundary. Default
// 1 = no accumulation (bit-identical to the pre-T16 DDP path).
let accum_steps: usize = flag(&args, "--accum-steps", 1).max(1);
let seq_len: usize = flag(&args, "--seq", 64);
let max_lr: f32 = flag(&args, "--max-lr", 3e-3);
let min_lr: f32 = flag(&args, "--min-lr", max_lr * 0.1);
@@ -150,6 +154,7 @@ fn main() {
let dcfg = DdpConfig {
seq_len,
batch_size: batch,
accum_steps,
steps,
schedule: LrSchedule {
max_lr,
@@ -167,8 +172,9 @@ fn main() {
};
println!(
"training: {steps} steps, seq {seq_len}, global batch {batch}, lr {max_lr:.1e}{min_lr:.1e}, \
eval every {eval_every}"
"training: {steps} steps, seq {seq_len}, global batch {batch} × accum {accum_steps} = \
effective global batch {}, lr {max_lr:.1e}{min_lr:.1e}, eval every {eval_every}",
batch * accum_steps
);
if bf16 {