train+ddp: micro-batch gradient accumulation (--accum-steps)

Accumulate grads over N micro-batches, then one AdamW step + zero_grad,
for an effective batch of N×micro at one micro-batch's activation cost.
Each micro-loss is scaled by 1/N before backward (the tape SUM-accumulates
the scaled grads) so the boundary grad equals a single step over an N×
batch. accum==1 skips the scale → bit-identical to the pre-T16 path.

DDP: the cross-rank all-reduce fires ONLY at the accumulation boundary
(intermediate micro-steps are local-only, no NCCL); the /world average is
orthogonal to the per-micro 1/N, so the boundary grad is the effective
global-batch mean. New --accum-steps flag in both train binaries; effective
batch is printed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

crates/xtrain-train/src/bin/train.rs

@@ -101,6 +101,10 @@ fn main() {
     // Optimization knobs.
     let steps: usize = flag(&args, "--steps", 2000);
     let batch_size: usize = flag(&args, "--batch", 8);
+    // Micro-batch gradient accumulation (Phase T16): effective batch =
+    // accum_steps × batch, at one micro-batch's activation-memory cost. Default 1
+    // = no accumulation (bit-identical to the pre-T16 path).
+    let accum_steps: usize = flag(&args, "--accum-steps", 1).max(1);
     let seq_len: usize = flag(&args, "--seq", 64);
     let max_lr: f32 = flag(&args, "--max-lr", 3e-3);
     let min_lr: f32 = flag(&args, "--min-lr", max_lr * 0.1);
@@ -201,6 +205,7 @@ fn main() {
     let tcfg = TrainConfig {
         seq_len,
         batch_size,
+        accum_steps,
         steps,
         schedule: LrSchedule {
             max_lr,
@@ -219,10 +224,13 @@ fn main() {
     };
 
     println!(
-        "training: {} steps, seq {}, batch {}, lr {:.1e}→{:.1e}, eval every {}",
+        "training: {} steps, seq {}, batch {} × accum {} = effective batch {}, \
+         lr {:.1e}→{:.1e}, eval every {}",
         tcfg.steps,
         tcfg.seq_len,
         tcfg.batch_size,
+        tcfg.accum_steps,
+        tcfg.batch_size * tcfg.accum_steps,
         tcfg.schedule.max_lr,
         tcfg.schedule.min_lr,
         tcfg.eval_every

crates/xtrain-train/src/train_loop.rs

@@ -27,6 +27,12 @@ use crate::schedule::LrSchedule;
 pub struct TrainConfig {
     pub seq_len: usize,
     pub batch_size: usize,
+    /// Micro-batch gradient accumulation (Phase T16): each optimizer step
+    /// accumulates grads over `accum_steps` micro-batches of `batch_size`
+    /// sequences, giving an EFFECTIVE batch of `accum_steps × batch_size` at the
+    /// activation-memory cost of a single micro-batch. `1` = no accumulation
+    /// (bit-identical to the pre-T16 path).
+    pub accum_steps: usize,
     pub steps: usize,
     pub schedule: LrSchedule,
     pub weight_decay: f32,
@@ -74,28 +80,43 @@ pub fn train(
     // Best-val checkpointing only kicks in when we actually evaluate.
     let track_best = valid.is_some() && cfg.eval_every > 0;
 
+    let accum = cfg.accum_steps.max(1);
     for step in 0..cfg.steps {
         let lr = cfg.schedule.lr(step);
 
-        // Sample `batch_size` sequences and run them as ONE batched forward/
-        // backward. The CE mean over all batch*seq rows is the batch-mean loss, so
-        // backward already yields the batch-mean gradient (clip pre-scale = 1.0).
-        let mut inputs = Vec::with_capacity(cfg.batch_size);
-        let mut targets_v = Vec::with_capacity(cfg.batch_size);
-        for _ in 0..cfg.batch_size {
-            let (input, target) = corpus.sample(cfg.seq_len, &mut rng);
-            inputs.push(input);
-            targets_v.push(target);
+        // Accumulate grads over `accum` micro-batches of `batch_size` sequences,
+        // then take ONE optimizer step (Phase T16). Each micro-batch is ONE batched
+        // forward/backward; its loss is the CE mean over batch*seq rows, so backward
+        // yields that micro-batch's mean grad. To make the SUM over `accum` micro-
+        // batches equal a single step over an `accum × batch` batch, each micro-loss
+        // is scaled by 1/accum before backward (the tape SUM-accumulates the scaled
+        // grads). `accum == 1` skips the scale entirely → bit-identical to pre-T16.
+        let mut step_loss_sum = 0.0f32;
+        for _ in 0..accum {
+            let mut inputs = Vec::with_capacity(cfg.batch_size);
+            let mut targets_v = Vec::with_capacity(cfg.batch_size);
+            for _ in 0..cfg.batch_size {
+                let (input, target) = corpus.sample(cfg.seq_len, &mut rng);
+                inputs.push(input);
+                targets_v.push(target);
+            }
+            let ids = batched_ids_tensor(&inputs, device);
+            let targets = batched_ids_tensor(&targets_v, device);
+            let loss = model.loss_batched(&ids, &targets, cfg.batch_size);
+            step_loss_sum += read_scalar(&loss);
+            if accum == 1 {
+                loss.backward();
+            } else {
+                xtrain_autodiff::ops::scale(&loss, 1.0 / accum as f32).backward();
+            }
+            tokens_seen += (cfg.batch_size * cfg.seq_len) as u64;
         }
-        let ids = batched_ids_tensor(&inputs, device);
-        let targets = batched_ids_tensor(&targets_v, device);
-        let loss = model.loss_batched(&ids, &targets, cfg.batch_size);
-        let step_loss = read_scalar(&loss);
-        loss.backward();
-        tokens_seen += (cfg.batch_size * cfg.seq_len) as u64;
+        // Reported loss = mean over the effective batch = mean of the raw micro
+        // losses (each is itself a micro-batch mean of equal size).
+        let step_loss = step_loss_sum / accum as f32;
         losses.push(step_loss);
 
-        // Backward already produced the batch-mean gradient — just clip it.
+        // Backward already produced the effective-batch mean gradient — just clip.
         let gnorm = clip_grad_norm_gpu(&params, cfg.max_grad_norm, 1.0);
         opt.step(lr, &params);
         for p in &params {