docs: T21 — record DDP-dropout wiring gap + fix (known-issues / evolution / dropout doc)

- known-issues.md: new "DDP-dropout wiring" Fixed entry (gap + fix +
  regression test), with the meta-lesson that op/single-GPU unit tests can
  miss launcher-level integration gaps — only the V9-PILOT end-to-end run on
  the real launcher path exposed it.
- 17-dropout.md: annotate the DDP-combination note with the T18 wiring gap
  and its T21 fix.
- evolution.md: T21 row (Infra) recording the fix + meta-lesson.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

crates/xtrain-distributed/tests/ddp_correctness.rs

@@ -38,17 +38,24 @@ fn test_config(vocab: usize) -> Config {
     cfg
 }
 
-/// Run `cfg`/`dcfg` as a 2-rank DDP job (the same launcher path as production) and
-/// return rank 0's (loss trace, final params on host, final `is_training()` flag).
-/// `cfg` carries the dropout prob; `dcfg` carries the loop knobs. Caller asserts.
-fn run_ddp2(
+/// Run `cfg`/`dcfg` as a DDP job over `devices` (the same launcher path as
+/// production — `DdpContext::init` + `train_rank` per rank) and return rank 0's
+/// (loss trace, final params on host, final `is_training()` flag). `cfg` carries
+/// the dropout prob; `dcfg` carries the loop knobs. Caller asserts.
+///
+/// `world == 1` is the deterministic path: `all_reduce_average_grads` short-circuits
+/// (no NCCL collective), so the run is bit-reproducible — used for the bit-identity
+/// gate. `world >= 2` exercises the real cross-rank NCCL all-reduce, which is not
+/// bit-reproducible run-to-run on this PCIe box (KI-5), so those gates use the same
+/// ULP/relative tolerances as the rest of this file.
+fn run_ddp(
+    devices: &[u32],
     cfg: Config,
     corpus: &Corpus,
     valid: Option<&Corpus>,
     dcfg: &DdpConfig,
 ) -> (Vec<f32>, Vec<Vec<f32>>, bool) {
-    let world = 2usize;
-    let devices = [0u32, 1u32];
+    let world = devices.len();
     let id = get_unique_id();
     let results: Vec<(Vec<f32>, Vec<Vec<f32>>, bool)> = std::thread::scope(|s| {
         let handles: Vec<_> = devices
@@ -427,21 +434,28 @@ fn ddp_throughput_scaling() {
     }
 }
 
-/// T21 regression: prove dropout is actually LIVE under DDP, and that p=0 is
-/// bit-identical to the no-dropout path. Guards the V9-PILOT launcher-wiring gap —
-/// `train_ddp` had no `--dropout` flag and `train_rank` never called `model.train()`,
-/// so under DDP every forward ran in the default eval mode and dropout was a silent
-/// identity regardless of config. Op/single-GPU tests never exercised dropout-under-
-/// DDP, so it slipped through; this test runs the real launcher path (`train_rank`).
+/// T21 regression: prove dropout is actually LIVE under DDP (with `p>0`), and that
+/// `p=0` is bit-identical to the no-dropout path. Guards the V9-PILOT launcher-
+/// wiring gap — `train_ddp` had no `--dropout` flag and `train_rank` never called
+/// `model.train()`, so under DDP every forward ran in the default eval mode and
+/// dropout was a silent identity regardless of config. Op/single-GPU tests never
+/// exercised dropout-under-DDP, so it slipped through; this test runs the REAL
+/// launcher path (`DdpContext::init` + `train_rank`).
 ///
-/// With dropout fixed across the 4 sub-runs, all three checks below would FAIL on the
-/// pre-T21 code: (a) the p>0 trace would be bit-identical to p=0 (model stuck in eval
-/// mode → identity), and (c) `is_training()` would be false after the run.
+/// On the pre-T21 code, both load-bearing gates FAIL: GATE B (p>0 trace would be
+/// bit-identical to p=0 — model stuck in eval mode → dropout is identity) and GATE C
+/// (`is_training()` would be false after the run).
+///
+/// Bit-identity (GATE A) is asserted at `world=1`, where `all_reduce_average_grads`
+/// short-circuits (no NCCL) so the run is deterministic. The cross-rank NCCL
+/// all-reduce (`world>=2`) is not bit-reproducible run-to-run on this PCIe box (KI-5,
+/// observed ≤~2.4e-7), so the `world=2` p=0-vs-no-dropout check (GATE A2) uses the
+/// same KI-5 ULP tolerance as the rest of this file, while GATE B's live-dropout
+/// signal (>1e-3) sits orders of magnitude above that noise floor.
 #[test]
 fn ddp_dropout_is_live_and_p0_bit_identical() {
-    let world = 2usize;
-    if device::device_count().unwrap_or(0) < world as i32 {
-        eprintln!("skip: need >= {world} GPUs");
+    if device::device_count().unwrap_or(0) < 2 {
+        eprintln!("skip: need >= 2 GPUs");
         return;
     }
 
@@ -474,62 +488,82 @@ fn ddp_dropout_is_live_and_p0_bit_identical() {
         ckpt_path: None,
     };
 
-    // (1) p=0 config — the no-dropout baseline. cfg.dropout defaults to 0.0.
-    let cfg_p0 = test_config(vocab);
-    assert_eq!(cfg_p0.dropout, 0.0, "baseline cfg must have dropout 0");
-    let (loss_p0, params_p0, _) = run_ddp2(cfg_p0, &corpus, Some(&valid), &base_dcfg);
-
-    // (2) Same config, dropout disabled by p=0 but explicitly set — must be the
-    //     SAME run (sanity: setting dropout=0 doesn't perturb anything).
-    let mut cfg_p0b = test_config(vocab);
-    cfg_p0b.dropout = 0.0;
-    let (loss_p0b, params_p0b, _) = run_ddp2(cfg_p0b, &corpus, Some(&valid), &base_dcfg);
-
-    // (3) Same config + data + seed, but dropout p=0.2 ON.
-    let mut cfg_p = test_config(vocab);
-    cfg_p.dropout = 0.2;
-    let (loss_p, _params_p, train_flag_p) = run_ddp2(cfg_p, &corpus, Some(&valid), &base_dcfg);
-
-    // GATE A — p=0 is bit-identical to the no-dropout path (regression guard).
-    // ops::dropout(p=0) is a clone no-op regardless of training mode, so these two
-    // runs must agree to the last bit on BOTH the loss trace and the final params.
-    let mut max_loss_diff = 0.0f32;
-    for (a, b) in loss_p0.iter().zip(&loss_p0b) {
-        max_loss_diff = max_loss_diff.max((a - b).abs());
-    }
-    let mut max_param_diff = 0.0f32;
-    for (a, b) in params_p0.iter().zip(&params_p0b) {
-        for (x, y) in a.iter().zip(b) {
-            max_param_diff = max_param_diff.max((x - y).abs());
-        }
-    }
+    // --- GATE A: bit-identity at world=1 (deterministic — no NCCL collective). ---
+    // The regression guard for `--dropout 0`: a p=0 run must be bit-for-bit the same
+    // as the no-dropout path, since ops::dropout(p=0) is a clone no-op regardless of
+    // training mode. At world=1, all_reduce_average_grads short-circuits, so the run
+    // is fully deterministic and bit-identity is the honest invariant (no NCCL noise).
+    let d1 = [0u32];
+    let cfg_nodrop = test_config(vocab); // cfg.dropout defaults to 0.0
+    assert_eq!(cfg_nodrop.dropout, 0.0, "baseline cfg must have dropout 0");
+    let mut cfg_p0 = test_config(vocab);
+    cfg_p0.dropout = 0.0; // explicitly set p=0 — must not perturb anything
+    let (loss_nd1, params_nd1, _) = run_ddp(&d1, cfg_nodrop, &corpus, Some(&valid), &base_dcfg);
+    let (loss_p01, params_p01, _) = run_ddp(&d1, cfg_p0, &corpus, Some(&valid), &base_dcfg);
+    let max_loss_diff_1 = loss_nd1
+        .iter()
+        .zip(&loss_p01)
+        .map(|(a, b)| (a - b).abs())
+        .fold(0.0f32, f32::max);
+    let max_param_diff_1 = params_nd1
+        .iter()
+        .zip(&params_p01)
+        .flat_map(|(a, b)| a.iter().zip(b).map(|(x, y)| (x - y).abs()))
+        .fold(0.0f32, f32::max);
     println!(
-        "T21 GATE A (p=0 bit-identical): max |loss diff| = {max_loss_diff:.3e}, \
-         max |param diff| = {max_param_diff:.3e}"
+        "T21 GATE A (world=1 p=0 bit-identical): max |loss diff| = {max_loss_diff_1:.3e}, \
+         max |param diff| = {max_param_diff_1:.3e}"
     );
     assert_eq!(
-        max_loss_diff, 0.0,
-        "p=0 DDP loss trace not bit-identical to no-dropout path"
+        max_loss_diff_1, 0.0,
+        "world=1 p=0 loss trace not bit-identical to no-dropout path"
     );
     assert_eq!(
-        max_param_diff, 0.0,
-        "p=0 DDP final params not bit-identical to no-dropout path"
+        max_param_diff_1, 0.0,
+        "world=1 p=0 final params not bit-identical to no-dropout path"
+    );
+
+    // --- world=2 runs: real cross-rank NCCL all-reduce (the production path). ---
+    let d2 = [0u32, 1u32];
+    let mut cfg_p0_w2 = test_config(vocab);
+    cfg_p0_w2.dropout = 0.0;
+    let mut cfg_p_w2 = test_config(vocab);
+    cfg_p_w2.dropout = 0.2;
+    let (loss_p0_2, _params_p0_2, _) = run_ddp(&d2, cfg_p0_w2, &corpus, Some(&valid), &base_dcfg);
+    let (loss_p_2, _params_p_2, train_flag_p) =
+        run_ddp(&d2, cfg_p_w2, &corpus, Some(&valid), &base_dcfg);
+
+    // GATE A2 — under DDP (world=2), p=0 matches a separate no-dropout baseline within
+    // NCCL's run-to-run ULP noise (KI-5; the all-reduce is not bit-reproducible). This
+    // confirms enabling dropout=0 doesn't perturb the DDP path beyond that noise floor.
+    let (loss_nd_2, _, _) = run_ddp(&d2, test_config(vocab), &corpus, Some(&valid), &base_dcfg);
+    let max_loss_diff_2 = loss_nd_2
+        .iter()
+        .zip(&loss_p0_2)
+        .map(|(a, b)| (a - b).abs())
+        .fold(0.0f32, f32::max);
+    println!("T21 GATE A2 (world=2 p=0 vs no-dropout, KI-5 noise): max |loss diff| = {max_loss_diff_2:.3e}");
+    assert!(
+        max_loss_diff_2 < 1e-6,
+        "world=2 p=0 diverged from no-dropout beyond NCCL noise: {max_loss_diff_2:.3e}"
     );
 
     // GATE B — dropout is LIVE with p>0 under DDP. If model.train() were not wired
-    // (the pre-T21 bug), the model would stay in eval mode and the p=0.2 forward
-    // would be IDENTITY → loss trace bit-identical to p=0. A real, sizeable
-    // difference proves dropout masks are actually applied during the training
-    // forward (and survive the mid-run eval flips, since model.train() is re-asserted
-    // each step). Inverted scaling + masking perturbs every step, so the gap is large.
-    let mut max_live_diff = 0.0f32;
-    for (a, b) in loss_p0.iter().zip(&loss_p) {
-        max_live_diff = max_live_diff.max((a - b).abs());
-    }
+    // (the pre-T21 bug), the model would stay in eval mode and the p=0.2 forward would
+    // be IDENTITY → loss trace bit-identical to p=0 (diff at the ~1e-7 NCCL noise
+    // floor). A difference orders of magnitude above that proves dropout masks are
+    // actually applied during the training forward — and that they survive the mid-run
+    // eval flips (model.train() is re-asserted each step). Inverted scaling + masking
+    // perturbs every step, so the gap is large (>1e-3 ≫ KI-5 noise ~2.4e-7).
+    let max_live_diff = loss_p0_2
+        .iter()
+        .zip(&loss_p_2)
+        .map(|(a, b)| (a - b).abs())
+        .fold(0.0f32, f32::max);
     println!(
-        "T21 GATE B (dropout live): p0[last]={:.6} p0.2[last]={:.6} max |loss diff| = {max_live_diff:.3e}",
-        loss_p0.last().unwrap(),
-        loss_p.last().unwrap()
+        "T21 GATE B (dropout live, world=2): p0[last]={:.6} p0.2[last]={:.6} max |loss diff| = {max_live_diff:.3e}",
+        loss_p0_2.last().unwrap(),
+        loss_p_2.last().unwrap()
     );
     assert!(
         max_live_diff > 1e-3,
@@ -547,7 +581,7 @@ fn ddp_dropout_is_live_and_p0_bit_identical() {
 
     // No NaN/Inf in the p>0 run (dropout converges normally under DDP).
     assert!(
-        loss_p.iter().all(|l| l.is_finite()),
+        loss_p_2.iter().all(|l| l.is_finite()),
         "p=0.2 DDP loss has non-finite values"
     );
 }