gahow/xtrain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixup! test: T21 — DDP-dropout regression (live under DDP + p=0 bit-identical)

Browse Source
This commit is contained in:
Gahow Wang
2026-06-18 21:15:05 +08:00
parent f7e893282a
commit 3c88fb7178
1 changed files with 19 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
crates/xtrain-distributed/tests/ddp_correctness.rs
View File

@@ -488,39 +488,45 @@ fn ddp_dropout_is_live_and_p0_bit_identical() {
ckpt_path: None,
};
// --- GATE A: bit-identity at world=1 (deterministic — no NCCL collective). ---
// --- GATE A: bit-identity at world=1, ONE step (the deterministic scope). ---
// The regression guard for `--dropout 0`: a p=0 run must be bit-for-bit the same
// as the no-dropout path, since ops::dropout(p=0) is a clone no-op regardless of
// training mode. At world=1, all_reduce_average_grads short-circuits, so the run
// is fully deterministic and bit-identity is the honest invariant (no NCCL noise).
// training mode. At world=1, all_reduce_average_grads short-circuits (no NCCL),
// so the only remaining non-determinism is the engine's atomicAdd backward
// reduction ORDER — run-to-run stable for a single forward+backward, but it
// compounds a few ULP over many AdamW steps (the known fresh-train md5 caveat).
// So the honest bit-identity scope is ONE step (no optimizer-state compounding):
// loss[0] is a pure forward and the params after one step are a pure function of
// that step's grads. dropout p=0 must not change either bit.
let d1 = [0u32];
let dcfg_1step = DdpConfig {
steps: 1,
eval_every: 0,
..base_dcfg.clone()
};
let cfg_nodrop = test_config(vocab); // cfg.dropout defaults to 0.0
assert_eq!(cfg_nodrop.dropout, 0.0, "baseline cfg must have dropout 0");
let mut cfg_p0 = test_config(vocab);
cfg_p0.dropout = 0.0; // explicitly set p=0 — must not perturb anything
let (loss_nd1, params_nd1, _) = run_ddp(&d1, cfg_nodrop, &corpus, Some(&valid), &base_dcfg);
let (loss_p01, params_p01, _) = run_ddp(&d1, cfg_p0, &corpus, Some(&valid), &base_dcfg);
let max_loss_diff_1 = loss_nd1
.iter()
.zip(&loss_p01)
.map(|(a, b)| (a - b).abs())
.fold(0.0f32, f32::max);
let (loss_nd1, params_nd1, _) = run_ddp(&d1, cfg_nodrop, &corpus, None, &dcfg_1step);
let (loss_p01, params_p01, _) = run_ddp(&d1, cfg_p0, &corpus, None, &dcfg_1step);
let max_loss_diff_1 = (loss_nd1[0] - loss_p01[0]).abs();
let max_param_diff_1 = params_nd1
.iter()
.zip(&params_p01)
.flat_map(|(a, b)| a.iter().zip(b).map(|(x, y)| (x - y).abs()))
.fold(0.0f32, f32::max);
println!(
"T21 GATE A (world=1 p=0 bit-identical): max |loss diff| = {max_loss_diff_1:.3e}, \
"T21 GATE A (world=1, 1 step, p=0 bit-identical): |loss diff| = {max_loss_diff_1:.3e}, \
max |param diff| = {max_param_diff_1:.3e}"
);
assert_eq!(
max_loss_diff_1, 0.0,
"world=1 p=0 loss trace not bit-identical to no-dropout path"
"world=1 p=0 step loss not bit-identical to no-dropout path"
);
assert_eq!(
max_param_diff_1, 0.0,
"world=1 p=0 final params not bit-identical to no-dropout path"
"world=1 p=0 post-step params not bit-identical to no-dropout path"
);
// --- world=2 runs: real cross-rank NCCL all-reduce (the production path). ---