gahow/xtrain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixup! test: T21 — DDP-dropout regression (live under DDP + p=0 bit-identical)

Browse Source
This commit is contained in:
Gahow Wang
2026-06-18 21:15:05 +08:00
parent f7e893282a
commit 41c25271e6
1 changed files with 36 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
crates/xtrain-distributed/tests/ddp_correctness.rs
View File

@@ -446,12 +446,15 @@ fn ddp_throughput_scaling() {
/// bit-identical to p=0 — model stuck in eval mode → dropout is identity) and GATE C
/// (`is_training()` would be false after the run).
///
/// Bit-identity (GATE A) is asserted at `world=1`, where `all_reduce_average_grads`
/// short-circuits (no NCCL) so the run is deterministic. The cross-rank NCCL
/// all-reduce (`world>=2`) is not bit-reproducible run-to-run on this PCIe box (KI-5,
/// observed ≤~2.4e-7), so the `world=2` p=0-vs-no-dropout check (GATE A2) uses the
/// same KI-5 ULP tolerance as the rest of this file, while GATE B's live-dropout
/// signal (>1e-3) sits orders of magnitude above that noise floor.
/// p=0 regression (GATE A) is checked at `world=1`, ONE step, where the NCCL
/// all-reduce short-circuits: the p=0 FORWARD is byte-identical to no-dropout so the
/// loss is BIT-IDENTICAL (== 0.0), and the post-step params match within the engine's
/// atomicAdd backward-reduction ULP floor (< 1e-7, dropout-independent — the
/// fresh-train md5 caveat). The cross-rank NCCL all-reduce (`world>=2`) is not
/// bit-reproducible run-to-run on this PCIe box (KI-5, observed ≤~2.4e-7), so the
/// `world=2` p=0-vs-no-dropout check (GATE A2) uses the same KI-5 ULP tolerance as the
/// rest of this file. GATE B's live-dropout signal (>1e-3) sits ~4 orders of magnitude
/// above every noise floor here, so it carries the load.
#[test]
fn ddp_dropout_is_live_and_p0_bit_identical() {
if device::device_count().unwrap_or(0) < 2 {
@@ -488,39 +491,48 @@ fn ddp_dropout_is_live_and_p0_bit_identical() {
ckpt_path: None,
};
// --- GATE A: bit-identity at world=1 (deterministic — no NCCL collective). ---
// The regression guard for `--dropout 0`: a p=0 run must be bit-for-bit the same
// as the no-dropout path, since ops::dropout(p=0) is a clone no-op regardless of
// training mode. At world=1, all_reduce_average_grads short-circuits, so the run
// is fully deterministic and bit-identity is the honest invariant (no NCCL noise).
// --- GATE A: p=0 == no-dropout at world=1, ONE step (the deterministic scope). ---
// The regression guard for `--dropout 0`. ops::dropout(p=0) returns x.clone() (a
// graph no-op) regardless of training mode, so the p=0 FORWARD graph is byte-for-
// byte the no-dropout forward → loss[0] must be BIT-IDENTICAL (the load-bearing
// claim, asserted == 0.0). At world=1 the NCCL all-reduce short-circuits, and one
// step has no optimizer-state compounding; the only residual non-determinism is
// the engine's atomicAdd backward-reduction ORDER (the documented fresh-train md5
// caveat — dropout-INDEPENDENT, present with or without the dropout op), which
// moves the post-step params by a single grad ULP. So params are checked against
// that tight reduction floor (< 1e-7), the same nature as the cross-rank KI-5
// tolerance used elsewhere in this file — not a dropout signal. GATE B (live) has
// a >1e-3 signal, ~4 orders of magnitude above this floor, so it carries the load.
let d1 = [0u32];
let dcfg_1step = DdpConfig {
steps: 1,
eval_every: 0,
..base_dcfg.clone()
};
let cfg_nodrop = test_config(vocab); // cfg.dropout defaults to 0.0
assert_eq!(cfg_nodrop.dropout, 0.0, "baseline cfg must have dropout 0");
let mut cfg_p0 = test_config(vocab);
cfg_p0.dropout = 0.0; // explicitly set p=0 — must not perturb anything
let (loss_nd1, params_nd1, _) = run_ddp(&d1, cfg_nodrop, &corpus, Some(&valid), &base_dcfg);
let (loss_p01, params_p01, _) = run_ddp(&d1, cfg_p0, &corpus, Some(&valid), &base_dcfg);
let max_loss_diff_1 = loss_nd1
.iter()
.zip(&loss_p01)
.map(|(a, b)| (a - b).abs())
.fold(0.0f32, f32::max);
let (loss_nd1, params_nd1, _) = run_ddp(&d1, cfg_nodrop, &corpus, None, &dcfg_1step);
let (loss_p01, params_p01, _) = run_ddp(&d1, cfg_p0, &corpus, None, &dcfg_1step);
let max_loss_diff_1 = (loss_nd1[0] - loss_p01[0]).abs();
let max_param_diff_1 = params_nd1
.iter()
.zip(&params_p01)
.flat_map(|(a, b)| a.iter().zip(b).map(|(x, y)| (x - y).abs()))
.fold(0.0f32, f32::max);
println!(
"T21 GATE A (world=1 p=0 bit-identical): max |loss diff| = {max_loss_diff_1:.3e}, \
max |param diff| = {max_param_diff_1:.3e}"
"T21 GATE A (world=1, 1 step, p=0 vs no-dropout): |loss diff| = {max_loss_diff_1:.3e} \
(bit-identical forward), max |param diff| = {max_param_diff_1:.3e} (atomicAdd floor)"
);
assert_eq!(
max_loss_diff_1, 0.0,
"world=1 p=0 loss trace not bit-identical to no-dropout path"
"world=1 p=0 forward loss not bit-identical to no-dropout path"
);
assert_eq!(
max_param_diff_1, 0.0,
"world=1 p=0 final params not bit-identical to no-dropout path"
assert!(
max_param_diff_1 < 1e-7,
"world=1 p=0 post-step params diverged from no-dropout beyond the atomicAdd \
reduction floor: {max_param_diff_1:.3e}"
);
// --- world=2 runs: real cross-rank NCCL all-reduce (the production path). ---