sft: assistant-only SFT (ignore-index CE) + chat-prompt greedy eval

Enable assistant-only supervised fine-tuning and a fixed chat-prompt eval path
used by the v12 SFT runs:

- cross_entropy ignores negative targets (-100 ignore-index), normalizing by
  valid rows instead of all rows; CUDA fwd/bwd skip t<0 (ops.rs, nn.cu).
- Corpus gains optional labels + load_sft_tsv_cached: two-column TSV is
  formatted as 'User: .. \nAssistant:' + answer + <|endoftext|>, prompt tokens
  masked to -100 while answer+EOS are supervised; i32 label cache alongside the
  u16 token cache; sample() retries windows that are fully masked; eval uses
  target_window so masking applies to val loss too (data.rs, train_loop.rs).
- train + train_ddp: --sft-tsv selects the TSV loader, --init-ckpt continues
  training from a base checkpoint.
- greedy_sample: --prompts-file/--prompt/--temperature for fixed chat-prompt
  generation eval.

Test fixtures updated for the new Corpus.labels field; dropout.rs carries
incidental rustfmt. Not rebuilt locally (no CUDA toolchain on this checkout);
correctness rests on the documented v12 base+SFT runs on the GPU box.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

crates/xtrain-model/tests/dropout.rs

@@ -66,10 +66,18 @@ fn tiny_cfg(dropout: f32) -> Config {
 fn batch_data(cfg: &Config, device: Device) -> (xtrain_tensor::Tensor, xtrain_tensor::Tensor) {
     let (batch, seq) = (3usize, 6usize);
     let seqs: Vec<Vec<i32>> = (0..batch)
-        .map(|b| (0..seq).map(|i| ((b * 7 + i * 3 + 1) % cfg.vocab) as i32).collect())
+        .map(|b| {
+            (0..seq)
+                .map(|i| ((b * 7 + i * 3 + 1) % cfg.vocab) as i32)
+                .collect()
+        })
         .collect();
     let tgts: Vec<Vec<i32>> = (0..batch)
-        .map(|b| (0..seq).map(|i| ((b * 5 + i * 2 + 2) % cfg.vocab) as i32).collect())
+        .map(|b| {
+            (0..seq)
+                .map(|i| ((b * 5 + i * 2 + 2) % cfg.vocab) as i32)
+                .collect()
+        })
         .collect();
     (
         batched_ids_tensor(&seqs, device),
@@ -94,7 +102,11 @@ fn fwd_bwd(
     let loss = m.loss_batched(ids, tgt, batch);
     let loss_val = host(&loss.value())[0];
     loss.backward();
-    let grads: Vec<Vec<f32>> = m.params().iter().map(|p| host(&p.grad().unwrap())).collect();
+    let grads: Vec<Vec<f32>> = m
+        .params()
+        .iter()
+        .map(|p| host(&p.grad().unwrap()))
+        .collect();
     (logits, loss_val, grads)
 }
 
@@ -186,7 +198,9 @@ fn recompute_with_dropout(dtype: DType, grad_tol: f32) {
 
     // Both models: same init, train mode, p=0.2. step_seed starts at 0 and bumps
     // to 1 on the first training forward in BOTH, so they draw the same masks.
-    let off = build(cfg, device).with_compute_dtype(dtype).with_training(true);
+    let off = build(cfg, device)
+        .with_compute_dtype(dtype)
+        .with_training(true);
     let on = build(cfg, device)
         .with_compute_dtype(dtype)
         .with_recompute(true)
@@ -194,11 +208,19 @@ fn recompute_with_dropout(dtype: DType, grad_tol: f32) {
 
     let off_loss = off.loss_batched(&ids, &tgt, batch);
     off_loss.backward();
-    let off_grads: Vec<Vec<f32>> = off.params().iter().map(|p| host(&p.grad().unwrap())).collect();
+    let off_grads: Vec<Vec<f32>> = off
+        .params()
+        .iter()
+        .map(|p| host(&p.grad().unwrap()))
+        .collect();
 
     let on_loss = on.loss_batched(&ids, &tgt, batch);
     on_loss.backward();
-    let on_grads: Vec<Vec<f32>> = on.params().iter().map(|p| host(&p.grad().unwrap())).collect();
+    let on_grads: Vec<Vec<f32>> = on
+        .params()
+        .iter()
+        .map(|p| host(&p.grad().unwrap()))
+        .collect();
 
     let mut max_rel = 0.0f32;
     for (a, b) in off_grads.iter().flatten().zip(on_grads.iter().flatten()) {
@@ -240,10 +262,18 @@ fn flash_plus_dropout_grad_check_fp32() {
     cfg.dropout = 0.2;
     let seq = 40usize;
     let seqs: Vec<Vec<i32>> = (0..batch)
-        .map(|b| (0..seq).map(|i| ((b * 7 + i * 3 + 1) % cfg.vocab) as i32).collect())
+        .map(|b| {
+            (0..seq)
+                .map(|i| ((b * 7 + i * 3 + 1) % cfg.vocab) as i32)
+                .collect()
+        })
         .collect();
     let tgts: Vec<Vec<i32>> = (0..batch)
-        .map(|b| (0..seq).map(|i| ((b * 5 + i * 2 + 2) % cfg.vocab) as i32).collect())
+        .map(|b| {
+            (0..seq)
+                .map(|i| ((b * 5 + i * 2 + 2) % cfg.vocab) as i32)
+                .collect()
+        })
         .collect();
     let ids = batched_ids_tensor(&seqs, device);
     let tgt = batched_ids_tensor(&tgts, device);
@@ -277,7 +307,16 @@ fn flash_plus_dropout_grad_check_fp32() {
     );
     // Same tolerances as the flash-vs-composed gate (flash.rs run_fp32): flash
     // differs from composed only by reduction order; dropout masks are identical.
-    assert!(logit_rel < 1e-3, "[F32] flash+dropout logits diverged: {logit_rel:.2e}");
-    assert!(loss_rel < 1e-3, "[F32] flash+dropout loss diverged: {loss_rel:.2e}");
-    assert!(grad_rel < 2e-2, "[F32] flash+dropout grads diverged: {grad_rel:.3e}");
+    assert!(
+        logit_rel < 1e-3,
+        "[F32] flash+dropout logits diverged: {logit_rel:.2e}"
+    );
+    assert!(
+        loss_rel < 1e-3,
+        "[F32] flash+dropout loss diverged: {loss_rel:.2e}"
+    );
+    assert!(
+        grad_rel < 2e-2,
+        "[F32] flash+dropout grads diverged: {grad_rel:.3e}"
+    );
 }