post-train: M1 — verifiable arithmetic task + SFT data generator
First post-training milestone (docs/18). Lands the verifiable task + its data
pipeline, all verified host-side (no CUDA); the SFT run itself reuses the
existing --sft-tsv path on the GPU box.
- task.rs: the shared task spec — two-operand integer arithmetic, answer in
\boxed{N}, with parse_boxed_answer + check_answer (exact-match rule-based
reward). One module reused by M1 (SFT data), M3 (DPO pairs), M4 (GRPO reward).
- gen_arith_task bin: writes arith_sft.tsv (--sft-tsv format) + held-out
arith_eval_prompts.txt (greedy_sample format) + arith_eval_gold.txt; train
deduped, eval disjoint from train.
- data.rs: extract assistant-only masking into a pure, testable sft_row()
(behavior-preserving; single-turn bit-identical to fbf4ac2).
Gate (verified locally, no_cuda): cargo test -p xtrain-train --lib = 9/9 pass
(masking, SFT-target self-consistency over 2000 samples, parser edges, seed
determinism); a 200/50 gen run = clean 2-col TSV, correct gold incl. negatives,
0 train/eval leakage. SFT training run + format-eval pending on dash5.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -124,10 +124,9 @@ impl Corpus {
|
||||
let answer = format!(" {assistant}\n<|endoftext|>");
|
||||
let prompt_ids: Vec<i32> = tok.encode(&prompt).into_iter().map(|t| t as i32).collect();
|
||||
let answer_ids: Vec<i32> = tok.encode(&answer).into_iter().map(|t| t as i32).collect();
|
||||
labels.extend(std::iter::repeat(-100).take(prompt_ids.len()));
|
||||
labels.extend(answer_ids.iter().copied());
|
||||
tokens.extend(prompt_ids);
|
||||
tokens.extend(answer_ids);
|
||||
let (row_tokens, row_labels) = sft_row(&prompt_ids, &answer_ids);
|
||||
tokens.extend(row_tokens);
|
||||
labels.extend(row_labels);
|
||||
}
|
||||
assert_eq!(tokens.len(), labels.len(), "SFT tokens/labels mismatch");
|
||||
write_u16_cache(&token_cache, &tokens);
|
||||
@@ -291,6 +290,20 @@ fn decode_tsv_escapes(s: &str) -> String {
|
||||
s.replace("\\n", "\n").replace("\\t", "\t")
|
||||
}
|
||||
|
||||
/// Build one SFT example's `(tokens, labels)` from already-tokenized prompt/answer
|
||||
/// ids: prompt tokens are masked to the ignore-index (`-100`, which `cross_entropy`
|
||||
/// skips) so only the answer + EOS tokens contribute to the loss. Pure (no tokenizer
|
||||
/// / no CUDA) so the assistant-only masking is unit-testable directly.
|
||||
fn sft_row(prompt_ids: &[i32], answer_ids: &[i32]) -> (Vec<i32>, Vec<i32>) {
|
||||
let mut tokens = Vec::with_capacity(prompt_ids.len() + answer_ids.len());
|
||||
tokens.extend_from_slice(prompt_ids);
|
||||
tokens.extend_from_slice(answer_ids);
|
||||
let mut labels = Vec::with_capacity(prompt_ids.len() + answer_ids.len());
|
||||
labels.extend(std::iter::repeat(-100).take(prompt_ids.len()));
|
||||
labels.extend_from_slice(answer_ids);
|
||||
(tokens, labels)
|
||||
}
|
||||
|
||||
/// Tiny LCG (same constants as the model tests' deterministic fill) so dataset
|
||||
/// sampling is reproducible from a single u64 seed.
|
||||
fn next_rand(state: &mut u64) -> u64 {
|
||||
@@ -299,3 +312,27 @@ fn next_rand(state: &mut u64) -> u64 {
|
||||
.wrapping_add(1442695040888963407);
|
||||
*state >> 16
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sft_row_masks_prompt_supervises_answer() {
|
||||
let prompt = [5, 6, 7];
|
||||
let answer = [8, 9]; // includes the EOS token in real use
|
||||
let (tokens, labels) = sft_row(&prompt, &answer);
|
||||
// Tokens are prompt then answer, in order.
|
||||
assert_eq!(tokens, vec![5, 6, 7, 8, 9]);
|
||||
// Prompt positions are ignore-index (-100); answer positions are supervised.
|
||||
assert_eq!(labels, vec![-100, -100, -100, 8, 9]);
|
||||
assert_eq!(tokens.len(), labels.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sft_row_handles_empty_answer() {
|
||||
let (tokens, labels) = sft_row(&[1, 2], &[]);
|
||||
assert_eq!(tokens, vec![1, 2]);
|
||||
assert_eq!(labels, vec![-100, -100]);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user