diff --git a/crates/xserv-tokenizer/src/bpe.rs b/crates/xserv-tokenizer/src/bpe.rs index 6752ace..6206874 100644 --- a/crates/xserv-tokenizer/src/bpe.rs +++ b/crates/xserv-tokenizer/src/bpe.rs @@ -12,6 +12,7 @@ pub struct Tokenizer { special_token_ids: HashMap, pre_tokenize_re: Regex, eos_token_id: Option, + eos_token_ids: Vec, byte_fallback: bool, } @@ -102,11 +103,27 @@ impl Tokenizer { decoder.resize(decoder.len().max(at.id as usize + 1), vec![]); decoder[at.id as usize] = at.content.as_bytes().to_vec(); } - let eos_token_id = special_tokens - .get("<|im_end|>") - .or_else(|| special_tokens.get("<|end_of_text|>")) - .or_else(|| special_tokens.get("<|endoftext|>")) - .copied(); + // End-of-generation tokens, in priority order. Families differ: + // Qwen uses <|im_end|>, Llama <|end_of_text|>, GPT-2 <|endoftext|>. + // gpt-oss (harmony) ends the assistant turn with <|return|> and also + // treats <|call|> (tool call) and <|endoftext|> as terminators + // (see generation_config.json eos_token_id = [200002, 199999, 200012]). + let eos_names = [ + "<|im_end|>", + "<|end_of_text|>", + "<|return|>", + "<|call|>", + "<|endoftext|>", + ]; + let mut eos_token_ids: Vec = Vec::new(); + for name in eos_names { + if let Some(&id) = special_tokens.get(name) { + if !eos_token_ids.contains(&id) { + eos_token_ids.push(id); + } + } + } + let eos_token_id = eos_token_ids.first().copied(); // Pre-tokenization regex let pre_tokenize_re = if byte_fallback { @@ -125,6 +142,7 @@ impl Tokenizer { special_token_ids, pre_tokenize_re, eos_token_id, + eos_token_ids, byte_fallback, } } @@ -249,6 +267,12 @@ impl Tokenizer { self.eos_token_id } + /// True if `id` is any end-of-generation token (a model may have several; + /// gpt-oss/harmony ends on <|return|>, <|call|>, or <|endoftext|>). + pub fn is_eos(&self, id: u32) -> bool { + self.eos_token_ids.contains(&id) + } + pub fn vocab_size(&self) -> usize { self.decoder.len() }