gahow/xserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

server: harmony termination via is_eos + TP repetition penalty

Browse Source 
Use tokenizer.is_eos() (multi-eos) for generation termination in both PP
and TP engines instead of a single eos id, so gpt-oss stops on <|return|>
/<|call|>/<|endoftext|>.

In the TP engine, optionally apply a repetition penalty on the greedy
decode path (XSERV_REP_PENALTY>1 over XSERV_REP_WINDOW recent tokens; off
by default) to break greedy repetition loops.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-31 00:56:33 +08:00
parent 99b212e6c1
commit 3c9d5e260e
2 changed files with 33 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/xserv-server/src/pp_engine.rs
View File

@@ -187,7 +187,6 @@ pub fn run_pp(model_dir: &Path, world: usize, max_seq_len: usize, rx: mpsc::Rece
let mut sc = build_stage(model_dir, &config, 0, world, 0, max_seq_len, id);
eprintln!("[pp-engine] ready (pp={world}, max_seq_len={max_seq_len})");
let eos = tokenizer.eos_token_id();
let n_workers = world - 1;
let next_peer = 1usize;
let broadcast = |txs: &[mpsc::Sender<PpCommand>], cmd: PpCommand| {
@@ -220,10 +219,10 @@ pub fn run_pp(model_dir: &Path, world: usize, max_seq_len: usize, rx: mpsc::Rece
let mut decode_buf: Vec<u8> = Vec::new();
let mut generated = 1usize;
emit_text(&tokenizer, &req, next, eos, &mut decode_buf);
emit_text(&tokenizer, &req, next, &mut decode_buf);
let finish = loop {
if eos == Some(next) {
if tokenizer.is_eos(next) {
break "stop";
}
if generated >= req.max_tokens {
@@ -235,7 +234,7 @@ pub fn run_pp(model_dir: &Path, world: usize, max_seq_len: usize, rx: mpsc::Rece
send_hidden(&sc, &x, next_peer);
next = token_rx.recv().expect("decode token");
generated += 1;
emit_text(&tokenizer, &req, next, eos, &mut decode_buf);
emit_text(&tokenizer, &req, next, &mut decode_buf);
};
let tail = tokenizer.flush_decode_stream(&mut decode_buf);
@@ -253,8 +252,8 @@ pub fn run_pp(model_dir: &Path, world: usize, max_seq_len: usize, rx: mpsc::Rece
}
/// Stream a token's decoded text to the client (EOS contributes no text).
fn emit_text(tokenizer: &Tokenizer, req: &GenerateRequest, token_id: u32, eos: Option<u32>, buf: &mut Vec<u8>) {
if eos == Some(token_id) {
fn emit_text(tokenizer: &Tokenizer, req: &GenerateRequest, token_id: u32, buf: &mut Vec<u8>) {
if tokenizer.is_eos(token_id) {
return;
}
let text = tokenizer.decode_token_stream(token_id, buf);