xserv-chat: support gpt-oss-20b with TP; fix GEMV precision bug
- Add ChatModel enum dispatching between Qwen3 and GptOss based on config.is_moe(), following the TP engine pattern. - Add --tp N flag for tensor-parallel inference (required for 39GB gpt-oss-20b which doesn't fit on a single 32GB GPU). - Add gpt-oss harmony chat template with channel/message format. - Replace hardcoded is_stop_token() with tokenizer.is_eos() for multi-model EOS support. - Restore gpt-oss hardcoded prompt template in server api.rs, lost during the Jinja template refactor. - Fix GEMV race condition: the K-split kernel zeroed the FP32 accumulator inside the kernel (block k=0) while other blocks atomicAdd'd concurrently. Pre-zero with cudaMemsetAsync instead. - Update benchmark docs with post-fix results. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -169,8 +169,10 @@ fn raise_exception(msg: String) -> Result<String, minijinja::Error> {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn build_prompt_hardcoded(messages: &[Message], model_type: &str) -> String {
|
||||
if model_type == "gpt_oss" {
|
||||
return build_prompt_gpt_oss(messages);
|
||||
}
|
||||
// Default: Qwen3 ChatML format
|
||||
let _ = model_type;
|
||||
let mut prompt = String::new();
|
||||
for msg in messages {
|
||||
match msg.role.as_str() {
|
||||
@@ -189,6 +191,41 @@ fn build_prompt_hardcoded(messages: &[Message], model_type: &str) -> String {
|
||||
prompt
|
||||
}
|
||||
|
||||
fn build_prompt_gpt_oss(messages: &[Message]) -> String {
|
||||
let mut prompt = String::new();
|
||||
prompt.push_str("<|start|>system<|message|>");
|
||||
prompt.push_str("You are a helpful assistant.\n\n# Valid channels: analysis, commentary, final. Channel must be included for every message.");
|
||||
prompt.push_str("<|end|>");
|
||||
let dev_instructions: String = messages
|
||||
.iter()
|
||||
.filter(|m| m.role == "system")
|
||||
.map(|m| m.content.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
if !dev_instructions.is_empty() {
|
||||
prompt.push_str("<|start|>developer<|message|># Instructions\n\n");
|
||||
prompt.push_str(&dev_instructions);
|
||||
prompt.push_str("<|end|>");
|
||||
}
|
||||
for msg in messages {
|
||||
match msg.role.as_str() {
|
||||
"user" => {
|
||||
prompt.push_str("<|start|>user<|message|>");
|
||||
prompt.push_str(&msg.content);
|
||||
prompt.push_str("<|end|>");
|
||||
}
|
||||
"assistant" => {
|
||||
prompt.push_str("<|start|>assistant<|channel|>final<|message|>");
|
||||
prompt.push_str(&msg.content);
|
||||
prompt.push_str("<|end|>");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
prompt.push_str("<|start|>assistant<|channel|>final<|message|>");
|
||||
prompt
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user