review cleanups: pp+gpt-oss guard, sparse GEMV asserts, warnings
- --pp with gpt-oss now fails with a clear message instead of a
cryptic missing-weight panic inside the Qwen3-only PP engine.
- Sparse GEMV wrappers assert K%16==0 (FP8) / K%32==0 (MXFP4) — the
uint4-vectorized kernels would silently drop a tail otherwise.
- Document the topk_ids buffer holding i32 under an F32 dtype label
(DType has no I32).
- Drop unused imports/locals and the cuBLASLt scale-mode constants
orphaned by the strided-batched FP8 rework (e631a71).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -65,6 +65,13 @@ async fn main() {
|
||||
std::process::exit(1);
|
||||
}
|
||||
let model_config = ModelConfig::from_file(&model_dir.join("config.json"));
|
||||
// gpt-oss is only implemented in the TP engine; route it there even at
|
||||
// tp=1 (single-rank world) so quantized models can serve on one GPU.
|
||||
let is_gpt_oss = model_config.model_type.as_deref() == Some("gpt_oss");
|
||||
if pp > 1 && is_gpt_oss {
|
||||
eprintln!("gpt-oss is not supported by the pipeline-parallel engine (Qwen3 only); use --tp instead");
|
||||
std::process::exit(1);
|
||||
}
|
||||
let model_max_seq_len = model_config.max_seq_len();
|
||||
if model_max_seq_len == 0 {
|
||||
eprintln!("model config has invalid max_seq_len=0");
|
||||
@@ -87,9 +94,6 @@ async fn main() {
|
||||
let (tx, rx) = mpsc::channel::<GenerateRequest>();
|
||||
|
||||
let model_dir_clone = model_dir.clone();
|
||||
// gpt-oss is only implemented in the TP engine; route it there even at
|
||||
// tp=1 (single-rank world) so quantized models can serve on one GPU.
|
||||
let is_gpt_oss = model_config.model_type.as_deref() == Some("gpt_oss");
|
||||
std::thread::spawn(move || {
|
||||
if pp > 1 {
|
||||
// Pipeline-parallel path: stage-0 coordinator + worker stage threads.
|
||||
|
||||
Reference in New Issue
Block a user