Align qwen27b baseline proposal with topology study
This commit is contained in:
@@ -3,20 +3,18 @@
|
||||
"diagnosis": "This model uses a long-context hybrid stack and fp8 quantization. The safest first measurement is to preserve the existing warmup, hybrid-model, chunked-prefill, and prefix-caching behavior from run_qwen27b.sh, while keeping a conservative sequence cap.",
|
||||
"config_patch": {
|
||||
"env_patch": {
|
||||
"VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
|
||||
"VLLM_ENABLE_TORCH_COMPILE": "1",
|
||||
"VLLM_USE_FLASHINFER_SAMPLER": "0",
|
||||
"VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1"
|
||||
"VLLM_ENABLE_TORCH_COMPILE": "1"
|
||||
},
|
||||
"flag_patch": {
|
||||
"tensor-parallel-size": 4,
|
||||
"data-parallel-size": 1,
|
||||
"expert-parallel-size": 1,
|
||||
"gpu-memory-utilization": 0.9,
|
||||
"block-size": 64,
|
||||
"max-num-batched-tokens": 8192,
|
||||
"max-num-seqs": 16,
|
||||
"enable-prefix-caching": true,
|
||||
"enable-chunked-prefill": true,
|
||||
"disable-cascade-attn": true
|
||||
"enable-chunked-prefill": true
|
||||
}
|
||||
},
|
||||
"expected_effects": [
|
||||
|
||||
Reference in New Issue
Block a user