Gahow Wang
445e491123
third_party/vllm/ now tracked in git for direct patch management.
Based on vLLM v0.18.1 release with one patch applied:
vllm/v1/core/sched/scheduler.py:
Replace fatal assert with graceful skip when KV transfer callback
arrives for an already-aborted request during PD disaggregated serving.
Future vLLM modifications should be made directly in third_party/vllm/
and committed normally. The patches/ directory is kept as documentation
of what changed from upstream.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
49 lines
1.4 KiB
YAML
49 lines
1.4 KiB
YAML
# Standard attention backend benchmark configuration
|
|
|
|
model:
|
|
num_layers: 32
|
|
num_q_heads: 32
|
|
num_kv_heads: 8 # GQA with 4:1 ratio
|
|
head_dim: 128
|
|
block_size: 16
|
|
|
|
batch_specs:
|
|
# Pure prefill
|
|
- "q512" # Small prefill (512 tokens)
|
|
- "q2k" # Medium prefill (2048 tokens)
|
|
- "q4k" # Large prefill (4096 tokens)
|
|
- "q8k" # Very large prefill (8192 tokens)
|
|
|
|
# Pure decode
|
|
- "8q1s1k" # 8 requests, 1k KV cache each
|
|
- "16q1s2k" # 16 requests, 2k KV cache each
|
|
- "32q1s1k" # 32 requests, 1k KV cache each
|
|
- "64q1s4k" # 64 requests, 4k KV cache each
|
|
|
|
# Mixed prefill/decode
|
|
- "2q2k_8q1s1k" # 2 prefill + 8 decode
|
|
- "4q1k_16q1s2k" # 4 prefill + 16 decode
|
|
- "2q4k_32q1s1k" # 2 large prefill + 32 decode
|
|
|
|
# Speculative decode (q <= 8)
|
|
- "16q2s1k" # 16 requests, 2 spec tokens, 1k KV cache
|
|
- "16q4s1k" # 16 requests, 4 spec tokens, 1k KV cache
|
|
- "16q8s1k" # 16 requests, 8 spec tokens, 1k KV cache
|
|
- "32q4s2k" # 32 requests, 4 spec tokens, 2k KV cache
|
|
- "8q8s4k" # 8 requests, 8 spec tokens, 4k KV cache
|
|
|
|
# Context extension (chunked prefill)
|
|
- "q1ks2k" # 1k query, 2k sequence
|
|
- "2q1ks4k" # 2 requests: 1k query, 4k sequence
|
|
|
|
# Available backends: FLASH_ATTN, TRITON_ATTN, FLASHINFER
|
|
backends:
|
|
- FLASH_ATTN
|
|
- TRITON_ATTN
|
|
- FLASHINFER
|
|
|
|
device: "cuda:0"
|
|
repeats: 5
|
|
warmup_iters: 3
|
|
profile_memory: false
|