gahow/agentic-pd-hybrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add transfer queue admission knobs

Browse Source
This commit is contained in:
Gahow Wang
2026-04-25 17:29:15 +00:00
parent fe583fb413
commit c928c7db23
3 changed files with 102 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/agentic_pd_hybrid/benchmark.py
View File

@@ -38,6 +38,8 @@ class BenchmarkConfig:
kvcache_seed_only_multiturn_sessions: bool = False
kvcache_prefill_backup_policy: str = "release-after-transfer"
kvcache_seed_max_inflight_decode: int | None = 3
kvcache_seed_max_decode_transfer_queue_reqs: int | None = None
kvcache_direct_max_decode_transfer_queue_reqs: int | None = None
kvcache_prefill_priority_eviction: bool = False
kvcache_prefill_direct_priority: int = -100
kvcache_prefill_normal_priority: int = 100
@@ -174,6 +176,12 @@ def run_live_benchmark(config: BenchmarkConfig) -> BenchmarkArtifacts:
kvcache_seed_max_inflight_decode=(
config.kvcache_seed_max_inflight_decode
),
kvcache_seed_max_decode_transfer_queue_reqs=(
config.kvcache_seed_max_decode_transfer_queue_reqs
),
kvcache_direct_max_decode_transfer_queue_reqs=(
config.kvcache_direct_max_decode_transfer_queue_reqs
),
kvcache_prefill_priority_eviction=(
config.kvcache_prefill_priority_eviction
),
@@ -220,6 +228,12 @@ def run_live_benchmark(config: BenchmarkConfig) -> BenchmarkArtifacts:
"kvcache_seed_max_inflight_decode": (
config.kvcache_seed_max_inflight_decode
),
"kvcache_seed_max_decode_transfer_queue_reqs": (
config.kvcache_seed_max_decode_transfer_queue_reqs
),
"kvcache_direct_max_decode_transfer_queue_reqs": (
config.kvcache_direct_max_decode_transfer_queue_reqs
),
"kvcache_prefill_priority_eviction": (
config.kvcache_prefill_priority_eviction
),