Gahow Wang
05592e6adc
Systematic study of prefill-decode disaggregation for agentic LLM workloads using production GLM-5.1 coder trace (2.1M requests, 71B input tokens). Key findings: - Cache-aware routing improves TPOT p90 by 15% and APC from 20.8% to 44.7% without PD separation, matching PD-Sep's decode isolation benefit - PD separation adds +72% TTFT overhead (KV transfer) with no TPOT gain when using the same cache-aware scheduler - Prefill remains compute-bound even at 95% KV cache reuse (AI >1000x vs decode AI <2), but absolute FLOPs drop 71% from cache hits - For agentic MoE workloads, cache-aware routing > PD separation Infrastructure: - Trace sampler preserving session structure + hash_ids for prefix sharing - Async trace replayer with streaming TTFT/TPOT/E2E measurement - Unified cache-aware + token-level load-balanced global scheduler proxy supporting both PD-colocated and PD-disaggregated (Mooncake/RDMA) modes - vLLM 0.18.1 scheduler patch for KV transfer abort race condition - Roofline analysis tool for prefill/decode compute characterization Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
97 lines
2.9 KiB
Bash
Executable File
97 lines
2.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# PD-Disaggregated serving via Mooncake (RDMA + DRAM KV pool).
|
|
#
|
|
# Architecture:
|
|
# Client → Proxy (port 8000)
|
|
# → Prefill (port 8010, TP=4, GPUs 0-3, bootstrap 8998)
|
|
# [prefill + store KV to DRAM pool via RDMA]
|
|
# → Decode (port 8020, TP=4, GPUs 4-7)
|
|
# [pull KV from DRAM pool via RDMA + decode]
|
|
#
|
|
# Usage: bash scripts/launch_pd_mooncake.sh
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
VENV="$PROJECT_DIR/.venv/bin"
|
|
VLLM="$VENV/vllm"
|
|
|
|
MODEL_PATH="${MODEL_PATH:-$HOME/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
|
PROXY_PORT=8000
|
|
PREFILL_PORT=8010
|
|
DECODE_PORT=8020
|
|
BOOTSTRAP_PORT=8998
|
|
|
|
PROXY_SCRIPT="$PROJECT_DIR/third_party/vllm/examples/online_serving/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py"
|
|
|
|
trap 'echo "Cleaning up..."; kill $(jobs -p) 2>/dev/null; wait 2>/dev/null' EXIT INT TERM
|
|
|
|
echo "=== PD-Disaggregated vLLM 0.18.1 (Mooncake/RDMA) ==="
|
|
echo " Model: $MODEL_PATH"
|
|
echo " Prefill: GPUs 0-3 (TP=4), port $PREFILL_PORT, bootstrap $BOOTSTRAP_PORT"
|
|
echo " Decode: GPUs 4-7 (TP=4), port $DECODE_PORT"
|
|
echo " Proxy: port $PROXY_PORT"
|
|
echo ""
|
|
|
|
# Step 1: Start prefill instance (KV producer)
|
|
echo "[1/3] Starting prefill instance..."
|
|
VLLM_MOONCAKE_BOOTSTRAP_PORT=$BOOTSTRAP_PORT \
|
|
CUDA_VISIBLE_DEVICES=0,1,2,3 \
|
|
$VLLM serve "$MODEL_PATH" \
|
|
--host 0.0.0.0 \
|
|
--port $PREFILL_PORT \
|
|
--tensor-parallel-size 4 \
|
|
--trust-remote-code \
|
|
--enable-prefix-caching \
|
|
--enforce-eager \
|
|
--dtype auto \
|
|
--gpu-memory-utilization 0.9 \
|
|
--kv-transfer-config \
|
|
'{"kv_connector":"MooncakeConnector","kv_role":"kv_producer"}' &
|
|
PREFILL_PID=$!
|
|
echo " Prefill PID=$PREFILL_PID"
|
|
|
|
# Step 2: Start decode instance (KV consumer)
|
|
echo "[2/3] Starting decode instance..."
|
|
CUDA_VISIBLE_DEVICES=4,5,6,7 \
|
|
$VLLM serve "$MODEL_PATH" \
|
|
--host 0.0.0.0 \
|
|
--port $DECODE_PORT \
|
|
--tensor-parallel-size 4 \
|
|
--trust-remote-code \
|
|
--enable-prefix-caching \
|
|
--enforce-eager \
|
|
--dtype auto \
|
|
--gpu-memory-utilization 0.8 \
|
|
--kv-transfer-config \
|
|
'{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer"}' &
|
|
DECODE_PID=$!
|
|
echo " Decode PID=$DECODE_PID"
|
|
|
|
# Wait for both instances
|
|
echo ""
|
|
echo "Waiting for instances..."
|
|
timeout 1200 bash -c "until curl -s localhost:$PREFILL_PORT/v1/models > /dev/null 2>&1; do sleep 5; done"
|
|
echo " Prefill ready!"
|
|
timeout 1200 bash -c "until curl -s localhost:$DECODE_PORT/v1/models > /dev/null 2>&1; do sleep 5; done"
|
|
echo " Decode ready!"
|
|
|
|
# Step 3: Start proxy (after instances are ready)
|
|
echo "[3/3] Starting proxy..."
|
|
$VENV/python "$PROXY_SCRIPT" \
|
|
--prefill "http://127.0.0.1:$PREFILL_PORT" "$BOOTSTRAP_PORT" \
|
|
--decode "http://127.0.0.1:$DECODE_PORT" \
|
|
--host 0.0.0.0 \
|
|
--port $PROXY_PORT &
|
|
PROXY_PID=$!
|
|
echo " Proxy PID=$PROXY_PID"
|
|
|
|
sleep 5
|
|
echo ""
|
|
echo "=== All ready ==="
|
|
echo " Send requests to: http://localhost:$PROXY_PORT"
|
|
echo ""
|
|
|
|
wait
|