MB5 driver: launcher, orchestrator, KV-pool timeline plotter
Three new files to drive the PD ratio sweep + per-request KV occupancy
capture, plus a deploy.sh update so the patched replayer rides along
to the fresh-venv host.
mb5_launch.sh
One script handles all four configs we plan to sweep:
CONFIG=8C / 6P+2D / 4P+4D / 2P+6D
- For 8C: 8 vLLM instances with kv_role=kv_both on GPU 0-7. Replayer
talks to them via the existing comma-separated round-robin in
replayer/replay.py — no proxy.
- For PD configs: kv_role=kv_producer for the P pool (with
VLLM_MOONCAKE_BOOTSTRAP_PORT) + kv_role=kv_consumer for the D pool,
routed by the official vLLM example
third_party/vllm/examples/online_serving/disaggregated_serving/
mooncake_connector/mooncake_connector_proxy.py — no policy choice
made by us, per user instruction to use the standard recipe.
- Applies instrument_kv_snapshot.py before launching so every
EngineCore writes its per-step KV snapshot to
$RUN_ROOT/kv_snapshots/mb5_kv_snapshot_pid<pid>.jsonl
- Reverts the patch on stop.
- Emits ENDPOINTS= line on stdout for the orchestrator to read.
mb5_run.sh
For each CONFIG × rep: launch, replay w600 trace via the existing
replayer, capture wall-clock, tear down, cool down 10 s. Defaults:
CONFIGS="8C 6P+2D 4P+4D 2P+6D"
REPS=3
TRACE=traces/w600_r0.0015_st30.jsonl
All artefacts go under $FRESH_ROOT/mb5_runs/$RUN_TAG_${config}_rep${rep}/
(vllm_logs/, kv_snapshots/, replay_metrics.jsonl, wall_clock_s.txt).
plot_kv_pool_timeline.py
Reads one or more mb5_kv_snapshot_pid*.jsonl files and renders a
stacked-area chart per file:
x = wall-clock since first snapshot
y = KV block count, stacked by per-request contribution
overlay: pool-total ceiling, 90% line, waiting-queue depth subplot
Bands are colored by a deterministic hash of request_id so individual
requests are visually tractable across the run.
This is the figure the user asked for — turns headline "PD-disagg is
10× worse" into a system-level picture of *where* the KV pool is
blocked, when, and by which requests.
deploy.sh
Also tar-syncs the local replayer/ dir to
/home/admin/cpfs/wjh/agentic-kv-fresh/replayer/ so mb5_run.sh can
`python -m replayer` against the patched (trace_span_s/amplification)
version, not the older copy under /home/admin/cpfs/wjh/agentic-kv/.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
201
microbench/fresh_setup/mb5_launch.sh
Executable file
201
microbench/fresh_setup/mb5_launch.sh
Executable file
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env bash
|
||||
# Launch vLLM instances + (optional) proxy for one MB5 config.
|
||||
#
|
||||
# CONFIG=8C : 8 kv_both instances on GPU 0-7 (replayer
|
||||
# talks directly via round-robin)
|
||||
# CONFIG=6P+2D : 6 kv_producer (GPU 0-5) + 2 kv_consumer (GPU 6-7)
|
||||
# CONFIG=4P+4D : 4 producer (GPU 0-3) + 4 consumer (GPU 4-7)
|
||||
# CONFIG=2P+6D : 2 producer (GPU 0-1) + 6 consumer (GPU 2-7)
|
||||
#
|
||||
# All configs use the fresh venv (vanilla vLLM 0.18.1 + Mooncake 0.3.11),
|
||||
# kv_both/kv_producer/kv_consumer roles, MB5 scheduler instrumentation
|
||||
# applied. PD configs are launched per the official vLLM example
|
||||
# (run_mooncake_connector.sh) — round-robin P / round-robin D via
|
||||
# mooncake_connector_proxy.py on PROXY_PORT.
|
||||
#
|
||||
# Usage:
|
||||
# CONFIG=4P+4D RUN_LABEL=run1 bash mb5_launch.sh start
|
||||
# bash mb5_launch.sh status
|
||||
# bash mb5_launch.sh stop
|
||||
#
|
||||
# After "start", grep "ENDPOINTS=" from this script's output to get the
|
||||
# URL(s) the replayer should target.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
FRESH_ROOT="/home/admin/cpfs/wjh/agentic-kv-fresh"
|
||||
VENV="${FRESH_ROOT}/.venv"
|
||||
MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
INSTRUMENT="${SCRIPT_DIR}/instrument_kv_snapshot.py"
|
||||
PROXY_SRC="${SCRIPT_DIR}/../../third_party/vllm/examples/online_serving/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py"
|
||||
|
||||
CONFIG="${CONFIG:-8C}"
|
||||
RUN_LABEL="${RUN_LABEL:-default}"
|
||||
|
||||
# All artefacts for this run live here
|
||||
RUN_ROOT="${FRESH_ROOT}/mb5_runs/${RUN_LABEL}_${CONFIG}"
|
||||
LOGS_DIR="${RUN_ROOT}/vllm_logs"
|
||||
SNAPSHOT_DIR="${RUN_ROOT}/kv_snapshots"
|
||||
|
||||
PROXY_PORT="${PROXY_PORT:-8100}"
|
||||
BASE_HTTP=8000
|
||||
BASE_BP=8998
|
||||
BASE_MASTER=29500
|
||||
|
||||
stop_all() {
|
||||
pkill -9 -f "mooncake_connector_proxy.py" 2>/dev/null || true
|
||||
pkill -9 -f "vllm serve" 2>/dev/null || true
|
||||
pkill -9 -f "EngineCore" 2>/dev/null || true
|
||||
sleep 3
|
||||
}
|
||||
|
||||
case "${1:-start}" in
|
||||
stop)
|
||||
stop_all
|
||||
python "${INSTRUMENT}" --revert --venv "${VENV}" 2>/dev/null || true
|
||||
exit 0;;
|
||||
status)
|
||||
ports=()
|
||||
case "${CONFIG}" in
|
||||
8C) for i in 0 1 2 3 4 5 6 7; do ports+=( $((BASE_HTTP+i)) ); done ;;
|
||||
*) ports=( ${PROXY_PORT} ) ;;
|
||||
esac
|
||||
for p in "${ports[@]}"; do
|
||||
if curl -sf "http://127.0.0.1:${p}/health" >/dev/null 2>&1 \
|
||||
|| curl -sf "http://127.0.0.1:${p}/v1/models" >/dev/null 2>&1; then
|
||||
echo "port ${p}: UP"
|
||||
else
|
||||
echo "port ${p}: DOWN"
|
||||
fi
|
||||
done
|
||||
exit 0;;
|
||||
start) ;;
|
||||
*) echo "Unknown command: $1"; exit 1;;
|
||||
esac
|
||||
|
||||
# --- parse CONFIG into (prefill_gpus, decode_gpus) ----------------
|
||||
case "${CONFIG}" in
|
||||
8C) ROLES="combined"; P_GPUS=""; D_GPUS=""; COMBINED_GPUS="0,1,2,3,4,5,6,7" ;;
|
||||
6P+2D) ROLES="pd"; P_GPUS="0,1,2,3,4,5"; D_GPUS="6,7" ;;
|
||||
4P+4D) ROLES="pd"; P_GPUS="0,1,2,3"; D_GPUS="4,5,6,7" ;;
|
||||
2P+6D) ROLES="pd"; P_GPUS="0,1"; D_GPUS="2,3,4,5,6,7" ;;
|
||||
*) echo "Unknown CONFIG=${CONFIG} (expected: 8C, 6P+2D, 4P+4D, 2P+6D)"; exit 1;;
|
||||
esac
|
||||
|
||||
stop_all
|
||||
mkdir -p "${LOGS_DIR}" "${SNAPSHOT_DIR}"
|
||||
source "${VENV}/bin/activate"
|
||||
|
||||
# Apply MB5 patch (idempotent — affects entire shared cpfs venv).
|
||||
python "${INSTRUMENT}" --apply --venv "${VENV}"
|
||||
|
||||
launch_vllm() {
|
||||
# $1 idx, $2 gpu, $3 port, $4 role, $5 bp_or_dash
|
||||
local idx="$1" gpu="$2" port="$3" role="$4" bp="$5"
|
||||
local cfg="{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"${role}\"}"
|
||||
local master=$((BASE_MASTER+idx))
|
||||
local env_bp_arg=""
|
||||
if [ -n "${bp}" ] && [ "${bp}" != "-" ]; then
|
||||
env_bp_arg="VLLM_MOONCAKE_BOOTSTRAP_PORT=${bp}"
|
||||
fi
|
||||
echo "[mb5] launching idx=${idx} gpu=${gpu} port=${port} role=${role} bp=${bp:-none}"
|
||||
PYTHONHASHSEED=42 \
|
||||
MB5_LOG_DIR="${SNAPSHOT_DIR}" \
|
||||
CUDA_VISIBLE_DEVICES="${gpu}" \
|
||||
MASTER_PORT="${master}" \
|
||||
${env_bp_arg} \
|
||||
nohup vllm serve "${MODEL}" \
|
||||
--host 0.0.0.0 --port "${port}" \
|
||||
--tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching \
|
||||
--dtype auto --gpu-memory-utilization 0.9 \
|
||||
--max-model-len 200000 \
|
||||
--max-num-batched-tokens 8192 \
|
||||
--kv-transfer-config "${cfg}" \
|
||||
--enable-prompt-tokens-details \
|
||||
> "${LOGS_DIR}/vllm_idx${idx}_gpu${gpu}_${role}.log" 2>&1 &
|
||||
disown
|
||||
}
|
||||
|
||||
idx=0
|
||||
proxy_args=()
|
||||
ENDPOINTS=""
|
||||
|
||||
case "${ROLES}" in
|
||||
combined)
|
||||
IFS=',' read -ra GPUS <<< "${COMBINED_GPUS}"
|
||||
for gpu in "${GPUS[@]}"; do
|
||||
port=$((BASE_HTTP+idx))
|
||||
bp=$((BASE_BP+idx))
|
||||
launch_vllm "${idx}" "${gpu}" "${port}" "kv_both" "${bp}"
|
||||
ENDPOINTS+="${ENDPOINTS:+,}http://127.0.0.1:${port}"
|
||||
idx=$((idx+1))
|
||||
sleep 1
|
||||
done
|
||||
;;
|
||||
pd)
|
||||
# producers
|
||||
IFS=',' read -ra PG <<< "${P_GPUS}"
|
||||
for gpu in "${PG[@]}"; do
|
||||
port=$((BASE_HTTP+idx))
|
||||
bp=$((BASE_BP+idx))
|
||||
launch_vllm "${idx}" "${gpu}" "${port}" "kv_producer" "${bp}"
|
||||
proxy_args+=( --prefill "http://127.0.0.1:${port}" "${bp}" )
|
||||
idx=$((idx+1))
|
||||
sleep 1
|
||||
done
|
||||
# consumers
|
||||
IFS=',' read -ra DG <<< "${D_GPUS}"
|
||||
for gpu in "${DG[@]}"; do
|
||||
port=$((BASE_HTTP+idx))
|
||||
launch_vllm "${idx}" "${gpu}" "${port}" "kv_consumer" "-"
|
||||
proxy_args+=( --decode "http://127.0.0.1:${port}" )
|
||||
idx=$((idx+1))
|
||||
sleep 1
|
||||
done
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "[mb5] waiting for all vllm /v1/models endpoints..."
|
||||
all_ports=()
|
||||
for ((p=0; p<idx; p++)); do all_ports+=( $((BASE_HTTP+p)) ); done
|
||||
for port in "${all_ports[@]}"; do
|
||||
tries=0
|
||||
while ! curl -sf "http://127.0.0.1:${port}/v1/models" >/dev/null 2>&1; do
|
||||
tries=$((tries+1))
|
||||
if [ ${tries} -gt 300 ]; then
|
||||
echo "[mb5] FATAL port ${port} did not come up in 10 min"
|
||||
tail -40 "${LOGS_DIR}/vllm_idx"*"_gpu"*".log" || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " port=${port} ready"
|
||||
done
|
||||
|
||||
if [ "${ROLES}" = "pd" ]; then
|
||||
echo "[mb5] launching mooncake_connector_proxy on ${PROXY_PORT}"
|
||||
nohup python "${PROXY_SRC}" "${proxy_args[@]}" --port "${PROXY_PORT}" --host 0.0.0.0 \
|
||||
> "${LOGS_DIR}/proxy.log" 2>&1 &
|
||||
disown
|
||||
# wait for proxy
|
||||
tries=0
|
||||
while ! curl -sf "http://127.0.0.1:${PROXY_PORT}/v1/models" >/dev/null 2>&1; do
|
||||
tries=$((tries+1))
|
||||
if [ ${tries} -gt 60 ]; then
|
||||
echo "[mb5] FATAL proxy did not come up in 2 min"
|
||||
tail -40 "${LOGS_DIR}/proxy.log" || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " proxy port=${PROXY_PORT} ready"
|
||||
ENDPOINTS="http://127.0.0.1:${PROXY_PORT}"
|
||||
fi
|
||||
|
||||
echo "[mb5] CONFIG=${CONFIG} RUN_LABEL=${RUN_LABEL} UP"
|
||||
echo "ENDPOINTS=${ENDPOINTS}"
|
||||
echo "RUN_ROOT=${RUN_ROOT}"
|
||||
echo "SNAPSHOT_DIR=${SNAPSHOT_DIR}"
|
||||
echo "VLLM_LOGS=${LOGS_DIR}"
|
||||
Reference in New Issue
Block a user