MB2: parameterize vLLM roles (kv_producer + kv_consumer default)
start_vllm_pair.sh ROLE_A / ROLE_B env vars (default kv_producer / kv_consumer for strict PD-disagg). Override to kv_both for the kv_both control. The role is injected into --kv-transfer-config so vLLM imposes the role restriction. mb2_kv_transfer.py --skip-verify flag drops step 3 (the plain completion sanity-check on the destination), required when the dst is kv_consumer-only since a kv_consumer instance refuses to serve a request without do_remote_prefill. The transfer-time itself is still measured from step 2 (do_remote_prefill on the consumer). Also: per-step client-side wall-clock timestamps (t_step1_client_unix, t_step2_client_unix, t_step2_end_unix) are now captured so the post-hoc breakdown analyzer can join with the per-instance JSONL logs on absolute time. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -29,6 +29,8 @@ BP_A=8998
|
||||
BP_B=8999
|
||||
MASTER_A=29500
|
||||
MASTER_B=29501
|
||||
ROLE_A="${ROLE_A:-kv_producer}" # or kv_both
|
||||
ROLE_B="${ROLE_B:-kv_consumer}" # or kv_both
|
||||
|
||||
MB2_LOG_ROOT="${FRESH_ROOT}/mb2_transfer_logs"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -78,8 +80,9 @@ fi
|
||||
mkdir -p "${MB2_LOG_ROOT}/A" "${MB2_LOG_ROOT}/B"
|
||||
|
||||
launch() {
|
||||
local idx="$1" gpu="$2" port="$3" bp="$4" master="$5"
|
||||
echo "[mb2] launching instance ${idx} on GPU ${gpu}, port ${port}, bp ${bp}"
|
||||
local idx="$1" gpu="$2" port="$3" bp="$4" master="$5" role="$6"
|
||||
local cfg="{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"${role}\"}"
|
||||
echo "[mb2] launching ${idx}: gpu=${gpu} port=${port} bp=${bp} role=${role}"
|
||||
PYTHONHASHSEED=42 \
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT="${bp}" \
|
||||
CUDA_VISIBLE_DEVICES="${gpu}" \
|
||||
@@ -91,15 +94,15 @@ launch() {
|
||||
--trust-remote-code --enable-prefix-caching \
|
||||
--dtype auto --gpu-memory-utilization 0.9 \
|
||||
--max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
--kv-transfer-config "${cfg}" \
|
||||
--enable-prompt-tokens-details \
|
||||
> "${LOGS_DIR}/vllm_${idx}_gpu${gpu}.log" 2>&1 &
|
||||
disown
|
||||
}
|
||||
|
||||
launch A "${GPU_A}" "${PORT_A}" "${BP_A}" "${MASTER_A}"
|
||||
launch A "${GPU_A}" "${PORT_A}" "${BP_A}" "${MASTER_A}" "${ROLE_A}"
|
||||
sleep 3
|
||||
launch B "${GPU_B}" "${PORT_B}" "${BP_B}" "${MASTER_B}"
|
||||
launch B "${GPU_B}" "${PORT_B}" "${BP_B}" "${MASTER_B}" "${ROLE_B}"
|
||||
|
||||
echo "[mb2] waiting for both /health endpoints..."
|
||||
for port in "${PORT_A}" "${PORT_B}"; do
|
||||
|
||||
Reference in New Issue
Block a user