Gahow Wang
297fed6e73
Validates the elastic_migration_v2 finding that kv_role=kv_both adds
TTFT p90 +45% even when PD-sep never fires. Replicates under
single-instance, synthetic, open-loop workload to disambiguate
mechanism cost from 8-instance feedback amplification.
Configurations (8):
plain, noop_connector, mooncake_{producer,consumer,both},
nixl_both, lmcache_only, multi_mooncake_lmcache.
Pre-flight verification gates risky configs (kv_consumer needs dummy
bootstrap, multi-connector composition, NoOp custom class loading).
Workload: two-phase sweep
Phase A: rate {0.5..32} req/s × shape (4096, 256), saturation criteria
Phase B: ref_safe rate × cartesian (input ∈ {512,4k,32k}, output ∈ {64,256,1024})
Step-timing patch enriches vLLM's existing AGENTIC_STEP_LOG_PATH emit
with step_duration_us and build_meta_us — directly measures per-step
substrate cost, not just user-visible TTFT/TPOT.
run_all.sh runs as 5-stage barrier:
0 pre-flight + apply patch
1 Phase A all configs
2 pick ref_safe / ref_load
3 Phase B all configs
4 revert patch + analyze + plot
Outputs aggregate.{json,csv}, MANIFEST.tsv, and 5 figures.
Estimated runtime: 4-5.5 hours on idle dash0 H20.
65 lines
2.0 KiB
Bash
Executable File
65 lines
2.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# Common environment for all connector_tax launch scripts.
|
|
#
|
|
# Usage: source common.sh
|
|
# Sets: MODEL_PATH, PYTHON, PORT, LOG_DIR, COMMON_VLLM_ARGS
|
|
|
|
set -euo pipefail
|
|
|
|
# Resolve project root (microbench/connector_tax/launch/common.sh → ../../..)
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" # microbench/connector_tax
|
|
MB_DIR="$(cd "$CT_DIR/.." && pwd)" # microbench
|
|
PROJ_DIR="$(cd "$MB_DIR/.." && pwd)" # repo root
|
|
|
|
VENV="$PROJ_DIR/.venv/bin"
|
|
PYTHON="${VENV}/python"
|
|
[[ -x "$PYTHON" ]] || PYTHON="$(command -v python3)"
|
|
|
|
MODEL_PATH="${MODEL_PATH:-$HOME/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
|
PORT="${PORT:-8000}"
|
|
GPU_ID="${GPU_ID:-0}"
|
|
|
|
# Per-run log dir (caller may override RUN_DIR)
|
|
RUN_DIR="${RUN_DIR:-$CT_DIR/results/_default}"
|
|
mkdir -p "$RUN_DIR"
|
|
LOG_DIR="$RUN_DIR"
|
|
|
|
# PYTHONPATH so `microbench.connector_tax.tools.noop_connector:NoOpConnector`
|
|
# resolves when launched from anywhere.
|
|
export PYTHONPATH="${PROJ_DIR}:${PYTHONPATH:-}"
|
|
|
|
# Step-timing log path (vLLM's existing scheduler emits when this is set;
|
|
# our patch enriches the record with timing fields)
|
|
export AGENTIC_STEP_LOG_PATH="${AGENTIC_STEP_LOG_PATH:-$LOG_DIR/engine_step.jsonl}"
|
|
|
|
# Common vLLM flags shared by all configs
|
|
COMMON_VLLM_ARGS=(
|
|
--model "$MODEL_PATH"
|
|
--host 0.0.0.0
|
|
--port "$PORT"
|
|
--tensor-parallel-size 1
|
|
--trust-remote-code
|
|
--enable-prefix-caching
|
|
--dtype auto
|
|
--gpu-memory-utilization 0.9
|
|
--max-model-len 200000
|
|
--no-enable-log-requests
|
|
)
|
|
|
|
# Wait until /v1/models returns 200, then return.
|
|
# Times out at $1 seconds (default 240).
|
|
wait_for_ready() {
|
|
local timeout="${1:-240}"
|
|
local i
|
|
for i in $(seq 1 "$timeout"); do
|
|
if curl -sf "http://127.0.0.1:$PORT/v1/models" >/dev/null 2>&1; then
|
|
echo "Server ready after ${i}s"
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "ERROR: Server did not become ready within ${timeout}s" >&2
|
|
return 1
|
|
}
|