diff --git a/scripts/bench.sh b/scripts/bench.sh index 2a64c44..1c301c3 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -36,6 +36,7 @@ MAX_SESSIONS=8 HEAVY_THRESHOLD=20000 NO_OFFLOAD=false OVERLOAD_FACTOR_ARG="" +MAX_BATCHED_TOKENS="" # Parse args while [[ $# -gt 0 ]]; do @@ -50,6 +51,7 @@ while [[ $# -gt 0 ]]; do --heavy-threshold) HEAVY_THRESHOLD="$2"; shift 2 ;; --no-offload) NO_OFFLOAD=true; shift ;; --overload-factor) OVERLOAD_FACTOR_ARG="$2"; shift 2 ;; + --max-batched-tokens) MAX_BATCHED_TOKENS="$2"; shift 2 ;; *) echo "Unknown: $1"; exit 1 ;; esac done @@ -80,6 +82,7 @@ cat > "$OUTDIR/config.json" << CONF "heavy_threshold": $HEAVY_THRESHOLD, "no_offload": "$NO_OFFLOAD", "overload_factor": "${OVERLOAD_FACTOR_ARG:-2.0}", + "max_batched_tokens": "${MAX_BATCHED_TOKENS:-default}", "timestamp": "$(date -Iseconds)", "hostname": "$(hostname)" } @@ -117,6 +120,12 @@ cleanup_gpu() { launch_instances() { echo "[launch] Starting $N_INSTANCES vLLM instances (mode=$MODE)..." + # Build optional vLLM args + local vllm_extra_args="" + if [ -n "$MAX_BATCHED_TOKENS" ]; then + vllm_extra_args="--max-num-batched-tokens $MAX_BATCHED_TOKENS" + fi + for i in $(seq 0 $((N_INSTANCES - 1))); do local port=$((BASE_PORT + i)) local master=$((29500 + i)) @@ -132,6 +141,7 @@ launch_instances() { --trust-remote-code --enable-prefix-caching --enforce-eager \ --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \ --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \ + $vllm_extra_args \ > "$logfile" 2>&1 & else MASTER_PORT=$master \ @@ -141,6 +151,7 @@ launch_instances() { --tensor-parallel-size 1 \ --trust-remote-code --enable-prefix-caching --enforce-eager \ --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \ + $vllm_extra_args \ > "$logfile" 2>&1 & fi @@ -313,7 +324,7 @@ print('=' * 70) echo "================================================================" echo " bench.sh: $TAG" -echo " mode=$MODE policy=$POLICY requests=$REQUESTS overload_factor=${OVERLOAD_FACTOR_ARG:-2.0}" +echo " mode=$MODE policy=$POLICY requests=$REQUESTS overload_factor=${OVERLOAD_FACTOR_ARG:-2.0} max_batched_tokens=${MAX_BATCHED_TOKENS:-default}" echo " $(date)" echo "================================================================"