Chunk-size ablation + comprehensive synthesis

max_num_batched_tokens sweep at 16 sessions (2048/4096/8192/16384):
- Default 8192 has best overall TPOT p90 (0.106) and E2E p50 (5.83)
- 16384: HEAVY TTFT -16%, HEAVY TPOT -17%, but overall worse (+18%)
- Smaller chunks (2048/4096) always worse (scheduler overhead)

bench.sh now supports --max-batched-tokens flag.

Updated elastic_hypotheses.md with H8 (high concurrency validated),
H9 (elastic RDMA at 16s rejected), and final synthesis.

Key conclusion: for agentic workloads, the dominant optimization is
cache-aware session-sticky routing (-60% TTFT, +24pp APC vs RR).
Neither PD-Sep, LMetric, elastic RDMA, nor chunk-size tuning provides
additional benefit beyond well-tuned routing.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/bench.sh

@@ -36,6 +36,7 @@ MAX_SESSIONS=8
 HEAVY_THRESHOLD=20000
 NO_OFFLOAD=false
 OVERLOAD_FACTOR_ARG=""
+MAX_BATCHED_TOKENS=""
 
 # Parse args
 while [[ $# -gt 0 ]]; do
@@ -50,6 +51,7 @@ while [[ $# -gt 0 ]]; do
         --heavy-threshold) HEAVY_THRESHOLD="$2"; shift 2 ;;
         --no-offload) NO_OFFLOAD=true; shift ;;
         --overload-factor) OVERLOAD_FACTOR_ARG="$2"; shift 2 ;;
+        --max-batched-tokens) MAX_BATCHED_TOKENS="$2"; shift 2 ;;
         *) echo "Unknown: $1"; exit 1 ;;
     esac
 done
@@ -80,6 +82,7 @@ cat > "$OUTDIR/config.json" << CONF
     "heavy_threshold": $HEAVY_THRESHOLD,
     "no_offload": "$NO_OFFLOAD",
     "overload_factor": "${OVERLOAD_FACTOR_ARG:-2.0}",
+    "max_batched_tokens": "${MAX_BATCHED_TOKENS:-default}",
     "timestamp": "$(date -Iseconds)",
     "hostname": "$(hostname)"
 }
@@ -117,6 +120,12 @@ cleanup_gpu() {
 launch_instances() {
     echo "[launch] Starting $N_INSTANCES vLLM instances (mode=$MODE)..."
 
+    # Build optional vLLM args
+    local vllm_extra_args=""
+    if [ -n "$MAX_BATCHED_TOKENS" ]; then
+        vllm_extra_args="--max-num-batched-tokens $MAX_BATCHED_TOKENS"
+    fi
+
     for i in $(seq 0 $((N_INSTANCES - 1))); do
         local port=$((BASE_PORT + i))
         local master=$((29500 + i))
@@ -132,6 +141,7 @@ launch_instances() {
                 --trust-remote-code --enable-prefix-caching --enforce-eager \
                 --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
                 --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
+                $vllm_extra_args \
                 > "$logfile" 2>&1 &
         else
             MASTER_PORT=$master \
@@ -141,6 +151,7 @@ launch_instances() {
                 --tensor-parallel-size 1 \
                 --trust-remote-code --enable-prefix-caching --enforce-eager \
                 --dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
+                $vllm_extra_args \
                 > "$logfile" 2>&1 &
         fi
 
@@ -313,7 +324,7 @@ print('=' * 70)
 
 echo "================================================================"
 echo "  bench.sh: $TAG"
-echo "  mode=$MODE  policy=$POLICY  requests=$REQUESTS  overload_factor=${OVERLOAD_FACTOR_ARG:-2.0}"
+echo "  mode=$MODE  policy=$POLICY  requests=$REQUESTS  overload_factor=${OVERLOAD_FACTOR_ARG:-2.0}  max_batched_tokens=${MAX_BATCHED_TOKENS:-default}"
 echo "  $(date)"
 echo "================================================================"