Layerwise under load: overlap benefit survives (bg=16)

mb7 with background decode load (8/instance). Critical-path transfer overhead
stays ~constant ~90ms for layerwise vs 158/239/749ms baseline (up to 7.9x at
32k), prefill not slowed, KV correct. Confirms the overlap holds on busy
instances. DESIGN.md updated with idle-vs-load table + the two blockers
(chunk-safety, concurrent-transfer safety) that the full 1200-req trace needs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-28 16:30:14 +08:00
parent fec50fa45d
commit e77bdcac5a
5 changed files with 387 additions and 9 deletions

View File

@@ -22,6 +22,7 @@ MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
GPUS=(${GPUS:-0 1})
SIZES="${SIZES:-8192,16384,32768}"
REPEATS="${REPEATS:-3}"
BG_LOAD="${BG_LOAD:-0}"
MAX_BATCHED="${MAX_BATCHED:-40960}" # >= max prompt => no chunked prefill
DATE="$(date +%Y%m%d_%H%M)"
OUTDIR="${OUTDIR:-$PROJ_DIR/outputs/mb7_${MODE}_${DATE}}"
@@ -100,7 +101,8 @@ echo "[run] mb7 --mode $MODE"
"$PYTHON" "$DRIVER" --mode "$MODE" \
--src-port "${PORTS[0]}" --dst-port "${PORTS[1]}" \
--src-bp "${BPS[0]}" --dst-bp "${BPS[1]}" \
--sizes "$SIZES" --repeats "$REPEATS" --out "$OUTDIR/mb7_result.json" \
--sizes "$SIZES" --repeats "$REPEATS" --bg-load "$BG_LOAD" \
--out "$OUTDIR/mb7_result.json" \
2>&1 | tee "$OUTDIR/mb7_run.txt"
echo "[done] $OUTDIR"