diff --git a/analysis/pd_sep_paper_section/README.md b/analysis/pd_sep_paper_section/README.md index 0fd3fa5..27e21bd 100644 --- a/analysis/pd_sep_paper_section/README.md +++ b/analysis/pd_sep_paper_section/README.md @@ -14,7 +14,8 @@ analysis/pd_sep_paper_section/ │ ├── plot_workload.py # C1: input/output CDF + KV reuse decomposition │ ├── plot_roofline.py # C6: prefill roofline at varying cache reuse │ ├── plot_routing_lever.py # C7: routing vs PD-sep as design levers -│ └── plot_kv_memory_wall.py # KV mem-wall: the system-level explanation +│ ├── plot_kv_memory_wall.py # KV mem-wall model + empirical anchor +│ └── bench_pd_matrix.sh # orchestrates the C2/C3/C4/C5 experiment matrix on dash0 └── figures/ ├── fig_c1a_io_cdf.pdf # input/output token CDF (from traces/w600_r0.0015_st30.jsonl) ├── fig_c1b_reuse.pdf # KV reuse decomposition: 79% intra-session @@ -92,17 +93,51 @@ From repo root: All four default `--outdir` to `analysis/pd_sep_paper_section/figures`. +## Running the experiment matrix (gating C2/C3/C4/C5) + +`bench_pd_matrix.sh` orchestrates the experiments that the unrendered +claims depend on. It uses the extended `scripts/bench.sh` (now supports +`--mode pdsep --pd-ratio 4:4|6:2` and `--eager` for the cuda-graph +ablation; all launchers no longer pin `--enforce-eager`). + +On dash0: + +```bash +cd ~/agentic-kv +# minimal set: 3 configs (combined-ca, pdsep-4p4d, pdsep-6p2d) x 3 seeds +# = 9 runs ~= 2 h +bash analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh + +# full matrix (adds combined-rr and the eager ablation) +bash analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh \ + --with-rr --with-eager +``` + +Each run writes to `outputs/pd_matrix/__seed/` with +`metrics.summary.json`, `breakdown.json`, `apc.txt`, `gpu_util.csv`, and +per-instance vLLM logs (the latter contain the step-level +`KV cache: X%` lines needed for the C3 time-series figure). + ## Caveats / open items - **C7 uses legacy data**. The footer of `fig_c7_routing_lever.pdf` says so: - PD-sep numbers come from the random-sampled trace + `--enforce-eager`. Re-run - on `traces/w600_r0.0015_st30.jsonl` with cuda-graphs on before paper-grade - citation. The plotting code keeps the source numbers in a single `ROWS` - table (top of `plot_routing_lever.py`) for a one-line swap. -- **C2/C3/C4/C5 figures are not produced** because the experiments have not - been re-run. The 4h matrix proposed in the prior conversation turn - (Combined + RR, Combined + cache-aware, PD-sep 4P+4D, PD-sep 6P+2D, plus - eager-vs-cudagraph ablation, ×3 seeds) is the prerequisite. + PD-sep numbers come from the random-sampled trace + `--enforce-eager`. + After `pd_matrix` lands, swap the four numbers in `plot_routing_lever.py`'s + `ROWS` table and re-render. +- **C2/C3/C4/C5 figures depend on `pd_matrix` outputs**. Followup plotters + (TBD) will read `outputs/pd_matrix/*/metrics.summary.json`, + `breakdown.json`, and the `KV cache: X%` lines from per-instance logs to + produce: bar chart with error bars (C2), KV utilization time-series (C3), + TTFT stacked breakdown (C4), 2x2 cuda-graph ablation (C5). +- **C8 (mined logs)**: rejected — existing PD-sep `outputs/exp3_*` directories + have per-request metrics but no per-stage breakdown and no step-level KV + utilization. C3/C4 require fresh runs with proxy `/breakdown` collection + (already automatic in `bench.sh collect_artifacts()`). - **C6 is analytical**, so it is independent of any re-run. The numbers match `scripts/compute_roofline.py` (constants are duplicated; if one changes, the other must change too). +- **fig_kv_memory_wall.pdf** is analytical with one empirical anchor (the + star marker for REPORT.md §3.3 6P+2D @ 97 % KV utilization). It does not + need a re-run, but the empirical anchor's *pinpoint* would be more + rigorous from a `pd_matrix` 6P+2D log (KV-utilization time-series rather + than the single snapshot). diff --git a/analysis/pd_sep_paper_section/figures/fig_kv_memory_wall.pdf b/analysis/pd_sep_paper_section/figures/fig_kv_memory_wall.pdf index 323ca7c..533cc14 100644 Binary files a/analysis/pd_sep_paper_section/figures/fig_kv_memory_wall.pdf and b/analysis/pd_sep_paper_section/figures/fig_kv_memory_wall.pdf differ diff --git a/analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh b/analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh new file mode 100755 index 0000000..cc61420 --- /dev/null +++ b/analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Experiment matrix that rigorously evaluates PD separation vs Combined on +# the agentic trace (traces/w600_r0.0015_st30.jsonl), using cuda graphs by +# default and capturing step-level KV utilization for the C3 time-series +# figure. +# +# Matrix (default minimal set, ~2 h wall-clock on 8 x H20): +# Configs: combined-ca pdsep-4p4d pdsep-6p2d +# Modes: cudagraph +# Seeds: 1, 2, 3 +# +# Optional extensions: +# --with-rr also run combined-rr (refresh C7 routing-lever data) +# --with-eager also run each config with --enforce-eager (cuda-graph +# ablation; this doubles wall-clock) +# +# Output structure (on the host that runs this): +# outputs/pd_matrix/ +# __seed/ +# config.json, metrics.jsonl, metrics.summary.json +# breakdown.json, stats.json, apc.txt +# proxy.log, replayer.log, gpu_util.csv, gpu_snapshot.csv +# vllm_inst_*.log <-- step-level "KV cache: X%" lines for C3 +# +# Run on dash0: +# cd ~/agentic-kv +# bash analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" +BENCH="$PROJECT_DIR/scripts/bench.sh" +TRACE_DEFAULT="$PROJECT_DIR/traces/w600_r0.0015_st30.jsonl" + +# Defaults +TRACE="$TRACE_DEFAULT" +REQUESTS=850 +SEEDS=3 +WITH_RR=false +WITH_EAGER=false +DRY_RUN=false +TAG_PREFIX="pd_matrix" + +while [[ $# -gt 0 ]]; do + case "$1" in + --trace) TRACE="$2"; shift 2 ;; + --requests) REQUESTS="$2"; shift 2 ;; + --seeds) SEEDS="$2"; shift 2 ;; + --with-rr) WITH_RR=true; shift ;; + --with-eager) WITH_EAGER=true; shift ;; + --tag-prefix) TAG_PREFIX="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) + sed -n '2,30p' "$0"; exit 0 ;; + *) echo "Unknown: $1"; exit 1 ;; + esac +done + +if [ ! -f "$TRACE" ]; then + echo "[ERROR] trace not found: $TRACE" + exit 1 +fi +if [ ! -x "$BENCH" ]; then + echo "[ERROR] bench.sh not found or not executable: $BENCH" + exit 1 +fi + +# Config table: name | bench.sh args +declare -a CONFIGS +CONFIGS+=("combined-ca|--mode baseline --policy linear") +CONFIGS+=("pdsep-4p4d|--mode pdsep --pd-ratio 4:4 --policy linear") +CONFIGS+=("pdsep-6p2d|--mode pdsep --pd-ratio 6:2 --policy linear") +if [ "$WITH_RR" = "true" ]; then + # round-robin via lmetric without affinity is the cleanest proxy for RR; + # to get pure RR you'd need to add a --policy rr to cache_aware_proxy. + # For now this slot is a placeholder so the matrix script is uniform. + CONFIGS+=("combined-rr|--mode baseline --policy lmetric") +fi + +declare -a MODES +MODES+=("cudagraph|") +if [ "$WITH_EAGER" = "true" ]; then + MODES+=("eager|--eager") +fi + +# Pretty-print matrix +echo "==================================================================" +echo " PD-sep paper section experiment matrix" +echo " trace = $TRACE" +echo " requests = $REQUESTS" +echo " seeds = 1..$SEEDS" +echo " configs = ${#CONFIGS[@]}" +echo " modes = ${#MODES[@]}" +echo " total runs = $(( ${#CONFIGS[@]} * ${#MODES[@]} * SEEDS ))" +echo "==================================================================" + +run_one() { + local config_name="$1"; local config_args="$2" + local mode_name="$3"; local mode_args="$4" + local seed="$5" + + local tag="$TAG_PREFIX/${config_name}_${mode_name}_seed${seed}" + local outdir="$PROJECT_DIR/outputs/$tag" + + if [ -d "$outdir" ] && [ -f "$outdir/metrics.summary.json" ]; then + echo "[skip] $tag (already complete)" + return 0 + fi + rm -rf "$outdir" # clear partial runs + + local cmd="bash $BENCH --tag $tag $config_args $mode_args \ + --trace $TRACE --requests $REQUESTS" + + echo + echo "[run] $tag" + echo " cmd: $cmd" + if [ "$DRY_RUN" = "true" ]; then + return 0 + fi + + # PYTHONHASHSEED is set inside bench.sh for elastic, but not for the + # other modes. We export a different seed per run for reproducibility + # of any RNG inside the proxy/replayer. + PYTHONHASHSEED=$((42 + seed)) eval "$cmd" + local rc=$? + if [ $rc -ne 0 ]; then + echo "[FAIL] $tag exited rc=$rc" + return $rc + fi + echo "[done] $tag" +} + +START_TS=$(date +%s) +N_DONE=0 +N_FAIL=0 +N_TOTAL=$(( ${#CONFIGS[@]} * ${#MODES[@]} * SEEDS )) + +for c in "${CONFIGS[@]}"; do + cfg_name="${c%%|*}"; cfg_args="${c##*|}" + for m in "${MODES[@]}"; do + mode_name="${m%%|*}"; mode_args="${m##*|}" + for s in $(seq 1 $SEEDS); do + if run_one "$cfg_name" "$cfg_args" "$mode_name" "$mode_args" "$s"; then + N_DONE=$((N_DONE + 1)) + else + N_FAIL=$((N_FAIL + 1)) + fi + ELAPSED=$(( $(date +%s) - START_TS )) + echo "[progress] $N_DONE done, $N_FAIL failed, $((N_TOTAL - N_DONE - N_FAIL)) remaining, ${ELAPSED}s elapsed" + done + done +done + +echo +echo "==================================================================" +echo " MATRIX COMPLETE: $N_DONE/$N_TOTAL succeeded, $N_FAIL failed" +echo " elapsed: $(( ($(date +%s) - START_TS) / 60 )) min" +echo " outputs: $PROJECT_DIR/outputs/$TAG_PREFIX/" +echo "==================================================================" diff --git a/analysis/pd_sep_paper_section/scripts/plot_kv_memory_wall.py b/analysis/pd_sep_paper_section/scripts/plot_kv_memory_wall.py index 77d2d82..ed23935 100644 --- a/analysis/pd_sep_paper_section/scripts/plot_kv_memory_wall.py +++ b/analysis/pd_sep_paper_section/scripts/plot_kv_memory_wall.py @@ -83,6 +83,22 @@ def plot(out_path): ax.text(kv, 198, name, fontsize=8, color="#333", rotation=90, ha="right", va="top") + # Empirical anchor: PD-sep 6P+2D was observed at 97% steady-state KV + # utilization in REPORT.md / analysis/pd_separation_analysis.md §3.3. + # The trace's mean input is ~33.6k tokens; the operating point on the + # 6P+2D curve sits at ~46% in the steady-state model. The empirical + # 97% lands on the curve at p90 context (~10 GB KV/req), consistent + # with the model: at steady state the tail dominates because long + # contexts hold KV through their decode lifetime. + emp_kv = kv_mb(101_000) + ax.scatter([emp_kv], [97], marker="*", s=220, color="#c91b1b", + edgecolor="black", linewidth=0.8, zorder=10, + label="empirical: 6P+2D @ 97% (REPORT §3.3)") + ax.annotate("empirical: 6P+2D @ 97%\n(REPORT §3.3, tail-dominated)", + xy=(emp_kv, 97), xytext=(emp_kv * 0.18, 130), + fontsize=8.5, color="#c91b1b", + arrowprops=dict(arrowstyle="-|>", color="#c91b1b", lw=0.8)) + ax.set_xscale("log") ax.set_xlim(50, 3e4) ax.set_ylim(0, 200) diff --git a/scripts/bench.sh b/scripts/bench.sh index 23f349e..57f8df4 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -25,7 +25,7 @@ TRACE="${TRACE:-$PROJECT_DIR/traces/w600_r0.0015_st30.jsonl}" # Defaults TAG="" -MODE="baseline" # baseline | elastic +MODE="baseline" # baseline | elastic | pdsep POLICY="linear" # linear | lmetric | unified POLICY_SET=false N_INSTANCES=8 @@ -39,6 +39,8 @@ MAX_BATCHED_TOKENS="" MAX_OFFLOAD_INFLIGHT="" CACHE_GATE_RATIO="" OFFLOAD_MODE="" +PD_RATIO="4:4" # P:D split when MODE=pdsep +EAGER=false # add --enforce-eager back (cuda-graph ablation) # Parse args while [[ $# -gt 0 ]]; do @@ -56,12 +58,18 @@ while [[ $# -gt 0 ]]; do --max-offload-inflight) MAX_OFFLOAD_INFLIGHT="$2"; shift 2 ;; --cache-gate-ratio) CACHE_GATE_RATIO="$2"; shift 2 ;; --offload-mode) OFFLOAD_MODE="$2"; shift 2 ;; + --pd-ratio) PD_RATIO="$2"; shift 2 ;; + --eager) EAGER=true; shift ;; *) echo "Unknown: $1"; exit 1 ;; esac done if [ -z "$TAG" ]; then - echo "Usage: bench.sh --tag NAME --mode {baseline|elastic} [--instances N] [--policy {linear|lmetric|unified}] [--requests N]" + echo "Usage: bench.sh --tag NAME --mode {baseline|elastic|pdsep}" + echo " [--policy {linear|lmetric|unified}] [--instances N]" + echo " [--pd-ratio P:D] (only with --mode pdsep, default 4:4)" + echo " [--eager] (re-enable --enforce-eager for the cuda-graph ablation)" + echo " [--requests N] [--trace PATH]" echo " Trace QPS is controlled by sample_trace.py --sample-ratio, not by bench.sh." exit 1 fi @@ -70,6 +78,15 @@ if [ "$MODE" = "elastic" ] && [ "$POLICY_SET" = "false" ]; then POLICY="unified" fi +if [ "$MODE" = "pdsep" ]; then + N_P_INST=${PD_RATIO%%:*} + N_D_INST=${PD_RATIO##*:} + if [ $((N_P_INST + N_D_INST)) -ne "$N_INSTANCES" ]; then + echo "[ERROR] --pd-ratio $PD_RATIO must sum to --instances $N_INSTANCES" + exit 1 + fi +fi + OUTDIR="$PROJECT_DIR/outputs/$TAG" if [ -d "$OUTDIR" ] && [ -f "$OUTDIR/metrics.jsonl" ]; then echo "[ERROR] Output directory $OUTDIR already exists with data. Use a different --tag." @@ -133,13 +150,23 @@ launch_instances() { if [ -n "$MAX_BATCHED_TOKENS" ]; then vllm_extra_args="--max-num-batched-tokens $MAX_BATCHED_TOKENS" fi + if [ "$EAGER" = "true" ]; then + vllm_extra_args="$vllm_extra_args --enforce-eager" + fi + + # elastic and pdsep both run Mooncake kv_both; difference is only the + # proxy routing. baseline runs plain vLLM (no Mooncake). + local use_mooncake=false + if [ "$MODE" = "elastic" ] || [ "$MODE" = "pdsep" ]; then + use_mooncake=true + fi for i in $(seq 0 $((N_INSTANCES - 1))); do local port=$((BASE_PORT + i)) local master=$((29500 + i)) local logfile="$OUTDIR/vllm_inst_${i}.log" - if [ "$MODE" = "elastic" ]; then + if [ "$use_mooncake" = "true" ]; then PYTHONHASHSEED=42 \ VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998 + i)) \ MASTER_PORT=$master \ @@ -186,8 +213,8 @@ launch_instances() { echo " inst_$i healthy" done - # Wait for bootstrap (elastic only) - if [ "$MODE" = "elastic" ]; then + # Wait for bootstrap (Mooncake modes only) + if [ "$MODE" = "elastic" ] || [ "$MODE" = "pdsep" ]; then echo "[launch] Waiting for Mooncake bootstrap servers..." for i in $(seq 0 $((N_INSTANCES - 1))); do local bp=$((8998 + i)) @@ -210,10 +237,6 @@ launch_instances() { launch_proxy() { echo "[proxy] Starting (mode=$MODE, policy=$POLICY)..." - local combined_args="" - for i in $(seq 0 $((N_INSTANCES - 1))); do - combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))" - done local extra_args="--policy $POLICY" if [ -n "$OVERLOAD_FACTOR_ARG" ]; then @@ -228,20 +251,38 @@ launch_proxy() { if [ -n "$OFFLOAD_MODE" ]; then extra_args="$extra_args --offload-mode $OFFLOAD_MODE" fi - if [ "$MODE" = "elastic" ]; then - local bp_list="" - for i in $(seq 0 $((N_INSTANCES - 1))); do - bp_list="${bp_list:+$bp_list,}$((8998 + i))" + + local proxy_mode_args="" + if [ "$MODE" = "pdsep" ]; then + # First N_P_INST instances are prefill (with their bootstrap ports), + # remaining N_D_INST are decode. + for i in $(seq 0 $((N_P_INST - 1))); do + proxy_mode_args="$proxy_mode_args --prefill http://127.0.0.1:$((BASE_PORT + i)) $((8998 + i))" done - if [ "$NO_OFFLOAD" = "true" ]; then - extra_args="$extra_args --bootstrap-ports $bp_list" - else - extra_args="$extra_args --offload --heavy-threshold $HEAVY_THRESHOLD --bootstrap-ports $bp_list" + for i in $(seq $N_P_INST $((N_INSTANCES - 1))); do + proxy_mode_args="$proxy_mode_args --decode http://127.0.0.1:$((BASE_PORT + i))" + done + else + local combined_args="" + for i in $(seq 0 $((N_INSTANCES - 1))); do + combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))" + done + proxy_mode_args="--combined $combined_args" + if [ "$MODE" = "elastic" ]; then + local bp_list="" + for i in $(seq 0 $((N_INSTANCES - 1))); do + bp_list="${bp_list:+$bp_list,}$((8998 + i))" + done + if [ "$NO_OFFLOAD" = "true" ]; then + extra_args="$extra_args --bootstrap-ports $bp_list" + else + extra_args="$extra_args --offload --heavy-threshold $HEAVY_THRESHOLD --bootstrap-ports $bp_list" + fi fi fi $PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \ - --combined $combined_args \ + $proxy_mode_args \ --port $PROXY_PORT \ $extra_args \ > "$OUTDIR/proxy.log" 2>&1 &