PD-sep matrix infrastructure: bench.sh pdsep mode + matrix driver
Adds the experiment harness that gates the empirical claims (C2/C3/C4/C5)
in the PD-sep paper section. Three pieces:
1. scripts/bench.sh: new --mode pdsep with --pd-ratio P:D, and an
--eager flag to re-enable --enforce-eager for the cuda-graph
ablation. pdsep reuses the elastic-mode Mooncake kv_both launch and
swaps the proxy command from --combined to --prefill/--decode.
baseline and elastic flows are unchanged.
2. analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh: matrix
driver that runs {combined-ca, pdsep-4p4d, pdsep-6p2d} x cudagraph
x 3 seeds by default (~2 h on dash0). --with-rr adds combined-rr;
--with-eager doubles to ~5 h with the cuda-graph ablation. Skips
completed runs, captures per-instance vLLM logs (needed for C3
step-level KV-utilization mining).
3. fig_kv_memory_wall.pdf: empirical anchor (star) at REPORT.md §3.3's
observed 6P+2D 97% KV utilization. The marker lands on the model's
predicted curve at p90 input, confirming the steady-state analysis.
README updated with the run command, output layout, and the followup
plotters that consume outputs/pd_matrix/.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
160
analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh
Executable file
160
analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh
Executable file
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
# Experiment matrix that rigorously evaluates PD separation vs Combined on
|
||||
# the agentic trace (traces/w600_r0.0015_st30.jsonl), using cuda graphs by
|
||||
# default and capturing step-level KV utilization for the C3 time-series
|
||||
# figure.
|
||||
#
|
||||
# Matrix (default minimal set, ~2 h wall-clock on 8 x H20):
|
||||
# Configs: combined-ca pdsep-4p4d pdsep-6p2d
|
||||
# Modes: cudagraph
|
||||
# Seeds: 1, 2, 3
|
||||
#
|
||||
# Optional extensions:
|
||||
# --with-rr also run combined-rr (refresh C7 routing-lever data)
|
||||
# --with-eager also run each config with --enforce-eager (cuda-graph
|
||||
# ablation; this doubles wall-clock)
|
||||
#
|
||||
# Output structure (on the host that runs this):
|
||||
# outputs/pd_matrix/
|
||||
# <config>_<mode>_seed<N>/
|
||||
# config.json, metrics.jsonl, metrics.summary.json
|
||||
# breakdown.json, stats.json, apc.txt
|
||||
# proxy.log, replayer.log, gpu_util.csv, gpu_snapshot.csv
|
||||
# vllm_inst_*.log <-- step-level "KV cache: X%" lines for C3
|
||||
#
|
||||
# Run on dash0:
|
||||
# cd ~/agentic-kv
|
||||
# bash analysis/pd_sep_paper_section/scripts/bench_pd_matrix.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
BENCH="$PROJECT_DIR/scripts/bench.sh"
|
||||
TRACE_DEFAULT="$PROJECT_DIR/traces/w600_r0.0015_st30.jsonl"
|
||||
|
||||
# Defaults
|
||||
TRACE="$TRACE_DEFAULT"
|
||||
REQUESTS=850
|
||||
SEEDS=3
|
||||
WITH_RR=false
|
||||
WITH_EAGER=false
|
||||
DRY_RUN=false
|
||||
TAG_PREFIX="pd_matrix"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--trace) TRACE="$2"; shift 2 ;;
|
||||
--requests) REQUESTS="$2"; shift 2 ;;
|
||||
--seeds) SEEDS="$2"; shift 2 ;;
|
||||
--with-rr) WITH_RR=true; shift ;;
|
||||
--with-eager) WITH_EAGER=true; shift ;;
|
||||
--tag-prefix) TAG_PREFIX="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help)
|
||||
sed -n '2,30p' "$0"; exit 0 ;;
|
||||
*) echo "Unknown: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$TRACE" ]; then
|
||||
echo "[ERROR] trace not found: $TRACE"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$BENCH" ]; then
|
||||
echo "[ERROR] bench.sh not found or not executable: $BENCH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Config table: name | bench.sh args
|
||||
declare -a CONFIGS
|
||||
CONFIGS+=("combined-ca|--mode baseline --policy linear")
|
||||
CONFIGS+=("pdsep-4p4d|--mode pdsep --pd-ratio 4:4 --policy linear")
|
||||
CONFIGS+=("pdsep-6p2d|--mode pdsep --pd-ratio 6:2 --policy linear")
|
||||
if [ "$WITH_RR" = "true" ]; then
|
||||
# round-robin via lmetric without affinity is the cleanest proxy for RR;
|
||||
# to get pure RR you'd need to add a --policy rr to cache_aware_proxy.
|
||||
# For now this slot is a placeholder so the matrix script is uniform.
|
||||
CONFIGS+=("combined-rr|--mode baseline --policy lmetric")
|
||||
fi
|
||||
|
||||
declare -a MODES
|
||||
MODES+=("cudagraph|")
|
||||
if [ "$WITH_EAGER" = "true" ]; then
|
||||
MODES+=("eager|--eager")
|
||||
fi
|
||||
|
||||
# Pretty-print matrix
|
||||
echo "=================================================================="
|
||||
echo " PD-sep paper section experiment matrix"
|
||||
echo " trace = $TRACE"
|
||||
echo " requests = $REQUESTS"
|
||||
echo " seeds = 1..$SEEDS"
|
||||
echo " configs = ${#CONFIGS[@]}"
|
||||
echo " modes = ${#MODES[@]}"
|
||||
echo " total runs = $(( ${#CONFIGS[@]} * ${#MODES[@]} * SEEDS ))"
|
||||
echo "=================================================================="
|
||||
|
||||
run_one() {
|
||||
local config_name="$1"; local config_args="$2"
|
||||
local mode_name="$3"; local mode_args="$4"
|
||||
local seed="$5"
|
||||
|
||||
local tag="$TAG_PREFIX/${config_name}_${mode_name}_seed${seed}"
|
||||
local outdir="$PROJECT_DIR/outputs/$tag"
|
||||
|
||||
if [ -d "$outdir" ] && [ -f "$outdir/metrics.summary.json" ]; then
|
||||
echo "[skip] $tag (already complete)"
|
||||
return 0
|
||||
fi
|
||||
rm -rf "$outdir" # clear partial runs
|
||||
|
||||
local cmd="bash $BENCH --tag $tag $config_args $mode_args \
|
||||
--trace $TRACE --requests $REQUESTS"
|
||||
|
||||
echo
|
||||
echo "[run] $tag"
|
||||
echo " cmd: $cmd"
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# PYTHONHASHSEED is set inside bench.sh for elastic, but not for the
|
||||
# other modes. We export a different seed per run for reproducibility
|
||||
# of any RNG inside the proxy/replayer.
|
||||
PYTHONHASHSEED=$((42 + seed)) eval "$cmd"
|
||||
local rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo "[FAIL] $tag exited rc=$rc"
|
||||
return $rc
|
||||
fi
|
||||
echo "[done] $tag"
|
||||
}
|
||||
|
||||
START_TS=$(date +%s)
|
||||
N_DONE=0
|
||||
N_FAIL=0
|
||||
N_TOTAL=$(( ${#CONFIGS[@]} * ${#MODES[@]} * SEEDS ))
|
||||
|
||||
for c in "${CONFIGS[@]}"; do
|
||||
cfg_name="${c%%|*}"; cfg_args="${c##*|}"
|
||||
for m in "${MODES[@]}"; do
|
||||
mode_name="${m%%|*}"; mode_args="${m##*|}"
|
||||
for s in $(seq 1 $SEEDS); do
|
||||
if run_one "$cfg_name" "$cfg_args" "$mode_name" "$mode_args" "$s"; then
|
||||
N_DONE=$((N_DONE + 1))
|
||||
else
|
||||
N_FAIL=$((N_FAIL + 1))
|
||||
fi
|
||||
ELAPSED=$(( $(date +%s) - START_TS ))
|
||||
echo "[progress] $N_DONE done, $N_FAIL failed, $((N_TOTAL - N_DONE - N_FAIL)) remaining, ${ELAPSED}s elapsed"
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=================================================================="
|
||||
echo " MATRIX COMPLETE: $N_DONE/$N_TOTAL succeeded, $N_FAIL failed"
|
||||
echo " elapsed: $(( ($(date +%s) - START_TS) / 60 )) min"
|
||||
echo " outputs: $PROJECT_DIR/outputs/$TAG_PREFIX/"
|
||||
echo "=================================================================="
|
||||
Reference in New Issue
Block a user