scripts: archive obsolete one-off shell/python scripts to legacy/ (D2, D3)
D2: run_benchmark.sh and run_experiments.sh still pass --time-scale and --max-inflight-sessions to the replayer, but those flags were removed when the project moved to trace-driven dispatch. The scripts cannot run as-is. D3: ~25 ad-hoc analyze_* / compare_* / profile_* / final_* scripts and a handful of single-experiment run_*.sh point at /home/admin/cpfs paths, deleted output directories, or a sampled trace file that no longer exists. Keep them in scripts/legacy/ for historical reference; the scripts that remain in scripts/ (analyze_trace, analyze_breakdown, analyze_cache_hit, analyze_eviction, compare_results, compute_roofline, sample_trace, analyze_agentic_patterns, simulate_cache_policies, plus launch_*.sh, gpu_monitor.sh, bench.sh) cover the current workflow. Adds scripts/legacy/README.md to document the archival policy.
This commit is contained in:
20
scripts/legacy/README.md
Normal file
20
scripts/legacy/README.md
Normal file
@@ -0,0 +1,20 @@
|
||||
# scripts/legacy
|
||||
|
||||
One-shot scripts kept for historical reference. They were tied to specific
|
||||
experiments (cluster paths, deleted output dirs, removed CLI flags such as
|
||||
`--time-scale` / `--max-inflight-sessions`) and are no longer expected to
|
||||
run as-is.
|
||||
|
||||
For new experiments use `scripts/bench.sh`. Pre-existing structured
|
||||
analyses still live in `scripts/` (e.g. `analyze_trace.py`,
|
||||
`analyze_breakdown.py`, `analyze_cache_hit.py`, `analyze_eviction.py`,
|
||||
`compare_results.py`, `compute_roofline.py`).
|
||||
|
||||
If you need to revive a legacy script, expect to:
|
||||
|
||||
- update hardcoded paths (cluster `/home/admin/cpfs/...`, deleted trace
|
||||
files, missing `outputs/<exp>/...` directories);
|
||||
- adapt to the current replayer CLI (`--time-scale` and
|
||||
`--max-inflight-sessions` were removed when methodology moved to
|
||||
trace-driven dispatch);
|
||||
- re-verify the assumptions documented in `REPORT.md`.
|
||||
230
scripts/legacy/ab_gpu_test.sh
Executable file
230
scripts/legacy/ab_gpu_test.sh
Executable file
@@ -0,0 +1,230 @@
|
||||
#!/bin/bash
|
||||
# A/B GPU utilization test: Combined vs PD-Sep
|
||||
# Each run: clean start → warm up → benchmark with GPU monitoring → collect
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
VENV="$PROJECT_DIR/.venv/bin"
|
||||
VLLM="$VENV/vllm"
|
||||
PYTHON="$VENV/python"
|
||||
MODEL="${MODEL_PATH:-$HOME/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
||||
TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
|
||||
|
||||
# Use 200 requests for faster iteration
|
||||
REQ_LIMIT=200
|
||||
MAX_SESSIONS=8
|
||||
MAX_CONCURRENT=16
|
||||
TIME_SCALE=20 # 2x faster to reduce wall time
|
||||
REQUEST_TIMEOUT=300
|
||||
|
||||
cleanup() {
|
||||
pkill -9 -f "gpu_monitor\|replayer\|cache_aware\|uvicorn" 2>/dev/null || true
|
||||
pkill -9 -f "vllm" 2>/dev/null || true
|
||||
sleep 5
|
||||
fuser /dev/nvidia* 2>/dev/null | tr " " "\n" | sort -u | xargs -r kill -9 2>/dev/null || true
|
||||
sleep 10
|
||||
}
|
||||
|
||||
wait_server() {
|
||||
local port=$1
|
||||
timeout 600 bash -c "until curl -s localhost:$port/v1/models >/dev/null 2>&1; do sleep 5; done"
|
||||
}
|
||||
|
||||
run_with_monitor() {
|
||||
local tag=$1
|
||||
local endpoint=$2
|
||||
local outdir="$PROJECT_DIR/outputs/gpu_ab_$tag"
|
||||
mkdir -p "$outdir"
|
||||
|
||||
# Start GPU monitor
|
||||
bash "$PROJECT_DIR/scripts/gpu_monitor.sh" "$outdir/gpu_util.csv" 5 &
|
||||
local MON_PID=$!
|
||||
|
||||
# Warm up: send 3 requests
|
||||
for i in 1 2 3; do
|
||||
curl -s -m 60 "$endpoint/v1/completions" -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"$MODEL\",\"prompt\":[100,200,300],\"max_tokens\":5,\"temperature\":0}" > /dev/null 2>&1
|
||||
done
|
||||
sleep 5
|
||||
|
||||
# Run benchmark
|
||||
$PYTHON -m replayer \
|
||||
--trace "$TRACE" \
|
||||
--output "$outdir/metrics.jsonl" \
|
||||
--endpoint "$endpoint" \
|
||||
--model "$MODEL" \
|
||||
--time-scale $TIME_SCALE \
|
||||
--max-inflight-sessions $MAX_SESSIONS \
|
||||
--concurrency-limit $MAX_CONCURRENT \
|
||||
--request-timeout $REQUEST_TIMEOUT \
|
||||
--request-limit $REQ_LIMIT \
|
||||
-v 2>&1 | tail -5
|
||||
|
||||
# Stop monitor
|
||||
kill $MON_PID 2>/dev/null
|
||||
wait $MON_PID 2>/dev/null || true
|
||||
|
||||
echo " GPU data: $outdir/gpu_util.csv ($(wc -l < "$outdir/gpu_util.csv") samples)"
|
||||
}
|
||||
|
||||
###############################################
|
||||
# Test A: Combined TP=1 DP=8 (cache-aware)
|
||||
###############################################
|
||||
echo "================================================================"
|
||||
echo " TEST A: Combined TP=1 DP=8 + cache-aware scheduler"
|
||||
echo "================================================================"
|
||||
cleanup
|
||||
|
||||
for i in $(seq 0 7); do
|
||||
port=$((8000+i))
|
||||
mport=$((29500+i))
|
||||
MASTER_PORT=$mport CUDA_VISIBLE_DEVICES=$i $VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port $port --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
> /tmp/combined_$i.log 2>&1 &
|
||||
sleep 1
|
||||
done
|
||||
|
||||
for i in $(seq 0 7); do wait_server $((8000+i)); done
|
||||
echo " All 8 instances ready"
|
||||
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 http://127.0.0.1:8003 \
|
||||
http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 http://127.0.0.1:8007 \
|
||||
--port 9090 > /tmp/proxy_combined.log 2>&1 &
|
||||
sleep 10
|
||||
|
||||
echo " Running benchmark..."
|
||||
run_with_monitor "combined" "http://localhost:9090"
|
||||
|
||||
###############################################
|
||||
# Test B: PD-Sep TP=1 4P+4D (cache-aware)
|
||||
###############################################
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " TEST B: PD-Sep TP=1 4P+4D + cache-aware scheduler (Mooncake)"
|
||||
echo "================================================================"
|
||||
cleanup
|
||||
|
||||
# 4 prefill
|
||||
for i in 0 1 2 3; do
|
||||
bp=$((8998+i))
|
||||
port=$((8010+i))
|
||||
mport=$((29500+i))
|
||||
MASTER_PORT=$mport VLLM_MOONCAKE_BOOTSTRAP_PORT=$bp CUDA_VISIBLE_DEVICES=$i \
|
||||
$VLLM serve "$MODEL" --host 0.0.0.0 --port $port --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_producer"}' \
|
||||
> /tmp/prefill_$i.log 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# 4 decode
|
||||
for i in 0 1 2 3; do
|
||||
gpu=$((4+i))
|
||||
port=$((8020+i))
|
||||
mport=$((29510+i))
|
||||
MASTER_PORT=$mport CUDA_VISIBLE_DEVICES=$gpu \
|
||||
$VLLM serve "$MODEL" --host 0.0.0.0 --port $port --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer","kv_load_failure_policy":"recompute"}' \
|
||||
> /tmp/decode_$i.log 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in 0 1 2 3; do wait_server $((8010+i)); done
|
||||
for i in 0 1 2 3; do wait_server $((8020+i)); done
|
||||
echo " All 8 instances ready"
|
||||
|
||||
# Wait for bootstrap
|
||||
for bp in 8998 8999 9000 9001; do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query >/dev/null 2>&1; do sleep 2; done"
|
||||
done
|
||||
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--prefill http://127.0.0.1:8010 8998 --prefill http://127.0.0.1:8011 8999 \
|
||||
--prefill http://127.0.0.1:8012 9000 --prefill http://127.0.0.1:8013 9001 \
|
||||
--decode http://127.0.0.1:8020 --decode http://127.0.0.1:8021 \
|
||||
--decode http://127.0.0.1:8022 --decode http://127.0.0.1:8023 \
|
||||
--port 9090 > /tmp/proxy_pdsep.log 2>&1 &
|
||||
sleep 15
|
||||
|
||||
echo " Running benchmark..."
|
||||
run_with_monitor "pdsep" "http://localhost:9090"
|
||||
|
||||
###############################################
|
||||
# Analyze
|
||||
###############################################
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " ANALYSIS"
|
||||
echo "================================================================"
|
||||
cleanup
|
||||
|
||||
$PYTHON << 'PYEOF'
|
||||
import csv, json, statistics
|
||||
|
||||
def analyze_gpu(path, label, gpu_groups=None):
|
||||
"""gpu_groups: dict of group_name -> list of gpu indices"""
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for r in reader:
|
||||
rows.append(r)
|
||||
|
||||
if not rows:
|
||||
print(f" {label}: no GPU data")
|
||||
return
|
||||
|
||||
# Group by GPU
|
||||
by_gpu = {}
|
||||
for r in rows:
|
||||
g = int(r["gpu"])
|
||||
by_gpu.setdefault(g, []).append(float(r["util_pct"]))
|
||||
|
||||
if gpu_groups:
|
||||
for gname, indices in gpu_groups.items():
|
||||
vals = []
|
||||
for i in indices:
|
||||
vals.extend(by_gpu.get(i, []))
|
||||
if vals:
|
||||
print(f" {label} {gname}: mean={statistics.fmean(vals):.1f}% p50={sorted(vals)[len(vals)//2]:.0f}% p90={sorted(vals)[int(0.9*len(vals))]:.0f}% max={max(vals):.0f}%")
|
||||
else:
|
||||
all_vals = []
|
||||
for vals in by_gpu.values():
|
||||
all_vals.extend(vals)
|
||||
if all_vals:
|
||||
print(f" {label} all GPUs: mean={statistics.fmean(all_vals):.1f}% p50={sorted(all_vals)[len(all_vals)//2]:.0f}% p90={sorted(all_vals)[int(0.9*len(all_vals))]:.0f}%")
|
||||
|
||||
# Per-GPU breakdown
|
||||
for g in sorted(by_gpu.keys()):
|
||||
vals = by_gpu[g]
|
||||
print(f" GPU {g}: mean={statistics.fmean(vals):.1f}% samples={len(vals)}")
|
||||
|
||||
def analyze_metrics(path, label):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
print(f" {label}: {len(ok)}/{len(rows)} OK")
|
||||
if ok:
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)]
|
||||
if ttfts: print(f" TTFT p50={p(ttfts,.5):.3f} p90={p(ttfts,.9):.3f}")
|
||||
if tpots: print(f" TPOT p50={p(tpots,.5):.3f} p90={p(tpots,.9):.3f}")
|
||||
|
||||
import os
|
||||
for tag, groups in [
|
||||
("combined", None),
|
||||
("pdsep", {"Prefill(0-3)": [0,1,2,3], "Decode(4-7)": [4,5,6,7]}),
|
||||
]:
|
||||
d = f"outputs/gpu_ab_{tag}"
|
||||
if os.path.exists(f"{d}/gpu_util.csv"):
|
||||
analyze_gpu(f"{d}/gpu_util.csv", tag.upper(), groups)
|
||||
if os.path.exists(f"{d}/metrics.jsonl"):
|
||||
analyze_metrics(f"{d}/metrics.jsonl", tag.upper())
|
||||
print()
|
||||
PYEOF
|
||||
78
scripts/legacy/analyze_3way.py
Normal file
78
scripts/legacy/analyze_3way.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""3-way GPU utilization + latency comparison."""
|
||||
import csv, json, statistics, os
|
||||
|
||||
def gpu_stats(path, groups):
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
by_gpu = {}
|
||||
for r in rows:
|
||||
g = int(r["gpu"])
|
||||
by_gpu.setdefault(g, []).append(float(r["util_pct"]))
|
||||
result = {}
|
||||
for gname, indices in groups.items():
|
||||
vals = []
|
||||
for i in indices:
|
||||
vals.extend(by_gpu.get(i, []))
|
||||
if vals:
|
||||
s = sorted(vals)
|
||||
p = lambda q: s[min(int(q*len(s)), len(s)-1)]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
result[gname] = {
|
||||
"mean": statistics.fmean(vals), "p50": p(.5), "p90": p(.9),
|
||||
"max": max(vals), "active_pct": nz*100//len(vals), "n": len(rows)//8
|
||||
}
|
||||
return result
|
||||
|
||||
def latency_stats(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {
|
||||
"ok": len(ok), "total": len(rows),
|
||||
"ttft50": p(ttfts,.5), "ttft90": p(ttfts,.9),
|
||||
"tpot50": p(tpots,.5), "tpot90": p(tpots,.9),
|
||||
"e2e50": p(lats,.5), "e2e90": p(lats,.9),
|
||||
}
|
||||
|
||||
configs = [
|
||||
("gpu_ab_combined", "Combined 8colo", {"All(0-7)": list(range(8))}),
|
||||
("gpu_ab_pdsep", "PD-Sep 4P+4D", {"P(0-3)": [0,1,2,3], "D(4-7)": [4,5,6,7], "All": list(range(8))}),
|
||||
("gpu_ab_6p2d", "PD-Sep 6P+2D", {"P(0-5)": list(range(6)), "D(6-7)": [6,7], "All": list(range(8))}),
|
||||
]
|
||||
|
||||
sep = "=" * 85
|
||||
print(sep)
|
||||
print(" 3-WAY COMPARISON: Combined vs 4P+4D vs 6P+2D")
|
||||
print(" Same cache-aware scheduler, 200 req, time_scale=20")
|
||||
print(sep)
|
||||
|
||||
print("\n GPU UTILIZATION:")
|
||||
fmt = " %-22s %7s %7s %7s %7s %7s"
|
||||
print(fmt % ("Config / Group", "Mean%", "P50%", "P90%", "Max%", "Active"))
|
||||
print(" " + "-" * 62)
|
||||
|
||||
for dirname, label, groups in configs:
|
||||
gpath = "outputs/%s/gpu_util.csv" % dirname
|
||||
if not os.path.exists(gpath):
|
||||
continue
|
||||
gs = gpu_stats(gpath, groups)
|
||||
for gname, s in gs.items():
|
||||
tag = "%s %s" % (label, gname)
|
||||
print(fmt % (tag[:22], "%.1f" % s["mean"], "%.0f" % s["p50"],
|
||||
"%.0f" % s["p90"], "%.0f" % s["max"], "%d%%" % s["active_pct"]))
|
||||
|
||||
print("\n LATENCY:")
|
||||
fmt2 = " %-17s %6s %8s %8s %8s %8s %8s"
|
||||
print(fmt2 % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50"))
|
||||
print(" " + "-" * 65)
|
||||
|
||||
for dirname, label, _ in configs:
|
||||
mpath = "outputs/%s/metrics.jsonl" % dirname
|
||||
if not os.path.exists(mpath):
|
||||
continue
|
||||
s = latency_stats(mpath)
|
||||
okn = "%d/%d" % (s["ok"], s["total"])
|
||||
print(fmt2 % (label[:17], okn, "%.3f" % s["ttft50"], "%.3f" % s["ttft90"],
|
||||
"%.3f" % s["tpot50"], "%.3f" % s["tpot90"], "%.3f" % s["e2e50"]))
|
||||
106
scripts/legacy/analyze_ablations.py
Normal file
106
scripts/legacy/analyze_ablations.py
Normal file
@@ -0,0 +1,106 @@
|
||||
"""4-way ablation analysis: Combined vs 4P4D vs 6P2D vs 6P2D-FnF."""
|
||||
import csv, json, statistics, os
|
||||
|
||||
def gpu_stats(path, groups):
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
by_gpu = {}
|
||||
for r in rows:
|
||||
g = int(r["gpu"])
|
||||
by_gpu.setdefault(g, []).append(float(r["util_pct"]))
|
||||
result = {}
|
||||
for gname, indices in groups.items():
|
||||
vals = []
|
||||
for i in indices:
|
||||
vals.extend(by_gpu.get(i, []))
|
||||
if vals:
|
||||
s = sorted(vals)
|
||||
p = lambda q: s[min(int(q*len(s)), len(s)-1)]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
result[gname] = {"mean": statistics.fmean(vals), "p50": p(.5), "p90": p(.9),
|
||||
"max": max(vals), "active": nz*100//len(vals)}
|
||||
return result
|
||||
|
||||
def lat_stats(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5), "e90": p(lats,.9)}
|
||||
|
||||
configs = [
|
||||
("gpu_ab_combined", "Combined 8colo", {"All": list(range(8))}),
|
||||
("gpu_ab_pdsep", "4P+4D await", {"P": [0,1,2,3], "D": [4,5,6,7], "All": list(range(8))}),
|
||||
("gpu_ab_6p2d", "6P+2D await", {"P": list(range(6)), "D": [6,7], "All": list(range(8))}),
|
||||
("gpu_ab_6p2d_fnf", "6P+2D fire-forget", {"P": list(range(6)), "D": [6,7], "All": list(range(8))}),
|
||||
]
|
||||
|
||||
sep = "=" * 90
|
||||
print(sep)
|
||||
print(" ABLATION RESULTS: GPU Utilization + Latency")
|
||||
print(" All use cache-aware + token-level LB scheduler")
|
||||
print(sep)
|
||||
|
||||
# GPU
|
||||
print("\n GPU UTILIZATION (All GPUs aggregate):")
|
||||
fmt = " %-20s %7s %7s %7s %7s %7s"
|
||||
print(fmt % ("Config", "Mean%", "P50%", "P90%", "Max%", "Active"))
|
||||
print(" " + "-" * 55)
|
||||
for dirname, label, groups in configs:
|
||||
gpath = "outputs/%s/gpu_util.csv" % dirname
|
||||
if not os.path.exists(gpath): continue
|
||||
gs = gpu_stats(gpath, groups)
|
||||
if "All" in gs:
|
||||
s = gs["All"]
|
||||
print(fmt % (label, "%.1f" % s["mean"], "%.0f" % s["p50"],
|
||||
"%.0f" % s["p90"], "%.0f" % s["max"], "%d%%" % s["active"]))
|
||||
|
||||
# P vs D breakdown for PD-Sep configs
|
||||
print("\n GPU UTILIZATION (P vs D breakdown):")
|
||||
for dirname, label, groups in configs:
|
||||
if dirname == "gpu_ab_combined": continue
|
||||
gpath = "outputs/%s/gpu_util.csv" % dirname
|
||||
if not os.path.exists(gpath): continue
|
||||
gs = gpu_stats(gpath, groups)
|
||||
parts = []
|
||||
if "P" in gs: parts.append("P:%.1f%%(%d%%act)" % (gs["P"]["mean"], gs["P"]["active"]))
|
||||
if "D" in gs: parts.append("D:%.1f%%(%d%%act)" % (gs["D"]["mean"], gs["D"]["active"]))
|
||||
print(" %-20s %s" % (label, " ".join(parts)))
|
||||
|
||||
# Latency
|
||||
print("\n LATENCY:")
|
||||
fmt2 = " %-20s %7s %8s %8s %8s %8s %8s"
|
||||
print(fmt2 % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50"))
|
||||
print(" " + "-" * 68)
|
||||
for dirname, label, _ in configs:
|
||||
mpath = "outputs/%s/metrics.jsonl" % dirname
|
||||
if not os.path.exists(mpath): continue
|
||||
s = lat_stats(mpath)
|
||||
print(fmt2 % (label, "%d/%d" % (s["ok"], s["n"]),
|
||||
"%.3f" % s["t50"], "%.3f" % s["t90"],
|
||||
"%.3f" % s["p50"], "%.3f" % s["p90"], "%.3f" % s["e50"]))
|
||||
|
||||
# Ablation conclusions
|
||||
print("\n" + sep)
|
||||
print(" ABLATION CONCLUSIONS")
|
||||
print(sep)
|
||||
print("""
|
||||
Ablation 1 — P/D ratio (6P+2D vs 4P+4D):
|
||||
TTFT: 1.99s -> 1.48s (-26%) More prefill GPUs = less queue
|
||||
TPOT: 0.075 -> 0.077 (~same) Decode still memory-bound
|
||||
Decode GPU util: 7.8% -> 19.0% (+143%) Less waste
|
||||
Verdict: HELPS — fewer decode GPUs is better for this workload
|
||||
|
||||
Ablation 2 — Fire-and-forget vs Await-prefill (on 6P+2D):
|
||||
TTFT: 1.48s -> 5.32s (+260%) WORSE — decode waits for KV internally
|
||||
TPOT: 0.066 -> 0.037 (-44%) BETTER — pipeline overlap helps decode
|
||||
Error: 6% -> 15% MORE errors from KV race conditions
|
||||
Verdict: HURTS overall — TTFT degradation outweighs TPOT gain
|
||||
|
||||
Overall: Combined 8colo remains best for single-machine agentic workload.
|
||||
PD-Sep optimizations (ratio tuning, scheduling) narrow the gap but don't close it.
|
||||
""")
|
||||
142
scripts/legacy/analyze_aggregation.py
Normal file
142
scripts/legacy/analyze_aggregation.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Analyze prefill aggregation strategy: 1 aggregator GPU + 7 combined GPUs."""
|
||||
import json
|
||||
from collections import defaultdict
|
||||
|
||||
rows = [json.loads(l) for l in open("traces/sampled_1000req_seed42.jsonl")]
|
||||
rows.sort(key=lambda r: float(r["timestamp"]))
|
||||
|
||||
BLOCK_SIZE = 512
|
||||
N_COMBINED = 7
|
||||
HEAVY_THRESHOLD = 20000
|
||||
|
||||
chat_to_session = {}
|
||||
sessions = defaultdict(list)
|
||||
for idx, r in enumerate(rows):
|
||||
cid = r["chat_id"]
|
||||
pid = r["parent_chat_id"]
|
||||
sid = r.get("session_id", str(cid) if pid < 0 else chat_to_session.get(pid, str(pid)))
|
||||
chat_to_session[cid] = str(sid)
|
||||
sessions[str(sid)].append((idx, r))
|
||||
|
||||
# Classify requests
|
||||
seen = defaultdict(set)
|
||||
session_inst = {}
|
||||
offloaded = []
|
||||
colocated = []
|
||||
total_transfer = 0
|
||||
|
||||
for idx, r in enumerate(rows):
|
||||
hids = r.get("hash_ids", [])
|
||||
il = r["input_length"]
|
||||
sid = str(r.get("session_id", chat_to_session.get(r["chat_id"], str(r["chat_id"]))))
|
||||
is_new = sid not in session_inst
|
||||
|
||||
if is_new:
|
||||
inst = hash(sid) % N_COMBINED
|
||||
session_inst[sid] = inst
|
||||
hit = 0
|
||||
for hid in hids:
|
||||
if hid in seen[inst]:
|
||||
hit += 1
|
||||
else:
|
||||
break
|
||||
new_tok = max(0, il - hit * BLOCK_SIZE)
|
||||
if new_tok >= HEAVY_THRESHOLD:
|
||||
offloaded.append({"idx": idx, "input": il, "new": new_tok, "sid": sid})
|
||||
total_transfer += il
|
||||
else:
|
||||
colocated.append({"idx": idx, "input": il, "new": new_tok})
|
||||
else:
|
||||
inst = session_inst[sid]
|
||||
hit = 0
|
||||
for hid in hids:
|
||||
if hid in seen[inst]:
|
||||
hit += 1
|
||||
else:
|
||||
break
|
||||
new_tok = max(0, il - hit * BLOCK_SIZE)
|
||||
colocated.append({"idx": idx, "input": il, "new": new_tok})
|
||||
|
||||
target = session_inst.get(sid, 0)
|
||||
for hid in hids:
|
||||
seen[target].add(hid)
|
||||
|
||||
total_reqs = len(rows)
|
||||
total_input = sum(r["input_length"] for r in rows)
|
||||
p = lambda v, q: v[min(int(q*len(v)), len(v)-1)] if v else 0
|
||||
|
||||
print("=" * 70)
|
||||
print(" PREFILL AGGREGATION: 1 Aggregator + 7 Combined")
|
||||
print("=" * 70)
|
||||
|
||||
print("\nRequest split:")
|
||||
print(" Offloaded (HEAVY new>=%dk): %d (%.0f%%)" % (
|
||||
HEAVY_THRESHOLD//1000, len(offloaded), len(offloaded)*100/total_reqs))
|
||||
print(" Colocated (rest): %d (%.0f%%)" % (
|
||||
len(colocated), len(colocated)*100/total_reqs))
|
||||
|
||||
off_sids = set(o["sid"] for o in offloaded)
|
||||
off_single = sum(1 for s in off_sids if len(sessions[s]) == 1)
|
||||
off_multi = len(off_sids) - off_single
|
||||
print("\nOffloaded sessions:")
|
||||
print(" Single-turn (transfer wasted): %d (%.0f%%)" % (
|
||||
off_single, off_single*100/max(len(off_sids),1)))
|
||||
print(" Multi-turn (future turns free): %d (%.0f%%)" % (
|
||||
off_multi, off_multi*100/max(len(off_sids),1)))
|
||||
|
||||
# Future turns saved from re-prefill
|
||||
future_turns_saved = 0
|
||||
future_tokens_saved = 0
|
||||
for s in off_sids:
|
||||
turns = sessions[s]
|
||||
if len(turns) > 1:
|
||||
for _, r in turns[1:]: # turn 2+
|
||||
future_turns_saved += 1
|
||||
# These get cache hit on combined instance
|
||||
future_tokens_saved += r["input_length"]
|
||||
|
||||
print("\n Future turns that get FREE cache hit (no re-prefill):")
|
||||
print(" Turns: %d, Tokens: %s" % (future_turns_saved, "{:,}".format(future_tokens_saved)))
|
||||
|
||||
print("\nKV transfer:")
|
||||
print(" Volume: %s tokens (%.1f%% of total input)" % (
|
||||
"{:,}".format(total_transfer), total_transfer*100/total_input))
|
||||
print(" This is a ONE-TIME cost per session, not per-turn")
|
||||
print(" vs PD-Sep: transfers EVERY turn (including warm ones)")
|
||||
|
||||
off_new = sorted([o["new"] for o in offloaded])
|
||||
print("\nAggregator workload:")
|
||||
print(" %d prefills, new_tokens p50=%d p90=%d" % (
|
||||
len(offloaded), p(off_new,.5), p(off_new,.9)))
|
||||
print(" Total new tokens: %s" % "{:,}".format(sum(off_new)))
|
||||
print(" Can batch concurrent heavy prefills for high GPU utilization")
|
||||
|
||||
colo_new = sorted([c["new"] for c in colocated])
|
||||
print("\nCombined instances workload (7 GPUs):")
|
||||
print(" %d requests, new_tokens p50=%d p90=%d" % (
|
||||
len(colocated), p(colo_new,.5), p(colo_new,.9)))
|
||||
print(" NO heavy prefills — only warm/medium + multi-turn decode")
|
||||
print(" TPOT should be better: no heavy prefill disruption")
|
||||
|
||||
# Compare with pure PD-Sep
|
||||
print("\n" + "=" * 70)
|
||||
print(" vs PURE PD-SEP: WHY THIS IS DIFFERENT")
|
||||
print("=" * 70)
|
||||
print("""
|
||||
Pure PD-Sep (4P+4D):
|
||||
- EVERY request: prefill on P, transfer KV, decode on D
|
||||
- Turn N+1: re-prefill on P (no cache), re-transfer to D
|
||||
- KV transfer: 100%% of requests, EVERY turn
|
||||
- Session affinity: BROKEN (P has no prior KV)
|
||||
|
||||
Prefill Aggregation (1 agg + 7 combined):
|
||||
- HEAVY cold start only: prefill on agg, transfer to combined
|
||||
- Turn N+1: COLOCATED on combined (cache hit ~80%%, zero transfer)
|
||||
- KV transfer: %.0f%% of requests, FIRST turn only
|
||||
- Session affinity: PRESERVED (combined holds all future KV)
|
||||
|
||||
Net: transfer volume reduced by ~%.0fx vs PD-Sep
|
||||
""" % (
|
||||
len(offloaded)*100/total_reqs,
|
||||
total_input / max(total_transfer, 1),
|
||||
))
|
||||
80
scripts/legacy/analyze_gpu_ab.py
Normal file
80
scripts/legacy/analyze_gpu_ab.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Analyze GPU utilization A/B test results."""
|
||||
import csv, json, statistics, os
|
||||
|
||||
def gpu_analysis(path, label, groups):
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
by_gpu = {}
|
||||
for r in rows:
|
||||
g = int(r["gpu"])
|
||||
by_gpu.setdefault(g, []).append(float(r["util_pct"]))
|
||||
|
||||
n = len(rows) // 8
|
||||
print(f"\n{'='*70}")
|
||||
print(f" {label} ({n} time points)")
|
||||
print(f"{'='*70}")
|
||||
for gname, indices in groups.items():
|
||||
vals = []
|
||||
for i in indices:
|
||||
vals.extend(by_gpu.get(i, []))
|
||||
if vals:
|
||||
s = sorted(vals)
|
||||
p = lambda q: s[min(int(q*len(s)), len(s)-1)]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
print(f" {gname}:")
|
||||
print(f" mean={statistics.fmean(vals):.1f}% p50={p(.5):.0f}% p90={p(.9):.0f}% max={max(vals):.0f}%")
|
||||
print(f" active_samples={nz}/{len(vals)} ({nz*100//len(vals)}%)")
|
||||
|
||||
for g in sorted(by_gpu.keys()):
|
||||
vals = by_gpu[g]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
print(f" GPU {g}: mean={statistics.fmean(vals):.1f}% max={max(vals):.0f}% active={nz*100//len(vals)}%")
|
||||
|
||||
def metrics_analysis(path, label):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
err = [r for r in rows if r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
|
||||
print(f"\n {label}: {len(ok)}/{len(rows)} OK, {len(err)} err")
|
||||
if ttfts: print(f" TTFT p50={p(ttfts,.5):.3f} p90={p(ttfts,.9):.3f}")
|
||||
if tpots: print(f" TPOT p50={p(tpots,.5):.3f} p90={p(tpots,.9):.3f}")
|
||||
if lats: print(f" E2E p50={p(lats,.5):.3f} p90={p(lats,.9):.3f}")
|
||||
|
||||
gpu_analysis("outputs/gpu_ab_combined/gpu_util.csv", "COMBINED TP=1 DP=8 (cache-aware)",
|
||||
{"All GPUs": list(range(8))})
|
||||
|
||||
gpu_analysis("outputs/gpu_ab_pdsep/gpu_util.csv", "PD-SEP TP=1 4P+4D (cache-aware Mooncake)",
|
||||
{"Prefill (GPU 0-3)": [0,1,2,3], "Decode (GPU 4-7)": [4,5,6,7], "All GPUs": list(range(8))})
|
||||
|
||||
print(f"\n{'='*70}")
|
||||
print(f" LATENCY COMPARISON")
|
||||
print(f"{'='*70}")
|
||||
metrics_analysis("outputs/gpu_ab_combined/metrics.jsonl", "COMBINED")
|
||||
metrics_analysis("outputs/gpu_ab_pdsep/metrics.jsonl", "PD-SEP")
|
||||
|
||||
print(f"\n{'='*70}")
|
||||
print(f" BOTTLENECK SUMMARY")
|
||||
print(f"{'='*70}")
|
||||
print("""
|
||||
1. DECODE GPU UNDERUTILIZATION
|
||||
PD-Sep decode GPUs: mean ~20%, max ~47%
|
||||
Combined GPUs: mean ~30%, max 100%
|
||||
-> Decode is memory-bound, GPU compute wasted on dedicated decode GPUs
|
||||
-> 4 GPUs reserved for decode never exceed 50% utilization
|
||||
|
||||
2. PREFILL GPU BURSTINESS
|
||||
PD-Sep prefill: high util when active (~86% p50), but idle ~48% of time
|
||||
Combined: more evenly distributed, active 64% of time
|
||||
-> await-prefill serializes P then D, creating idle gaps between requests
|
||||
|
||||
3. KV TRANSFER OVERHEAD
|
||||
TTFT(PD-Sep) - TTFT(Combined) = pure KV transfer + proxy routing cost
|
||||
This penalty grows with input length (more KV to transfer)
|
||||
|
||||
4. RESOURCE PARTITIONING INEFFICIENCY
|
||||
PD-Sep: fixed 4P+4D split cannot adapt to workload phase
|
||||
Combined: 8 GPUs flexibly serve both P and D based on demand
|
||||
""")
|
||||
96
scripts/legacy/analyze_h4_results.py
Normal file
96
scripts/legacy/analyze_h4_results.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Analyze H4 cache-aware gate experiment results."""
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
outdir = sys.argv[1] if len(sys.argv) > 1 else "outputs/h4_cache_gate"
|
||||
|
||||
rows = [json.loads(l) for l in open(f"{outdir}/metrics.jsonl")]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
fail = [r for r in rows if r.get("error")]
|
||||
p = lambda v, q: sorted(v)[min(int(q * len(v)), len(v) - 1)] if v else 0
|
||||
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"] > 0])
|
||||
e2es = sorted([r["latency_s"] for r in ok])
|
||||
|
||||
print("=" * 70)
|
||||
print("H4 Cache-Aware Offload Gate Results")
|
||||
print("=" * 70)
|
||||
print(f"OK={len(ok)}/{len(rows)} TTFT50={p(ttfts,.5):.3f} TTFT90={p(ttfts,.9):.3f} TPOT90={p(tpots,.9):.4f} E2E50={p(e2es,.5):.3f} E2E90={p(e2es,.9):.3f}")
|
||||
|
||||
# Per-class breakdown
|
||||
for lo, hi, cl in [(0, 5000, "WARM"), (5000, 20000, "MED"), (20000, 200000, "HEAVY")]:
|
||||
sub = [r for r in ok if lo <= r["input_length"] < hi and r.get("ttft_s")]
|
||||
if sub:
|
||||
t = sorted([r["ttft_s"] for r in sub])
|
||||
tp = sorted([r["tpot_s"] for r in sub if r.get("tpot_s") and r["tpot_s"] > 0])
|
||||
e = sorted([r["latency_s"] for r in sub])
|
||||
print(f" {cl:6s} n={len(sub):3d} TTFT50={p(t,.5):.3f} TTFT90={p(t,.9):.3f} TPOT90={p(tp,.9):.4f} E2E50={p(e,.5):.3f} E2E90={p(e,.9):.3f}")
|
||||
|
||||
# Route distribution from breakdown
|
||||
try:
|
||||
bd = json.load(open(f"{outdir}/breakdown.json"))
|
||||
rc = Counter(b.get("route_class", "") for b in bd)
|
||||
print(f"\nRoute class distribution:")
|
||||
for cls, cnt in sorted(rc.items()):
|
||||
print(f" {cls}: {cnt}")
|
||||
|
||||
heavy = [b for b in bd if b.get("route_class", "").startswith("HEAVY")]
|
||||
reasons = Counter(b.get("offload_reason", "") for b in heavy)
|
||||
print(f"\nHEAVY offload reasons: {dict(reasons)}")
|
||||
|
||||
colo = [b for b in bd if b.get("route_class") == "HEAVY_COLO"]
|
||||
offloaded = [b for b in bd if b.get("route_class") == "HEAVY_OFFLOAD"]
|
||||
print(f"\nHEAVY_COLO (cold, no RDMA): {len(colo)}")
|
||||
print(f"HEAVY_OFFLOAD (cached, RDMA): {len(offloaded)}")
|
||||
|
||||
# Cache ratio distribution for HEAVY
|
||||
print("\nCache ratio distribution for HEAVY:")
|
||||
for b in heavy:
|
||||
cr = b.get("cache_ratio", b.get("cache_hit", 0) / max(b.get("input_length", 1), 1))
|
||||
cls = b.get("route_class", "")
|
||||
reason = b.get("offload_reason", "")
|
||||
# Don't print individual ones, summarize
|
||||
|
||||
ratios = [b.get("cache_ratio", b.get("cache_hit", 0) / max(b.get("input_length", 1), 1)) for b in heavy]
|
||||
if ratios:
|
||||
ratios.sort()
|
||||
print(f" min={min(ratios):.2f} p50={p(ratios,.5):.2f} mean={sum(ratios)/len(ratios):.2f} max={max(ratios):.2f}")
|
||||
print(f" >=0.3 (would offload): {sum(1 for r in ratios if r >= 0.3)}")
|
||||
print(f" <0.3 (stays colo): {sum(1 for r in ratios if r < 0.3)}")
|
||||
|
||||
# TTFT comparison: HEAVY_COLO timing
|
||||
if colo:
|
||||
colo_ttft = sorted([b["t_first_token"] - b["t_proxy_recv"] for b in colo if b.get("t_first_token")])
|
||||
if colo_ttft:
|
||||
print(f"\n HEAVY_COLO TTFT: p50={p(colo_ttft,.5):.2f}s p90={p(colo_ttft,.9):.2f}s")
|
||||
if offloaded:
|
||||
off_ttft = sorted([b["t_first_token"] - b["t_proxy_recv"] for b in offloaded if b.get("t_first_token")])
|
||||
if off_ttft:
|
||||
print(f" HEAVY_OFFLOAD TTFT: p50={p(off_ttft,.5):.2f}s p90={p(off_ttft,.9):.2f}s")
|
||||
pf = [b["t_prefill_done"] - b["t_prefill_sent"] for b in offloaded if b.get("t_prefill_done") and b.get("t_prefill_sent")]
|
||||
kv = [b["t_first_token"] - b["t_prefill_done"] for b in offloaded if b.get("t_first_token") and b.get("t_prefill_done")]
|
||||
if pf:
|
||||
pf.sort()
|
||||
print(f" Offload prefill: p50={p(pf,.5):.2f}s p90={p(pf,.9):.2f}s")
|
||||
if kv:
|
||||
kv.sort()
|
||||
print(f" Offload KV xfer: p50={p(kv,.5):.2f}s p90={p(kv,.9):.2f}s")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Breakdown analysis error: {e}")
|
||||
|
||||
if fail:
|
||||
print(f"\nFailed requests ({len(fail)}):")
|
||||
for r in fail[:5]:
|
||||
print(f" input={r['input_length']} error={str(r['error'])[:80]}")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("Comparison with all prior experiments")
|
||||
print("=" * 70)
|
||||
print("Baseline 8C plain: OK=198/200 TTFT50=1.075 TTFT90=9.384 TPOT90=0.0761 E2E50=5.075")
|
||||
print("Phase0A 7C kv_both: OK=198/200 TTFT50=1.073 TPOT90=0.0738 E2E50=5.096")
|
||||
print("V2 all-offload: OK=179/185 TTFT50=0.762 TPOT90=0.0746 E2E50=4.628")
|
||||
print(f"H4 cache-aware gate: OK={len(ok)}/{len(rows)} TTFT50={p(ttfts,.5):.3f} TTFT90={p(ttfts,.9):.3f} TPOT90={p(tpots,.9):.4f} E2E50={p(e2es,.5):.3f}")
|
||||
60
scripts/legacy/analyze_h5_rdma.py
Normal file
60
scripts/legacy/analyze_h5_rdma.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""H5: RDMA transfer breakdown analysis from V2 offload data."""
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
bd_path = sys.argv[1] if len(sys.argv) > 1 else "outputs/v2_offload/breakdown.json"
|
||||
bd = json.load(open(bd_path))
|
||||
offloaded = [b for b in bd if b.get("route_class") == "HEAVY_OFFLOAD"]
|
||||
|
||||
records = []
|
||||
for b in offloaded:
|
||||
keys = ["t_prefill_sent", "t_prefill_done", "t_first_token", "t_done", "t_proxy_recv"]
|
||||
if not all(k in b for k in keys):
|
||||
continue
|
||||
records.append({
|
||||
"il": b["input_length"],
|
||||
"ch": b.get("cache_hit", 0),
|
||||
"kv": b["t_first_token"] - b["t_prefill_done"],
|
||||
"pf": b["t_prefill_done"] - b["t_prefill_sent"],
|
||||
"dc": b["t_done"] - b["t_first_token"],
|
||||
"ttft": b["t_first_token"] - b["t_proxy_recv"],
|
||||
})
|
||||
|
||||
print(f"Records with full timing: {len(records)}")
|
||||
|
||||
# Concurrency effect
|
||||
low_kv = [r for r in records if r["kv"] < 1.5]
|
||||
high_kv = [r for r in records if r["kv"] >= 1.5]
|
||||
print("\n=== Concurrency Effect on KV Transfer ===")
|
||||
if low_kv:
|
||||
print(f" Low KV (<1.5s): n={len(low_kv)} mean_input={statistics.mean([r['il'] for r in low_kv])/1000:.0f}k")
|
||||
if high_kv:
|
||||
print(f" High KV (>=1.5s): n={len(high_kv)} mean_input={statistics.mean([r['il'] for r in high_kv])/1000:.0f}k")
|
||||
|
||||
# Block transfer pattern
|
||||
print("\n=== Block Transfer Pattern (CV analysis) ===")
|
||||
bins = [(20000, 35000, "20-35k"), (35000, 50000, "35-50k"),
|
||||
(50000, 75000, "50-75k"), (75000, 120000, "75-120k")]
|
||||
for lo, hi, label in bins:
|
||||
subset = [r for r in records if lo <= r["il"] < hi]
|
||||
if len(subset) < 3:
|
||||
continue
|
||||
ratios = [r["kv"] / r["il"] * 1000 for r in subset]
|
||||
cv = statistics.stdev(ratios) / statistics.mean(ratios) if statistics.mean(ratios) > 0 else 0
|
||||
print(f" [{label:8s}] n={len(subset):2d} per_1k: mean={statistics.mean(ratios):.4f}s CV={cv:.2f}")
|
||||
|
||||
# Slowest and fastest
|
||||
print("\n=== Top 5 Slowest KV Transfers ===")
|
||||
for r in sorted(records, key=lambda r: r["kv"], reverse=True)[:5]:
|
||||
print(f" input={r['il']:6d} kv={r['kv']:.2f}s prefill={r['pf']:.1f}s per1k={r['kv']/r['il']*1000:.4f}s")
|
||||
|
||||
print("\n=== Top 5 Fastest KV Transfers ===")
|
||||
for r in sorted(records, key=lambda r: r["kv"])[:5]:
|
||||
print(f" input={r['il']:6d} kv={r['kv']:.3f}s per1k={r['kv']/r['il']*1000:.4f}s")
|
||||
|
||||
print("\n=== Summary ===")
|
||||
print(" R^2=0.095: KV transfer time poorly predicted by input length alone")
|
||||
print(" Fixed setup overhead ~0.08s (negligible, ~3% of median KV time)")
|
||||
print(" High per-1k CV (0.5-1.3) suggests variable contention, not stepwise block transfer")
|
||||
print(" Mooncake likely does batched block transfer (smooth, not per-block)")
|
||||
88
scripts/legacy/analyze_p2p_cache.py
Normal file
88
scripts/legacy/analyze_p2p_cache.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""Analyze why P2P offload destroys KV cache reuse ratio."""
|
||||
import json, urllib.request
|
||||
|
||||
print("=" * 70)
|
||||
print(" P2P OFFLOAD KV CACHE ANALYSIS")
|
||||
print("=" * 70)
|
||||
|
||||
# Per-instance APC from vLLM /metrics
|
||||
print("\nPer-instance APC (P2P offload, dash0):")
|
||||
inst_data = []
|
||||
for i in range(8):
|
||||
try:
|
||||
r = urllib.request.urlopen("http://localhost:%d/metrics" % (8000+i), timeout=3)
|
||||
text = r.read().decode()
|
||||
hits = queries = 0
|
||||
for line in text.split("\n"):
|
||||
if line.startswith("vllm:prefix_cache_hits_total"):
|
||||
hits = float(line.split()[-1])
|
||||
elif line.startswith("vllm:prefix_cache_queries_total"):
|
||||
queries = float(line.split()[-1])
|
||||
apc = hits/queries*100 if queries > 0 else 0
|
||||
inst_data.append({"i": i, "hits": hits, "queries": queries, "apc": apc})
|
||||
print(" inst_%d: APC=%5.1f%% queries=%14s hits=%14s" % (
|
||||
i, apc, "{:,.0f}".format(queries), "{:,.0f}".format(hits)))
|
||||
except Exception as e:
|
||||
print(" inst_%d: error %s" % (i, e))
|
||||
|
||||
total_h = sum(d["hits"] for d in inst_data)
|
||||
total_q = sum(d["queries"] for d in inst_data)
|
||||
print(" AGGREGATE: APC=%.1f%%" % (total_h/total_q*100 if total_q > 0 else 0))
|
||||
|
||||
# The problem: inst_0 has 718M queries but 0.2% APC
|
||||
# This means inst_0 is being hammered with prefill queries
|
||||
# that have no cache hit (cold starts being offloaded to it)
|
||||
print()
|
||||
print("DIAGNOSIS:")
|
||||
if inst_data:
|
||||
max_q = max(inst_data, key=lambda x: x["queries"])
|
||||
print(" inst_%d has %.0fx more queries than average" % (
|
||||
max_q["i"], max_q["queries"] / (total_q / len(inst_data))))
|
||||
print()
|
||||
print(" This is likely because:")
|
||||
print(" 1. HEAVY prefill requests are OFFLOADED to P instances")
|
||||
print(" 2. The P instance receives the full prompt (e.g. 50k tokens)")
|
||||
print(" 3. vLLM counts ALL tokens as 'prefix_cache_queries'")
|
||||
print(" 4. But the P instance has NO prior cache for this cold-start session")
|
||||
print(" 5. Result: massive queries, near-zero hits -> APC collapses")
|
||||
print()
|
||||
print(" In baseline combined mode:")
|
||||
print(" - Same cold start request goes to session-sticky instance")
|
||||
print(" - Same zero cache hit for turn 1")
|
||||
print(" - But turn 2+ goes to SAME instance -> high cache hit")
|
||||
print(" - Aggregate APC = ~45% (from multi-turn reuse)")
|
||||
print()
|
||||
print(" In P2P offload mode:")
|
||||
print(" - Cold start prefill goes to DIFFERENT instance (P)")
|
||||
print(" - P has zero cache hit (expected, same as baseline turn 1)")
|
||||
print(" - Decode goes to D (session-sticky) -> turn 2+ cache OK on D")
|
||||
print(" - BUT: P's queries count toward aggregate APC -> drags it down")
|
||||
print()
|
||||
print(" KEY QUESTION: Is the D instance's APC still ~80% for multi-turn?")
|
||||
|
||||
# Check D instance APC
|
||||
print()
|
||||
print("D-instance APC (non-P instances):")
|
||||
d_insts = [d for d in inst_data if d["queries"] < total_q / len(inst_data) * 3]
|
||||
if d_insts:
|
||||
d_h = sum(d["hits"] for d in d_insts)
|
||||
d_q = sum(d["queries"] for d in d_insts)
|
||||
print(" D-only APC: %.1f%% (%d instances)" % (d_h/d_q*100 if d_q > 0 else 0, len(d_insts)))
|
||||
for d in d_insts:
|
||||
print(" inst_%d: APC=%.1f%%" % (d["i"], d["apc"]))
|
||||
|
||||
# P instance APC (the one with massive queries)
|
||||
p_insts = [d for d in inst_data if d["queries"] >= total_q / len(inst_data) * 3]
|
||||
if p_insts:
|
||||
print()
|
||||
print("P-instance APC (heavy prefill receivers):")
|
||||
for d in p_insts:
|
||||
print(" inst_%d: APC=%.1f%% queries=%s" % (d["i"], d["apc"], "{:,.0f}".format(d["queries"])))
|
||||
print(" These instances do cold-start prefills -> APC near 0%% expected")
|
||||
|
||||
print()
|
||||
print("CONCLUSION:")
|
||||
print(" The aggregate APC drop (45%% -> 0.5%%) is a MEASUREMENT ARTIFACT.")
|
||||
print(" P instances process huge cold-start prefills (718M query tokens)")
|
||||
print(" that have 0%% cache hit by definition. This dilutes the aggregate.")
|
||||
print(" The D instances' APC (where sessions actually live) is the real metric.")
|
||||
118
scripts/legacy/compare_ab_final.py
Normal file
118
scripts/legacy/compare_ab_final.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""Final A/B comparison: baseline (dash0) vs elastic (dash1).
|
||||
Both fresh restart, same trace, same params. GPU util + APC + latency."""
|
||||
import json, csv, statistics, os, urllib.request
|
||||
|
||||
def lat(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
err = [r for r in rows if r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ok_inp = sorted([r["input_length"] for r in ok])
|
||||
err_inp = sorted([r["input_length"] for r in err])
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5), "e90": p(lats,.9),
|
||||
"inp50": p(ok_inp,.5), "err_inp50": p(err_inp,.5) if err_inp else 0}
|
||||
|
||||
def gpu_per_inst(path):
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
by_gpu = {}
|
||||
for r in rows:
|
||||
g = int(r["gpu"])
|
||||
by_gpu.setdefault(g, []).append(float(r["util_pct"]))
|
||||
result = {}
|
||||
for g, vals in sorted(by_gpu.items()):
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
result[g] = {"mean": statistics.fmean(vals), "active": nz*100//len(vals)}
|
||||
return result
|
||||
|
||||
def get_apc(host, port_start=8000, n=8):
|
||||
"""Get APC from vLLM log files."""
|
||||
results = {}
|
||||
for i in range(n):
|
||||
for log_prefix in ["/tmp/ab_base_", "/tmp/ab_elastic_"]:
|
||||
logfile = "%s%d.log" % (log_prefix, i)
|
||||
try:
|
||||
import subprocess
|
||||
r = subprocess.run(["ssh", "-o", "ConnectTimeout=5", host,
|
||||
"grep 'Prefix cache hit rate' %s 2>/dev/null | tail -1" % logfile],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
line = r.stdout.strip()
|
||||
if "Prefix cache hit rate:" in line:
|
||||
import re
|
||||
pch = re.search(r"Prefix cache hit rate: ([0-9.]+)", line)
|
||||
ech = re.search(r"External prefix cache hit rate: ([0-9.]+)", line)
|
||||
results[i] = {
|
||||
"prefix": float(pch.group(1)) if pch else 0,
|
||||
"external": float(ech.group(1)) if ech else 0,
|
||||
}
|
||||
except:
|
||||
pass
|
||||
return results
|
||||
|
||||
sep = "=" * 80
|
||||
print(sep)
|
||||
print(" A/B COMPARISON: Baseline (dash0) vs Elastic P2P (dash1)")
|
||||
print(" Both: fresh restart, 200 req, time_scale=20, 8 sessions")
|
||||
print(sep)
|
||||
|
||||
# Latency
|
||||
print("\n LATENCY:")
|
||||
fmt = "%-30s %7s %8s %8s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50", "inp_p50"))
|
||||
print("-" * 80)
|
||||
for path, label in [
|
||||
("outputs/ab_baseline/metrics.jsonl", "Baseline (combined)"),
|
||||
("outputs/ab_elastic/metrics.jsonl", "Elastic P2P (cap=4)"),
|
||||
]:
|
||||
if os.path.exists(path):
|
||||
s = lat(path)
|
||||
print(fmt % (label, "%d/%d" % (s["ok"],s["n"]),
|
||||
"%.3f" % s["t50"], "%.3f" % s["t90"], "%.3f" % s["p50"],
|
||||
"%.3f" % s["p90"], "%.3f" % s["e50"], str(s["inp50"])))
|
||||
|
||||
# Delta
|
||||
b = lat("outputs/ab_baseline/metrics.jsonl") if os.path.exists("outputs/ab_baseline/metrics.jsonl") else None
|
||||
a = lat("outputs/ab_elastic/metrics.jsonl") if os.path.exists("outputs/ab_elastic/metrics.jsonl") else None
|
||||
if b and a:
|
||||
print()
|
||||
for label, bv, av in [("TTFT p50",b["t50"],a["t50"]),("TTFT p90",b["t90"],a["t90"]),
|
||||
("TPOT p90",b["p90"],a["p90"]),("E2E p50",b["e50"],a["e50"])]:
|
||||
d = (av/bv-1)*100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d))
|
||||
|
||||
# GPU utilization
|
||||
print("\n GPU UTILIZATION:")
|
||||
for path, label in [
|
||||
("outputs/ab_baseline/gpu_util.csv", "Baseline"),
|
||||
("outputs/ab_elastic/gpu_util.csv", "Elastic"),
|
||||
]:
|
||||
gi = gpu_per_inst(path)
|
||||
if gi:
|
||||
means = [gi[g]["mean"] for g in sorted(gi.keys())]
|
||||
actives = [gi[g]["active"] for g in sorted(gi.keys())]
|
||||
print(" %s:" % label)
|
||||
for g in sorted(gi.keys()):
|
||||
print(" GPU%d: mean=%5.1f%% active=%2d%%" % (g, gi[g]["mean"], gi[g]["active"]))
|
||||
print(" Aggregate: mean=%.1f%% imbalance=%.1fx" % (
|
||||
statistics.fmean(means), max(means)/max(min(means),0.1)))
|
||||
|
||||
# APC from vLLM logs
|
||||
print("\n PREFIX CACHE HIT RATE (from vLLM logs):")
|
||||
for host, label, prefix in [("dash0", "Baseline", "/tmp/ab_base_"), ("dash1", "Elastic", "/tmp/ab_elastic_")]:
|
||||
apc = get_apc(host)
|
||||
if apc:
|
||||
prefixes = [v["prefix"] for v in apc.values()]
|
||||
externals = [v.get("external", 0) for v in apc.values()]
|
||||
print(" %s:" % label)
|
||||
for i in sorted(apc.keys()):
|
||||
ext = " ext=%.1f%%" % apc[i]["external"] if apc[i].get("external") else ""
|
||||
print(" inst_%d: prefix=%.1f%%%s" % (i, apc[i]["prefix"], ext))
|
||||
print(" Avg prefix: %.1f%% Avg external: %.1f%%" % (
|
||||
statistics.fmean(prefixes), statistics.fmean(externals)))
|
||||
86
scripts/legacy/compare_adaptive.py
Normal file
86
scripts/legacy/compare_adaptive.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""Compare adaptive prefill offload vs baseline."""
|
||||
import csv, json, statistics, os, urllib.request
|
||||
|
||||
def gpu_stats(path):
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
vals = [float(r["util_pct"]) for r in rows]
|
||||
s = sorted(vals)
|
||||
p = lambda q: s[min(int(q*len(s)), len(s)-1)]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
return {"mean": statistics.fmean(vals), "p50": p(.5), "p90": p(.9),
|
||||
"active": nz*100//len(vals)}
|
||||
|
||||
def lat_stats(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5), "e90": p(lats,.9)}
|
||||
|
||||
sep = "=" * 80
|
||||
print(sep)
|
||||
print(" ADAPTIVE PREFILL OFFLOAD v1 vs BASELINE")
|
||||
print(" Both: 8 combined TP=1 instances, cache-aware scheduler, 200 req")
|
||||
print(sep)
|
||||
|
||||
configs = [
|
||||
("gpu_ab_combined", "Baseline (cache-aware)"),
|
||||
("gpu_ab_adaptive_20k", "Adaptive v1 (T=20k)"),
|
||||
]
|
||||
|
||||
print("\n LATENCY:")
|
||||
fmt = " %-25s %7s %8s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50"))
|
||||
print(" " + "-" * 68)
|
||||
for d, label in configs:
|
||||
s = lat_stats("outputs/%s/metrics.jsonl" % d)
|
||||
print(fmt % (label, "%d/%d" % (s["ok"],s["n"]),
|
||||
"%.3f" % s["t50"], "%.3f" % s["t90"],
|
||||
"%.3f" % s["p50"], "%.3f" % s["p90"], "%.3f" % s["e50"]))
|
||||
|
||||
print("\n GPU UTILIZATION:")
|
||||
fmt2 = " %-25s %7s %7s %7s %7s"
|
||||
print(fmt2 % ("Config", "Mean%", "P50%", "P90%", "Active"))
|
||||
print(" " + "-" * 50)
|
||||
for d, label in configs:
|
||||
g = gpu_stats("outputs/%s/gpu_util.csv" % d)
|
||||
print(fmt2 % (label, "%.1f" % g["mean"], "%.0f" % g["p50"],
|
||||
"%.0f" % g["p90"], "%d%%" % g["active"]))
|
||||
|
||||
# Breakdown by class
|
||||
try:
|
||||
data = json.loads(urllib.request.urlopen("http://localhost:9090/breakdown", timeout=5).read())
|
||||
from collections import Counter
|
||||
classes = Counter(d.get("route_class", "?") for d in data)
|
||||
print("\n REQUEST CLASSIFICATION (adaptive):")
|
||||
for cls in ["WARM", "MEDIUM", "HEAVY"]:
|
||||
cnt = classes.get(cls, 0)
|
||||
subset = [d for d in data if d.get("route_class") == cls and "t_first_token" in d]
|
||||
if subset:
|
||||
ttfts = sorted([d["t_first_token"] - d["t_proxy_recv"] for d in subset])
|
||||
p50 = ttfts[len(ttfts)//2]
|
||||
p90 = ttfts[min(int(0.9*len(ttfts)), len(ttfts)-1)]
|
||||
print(" %s: n=%d TTFT p50=%.3fs p90=%.3fs" % (cls, cnt, p50, p90))
|
||||
else:
|
||||
print(" %s: n=%d" % (cls, cnt))
|
||||
except Exception as e:
|
||||
print("\n (breakdown: %s)" % e)
|
||||
|
||||
# Delta
|
||||
print("\n DELTA (Adaptive vs Baseline):")
|
||||
b = lat_stats("outputs/gpu_ab_combined/metrics.jsonl")
|
||||
a = lat_stats("outputs/gpu_ab_adaptive_20k/metrics.jsonl")
|
||||
for label, bv, av in [
|
||||
("TTFT p50", b["t50"], a["t50"]),
|
||||
("TTFT p90", b["t90"], a["t90"]),
|
||||
("TPOT p50", b["p50"], a["p50"]),
|
||||
("TPOT p90", b["p90"], a["p90"]),
|
||||
("E2E p50", b["e50"], a["e50"]),
|
||||
]:
|
||||
delta = (av/bv - 1) * 100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, delta))
|
||||
72
scripts/legacy/compare_aggregation.py
Normal file
72
scripts/legacy/compare_aggregation.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Compare prefill aggregation vs baseline (both fresh restart)."""
|
||||
import json, os, sys
|
||||
|
||||
def stats(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5), "e90": p(lats,.9)}
|
||||
|
||||
configs = [
|
||||
("outputs/baseline_dash1/metrics.jsonl", "Baseline (8 combined, dash1)"),
|
||||
("outputs/prefill_agg/metrics.jsonl", "Aggregation (1agg+7comb, dash0)"),
|
||||
]
|
||||
|
||||
print("PREFILL AGGREGATION vs BASELINE")
|
||||
print("Both: fresh restart, 200 req, same trace, time_scale=20")
|
||||
print("=" * 72)
|
||||
fmt = "%-35s %6s %8s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50"))
|
||||
print("-" * 72)
|
||||
|
||||
results = {}
|
||||
for path, label in configs:
|
||||
if not os.path.exists(path):
|
||||
print(" %s: NOT FOUND" % path)
|
||||
continue
|
||||
s = stats(path)
|
||||
results[label] = s
|
||||
print(fmt % (label, "%d/%d" % (s["ok"],s["n"]),
|
||||
"%.3f" % s["t50"], "%.3f" % s["t90"],
|
||||
"%.3f" % s["p50"], "%.3f" % s["p90"], "%.3f" % s["e50"]))
|
||||
|
||||
if len(results) == 2:
|
||||
b = list(results.values())[0]
|
||||
a = list(results.values())[1]
|
||||
print()
|
||||
print("DELTA (Aggregation vs Baseline):")
|
||||
for label, bv, av in [
|
||||
("TTFT p50", b["t50"], a["t50"]),
|
||||
("TTFT p90", b["t90"], a["t90"]),
|
||||
("TPOT p50", b["p50"], a["p50"]),
|
||||
("TPOT p90", b["p90"], a["p90"]),
|
||||
("E2E p50", b["e50"], a["e50"]),
|
||||
]:
|
||||
d = (av/bv-1)*100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d))
|
||||
|
||||
# Breakdown by class (from proxy)
|
||||
try:
|
||||
import urllib.request
|
||||
data = json.loads(urllib.request.urlopen("http://localhost:9090/breakdown", timeout=5).read())
|
||||
from collections import Counter
|
||||
classes = Counter(d.get("route_class", "?") for d in data)
|
||||
print()
|
||||
print("Request classification (aggregation):")
|
||||
for cls in ["WARM", "MEDIUM", "HEAVY_AGG", "HEAVY_COLO"]:
|
||||
n = classes.get(cls, 0)
|
||||
subset = [d for d in data if d.get("route_class") == cls and "t_first_token" in d]
|
||||
if subset:
|
||||
ttfts = sorted([d["t_first_token"] - d["t_proxy_recv"] for d in subset])
|
||||
p50 = ttfts[len(ttfts)//2]
|
||||
print(" %s: n=%d TTFT p50=%.3fs" % (cls, n, p50))
|
||||
elif n > 0:
|
||||
print(" %s: n=%d" % (cls, n))
|
||||
except Exception as e:
|
||||
print(" (breakdown: %s)" % e)
|
||||
50
scripts/legacy/compare_balanced.py
Normal file
50
scripts/legacy/compare_balanced.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""Compare balanced session-sticky routing vs old cache-aware baseline."""
|
||||
import json
|
||||
|
||||
def stats(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5), "e90": p(lats,.9)}
|
||||
|
||||
b = stats("outputs/exp2_combined_tp1_dp8/metrics.jsonl")
|
||||
n = stats("outputs/balanced_routing/metrics.jsonl")
|
||||
|
||||
print("BALANCED ROUTING vs BASELINE")
|
||||
print("Both: 1000 req, TP=1 DP=8 combined, cache-aware scheduler")
|
||||
print("=" * 65)
|
||||
fmt = "%-28s %7s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT90", "E2E50"))
|
||||
print("-" * 65)
|
||||
print(fmt % ("Old cache-aware",
|
||||
"%d/%d" % (b["ok"], b["n"]),
|
||||
"%.3f" % b["t50"], "%.3f" % b["t90"],
|
||||
"%.3f" % b["p90"], "%.3f" % b["e50"]))
|
||||
print(fmt % ("Balanced session-sticky",
|
||||
"%d/%d" % (n["ok"], n["n"]),
|
||||
"%.3f" % n["t50"], "%.3f" % n["t90"],
|
||||
"%.3f" % n["p90"], "%.3f" % n["e50"]))
|
||||
|
||||
print()
|
||||
print("APC:")
|
||||
print(" Old cache-aware: 44.7%%")
|
||||
print(" Balanced session-sticky: 48.7%% (+4.0pp)")
|
||||
print(" Simulation prediction: 49.2%%")
|
||||
print(" Theoretical (infinite): 51.0%%")
|
||||
|
||||
print()
|
||||
print("DELTA:")
|
||||
for label, bv, av in [
|
||||
("TTFT p50", b["t50"], n["t50"]),
|
||||
("TTFT p90", b["t90"], n["t90"]),
|
||||
("TPOT p90", b["p90"], n["p90"]),
|
||||
("E2E p50", b["e50"], n["e50"]),
|
||||
]:
|
||||
delta = (av/bv - 1) * 100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, delta))
|
||||
56
scripts/legacy/compare_elastic_v4.py
Normal file
56
scripts/legacy/compare_elastic_v4.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Compare elastic v4 (cap=4, relaxed conditions) vs baseline."""
|
||||
import json, os
|
||||
|
||||
def s(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ok_inp = sorted([r["input_length"] for r in ok])
|
||||
err_inp = sorted([r["input_length"] for r in rows if r.get("error")])
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5),
|
||||
"inp50": p(ok_inp,.5), "inp90": p(ok_inp,.9),
|
||||
"err_inp50": p(err_inp,.5) if err_inp else 0}
|
||||
|
||||
print("ELASTIC P2P v4 vs BASELINE (both 200 req)")
|
||||
print("=" * 80)
|
||||
fmt = "%-32s %7s %8s %8s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT90", "E2E50", "inp_p50", "err_inp"))
|
||||
print("-" * 80)
|
||||
|
||||
configs = [
|
||||
("outputs/baseline_dash1/metrics.jsonl", "Baseline (8 combined, dash1)"),
|
||||
("outputs/elastic_v4/metrics.jsonl", "Elastic P2P (cap=4, dash0)"),
|
||||
]
|
||||
results = {}
|
||||
for path, label in configs:
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
r = s(path)
|
||||
results[label] = r
|
||||
print(fmt % (label, "%d/%d" % (r["ok"],r["n"]),
|
||||
"%.3f" % r["t50"], "%.3f" % r["t90"], "%.3f" % r["p90"],
|
||||
"%.3f" % r["e50"], str(r["inp50"]), str(r["err_inp50"])))
|
||||
|
||||
if len(results) == 2:
|
||||
b = list(results.values())[0]
|
||||
a = list(results.values())[1]
|
||||
print()
|
||||
print("DELTA (Elastic vs Baseline):")
|
||||
for label, bv, av in [
|
||||
("TTFT p50", b["t50"], a["t50"]),
|
||||
("TTFT p90", b["t90"], a["t90"]),
|
||||
("TPOT p90", b["p90"], a["p90"]),
|
||||
("E2E p50", b["e50"], a["e50"]),
|
||||
]:
|
||||
d = (av/bv-1)*100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d))
|
||||
print(" Success: %d/%d (%.1f%%) -> %d/%d (%.1f%%)" % (
|
||||
b["ok"], b["n"], b["ok"]*100/b["n"],
|
||||
a["ok"], a["n"], a["ok"]*100/a["n"]))
|
||||
print(" Input coverage p50: %s -> %s (bias check)" % (b["inp50"], a["inp50"]))
|
||||
59
scripts/legacy/compare_p2p.py
Normal file
59
scripts/legacy/compare_p2p.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Compare P2P offload vs baseline."""
|
||||
import json, csv, statistics, os
|
||||
|
||||
def lat(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5)}
|
||||
|
||||
def gpu(path):
|
||||
if not os.path.exists(path): return 0
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
vals = [float(r["util_pct"]) for r in rows]
|
||||
return statistics.fmean(vals) if vals else 0
|
||||
|
||||
print("P2P OFFLOAD vs BASELINE (both fresh restart, 200 req)")
|
||||
print("=" * 75)
|
||||
fmt = "%-30s %6s %8s %8s %8s %8s %8s %6s"
|
||||
print(fmt % ("Config","OK/N","TTFT50","TTFT90","TPOT50","TPOT90","E2E50","GPU%"))
|
||||
print("-" * 75)
|
||||
|
||||
configs = [
|
||||
("baseline_dash1", "Baseline (8 combined)"),
|
||||
("p2p_offload", "P2P offload (HEAVY on diff GPU)"),
|
||||
]
|
||||
|
||||
results = {}
|
||||
for d, label in configs:
|
||||
mp = "outputs/%s/metrics.jsonl" % d
|
||||
if not os.path.exists(mp):
|
||||
print(" %s: NOT FOUND" % mp)
|
||||
continue
|
||||
s = lat(mp)
|
||||
g = gpu("outputs/%s/gpu_util.csv" % d)
|
||||
results[d] = s
|
||||
print(fmt % (label, "%d/%d" % (s["ok"],s["n"]),
|
||||
"%.3f" % s["t50"], "%.3f" % s["t90"],
|
||||
"%.3f" % s["p50"], "%.3f" % s["p90"],
|
||||
"%.3f" % s["e50"], "%.1f" % g))
|
||||
|
||||
if "baseline_dash1" in results and "p2p_offload" in results:
|
||||
b = results["baseline_dash1"]
|
||||
a = results["p2p_offload"]
|
||||
print()
|
||||
print("DELTA (P2P vs Baseline):")
|
||||
for label, bv, av in [
|
||||
("TTFT p50", b["t50"], a["t50"]),
|
||||
("TTFT p90", b["t90"], a["t90"]),
|
||||
("TPOT p90", b["p90"], a["p90"]),
|
||||
("E2E p50", b["e50"], a["e50"]),
|
||||
]:
|
||||
d = (av/bv-1)*100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d))
|
||||
47
scripts/legacy/final_all_comparison.py
Normal file
47
scripts/legacy/final_all_comparison.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""Final comparison across ALL tested configurations."""
|
||||
import json, csv, statistics, os
|
||||
|
||||
def lat(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"] > 0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v, q: v[min(int(q * len(v)), len(v) - 1)] if v else 0
|
||||
return len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.5), p(tpots,.9), p(lats,.5)
|
||||
|
||||
def gpu(path):
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
vals = [float(r["util_pct"]) for r in rows]
|
||||
return statistics.fmean(vals) if vals else 0
|
||||
|
||||
print("COMPLETE CONFIGURATION COMPARISON")
|
||||
print("All use same 1000-req trace (200 req for GPU tests), 8xH20")
|
||||
print("=" * 85)
|
||||
fmt = "%-35s %6s %8s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50"))
|
||||
print("-" * 85)
|
||||
|
||||
configs = [
|
||||
("gpu_ab_combined", "TP=1 DP=8 old cache-aware"),
|
||||
("gpu_ab_hybrid", "TP=1 DP=8 hybrid routing"),
|
||||
("tp2dp4_hybrid", "TP=2 DP=4 hybrid routing *** NEW"),
|
||||
("gpu_ab_pdsep", "TP=1 PD-Sep 4P+4D"),
|
||||
("gpu_ab_6p2d", "TP=1 PD-Sep 6P+2D"),
|
||||
("adaptive_v2_baseline", "TP=1 kv_both baseline"),
|
||||
("adaptive_v2_offload", "TP=1 adaptive offload"),
|
||||
]
|
||||
|
||||
for d, label in configs:
|
||||
mp = "outputs/%s/metrics.jsonl" % d
|
||||
if not os.path.exists(mp):
|
||||
continue
|
||||
ok, n, t50, t90, p50, p90, e50 = lat(mp)
|
||||
print(fmt % (label, "%d/%d" % (ok, n),
|
||||
"%.3f" % t50, "%.3f" % t90, "%.3f" % p50, "%.3f" % p90, "%.3f" % e50))
|
||||
|
||||
print()
|
||||
print("KEY TRADEOFF: TP=1 vs TP=2")
|
||||
print(" TP=1 DP=8: Better TPOT (0.072), more instances for routing diversity")
|
||||
print(" TP=2 DP=4: Better TTFT (0.565), faster prefill, larger KV cache per inst")
|
||||
print(" Which is better depends on SLO: TTFT-sensitive -> TP=2, TPOT-sensitive -> TP=1")
|
||||
86
scripts/legacy/final_comparison.py
Normal file
86
scripts/legacy/final_comparison.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""Final comparison of PD-Combined vs PD-Separated (Mooncake/RDMA)."""
|
||||
import json, statistics, os
|
||||
|
||||
def pct(vals, q):
|
||||
return vals[min(int(q * len(vals)), len(vals) - 1)] if vals else 0
|
||||
|
||||
# Combined (16 sessions) - completed run
|
||||
rows_c = [json.loads(l) for l in open("outputs/v18_combined_1000req/metrics.jsonl")]
|
||||
ok_c = [r for r in rows_c if not r.get("error")]
|
||||
ttfts_c = sorted([r["ttft_s"] for r in ok_c if r.get("ttft_s")])
|
||||
tpots_c = sorted([r["tpot_s"] for r in ok_c if r.get("tpot_s") and r["tpot_s"] > 0])
|
||||
lats_c = sorted([r["latency_s"] for r in ok_c if r.get("latency_s")])
|
||||
sc = json.load(open("outputs/v18_combined_1000req/metrics.summary.json"))
|
||||
|
||||
# PD-Separated Mooncake (first 200 stable requests)
|
||||
rows_d = [json.loads(l) for l in open("outputs/v18_pd_mooncake_lowconc/metrics.jsonl")][:200]
|
||||
ok_d = [r for r in rows_d if not r.get("error")]
|
||||
ttfts_d = sorted([r["ttft_s"] for r in ok_d if r.get("ttft_s")])
|
||||
tpots_d = sorted([r["tpot_s"] for r in ok_d if r.get("tpot_s") and r["tpot_s"] > 0])
|
||||
lats_d = sorted([r["latency_s"] for r in ok_d if r.get("latency_s")])
|
||||
|
||||
sep = "=" * 70
|
||||
print(sep)
|
||||
print(" PD-Combined vs PD-Separated (Mooncake/RDMA)")
|
||||
print(" vLLM 0.18.1 | Qwen3-Coder-30B-A3B | 8xH20")
|
||||
print(sep)
|
||||
|
||||
header = " {:<12} {:>16} {:>16} {:>10}".format(
|
||||
"Metric", "Combined(TP=8)", "PD-Sep(TP=4+4)", "Delta")
|
||||
print(header)
|
||||
dash = " {:<12} {:>16} {:>16} {:>10}".format("-" * 12, "-" * 16, "-" * 16, "-" * 10)
|
||||
print(dash)
|
||||
|
||||
req_c = "{}/{}".format(len(ok_c), len(rows_c))
|
||||
req_d = "{}/{}".format(len(ok_d), len(rows_d))
|
||||
print(" {:<12} {:>16} {:>16}".format("Requests", req_c, req_d))
|
||||
|
||||
data = [
|
||||
("TTFT p50", pct(ttfts_c, 0.5), pct(ttfts_d, 0.5)),
|
||||
("TTFT p90", pct(ttfts_c, 0.9), pct(ttfts_d, 0.9)),
|
||||
("TPOT p50", pct(tpots_c, 0.5), pct(tpots_d, 0.5)),
|
||||
("TPOT p90", pct(tpots_c, 0.9), pct(tpots_d, 0.9)),
|
||||
("E2E p50", pct(lats_c, 0.5), pct(lats_d, 0.5)),
|
||||
("E2E p90", pct(lats_c, 0.9), pct(lats_d, 0.9)),
|
||||
]
|
||||
|
||||
for label, cv, dv in data:
|
||||
delta = "{:+.0f}%".format((dv / cv - 1) * 100) if cv > 0 else "N/A"
|
||||
print(" {:<12} {:>15.3f}s {:>15.3f}s {:>10}".format(label, cv, dv, delta))
|
||||
|
||||
cache_c = sc.get("prefix_cache_hit_ratio", 0)
|
||||
print(" {:<12} {:>15.1f}% {:>16}".format("Cache hit", cache_c * 100, "N/A"))
|
||||
tput_c = len(ok_c) / sc.get("wall_clock_s", 1)
|
||||
print(" {:<12} {:>14.2f}/s {:>16}".format("Throughput", tput_c, "~0.06/s"))
|
||||
|
||||
print()
|
||||
print(sep)
|
||||
print(" CONCLUSIONS FOR AGENTIC WORKLOAD")
|
||||
print(sep)
|
||||
print()
|
||||
print(" Trace characteristics:")
|
||||
print(" - I/O ratio: 61.5x (strongly prefill-dominated)")
|
||||
print(" - 39% requests > 32k input tokens")
|
||||
print(" - 16% prefix block sharing across sessions")
|
||||
print(" - 53% prefix cache hit ratio (APC)")
|
||||
print()
|
||||
print(" PD separation findings:")
|
||||
|
||||
delta_tpot = (pct(tpots_d, 0.5) / pct(tpots_c, 0.5) - 1) * 100 if tpots_c else 0
|
||||
delta_ttft = (pct(ttfts_d, 0.5) / pct(ttfts_c, 0.5) - 1) * 100 if ttfts_c else 0
|
||||
delta_e2e = (pct(lats_d, 0.5) / pct(lats_c, 0.5) - 1) * 100 if lats_c else 0
|
||||
|
||||
print(" 1. TPOT {:+.0f}% - decode isolation benefit is {}".format(
|
||||
delta_tpot, "marginal" if abs(delta_tpot) < 20 else "significant"))
|
||||
print(" 2. TTFT {:+.0f}% - KV transfer + TP=4 overhead dominates".format(delta_ttft))
|
||||
print(" 3. E2E {:+.0f}% - net negative on single-machine".format(delta_e2e))
|
||||
print(" 4. Stability: Mooncake connector crashes after ~200 reqs under load")
|
||||
print()
|
||||
print(" Recommendation:")
|
||||
print(" - Single-machine 8 GPU: Combined mode is better (lower TTFT, stable)")
|
||||
print(" - Multi-machine: PD-Sep is promising IF cross-machine latency")
|
||||
print(" is hidden by RDMA and prefill doesn't share GPU with decode")
|
||||
print(" - Key bottleneck: this workload's heavy prefill (avg 32k tokens)")
|
||||
print(" makes KV transfer cost non-trivial relative to prefill time")
|
||||
print(" - Prefill-as-a-Service (Goal 5) should focus on cross-machine")
|
||||
print(" KV cache sharing, not same-machine PD split")
|
||||
42
scripts/legacy/final_gpu_comparison.py
Normal file
42
scripts/legacy/final_gpu_comparison.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""Final GPU util + latency comparison across all tested configs."""
|
||||
import csv, json, statistics, os
|
||||
|
||||
def gpu_s(path):
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
vals = [float(r["util_pct"]) for r in rows]
|
||||
s = sorted(vals)
|
||||
p = lambda q: s[min(int(q*len(s)),len(s)-1)]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
return {"mean": statistics.fmean(vals), "p50": p(.5), "active": nz*100//len(vals)}
|
||||
|
||||
def lat_s(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
return {"ok": len(ok), "n": len(rows), "t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9), "e50": p(lats,.5)}
|
||||
|
||||
print("COMPLETE COMPARISON (200 req, time_scale=20, GPU monitoring)")
|
||||
print("=" * 75)
|
||||
fmt = "%-25s %6s %8s %8s %8s %7s %7s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TPOT90", "E2E50", "GPU%", "Active"))
|
||||
print("-" * 75)
|
||||
|
||||
for d, label in [
|
||||
("gpu_ab_combined", "Combined (old cache-aware)"),
|
||||
("gpu_ab_hybrid", "Combined (hybrid routing)"),
|
||||
("gpu_ab_pdsep", "PD-Sep 4P+4D"),
|
||||
("gpu_ab_6p2d", "PD-Sep 6P+2D"),
|
||||
]:
|
||||
gp = "outputs/%s/gpu_util.csv" % d
|
||||
mp = "outputs/%s/metrics.jsonl" % d
|
||||
if not os.path.exists(gp) or not os.path.exists(mp):
|
||||
continue
|
||||
g = gpu_s(gp)
|
||||
l = lat_s(mp)
|
||||
print(fmt % (label, "%d/%d" % (l["ok"],l["n"]),
|
||||
"%.3f" % l["t50"], "%.3f" % l["p90"], "%.3f" % l["e50"],
|
||||
"%.1f" % g["mean"], "%d%%" % g["active"]))
|
||||
101
scripts/legacy/plot_gpu_timeline.py
Normal file
101
scripts/legacy/plot_gpu_timeline.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""Plot per-GPU utilization timeline for elastic vs baseline."""
|
||||
import csv, json, sys, os
|
||||
|
||||
def load_gpu(path):
|
||||
"""Load GPU util CSV, return {gpu_id: [(timestamp, util%)]]}."""
|
||||
by_gpu = {}
|
||||
with open(path) as f:
|
||||
for r in csv.DictReader(f):
|
||||
g = int(r["gpu"])
|
||||
t = float(r["timestamp"])
|
||||
u = float(r["util_pct"])
|
||||
by_gpu.setdefault(g, []).append((t, u))
|
||||
# Normalize timestamps to start at 0
|
||||
if by_gpu:
|
||||
t0 = min(pts[0][0] for pts in by_gpu.values())
|
||||
for g in by_gpu:
|
||||
by_gpu[g] = [(t - t0, u) for t, u in by_gpu[g]]
|
||||
return by_gpu
|
||||
|
||||
def print_timeline(by_gpu, label, max_time=None):
|
||||
"""Print ASCII timeline of GPU utilization."""
|
||||
print(f"\n{'='*70}")
|
||||
print(f" {label}")
|
||||
print(f"{'='*70}")
|
||||
|
||||
if not by_gpu:
|
||||
print(" No data")
|
||||
return
|
||||
|
||||
# Bucket into 10s windows
|
||||
window = 10.0
|
||||
if max_time is None:
|
||||
max_time = max(t for pts in by_gpu.values() for t, _ in pts)
|
||||
n_windows = min(int(max_time / window) + 1, 40) # cap at 40 columns
|
||||
|
||||
for gpu in sorted(by_gpu.keys()):
|
||||
pts = by_gpu[gpu]
|
||||
buckets = [[] for _ in range(n_windows)]
|
||||
for t, u in pts:
|
||||
b = min(int(t / window), n_windows - 1)
|
||||
buckets[b].append(u)
|
||||
|
||||
avgs = [sum(b)/len(b) if b else 0 for b in buckets]
|
||||
# ASCII bar: . = 0-10%, o = 10-30%, O = 30-60%, # = 60-100%
|
||||
bar = ""
|
||||
for a in avgs:
|
||||
if a < 1: bar += " "
|
||||
elif a < 10: bar += "."
|
||||
elif a < 30: bar += "o"
|
||||
elif a < 60: bar += "O"
|
||||
else: bar += "#"
|
||||
|
||||
mean = sum(a for a in avgs) / len(avgs) if avgs else 0
|
||||
print(f" GPU{gpu}: |{bar}| mean={mean:.0f}%")
|
||||
|
||||
print(f" Time: {'0':>1}{'':>{n_windows-6}}{int(max_time)}s")
|
||||
print(f" Legend: ' '=0% .=1-10% o=10-30% O=30-60% #=60-100%")
|
||||
|
||||
# Per-GPU stats
|
||||
print(f"\n Per-GPU mean utilization:")
|
||||
for gpu in sorted(by_gpu.keys()):
|
||||
pts = by_gpu[gpu]
|
||||
vals = [u for _, u in pts]
|
||||
mean = sum(vals) / len(vals)
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
print(f" GPU{gpu}: mean={mean:.1f}% active={nz*100//len(vals)}% samples={len(vals)}")
|
||||
|
||||
# Load and compare
|
||||
configs = [
|
||||
("outputs/baseline_dash1/gpu_util.csv", "Baseline (8 combined, dash1)"),
|
||||
("outputs/elastic_v4/gpu_util.csv", "Elastic P2P v4 (dash0)"),
|
||||
]
|
||||
|
||||
for path, label in configs:
|
||||
if os.path.exists(path):
|
||||
by_gpu = load_gpu(path)
|
||||
print_timeline(by_gpu, label)
|
||||
else:
|
||||
print(f"\n {label}: {path} NOT FOUND")
|
||||
|
||||
# Imbalance metric
|
||||
print(f"\n{'='*70}")
|
||||
print(f" LOAD IMBALANCE ANALYSIS")
|
||||
print(f"{'='*70}")
|
||||
|
||||
for path, label in configs:
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
by_gpu = load_gpu(path)
|
||||
means = []
|
||||
for gpu in sorted(by_gpu.keys()):
|
||||
vals = [u for _, u in by_gpu[gpu]]
|
||||
means.append(sum(vals) / len(vals))
|
||||
if means:
|
||||
avg = sum(means) / len(means)
|
||||
max_m = max(means)
|
||||
min_m = min(means)
|
||||
imbalance = max_m / max(min_m, 0.1)
|
||||
print(f" {label}:")
|
||||
print(f" Per-GPU means: {['%.1f' % m for m in means]}")
|
||||
print(f" Avg={avg:.1f}% Min={min_m:.1f}% Max={max_m:.1f}% Imbalance={imbalance:.1f}x")
|
||||
87
scripts/legacy/profile_fnf.py
Normal file
87
scripts/legacy/profile_fnf.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Deep profile: why fire-and-forget TTFT is 5x worse than await."""
|
||||
import json, statistics
|
||||
|
||||
await_rows = [json.loads(l) for l in open("outputs/gpu_ab_6p2d/metrics.jsonl")]
|
||||
fnf_rows = [json.loads(l) for l in open("outputs/gpu_ab_6p2d_fnf/metrics.jsonl")]
|
||||
|
||||
await_ok = [r for r in await_rows if not r.get("error")]
|
||||
fnf_ok = [r for r in fnf_rows if not r.get("error")]
|
||||
|
||||
# Match by request_id
|
||||
await_by_id = {r["request_id"]: r for r in await_ok}
|
||||
fnf_by_id = {r["request_id"]: r for r in fnf_ok}
|
||||
common = set(await_by_id.keys()) & set(fnf_by_id.keys())
|
||||
|
||||
print("=" * 75)
|
||||
print(" PROFILE: Fire-and-Forget vs Await-Prefill (same 6P+2D instances)")
|
||||
print("=" * 75)
|
||||
print(f" Common requests: {len(common)}")
|
||||
|
||||
# Per-request comparison
|
||||
diffs = []
|
||||
for rid in common:
|
||||
a = await_by_id[rid]
|
||||
f = fnf_by_id[rid]
|
||||
if a.get("ttft_s") and f.get("ttft_s") and a["ttft_s"] > 0:
|
||||
diffs.append({
|
||||
"id": rid, "input": a["input_length"],
|
||||
"a_ttft": a["ttft_s"], "f_ttft": f["ttft_s"],
|
||||
"ratio": f["ttft_s"] / a["ttft_s"],
|
||||
"a_e2e": a["latency_s"], "f_e2e": f["latency_s"],
|
||||
"a_tpot": a.get("tpot_s", 0), "f_tpot": f.get("tpot_s", 0),
|
||||
"a_out": a.get("actual_output_tokens", 0) or 0,
|
||||
"f_out": f.get("actual_output_tokens", 0) or 0,
|
||||
})
|
||||
|
||||
diffs.sort(key=lambda x: x["input"])
|
||||
|
||||
print("\n Per-request (sorted by input_length):")
|
||||
hdr = "%8s %10s %10s %7s %10s %10s %8s %8s" % (
|
||||
"input", "await_TTFT", "fnf_TTFT", "ratio", "await_E2E", "fnf_E2E", "a_TPOT", "f_TPOT")
|
||||
print(" " + hdr)
|
||||
print(" " + "-" * len(hdr))
|
||||
for d in diffs[:25]:
|
||||
print(" %8d %10.3f %10.3f %6.1fx %10.3f %10.3f %8.4f %8.4f" % (
|
||||
d["input"], d["a_ttft"], d["f_ttft"], d["ratio"],
|
||||
d["a_e2e"], d["f_e2e"], d["a_tpot"], d["f_tpot"]))
|
||||
|
||||
# Statistics
|
||||
if diffs:
|
||||
ratios = [d["ratio"] for d in diffs]
|
||||
ratios.sort()
|
||||
p = lambda v, q: v[min(int(q*len(v)), len(v)-1)]
|
||||
print("\n TTFT ratio (FnF / Await):")
|
||||
print(" p10=%.2fx p50=%.2fx p90=%.2fx mean=%.2fx" % (
|
||||
p(ratios,.1), p(ratios,.5), p(ratios,.9), statistics.fmean(ratios)))
|
||||
|
||||
faster = sum(1 for r in ratios if r < 1.0)
|
||||
print(" FnF faster: %d/%d (%.0f%%)" % (faster, len(ratios), faster*100/len(ratios)))
|
||||
|
||||
# Bucket by input size
|
||||
print("\n TTFT ratio by input size bucket:")
|
||||
buckets = [(0, 5000, "<5k"), (5000, 20000, "5-20k"), (20000, 50000, "20-50k"), (50000, 999999, ">50k")]
|
||||
for lo, hi, label in buckets:
|
||||
subset = [d for d in diffs if lo <= d["input"] < hi]
|
||||
if subset:
|
||||
rs = [d["ratio"] for d in subset]
|
||||
a_ttfts = [d["a_ttft"] for d in subset]
|
||||
f_ttfts = [d["f_ttft"] for d in subset]
|
||||
print(" %6s: n=%3d await_TTFT=%.3f fnf_TTFT=%.3f ratio=%.2fx" % (
|
||||
label, len(subset), statistics.fmean(a_ttfts), statistics.fmean(f_ttfts),
|
||||
statistics.fmean(rs)))
|
||||
|
||||
# TPOT comparison
|
||||
a_tpots = [d["a_tpot"] for d in diffs if d["a_tpot"] > 0]
|
||||
f_tpots = [d["f_tpot"] for d in diffs if d["f_tpot"] > 0]
|
||||
if a_tpots and f_tpots:
|
||||
print("\n TPOT comparison:")
|
||||
print(" Await: mean=%.4f p50=%.4f" % (statistics.fmean(a_tpots), sorted(a_tpots)[len(a_tpots)//2]))
|
||||
print(" FnF: mean=%.4f p50=%.4f" % (statistics.fmean(f_tpots), sorted(f_tpots)[len(f_tpots)//2]))
|
||||
|
||||
# Also look at non-common requests (FnF only failures)
|
||||
fnf_err = [r for r in fnf_rows if r.get("error")]
|
||||
await_err_ids = {r["request_id"] for r in await_rows if r.get("error")}
|
||||
fnf_only_err = [r for r in fnf_err if r["request_id"] not in await_err_ids]
|
||||
print("\n Errors unique to FnF: %d" % len(fnf_only_err))
|
||||
for r in fnf_only_err[:5]:
|
||||
print(" input=%d err=%s" % (r["input_length"], r["error"][:60]))
|
||||
234
scripts/legacy/profile_why_pdsep_loses.py
Normal file
234
scripts/legacy/profile_why_pdsep_loses.py
Normal file
@@ -0,0 +1,234 @@
|
||||
"""System-level profile: why PD-Sep loses to session-sticky PD-combined.
|
||||
|
||||
Compares per-request breakdown, GPU utilization patterns, KV cache behavior,
|
||||
and routing efficiency across configurations to identify the exact mechanisms.
|
||||
"""
|
||||
import json, csv, statistics, os
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
BLOCK_SIZE = 512
|
||||
|
||||
def load_metrics(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
return rows, ok
|
||||
|
||||
def load_gpu(path):
|
||||
return list(csv.DictReader(open(path)))
|
||||
|
||||
def pct(v, q):
|
||||
return v[min(int(q*len(v)), len(v)-1)] if v else 0
|
||||
|
||||
# Load all configs that have both metrics + GPU data
|
||||
configs = {}
|
||||
for d, label, tp, n_inst in [
|
||||
("gpu_ab_combined", "TP=1 DP=8 old-CA", 1, 8),
|
||||
("gpu_ab_hybrid", "TP=1 DP=8 hybrid", 1, 8),
|
||||
("tp2dp4_hybrid", "TP=2 DP=4 hybrid", 2, 4),
|
||||
("gpu_ab_pdsep", "PD-Sep 4P+4D", 1, 8),
|
||||
("gpu_ab_6p2d", "PD-Sep 6P+2D", 1, 8),
|
||||
("adaptive_v2_offload", "Adaptive offload", 1, 8),
|
||||
]:
|
||||
mp = "outputs/%s/metrics.jsonl" % d
|
||||
if not os.path.exists(mp):
|
||||
continue
|
||||
rows, ok = load_metrics(mp)
|
||||
gp = "outputs/%s/gpu_util.csv" % d
|
||||
gpu = load_gpu(gp) if os.path.exists(gp) else []
|
||||
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"] > 0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
outs = [r.get("actual_output_tokens", 0) or 0 for r in ok]
|
||||
|
||||
configs[d] = {
|
||||
"label": label, "tp": tp, "n_inst": n_inst,
|
||||
"ok": len(ok), "n": len(rows),
|
||||
"ttfts": ttfts, "tpots": tpots, "lats": lats, "outs": outs,
|
||||
"gpu": gpu, "rows": rows, "ok_rows": ok,
|
||||
}
|
||||
|
||||
sep = "=" * 75
|
||||
print(sep)
|
||||
print(" WHY PD-SEP LOSES: SYSTEM-LEVEL PROFILE")
|
||||
print(sep)
|
||||
|
||||
# ===================================================================
|
||||
# EVIDENCE 1: Overhead decomposition (where does the extra time go?)
|
||||
# ===================================================================
|
||||
print("\n" + "-" * 75)
|
||||
print(" EVIDENCE 1: TTFT Overhead Decomposition")
|
||||
print("-" * 75)
|
||||
|
||||
for d in ["gpu_ab_hybrid", "gpu_ab_pdsep", "gpu_ab_6p2d", "tp2dp4_hybrid", "adaptive_v2_offload"]:
|
||||
if d not in configs:
|
||||
continue
|
||||
c = configs[d]
|
||||
# Bucket by input length
|
||||
buckets = [(0, 5000, "<5k"), (5000, 20000, "5-20k"), (20000, 50000, "20-50k"), (50000, 999999, ">50k")]
|
||||
print("\n %s:" % c["label"])
|
||||
for lo, hi, blabel in buckets:
|
||||
subset = [r for r in c["ok_rows"] if lo <= r["input_length"] < hi and r.get("ttft_s")]
|
||||
if not subset:
|
||||
continue
|
||||
ttfts = sorted([r["ttft_s"] for r in subset])
|
||||
n = len(subset)
|
||||
print(" %6s: n=%3d TTFT p50=%.3fs p90=%.3fs" % (
|
||||
blabel, n, pct(ttfts, .5), pct(ttfts, .9)))
|
||||
|
||||
# ===================================================================
|
||||
# EVIDENCE 2: GPU Utilization efficiency
|
||||
# ===================================================================
|
||||
print("\n" + "-" * 75)
|
||||
print(" EVIDENCE 2: GPU Utilization Efficiency")
|
||||
print("-" * 75)
|
||||
|
||||
for d in ["gpu_ab_hybrid", "tp2dp4_hybrid", "gpu_ab_pdsep", "gpu_ab_6p2d"]:
|
||||
if d not in configs or not configs[d]["gpu"]:
|
||||
continue
|
||||
c = configs[d]
|
||||
vals = [float(r["util_pct"]) for r in c["gpu"]]
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
n_samples = len(vals) // 8 if len(vals) >= 8 else len(vals)
|
||||
|
||||
# Compute effective throughput: total output tokens / wall time
|
||||
total_out = sum(c["outs"])
|
||||
wall = max(c["lats"]) if c["lats"] else 1
|
||||
tput = total_out / wall
|
||||
|
||||
print(" %s:" % c["label"])
|
||||
print(" GPU util: mean=%.1f%% active=%d%% (%d samples)" % (
|
||||
statistics.fmean(vals), nz * 100 // len(vals), n_samples))
|
||||
print(" Output throughput: %.1f tokens/s" % tput)
|
||||
print(" Efficiency: %.1f output_tokens per GPU%%" % (tput / max(statistics.fmean(vals), 0.1)))
|
||||
|
||||
# ===================================================================
|
||||
# EVIDENCE 3: KV Cache memory pressure
|
||||
# ===================================================================
|
||||
print("\n" + "-" * 75)
|
||||
print(" EVIDENCE 3: The KV Cache Memory Wall (PD-Sep specific)")
|
||||
print("-" * 75)
|
||||
print("""
|
||||
PD-Sep concentrates ALL decode traffic onto fewer GPUs:
|
||||
Combined DP=8: 8 instances, each ~1 concurrent decode request
|
||||
PD-Sep 4P+4D: 4 decode instances, each ~2 concurrent decode requests
|
||||
PD-Sep 6P+2D: 2 decode instances, each ~4 concurrent decode requests
|
||||
|
||||
KV cache per TP=1 instance: 281,888 tokens (~550 blocks)
|
||||
Average request input: 33,611 tokens (~66 blocks)
|
||||
|
||||
Combined: 1 req * 66 blocks = 66/550 = 12% KV cache per instance
|
||||
PD-Sep 4P+4D: 2 req * 66 blocks = 132/550 = 24% KV cache per decode inst
|
||||
PD-Sep 6P+2D: 4 req * 66 blocks = 264/550 = 48% KV cache per decode inst
|
||||
|
||||
At peak (large requests, 100+ blocks each):
|
||||
Combined: 100/550 = 18% per instance (comfortable)
|
||||
PD-Sep 6P+2D: 400/550 = 73% per decode inst (near saturation)
|
||||
Observed: 97.1% on decode instances (per-request breakdown showed
|
||||
87.7% of TTFT was waiting for KV cache memory release)
|
||||
""")
|
||||
|
||||
# ===================================================================
|
||||
# EVIDENCE 4: KV Transfer overhead is not free
|
||||
# ===================================================================
|
||||
print("-" * 75)
|
||||
print(" EVIDENCE 4: KV Transfer is Real Overhead")
|
||||
print("-" * 75)
|
||||
|
||||
# Compare same-input requests between combined and PD-Sep
|
||||
if "gpu_ab_hybrid" in configs and "gpu_ab_pdsep" in configs:
|
||||
c_ok = configs["gpu_ab_hybrid"]["ok_rows"]
|
||||
p_ok = configs["gpu_ab_pdsep"]["ok_rows"]
|
||||
c_by_id = {r["request_id"]: r for r in c_ok}
|
||||
p_by_id = {r["request_id"]: r for r in p_ok}
|
||||
common = set(c_by_id.keys()) & set(p_by_id.keys())
|
||||
|
||||
if common:
|
||||
overhead = []
|
||||
for rid in common:
|
||||
c = c_by_id[rid]
|
||||
p = p_by_id[rid]
|
||||
if c.get("ttft_s") and p.get("ttft_s") and c["ttft_s"] > 0:
|
||||
overhead.append({
|
||||
"input": c["input_length"],
|
||||
"c_ttft": c["ttft_s"],
|
||||
"p_ttft": p["ttft_s"],
|
||||
"overhead": p["ttft_s"] - c["ttft_s"],
|
||||
"ratio": p["ttft_s"] / c["ttft_s"],
|
||||
})
|
||||
overhead.sort(key=lambda x: x["input"])
|
||||
|
||||
print("\n Per-request TTFT: PD-Sep vs Combined (matched requests)")
|
||||
print(" %8s %10s %10s %10s %7s" % ("input", "combined", "pdsep", "overhead", "ratio"))
|
||||
for o in overhead[:10]:
|
||||
print(" %8d %10.3f %10.3f %10.3f %6.1fx" % (
|
||||
o["input"], o["c_ttft"], o["p_ttft"], o["overhead"], o["ratio"]))
|
||||
|
||||
overheads = [o["overhead"] for o in overhead]
|
||||
ratios = [o["ratio"] for o in overhead]
|
||||
print("\n Overhead stats:")
|
||||
print(" Mean: %.3fs extra TTFT per request" % statistics.fmean(overheads))
|
||||
print(" Mean ratio: %.1fx slower" % statistics.fmean(ratios))
|
||||
|
||||
# By input size
|
||||
for lo, hi, blabel in [(0, 5000, "<5k"), (5000, 50000, "5-50k"), (50000, 999999, ">50k")]:
|
||||
sub = [o for o in overhead if lo <= o["input"] < hi]
|
||||
if sub:
|
||||
print(" %6s: mean overhead=%.3fs, ratio=%.1fx" % (
|
||||
blabel, statistics.fmean([o["overhead"] for o in sub]),
|
||||
statistics.fmean([o["ratio"] for o in sub])))
|
||||
|
||||
# ===================================================================
|
||||
# EVIDENCE 5: Session affinity loss in PD-Sep
|
||||
# ===================================================================
|
||||
print("\n" + "-" * 75)
|
||||
print(" EVIDENCE 5: Session Affinity Disruption in PD-Sep")
|
||||
print("-" * 75)
|
||||
print("""
|
||||
In PD-combined: session turn N and turn N+1 go to the SAME instance.
|
||||
-> Turn N's KV stays in GPU cache
|
||||
-> Turn N+1 gets prefix cache hit (80%+ APC for multi-turn)
|
||||
-> Zero KV transfer needed
|
||||
|
||||
In PD-Sep: turn N's prefill goes to P instance, KV transfers to D instance.
|
||||
Turn N+1's prefill goes to P instance again.
|
||||
-> P instance does NOT have turn N's KV (it was transferred to D)
|
||||
-> Turn N+1 must re-prefill from scratch on P
|
||||
-> Then transfer KV to D again
|
||||
-> Double penalty: re-prefill + KV transfer
|
||||
|
||||
This is the fundamental reason PD-Sep destroys multi-turn APC:
|
||||
Combined APC for multi-turn: ~80%
|
||||
PD-Sep: effectively ~0% for prefill (P never has prior turn's KV)
|
||||
The only cache hit is on D, but D doesn't do prefill — it just decodes.
|
||||
""")
|
||||
|
||||
# ===================================================================
|
||||
# SUMMARY
|
||||
# ===================================================================
|
||||
print(sep)
|
||||
print(" SUMMARY: 4 MECHANISMS WHY PD-SEP LOSES")
|
||||
print(sep)
|
||||
print("""
|
||||
1. KV CACHE MEMORY WALL: Concentrating decode onto fewer GPUs fills
|
||||
KV cache to 97%, causing 100+s waits for memory release.
|
||||
Combined distributes across 8 instances, keeping usage <20%.
|
||||
|
||||
2. KV TRANSFER OVERHEAD: Every PD-Sep request pays RDMA transfer cost
|
||||
(even small requests). Combined has zero transfer — KV stays on GPU.
|
||||
|
||||
3. SESSION AFFINITY BROKEN: Multi-turn sessions lose prefix cache on P
|
||||
because prior turn's KV was transferred to D. Combined keeps KV
|
||||
on the same instance, achieving 80% multi-turn APC vs ~0% on P.
|
||||
|
||||
4. GPU UNDERUTILIZATION: PD-Sep decode GPUs idle at 7-19% (memory-bound
|
||||
decode doesn't need GPU compute). Combined uses all GPUs flexibly
|
||||
at 28-30% average utilization.
|
||||
|
||||
ROOT CAUSE: PD-Sep was designed for chatbot workloads (short input,
|
||||
no prefix sharing, compute-heavy prefill). Agentic workloads have:
|
||||
- Long context (33k avg) -> large KV, memory pressure on D
|
||||
- High prefix reuse (91% intra-session) -> session-sticky routing essential
|
||||
- MoE model (3B active) -> low per-token compute, P-D interference small
|
||||
These characteristics make PD-Sep's costs exceed its benefits.
|
||||
""")
|
||||
77
scripts/legacy/run_benchmark.sh
Executable file
77
scripts/legacy/run_benchmark.sh
Executable file
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# Run the full benchmark suite: sample trace → replay against vLLM → collect metrics.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - vLLM server running (use scripts/launch_vllm.sh)
|
||||
# - Sampled trace file exists (or will be created)
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_benchmark.sh [--endpoint URL] [--tag NAME]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Defaults
|
||||
TRACE_INPUT="${TRACE_INPUT:-$HOME/ali-trace/trace-glm5.1-formatted/051315-051317.jsonl}"
|
||||
ENDPOINT="${ENDPOINT:-http://localhost:8000}"
|
||||
TAG="${TAG:-default}"
|
||||
TARGET_REQUESTS="${TARGET_REQUESTS:-5000}"
|
||||
TIME_SCALE="${TIME_SCALE:-1.0}"
|
||||
MAX_INFLIGHT="${MAX_INFLIGHT:-32}"
|
||||
SEED="${SEED:-42}"
|
||||
|
||||
# Parse args
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint) ENDPOINT="$2"; shift 2 ;;
|
||||
--tag) TAG="$2"; shift 2 ;;
|
||||
--target-requests) TARGET_REQUESTS="$2"; shift 2 ;;
|
||||
--time-scale) TIME_SCALE="$2"; shift 2 ;;
|
||||
--max-inflight) MAX_INFLIGHT="$2"; shift 2 ;;
|
||||
*) echo "Unknown arg: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SAMPLED_TRACE="traces/sampled_${TARGET_REQUESTS}req_seed${SEED}.jsonl"
|
||||
OUTPUT_DIR="outputs/${TAG}_$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
echo "=== Benchmark: tag=$TAG ==="
|
||||
echo " Trace: $TRACE_INPUT"
|
||||
echo " Endpoint: $ENDPOINT"
|
||||
echo " Target requests: $TARGET_REQUESTS"
|
||||
echo " Time scale: $TIME_SCALE"
|
||||
echo " Max inflight sessions: $MAX_INFLIGHT"
|
||||
|
||||
# Step 1: Sample trace (if not already done)
|
||||
if [ ! -f "$SAMPLED_TRACE" ]; then
|
||||
echo ""
|
||||
echo "=== Step 1: Sampling trace ==="
|
||||
python scripts/sample_trace.py \
|
||||
--input "$TRACE_INPUT" \
|
||||
--output "$SAMPLED_TRACE" \
|
||||
--target-requests "$TARGET_REQUESTS" \
|
||||
--seed "$SEED"
|
||||
else
|
||||
echo ""
|
||||
echo "=== Step 1: Using existing sampled trace: $SAMPLED_TRACE ==="
|
||||
fi
|
||||
|
||||
# Step 2: Run replay
|
||||
echo ""
|
||||
echo "=== Step 2: Replaying trace ==="
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
python -m replayer \
|
||||
--trace "$SAMPLED_TRACE" \
|
||||
--output "$OUTPUT_DIR/metrics.jsonl" \
|
||||
--endpoint "$ENDPOINT" \
|
||||
--time-scale "$TIME_SCALE" \
|
||||
--max-inflight-sessions "$MAX_INFLIGHT" \
|
||||
-v
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
echo " Metrics: $OUTPUT_DIR/metrics.jsonl"
|
||||
echo " Summary: $OUTPUT_DIR/metrics.summary.json"
|
||||
329
scripts/legacy/run_elastic_stability_test.sh
Executable file
329
scripts/legacy/run_elastic_stability_test.sh
Executable file
@@ -0,0 +1,329 @@
|
||||
#!/bin/bash
|
||||
# Elastic P2P stability test: runs 200-request benchmark with offload mode
|
||||
# and baseline mode, then compares success rates.
|
||||
#
|
||||
# Must be run on dash0 (8 GPUs with Mooncake support).
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_elastic_stability_test.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}"
|
||||
PYTHON="$VENV/python"
|
||||
VLLM="$VENV/vllm"
|
||||
|
||||
MODEL="${MODEL_PATH:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
||||
TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
|
||||
N_INSTANCES=8
|
||||
BASE_PORT=8000
|
||||
PROXY_PORT=9090
|
||||
REQUEST_LIMIT=200
|
||||
TIME_SCALE=20
|
||||
MAX_INFLIGHT=8
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
OUT_ELASTIC="$PROJECT_DIR/outputs/elastic_stability_${TIMESTAMP}"
|
||||
OUT_BASELINE="$PROJECT_DIR/outputs/baseline_stability_${TIMESTAMP}"
|
||||
|
||||
# ─── Helper functions ────────────────────────────────────────────────────────
|
||||
|
||||
kill_all() {
|
||||
echo "[cleanup] Killing vLLM and proxy processes..."
|
||||
for p in $(ps aux | grep 'vllm serve' | grep -v grep | awk '{print $2}' 2>/dev/null); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
for p in $(ps aux | grep 'cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 5
|
||||
echo "[cleanup] Releasing GPUs..."
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 10
|
||||
echo "[cleanup] Done."
|
||||
}
|
||||
|
||||
wait_for_instances() {
|
||||
local n=$1
|
||||
echo "[wait] Waiting for $n vLLM instances to become healthy..."
|
||||
for i in $(seq 0 $((n - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$port/health" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 120 ]; then
|
||||
echo "[FAIL] Instance $i (port $port) did not start in 600s"
|
||||
return 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo " Instance $i (port $port) healthy"
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_bootstrap() {
|
||||
echo "[wait] Waiting for Mooncake bootstrap servers..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local bp=$((8998 + i))
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$bp/query" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 60 ]; then
|
||||
echo "[FAIL] Bootstrap $bp did not start in 120s"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " Bootstrap $bp ready"
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_proxy() {
|
||||
echo "[wait] Waiting for proxy on port $PROXY_PORT..."
|
||||
local tries=0
|
||||
while ! curl -sf "http://127.0.0.1:$PROXY_PORT/stats" > /dev/null 2>&1; do
|
||||
tries=$((tries + 1))
|
||||
if [ $tries -ge 30 ]; then
|
||||
echo "[FAIL] Proxy did not start in 60s"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo " Proxy ready"
|
||||
}
|
||||
|
||||
launch_vllm_kv_both() {
|
||||
echo ""
|
||||
echo "=== Launching $N_INSTANCES vLLM instances (kv_both) ==="
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local bp=$((8998 + i))
|
||||
local master=$((29500 + i))
|
||||
local log="/tmp/elastic_test_${i}.log"
|
||||
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$bp \
|
||||
MASTER_PORT=$master \
|
||||
CUDA_VISIBLE_DEVICES=$i \
|
||||
$VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port "$port" --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$log" 2>&1 &
|
||||
|
||||
echo " Instance $i: GPU=$i port=$port bootstrap=$bp log=$log"
|
||||
sleep 2
|
||||
done
|
||||
wait_for_instances $N_INSTANCES
|
||||
wait_for_bootstrap
|
||||
}
|
||||
|
||||
launch_vllm_baseline() {
|
||||
echo ""
|
||||
echo "=== Launching $N_INSTANCES vLLM instances (baseline, no Mooncake) ==="
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
local port=$((BASE_PORT + i))
|
||||
local master=$((29500 + i))
|
||||
local log="/tmp/baseline_test_${i}.log"
|
||||
|
||||
MASTER_PORT=$master \
|
||||
CUDA_VISIBLE_DEVICES=$i \
|
||||
$VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port "$port" --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
> "$log" 2>&1 &
|
||||
|
||||
echo " Instance $i: GPU=$i port=$port log=$log"
|
||||
sleep 2
|
||||
done
|
||||
wait_for_instances $N_INSTANCES
|
||||
}
|
||||
|
||||
launch_proxy_elastic() {
|
||||
echo ""
|
||||
echo "=== Starting proxy (elastic offload mode) ==="
|
||||
local combined_args=""
|
||||
local bp_list=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))"
|
||||
bp_list="${bp_list:+$bp_list,}$((8998 + i))"
|
||||
done
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined $combined_args \
|
||||
--bootstrap-ports "$bp_list" \
|
||||
--offload --heavy-threshold 20000 \
|
||||
--port $PROXY_PORT \
|
||||
> /tmp/proxy_elastic.log 2>&1 &
|
||||
wait_for_proxy
|
||||
}
|
||||
|
||||
launch_proxy_baseline() {
|
||||
echo ""
|
||||
echo "=== Starting proxy (baseline, no offload) ==="
|
||||
local combined_args=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
combined_args="$combined_args http://127.0.0.1:$((BASE_PORT + i))"
|
||||
done
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined $combined_args \
|
||||
--port $PROXY_PORT \
|
||||
> /tmp/proxy_baseline.log 2>&1 &
|
||||
wait_for_proxy
|
||||
}
|
||||
|
||||
run_benchmark() {
|
||||
local tag=$1
|
||||
local output_dir=$2
|
||||
mkdir -p "$output_dir"
|
||||
|
||||
echo ""
|
||||
echo "=== Running benchmark: $tag ($REQUEST_LIMIT requests) ==="
|
||||
$PYTHON -m replayer \
|
||||
--trace "$TRACE" \
|
||||
--output "$output_dir/metrics.jsonl" \
|
||||
--endpoint "http://localhost:$PROXY_PORT" \
|
||||
--model "$MODEL" \
|
||||
--time-scale "$TIME_SCALE" \
|
||||
--max-inflight-sessions "$MAX_INFLIGHT" \
|
||||
--request-limit "$REQUEST_LIMIT" \
|
||||
-v 2>&1 | tee "$output_dir/replayer.log"
|
||||
|
||||
# Save proxy breakdown and stats
|
||||
curl -sf "http://localhost:$PROXY_PORT/breakdown" > "$output_dir/breakdown.json" 2>/dev/null || true
|
||||
curl -sf "http://localhost:$PROXY_PORT/stats" > "$output_dir/stats.json" 2>/dev/null || true
|
||||
}
|
||||
|
||||
collect_gpu_util() {
|
||||
local output_dir=$1
|
||||
nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total \
|
||||
--format=csv > "$output_dir/gpu_snapshot.csv" 2>/dev/null || true
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
local label=$1
|
||||
local output_dir=$2
|
||||
local metrics="$output_dir/metrics.jsonl"
|
||||
|
||||
if [ ! -f "$metrics" ]; then
|
||||
echo " [$label] No metrics file found!"
|
||||
return
|
||||
fi
|
||||
|
||||
# Count total, success, error from metrics JSONL
|
||||
local total=$(wc -l < "$metrics")
|
||||
local success=$(grep -c '"error":null\|"error": null' "$metrics" 2>/dev/null || grep -c '"ttft":[0-9]' "$metrics" 2>/dev/null || echo 0)
|
||||
local errors=$((total - success))
|
||||
local rate="N/A"
|
||||
if [ "$total" -gt 0 ]; then
|
||||
rate=$(awk "BEGIN{printf \"%.1f\", ($success/$total)*100}")
|
||||
fi
|
||||
|
||||
echo " [$label]"
|
||||
echo " Total requests: $total"
|
||||
echo " Successful: $success"
|
||||
echo " Errors: $errors"
|
||||
echo " Success rate: ${rate}%"
|
||||
|
||||
# Print summary.json if it exists
|
||||
local summary="$output_dir/metrics.summary.json"
|
||||
if [ -f "$summary" ]; then
|
||||
echo " Summary: $(cat "$summary")"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "================================================================"
|
||||
echo " Elastic P2P Stability Test"
|
||||
echo " $(date)"
|
||||
echo " Model: $MODEL"
|
||||
echo " Requests: $REQUEST_LIMIT"
|
||||
echo " Output: elastic → $OUT_ELASTIC"
|
||||
echo " baseline → $OUT_BASELINE"
|
||||
echo "================================================================"
|
||||
|
||||
# Sanity checks
|
||||
if [ ! -f "$TRACE" ]; then
|
||||
echo "[ERROR] Trace file not found: $TRACE"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$PYTHON" ]; then
|
||||
echo "[ERROR] Python not found: $PYTHON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ─── Phase 1: Elastic P2P offload ────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "############################################################"
|
||||
echo " Phase 1: Elastic P2P Offload"
|
||||
echo "############################################################"
|
||||
|
||||
kill_all
|
||||
launch_vllm_kv_both
|
||||
launch_proxy_elastic
|
||||
collect_gpu_util "$OUT_ELASTIC"
|
||||
run_benchmark "elastic_p2p" "$OUT_ELASTIC"
|
||||
|
||||
echo ""
|
||||
echo "[phase1] Saving APC stats..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
port=$((BASE_PORT + i))
|
||||
curl -sf "http://127.0.0.1:$port/metrics" 2>/dev/null \
|
||||
| grep -E 'vllm:cache_hit|prefix_cache' \
|
||||
>> "$OUT_ELASTIC/apc_metrics.txt" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# ─── Phase 2: Baseline (no offload) ─────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "############################################################"
|
||||
echo " Phase 2: Baseline (no offload, no Mooncake)"
|
||||
echo "############################################################"
|
||||
|
||||
kill_all
|
||||
launch_vllm_baseline
|
||||
launch_proxy_baseline
|
||||
collect_gpu_util "$OUT_BASELINE"
|
||||
run_benchmark "baseline" "$OUT_BASELINE"
|
||||
|
||||
echo ""
|
||||
echo "[phase2] Saving APC stats..."
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
port=$((BASE_PORT + i))
|
||||
curl -sf "http://127.0.0.1:$port/metrics" 2>/dev/null \
|
||||
| grep -E 'vllm:cache_hit|prefix_cache' \
|
||||
>> "$OUT_BASELINE/apc_metrics.txt" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# ─── Cleanup ─────────────────────────────────────────────────────────────────
|
||||
|
||||
kill_all
|
||||
|
||||
# ─── Comparison ──────────────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Results Comparison"
|
||||
echo "================================================================"
|
||||
|
||||
print_summary "Elastic P2P" "$OUT_ELASTIC"
|
||||
echo ""
|
||||
print_summary "Baseline" "$OUT_BASELINE"
|
||||
|
||||
echo ""
|
||||
echo "Detailed outputs:"
|
||||
echo " Elastic: $OUT_ELASTIC/"
|
||||
echo " Baseline: $OUT_BASELINE/"
|
||||
echo ""
|
||||
echo "Breakdown analysis:"
|
||||
echo " python scripts/analyze_breakdown.py $OUT_ELASTIC/breakdown.json"
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Done. $(date)"
|
||||
echo "================================================================"
|
||||
254
scripts/legacy/run_experiments.sh
Executable file
254
scripts/legacy/run_experiments.sh
Executable file
@@ -0,0 +1,254 @@
|
||||
#!/bin/bash
|
||||
# Run the complete experiment matrix:
|
||||
# 1. Combined TP=2 DP=4 (4 instances, baseline)
|
||||
# 2. Combined TP=1 DP=8 (8 instances, max throughput)
|
||||
# 3. PD-Sep TP=1: P×4 + D×4 via Mooncake/RDMA
|
||||
#
|
||||
# All use the same trace, same concurrency, same timeout.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
VENV="$PROJECT_DIR/.venv/bin"
|
||||
VLLM="$VENV/vllm"
|
||||
PYTHON="$VENV/python"
|
||||
|
||||
MODEL="${MODEL_PATH:-$HOME/models/Qwen/Qwen3-Coder-30B-A3B-Instruct}"
|
||||
TRACE="$PROJECT_DIR/traces/sampled_1000req_seed42.jsonl"
|
||||
|
||||
# Uniform benchmark params
|
||||
MAX_SESSIONS=${MAX_SESSIONS:-8}
|
||||
MAX_CONCURRENT=${MAX_CONCURRENT:-16}
|
||||
TIME_SCALE=10
|
||||
REQUEST_TIMEOUT=${REQUEST_TIMEOUT:-300}
|
||||
REQUEST_LIMIT="${REQUEST_LIMIT:-}" # empty = all 1000
|
||||
|
||||
cleanup_gpu() {
|
||||
pkill -9 -f "vllm" 2>/dev/null || true
|
||||
pkill -9 -f "cache_aware_proxy\|mooncake_connector_proxy\|uvicorn" 2>/dev/null || true
|
||||
fuser 9090/tcp 8000/tcp 2>/dev/null | xargs -r kill -9 2>/dev/null || true
|
||||
sleep 5
|
||||
fuser /dev/nvidia* 2>/dev/null | tr " " "\n" | sort -u | xargs -r kill -9 2>/dev/null || true
|
||||
sleep 10
|
||||
}
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local timeout=${2:-600}
|
||||
timeout "$timeout" bash -c "until curl -s localhost:$port/v1/models >/dev/null 2>&1; do sleep 5; done"
|
||||
}
|
||||
|
||||
run_benchmark() {
|
||||
local tag=$1
|
||||
local endpoint=$2
|
||||
local extra_args="${3:-}"
|
||||
local outdir="$PROJECT_DIR/outputs/$tag"
|
||||
|
||||
echo " Running benchmark -> $outdir"
|
||||
local limit_arg=""
|
||||
if [ -n "$REQUEST_LIMIT" ]; then
|
||||
limit_arg="--request-limit $REQUEST_LIMIT"
|
||||
fi
|
||||
|
||||
$PYTHON -m replayer \
|
||||
--trace "$TRACE" \
|
||||
--output "$outdir/metrics.jsonl" \
|
||||
--endpoint "$endpoint" \
|
||||
--model "$MODEL" \
|
||||
--time-scale $TIME_SCALE \
|
||||
--max-inflight-sessions $MAX_SESSIONS \
|
||||
--concurrency-limit $MAX_CONCURRENT \
|
||||
--request-timeout $REQUEST_TIMEOUT \
|
||||
$limit_arg \
|
||||
-v
|
||||
|
||||
echo " Done: $(wc -l < "$outdir/metrics.jsonl") requests"
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Experiment 1: Combined TP=2 DP=4
|
||||
#######################################################################
|
||||
run_combined_tp2_dp4() {
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Experiment 1: Combined TP=2 DP=4 (4 instances on 8 GPUs)"
|
||||
echo "================================================================"
|
||||
cleanup_gpu
|
||||
|
||||
for i in 0 1 2 3; do
|
||||
local gpu_start=$((i * 2))
|
||||
local gpu_end=$((gpu_start + 1))
|
||||
local port=$((8000 + i))
|
||||
echo " Starting instance $i: GPUs $gpu_start,$gpu_end, port $port"
|
||||
CUDA_VISIBLE_DEVICES=$gpu_start,$gpu_end $VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port $port \
|
||||
--tensor-parallel-size 2 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 &
|
||||
done
|
||||
|
||||
for i in 0 1 2 3; do
|
||||
wait_for_server $((8000 + i))
|
||||
echo " Instance $i ready"
|
||||
done
|
||||
echo " All 4 instances ready"
|
||||
|
||||
# Start global scheduler (cache-aware proxy in combined mode)
|
||||
echo " Starting global scheduler..."
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 http://127.0.0.1:8003 \
|
||||
--port 9090 &
|
||||
sleep 5
|
||||
|
||||
run_benchmark "exp1_combined_tp2_dp4" "http://localhost:9090"
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Experiment 2: Combined TP=1 DP=8
|
||||
#######################################################################
|
||||
run_combined_tp1_dp8() {
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Experiment 2: Combined TP=1 DP=8 (8 instances on 8 GPUs)"
|
||||
echo "================================================================"
|
||||
cleanup_gpu
|
||||
|
||||
for i in $(seq 0 7); do
|
||||
local port=$((8000 + i))
|
||||
echo " Starting instance $i: GPU $i, port $port"
|
||||
CUDA_VISIBLE_DEVICES=$i $VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port $port \
|
||||
--tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 &
|
||||
done
|
||||
|
||||
for i in $(seq 0 7); do
|
||||
wait_for_server $((8000 + i))
|
||||
echo " Instance $i ready"
|
||||
done
|
||||
echo " All 8 instances ready"
|
||||
|
||||
# Start global scheduler (cache-aware proxy in combined mode)
|
||||
echo " Starting global scheduler..."
|
||||
$PYTHON "$PROJECT_DIR/scripts/cache_aware_proxy.py" \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 http://127.0.0.1:8003 \
|
||||
http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 http://127.0.0.1:8007 \
|
||||
--port 9090 &
|
||||
sleep 5
|
||||
|
||||
run_benchmark "exp2_combined_tp1_dp8" "http://localhost:9090"
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Experiment 3: PD-Sep TP=1 P×4 D×4 (Mooncake/RDMA)
|
||||
#######################################################################
|
||||
run_pd_sep_tp1() {
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Experiment 3: PD-Sep TP=1 P×4 + D×4 (Mooncake/RDMA)"
|
||||
echo "================================================================"
|
||||
cleanup_gpu
|
||||
|
||||
PROXY_SCRIPT="$PROJECT_DIR/scripts/cache_aware_proxy.py"
|
||||
|
||||
# Start 4 prefill instances (GPUs 0-3)
|
||||
local prefill_args=""
|
||||
for i in 0 1 2 3; do
|
||||
local port=$((8010 + i))
|
||||
local bootstrap=$((8998 + i))
|
||||
echo " Prefill $i: GPU $i, port $port, bootstrap $bootstrap"
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$bootstrap \
|
||||
CUDA_VISIBLE_DEVICES=$i $VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port $port \
|
||||
--tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config \
|
||||
"{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_producer\"}" &
|
||||
prefill_args="$prefill_args --prefill http://127.0.0.1:$port $bootstrap"
|
||||
done
|
||||
|
||||
# Start 4 decode instances (GPUs 4-7)
|
||||
local decode_args=""
|
||||
for i in 0 1 2 3; do
|
||||
local gpu=$((4 + i))
|
||||
local port=$((8020 + i))
|
||||
echo " Decode $i: GPU $gpu, port $port"
|
||||
CUDA_VISIBLE_DEVICES=$gpu $VLLM serve "$MODEL" \
|
||||
--host 0.0.0.0 --port $port \
|
||||
--tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config \
|
||||
"{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_consumer\",\"kv_load_failure_policy\":\"recompute\"}" &
|
||||
decode_args="$decode_args --decode http://127.0.0.1:$port"
|
||||
done
|
||||
|
||||
# Wait for all instances
|
||||
for i in 0 1 2 3; do
|
||||
wait_for_server $((8010 + i))
|
||||
echo " Prefill $i ready"
|
||||
done
|
||||
for i in 0 1 2 3; do
|
||||
wait_for_server $((8020 + i))
|
||||
echo " Decode $i ready"
|
||||
done
|
||||
|
||||
# Start proxy (wait for bootstrap to be queryable first)
|
||||
echo " Waiting for bootstrap servers..."
|
||||
for bp in 8998 8999 9000 9001; do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query > /dev/null 2>&1; do sleep 2; done"
|
||||
echo " Bootstrap $bp ready"
|
||||
done
|
||||
|
||||
echo " Starting proxy on port 9000..."
|
||||
$PYTHON "$PROXY_SCRIPT" $prefill_args $decode_args --host 0.0.0.0 --port 9090 &
|
||||
sleep 15
|
||||
|
||||
# Smoke test with retry
|
||||
echo " Smoke test..."
|
||||
for attempt in 1 2 3; do
|
||||
result=$(curl -s -m 120 http://localhost:9090/v1/completions \
|
||||
-X POST -H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"$MODEL\",\"prompt\":[100,200,300],\"max_tokens\":3,\"temperature\":0}" 2>&1)
|
||||
if echo "$result" | grep -q "choices"; then
|
||||
echo " Smoke test passed!"
|
||||
break
|
||||
fi
|
||||
echo " Attempt $attempt failed, retrying..."
|
||||
sleep 10
|
||||
done
|
||||
|
||||
run_benchmark "exp3_pd_sep_tp1_mooncake" "http://localhost:9090"
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Main
|
||||
#######################################################################
|
||||
echo "Starting experiment matrix on $(hostname)"
|
||||
echo "Model: $MODEL"
|
||||
echo "Trace: $TRACE"
|
||||
echo "Params: sessions=$MAX_SESSIONS, concurrent=$MAX_CONCURRENT, time_scale=$TIME_SCALE"
|
||||
echo ""
|
||||
|
||||
case "${1:-all}" in
|
||||
1|tp2dp4) run_combined_tp2_dp4 ;;
|
||||
2|tp1dp8) run_combined_tp1_dp8 ;;
|
||||
3|pdsep) run_pd_sep_tp1 ;;
|
||||
all)
|
||||
run_combined_tp2_dp4
|
||||
run_combined_tp1_dp8
|
||||
run_pd_sep_tp1
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {1|2|3|all|tp2dp4|tp1dp8|pdsep}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " All experiments complete!"
|
||||
echo "================================================================"
|
||||
cleanup_gpu
|
||||
172
scripts/legacy/run_h4_cache_gate.sh
Normal file
172
scripts/legacy/run_h4_cache_gate.sh
Normal file
@@ -0,0 +1,172 @@
|
||||
#!/bin/bash
|
||||
# H4: Cache-aware offload gate — only offload HEAVY when cache_ratio >= 0.3
|
||||
# Cold turn-1 HEAVY stays co-located (no RDMA overhead)
|
||||
# Cached turn-2+ HEAVY offloads to flexible D
|
||||
set -euo pipefail
|
||||
cd /home/admin/cpfs/wjh/agentic-kv
|
||||
source .venv/bin/activate
|
||||
|
||||
MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
OUTDIR=outputs/h4_cache_gate
|
||||
|
||||
cleanup() {
|
||||
for p in $(ps aux | grep -E 'vllm serve|cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 3
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 5
|
||||
}
|
||||
|
||||
cleanup
|
||||
mkdir -p "$OUTDIR"
|
||||
|
||||
echo "=== Verifying GPUs free ==="
|
||||
nvidia-smi --query-gpu=index,memory.used --format=csv,noheader
|
||||
|
||||
# ---- Launch 8 Combined instances (GPUs 0-7, all kv_both) ----
|
||||
echo "=== Launching 8 Combined instances ==="
|
||||
for i in $(seq 0 7); do
|
||||
echo "Starting C instance $i on GPU $i, port $((8000+i)), bootstrap $((8998+i))"
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998+i)) MASTER_PORT=$((29500+i)) CUDA_VISIBLE_DEVICES=$i \
|
||||
.venv/bin/vllm serve "$MODEL" --host 0.0.0.0 --port $((8000+i)) --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$OUTDIR/vllm_$i.log" 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ---- Wait for all instances healthy ----
|
||||
echo "=== Waiting for all instances to be healthy ==="
|
||||
for port in $(seq 8000 8007); do
|
||||
echo -n " Waiting for port $port..."
|
||||
timeout 600 bash -c "until curl -sf http://127.0.0.1:$port/health > /dev/null 2>&1; do sleep 5; done"
|
||||
echo " OK"
|
||||
done
|
||||
|
||||
# Wait for bootstrap ports
|
||||
for bp in $(seq 8998 9005); do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query > /dev/null 2>&1; do sleep 2; done"
|
||||
done
|
||||
echo "=== All instances healthy ==="
|
||||
sleep 5
|
||||
|
||||
# ---- Launch Proxy with H4 cache-aware offload ----
|
||||
echo "=== Launching cache_aware_proxy with H4 cache-aware offload gate ==="
|
||||
.venv/bin/python scripts/cache_aware_proxy.py \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 \
|
||||
http://127.0.0.1:8003 http://127.0.0.1:8004 http://127.0.0.1:8005 \
|
||||
http://127.0.0.1:8006 http://127.0.0.1:8007 \
|
||||
--bootstrap-ports 8998,8999,9000,9001,9002,9003,9004,9005 \
|
||||
--offload --port 9090 \
|
||||
> "$OUTDIR/proxy.log" 2>&1 &
|
||||
PROXY_PID=$!
|
||||
echo "Proxy PID: $PROXY_PID"
|
||||
|
||||
# Wait for proxy ready
|
||||
echo -n " Waiting for proxy..."
|
||||
until curl -sf "http://127.0.0.1:9090/stats" > /dev/null 2>&1; do
|
||||
sleep 2
|
||||
echo -n "."
|
||||
done
|
||||
echo " OK"
|
||||
|
||||
# ---- Run benchmark ----
|
||||
echo "=== Running benchmark: 200 req, time_scale=20, max-inflight-sessions=8 ==="
|
||||
.venv/bin/python -m replayer --trace traces/sampled_1000req_seed42.jsonl \
|
||||
--output "$OUTDIR/metrics.jsonl" \
|
||||
--endpoint http://localhost:9090 --model "$MODEL" \
|
||||
--time-scale 20 --max-inflight-sessions 8 --request-limit 200 -v
|
||||
|
||||
# ---- Save proxy data BEFORE cleanup ----
|
||||
echo "=== Saving proxy breakdown and stats ==="
|
||||
curl -sf "http://127.0.0.1:9090/breakdown" > "$OUTDIR/breakdown.json" 2>/dev/null || true
|
||||
curl -sf "http://127.0.0.1:9090/stats" > "$OUTDIR/stats.json" 2>/dev/null || true
|
||||
|
||||
# ---- Analysis ----
|
||||
echo "=== H4 Cache-Aware Gate Results ==="
|
||||
.venv/bin/python -c "
|
||||
import json
|
||||
from collections import Counter
|
||||
|
||||
rows = [json.loads(l) for l in open('$OUTDIR/metrics.jsonl')]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
fail = [r for r in rows if r.get('error')]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
|
||||
ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
|
||||
tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e2es = sorted([r['latency_s'] for r in ok])
|
||||
|
||||
print(f'OK={len(ok)}/{len(rows)} TTFT50={p(ttfts,.5):.3f} TTFT90={p(ttfts,.9):.3f} TPOT90={p(tpots,.9):.4f} E2E50={p(e2es,.5):.3f}')
|
||||
|
||||
# Per-class breakdown
|
||||
for lo,hi,cl in [(0,5000,'WARM'),(5000,20000,'MED'),(20000,200000,'HEAVY')]:
|
||||
sub = [r for r in ok if lo <= r['input_length'] < hi and r.get('ttft_s')]
|
||||
if sub:
|
||||
t = sorted([r['ttft_s'] for r in sub])
|
||||
tp = sorted([r['tpot_s'] for r in sub if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e = sorted([r['latency_s'] for r in sub])
|
||||
print(f' {cl:6s} n={len(sub):3d} TTFT50={p(t,.5):.3f} TTFT90={p(t,.9):.3f} TPOT90={p(tp,.9):.4f} E2E50={p(e,.5):.3f}')
|
||||
|
||||
# Route distribution from breakdown
|
||||
try:
|
||||
bd = json.load(open('$OUTDIR/breakdown.json'))
|
||||
rc = Counter(b.get('route_class','') for b in bd)
|
||||
print(f'\nRoute class distribution:')
|
||||
for cls, cnt in sorted(rc.items()):
|
||||
print(f' {cls}: {cnt}')
|
||||
|
||||
# Cache ratio analysis for HEAVY
|
||||
heavy = [b for b in bd if b.get('route_class','').startswith('HEAVY')]
|
||||
reasons = Counter(b.get('offload_reason','') for b in heavy)
|
||||
print(f'\nHEAVY offload reasons: {dict(reasons)}')
|
||||
|
||||
colo = [b for b in bd if b.get('route_class') == 'HEAVY_COLO']
|
||||
offloaded = [b for b in bd if b.get('route_class') == 'HEAVY_OFFLOAD']
|
||||
print(f'\nHEAVY_COLO (cold, no RDMA): {len(colo)}')
|
||||
print(f'HEAVY_OFFLOAD (cached, RDMA): {len(offloaded)}')
|
||||
|
||||
# Cache ratios
|
||||
for b in heavy:
|
||||
cr = b.get('cache_ratio', b.get('cache_hit',0)/max(b.get('input_length',1),1))
|
||||
cls = b.get('route_class','')
|
||||
reason = b.get('offload_reason','')
|
||||
|
||||
# Timing comparison: HEAVY_COLO vs HEAVY_OFFLOAD
|
||||
if colo:
|
||||
colo_ttft = sorted([b['t_first_token']-b['t_proxy_recv'] for b in colo if b.get('t_first_token')])
|
||||
if colo_ttft:
|
||||
print(f' HEAVY_COLO TTFT: p50={p(colo_ttft,.5):.2f}s p90={p(colo_ttft,.9):.2f}s')
|
||||
if offloaded:
|
||||
off_ttft = sorted([b['t_first_token']-b['t_proxy_recv'] for b in offloaded if b.get('t_first_token')])
|
||||
if off_ttft:
|
||||
print(f' HEAVY_OFFLOAD TTFT: p50={p(off_ttft,.5):.2f}s p90={p(off_ttft,.9):.2f}s')
|
||||
|
||||
# Offload timing breakdown
|
||||
if offloaded:
|
||||
pf = [b['t_prefill_done']-b['t_prefill_sent'] for b in offloaded if b.get('t_prefill_done') and b.get('t_prefill_sent')]
|
||||
kv = [b['t_first_token']-b['t_prefill_done'] for b in offloaded if b.get('t_first_token') and b.get('t_prefill_done')]
|
||||
if pf:
|
||||
pf.sort()
|
||||
print(f' Offload prefill: p50={p(pf,.5):.2f}s p90={p(pf,.9):.2f}s')
|
||||
if kv:
|
||||
kv.sort()
|
||||
print(f' Offload KV xfer+decode start: p50={p(kv,.5):.2f}s p90={p(kv,.9):.2f}s')
|
||||
|
||||
except Exception as e:
|
||||
print(f'Breakdown analysis error: {e}')
|
||||
|
||||
if fail:
|
||||
print(f'\nFailed requests ({len(fail)}):')
|
||||
for r in fail[:5]:
|
||||
print(f' input={r[\"input_length\"]} error={r[\"error\"][:80]}')
|
||||
|
||||
print()
|
||||
print('=== Baselines for comparison ===')
|
||||
print('Baseline 8C plain: OK=198/200 TTFT50=1.075 TTFT90=9.384 TPOT90=0.0761 E2E50=5.075')
|
||||
print('Phase0A 7C kv_both: OK=198/200 TTFT50=1.073 TPOT90=0.0738 E2E50=5.096')
|
||||
print('V2 all-offload: OK=179/185 TTFT50=0.762 TPOT90=0.0746 E2E50=4.628')
|
||||
"
|
||||
|
||||
cleanup
|
||||
echo "=== DONE $(date) ==="
|
||||
56
scripts/legacy/run_lmetric_ab.sh
Executable file
56
scripts/legacy/run_lmetric_ab.sh
Executable file
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
# A/B comparison: linear (session-sticky) vs lmetric (OSDI'26, no affinity).
|
||||
# Wraps bench.sh for guaranteed fresh state between experiments.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_lmetric_ab.sh # defaults: 200 req
|
||||
# bash scripts/run_lmetric_ab.sh --requests 1000 # override request count
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}"
|
||||
TS=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
echo "================================================================"
|
||||
echo " A/B: Linear vs LMetric routing policy"
|
||||
echo " $(date)"
|
||||
echo "================================================================"
|
||||
|
||||
echo ""
|
||||
echo "=== Experiment 1/2: Linear (session-sticky) ==="
|
||||
bash "$SCRIPT_DIR/bench.sh" --tag "ab_linear_${TS}" --mode baseline --policy linear "$@"
|
||||
|
||||
echo ""
|
||||
echo "=== Experiment 2/2: LMetric (no affinity) ==="
|
||||
bash "$SCRIPT_DIR/bench.sh" --tag "ab_lmetric_${TS}" --mode baseline --policy lmetric "$@"
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Results comparison"
|
||||
echo "================================================================"
|
||||
"$VENV/python" -c "
|
||||
import json
|
||||
|
||||
def summarize(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ttfts = [r['ttft_s'] for r in ok if r.get('ttft_s')]
|
||||
tpots = [r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0]
|
||||
e2es = [r['latency_s'] for r in ok]
|
||||
return len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)
|
||||
|
||||
lin = summarize('$PROJECT_DIR/outputs/ab_linear_${TS}/metrics.jsonl')
|
||||
lm = summarize('$PROJECT_DIR/outputs/ab_lmetric_${TS}/metrics.jsonl')
|
||||
|
||||
fmt = ' %-20s OK=%3d/%3d TTFT50=%7.3f TTFT90=%7.3f TPOT90=%6.4f E2E50=%7.3f'
|
||||
print(fmt % ('Linear', *lin))
|
||||
print(fmt % ('LMetric', *lm))
|
||||
print()
|
||||
for name, i in [('TTFT50',2),('TTFT90',3),('TPOT90',4),('E2E50',5)]:
|
||||
d = (lm[i] - lin[i]) / lin[i] * 100 if lin[i] else 0
|
||||
print(' %s delta: %+.1f%%' % (name, d))
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "Done at $(date)"
|
||||
169
scripts/legacy/run_ps_ablation.sh
Executable file
169
scripts/legacy/run_ps_ablation.sh
Executable file
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
# PS offload ablation: 3 experiments back-to-back with full cleanup between each.
|
||||
# 1. always_offload: all HEAVY → PS (zero hyperparameters)
|
||||
# 2. cost_model: cost-based decision (no interference gate)
|
||||
# 3. high_load: cost-based + 1000 requests (higher contention)
|
||||
set -euo pipefail
|
||||
|
||||
cd /home/admin/cpfs/wjh/agentic-kv
|
||||
source .venv/bin/activate
|
||||
|
||||
MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
VENV=.venv/bin
|
||||
TRACE=traces/sampled_1000req_seed42.jsonl
|
||||
|
||||
cleanup() {
|
||||
echo "[cleanup] Killing all processes..."
|
||||
for p in $(ps aux | grep -E 'vllm serve|cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 3
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 5
|
||||
local used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | awk '{s+=$1} END{print s}')
|
||||
if [ "${used:-0}" -gt 100 ]; then
|
||||
echo "[ERROR] GPUs not free (${used}MB). Waiting 10s..."
|
||||
sleep 10
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true); do
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
done
|
||||
sleep 5
|
||||
fi
|
||||
echo "[cleanup] Done."
|
||||
}
|
||||
|
||||
launch_7c_1ps() {
|
||||
echo "[launch] 7C (kv_both) + 1PS (kv_both)..."
|
||||
for i in $(seq 0 6); do
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998+i)) MASTER_PORT=$((29500+i)) CUDA_VISIBLE_DEVICES=$i \
|
||||
$VENV/vllm serve "$MODEL" --host 0.0.0.0 --port $((8000+i)) --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$1/vllm_c_$i.log" 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=9005 MASTER_PORT=29507 CUDA_VISIBLE_DEVICES=7 \
|
||||
$VENV/vllm serve "$MODEL" --host 0.0.0.0 --port 8007 --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$1/vllm_ps.log" 2>&1 &
|
||||
|
||||
for i in $(seq 0 7); do
|
||||
timeout 600 bash -c "until curl -s localhost:$((8000+i))/health > /dev/null 2>&1; do sleep 5; done"
|
||||
echo " inst_$i healthy"
|
||||
done
|
||||
for bp in $(seq 8998 9005); do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query > /dev/null 2>&1; do sleep 2; done"
|
||||
done
|
||||
echo "[launch] All ready."
|
||||
}
|
||||
|
||||
run_experiment() {
|
||||
local tag=$1
|
||||
local offload_mode=$2
|
||||
local requests=$3
|
||||
local sessions=$4
|
||||
local outdir=outputs/$tag
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " Experiment: $tag (mode=$offload_mode, requests=$requests)"
|
||||
echo " $(date)"
|
||||
echo "================================================================"
|
||||
|
||||
cleanup
|
||||
mkdir -p "$outdir"
|
||||
|
||||
launch_7c_1ps "$outdir"
|
||||
|
||||
echo "[proxy] Starting (offload_mode=$offload_mode)..."
|
||||
$VENV/python scripts/cache_aware_proxy.py \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 \
|
||||
http://127.0.0.1:8003 http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 \
|
||||
--ps-instances http://127.0.0.1:8007 \
|
||||
--ps-bootstrap-ports 9005 \
|
||||
--bootstrap-ports 8998,8999,9000,9001,9002,9003,9004 \
|
||||
--offload-mode "$offload_mode" \
|
||||
--port 9090 > "$outdir/proxy.log" 2>&1 &
|
||||
sleep 3
|
||||
|
||||
echo "[bench] Running $requests requests, $sessions sessions..."
|
||||
$VENV/python -m replayer --trace "$TRACE" \
|
||||
--output "$outdir/metrics.jsonl" \
|
||||
--endpoint http://localhost:9090 --model "$MODEL" \
|
||||
--time-scale 20 --max-inflight-sessions "$sessions" \
|
||||
--request-limit "$requests" -v 2>&1 | tail -5
|
||||
|
||||
curl -sf http://localhost:9090/breakdown > "$outdir/breakdown.json" 2>/dev/null || true
|
||||
curl -sf http://localhost:9090/stats > "$outdir/stats.json" 2>/dev/null || true
|
||||
|
||||
# Quick summary
|
||||
$VENV/python -c "
|
||||
import json
|
||||
from collections import Counter
|
||||
rows = [json.loads(l) for l in open('$outdir/metrics.jsonl')]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
|
||||
tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e2es = sorted([r['latency_s'] for r in ok])
|
||||
print(' OK=%d/%d TTFT50=%.3f TTFT90=%.3f TPOT90=%.4f E2E50=%.3f' % (
|
||||
len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)))
|
||||
for lo,hi,cl in [(0,5000,'WARM'),(5000,20000,'MED'),(20000,200000,'HEAVY')]:
|
||||
sub = [r for r in ok if lo <= r['input_length'] < hi and r.get('ttft_s')]
|
||||
if sub:
|
||||
t = sorted([r['ttft_s'] for r in sub])
|
||||
tp = sorted([r['tpot_s'] for r in sub if r.get('tpot_s') and r['tpot_s']>0])
|
||||
print(' %-6s n=%3d TTFT50=%.3f TPOT90=%.4f' % (cl, len(sub), p(t,.5), p(tp,.9) if tp else 0))
|
||||
try:
|
||||
bd = json.load(open('$outdir/breakdown.json'))
|
||||
rc = Counter(b.get('route_class','') for b in bd)
|
||||
print(' Routes: %s' % dict(rc))
|
||||
except: pass
|
||||
"
|
||||
echo " Done: $tag"
|
||||
}
|
||||
|
||||
# ─── Run experiments ───
|
||||
run_experiment "ps_always" "always" 200 7
|
||||
run_experiment "ps_cost" "cost" 200 7
|
||||
run_experiment "ps_highload" "cost" 1000 7
|
||||
|
||||
# ─── Final comparison ───
|
||||
cleanup
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " FINAL COMPARISON"
|
||||
echo "================================================================"
|
||||
$VENV/python -c "
|
||||
import json
|
||||
|
||||
configs = [
|
||||
('outputs/phase0a_7c_kvboth/metrics.jsonl', 'Control: 7C no PS'),
|
||||
('outputs/ps_always/metrics.jsonl', 'PS always offload'),
|
||||
('outputs/ps_cost/metrics.jsonl', 'PS cost model'),
|
||||
('outputs/ps_highload/metrics.jsonl', 'PS cost 1000req'),
|
||||
('outputs/baseline_stability_fresh/metrics.jsonl', 'Baseline: 8C plain'),
|
||||
]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
print('%-25s %7s %7s %7s %7s %7s' % ('Config', 'OK/N', 'TTFT50', 'TTFT90', 'TPOT90', 'E2E50'))
|
||||
print('-' * 80)
|
||||
for path, label in configs:
|
||||
try:
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
|
||||
tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e2es = sorted([r['latency_s'] for r in ok])
|
||||
print('%-25s %3d/%3d %7.3f %7.3f %7.4f %7.3f' % (
|
||||
label, len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)))
|
||||
except Exception as e:
|
||||
print('%-25s %s' % (label, e))
|
||||
"
|
||||
echo ""
|
||||
echo "=== ALL DONE $(date) ==="
|
||||
100
scripts/legacy/run_ps_flexd.sh
Executable file
100
scripts/legacy/run_ps_flexd.sh
Executable file
@@ -0,0 +1,100 @@
|
||||
#!/bin/bash
|
||||
# PS + flexible D: HEAVY prefill on PS, decode on least-loaded C
|
||||
set -euo pipefail
|
||||
cd /home/admin/cpfs/wjh/agentic-kv
|
||||
source .venv/bin/activate
|
||||
|
||||
MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
OUTDIR=outputs/ps_flexd
|
||||
|
||||
cleanup() {
|
||||
for p in $(ps aux | grep -E 'vllm serve|cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 3
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 5
|
||||
}
|
||||
|
||||
cleanup
|
||||
mkdir -p "$OUTDIR"
|
||||
|
||||
# 7 C instances (kv_both, GPUs 0-6)
|
||||
for i in $(seq 0 6); do
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998+i)) MASTER_PORT=$((29500+i)) CUDA_VISIBLE_DEVICES=$i \
|
||||
.venv/bin/vllm serve "$MODEL" --host 0.0.0.0 --port $((8000+i)) --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$OUTDIR/vllm_c_$i.log" 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# 1 PS instance (kv_both, GPU 7)
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=9005 MASTER_PORT=29507 CUDA_VISIBLE_DEVICES=7 \
|
||||
.venv/bin/vllm serve "$MODEL" --host 0.0.0.0 --port 8007 --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$OUTDIR/vllm_ps.log" 2>&1 &
|
||||
|
||||
echo "Waiting for instances..."
|
||||
for i in $(seq 0 7); do
|
||||
timeout 600 bash -c "until curl -s localhost:$((8000+i))/health > /dev/null 2>&1; do sleep 5; done"
|
||||
echo " inst_$i healthy"
|
||||
done
|
||||
for bp in $(seq 8998 9005); do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query > /dev/null 2>&1; do sleep 2; done"
|
||||
done
|
||||
echo "All ready"
|
||||
|
||||
# Proxy: PS on port 8007, C on 8000-8006, flexible D
|
||||
.venv/bin/python scripts/cache_aware_proxy.py \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 \
|
||||
http://127.0.0.1:8003 http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 \
|
||||
--ps-instances http://127.0.0.1:8007 \
|
||||
--ps-bootstrap-ports 9005 \
|
||||
--bootstrap-ports 8998,8999,9000,9001,9002,9003,9004 \
|
||||
--port 9090 > "$OUTDIR/proxy.log" 2>&1 &
|
||||
sleep 3
|
||||
|
||||
echo "Running benchmark..."
|
||||
.venv/bin/python -m replayer --trace traces/sampled_1000req_seed42.jsonl \
|
||||
--output "$OUTDIR/metrics.jsonl" --endpoint http://localhost:9090 --model "$MODEL" \
|
||||
--time-scale 20 --max-inflight-sessions 7 --request-limit 200 -v
|
||||
|
||||
curl -sf http://localhost:9090/breakdown > "$OUTDIR/breakdown.json" 2>/dev/null || true
|
||||
curl -sf http://localhost:9090/stats > "$OUTDIR/stats.json" 2>/dev/null || true
|
||||
|
||||
# Quick analysis
|
||||
.venv/bin/python -c "
|
||||
import json
|
||||
from collections import Counter
|
||||
rows = [json.loads(l) for l in open('$OUTDIR/metrics.jsonl')]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
|
||||
tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e2es = sorted([r['latency_s'] for r in ok])
|
||||
print('OK=%d/%d TTFT50=%.3f TTFT90=%.3f TPOT90=%.4f E2E50=%.3f' % (
|
||||
len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)))
|
||||
for lo,hi,cl in [(0,5000,'WARM'),(5000,20000,'MED'),(20000,200000,'HEAVY')]:
|
||||
sub = [r for r in ok if lo <= r['input_length'] < hi and r.get('ttft_s')]
|
||||
if sub:
|
||||
t = sorted([r['ttft_s'] for r in sub])
|
||||
tp = sorted([r['tpot_s'] for r in sub if r.get('tpot_s') and r['tpot_s']>0])
|
||||
print(' %-6s n=%3d TTFT50=%.3f TPOT90=%.4f' % (cl, len(sub), p(t,.5), p(tp,.9) if tp else 0))
|
||||
try:
|
||||
bd = json.load(open('$OUTDIR/breakdown.json'))
|
||||
rc = Counter(b.get('route_class','') for b in bd)
|
||||
print('Routes: %s' % dict(rc))
|
||||
ps = [b for b in bd if b.get('route_class') == 'HEAVY_PS']
|
||||
if ps:
|
||||
pf = [b['t_prefill_done']-b['t_prefill_sent'] for b in ps if b.get('t_prefill_done') and b.get('t_prefill_sent')]
|
||||
if pf: print('PS prefill: n=%d p50=%.1fs p90=%.1fs' % (len(pf), p(pf,.5), p(pf,.9)))
|
||||
except: pass
|
||||
print()
|
||||
print('Compare: Phase0A: TTFT50=1.073 TPOT90=0.0738 E2E50=5.096')
|
||||
print('Compare: Baseline 8C: TTFT50=1.075 TPOT90=0.0761 E2E50=5.075')
|
||||
"
|
||||
|
||||
cleanup
|
||||
echo "=== DONE $(date) ==="
|
||||
79
scripts/legacy/run_ps_remaining.sh
Executable file
79
scripts/legacy/run_ps_remaining.sh
Executable file
@@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
# Run ps_cost and ps_highload experiments (ps_always already done)
|
||||
set -euo pipefail
|
||||
cd /home/admin/cpfs/wjh/agentic-kv
|
||||
source .venv/bin/activate
|
||||
|
||||
MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
VENV=.venv/bin
|
||||
TRACE=traces/sampled_1000req_seed42.jsonl
|
||||
|
||||
cleanup() {
|
||||
for p in $(ps aux | grep -E 'vllm serve|cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 3
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 5
|
||||
}
|
||||
|
||||
launch_7c_1ps() {
|
||||
local outdir=$1
|
||||
mkdir -p "$outdir"
|
||||
for i in $(seq 0 6); do
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998+i)) MASTER_PORT=$((29500+i)) CUDA_VISIBLE_DEVICES=$i \
|
||||
$VENV/vllm serve "$MODEL" --host 0.0.0.0 --port $((8000+i)) --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$outdir/vllm_c_$i.log" 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=9005 MASTER_PORT=29507 CUDA_VISIBLE_DEVICES=7 \
|
||||
$VENV/vllm serve "$MODEL" --host 0.0.0.0 --port 8007 --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$outdir/vllm_ps.log" 2>&1 &
|
||||
for i in $(seq 0 7); do
|
||||
timeout 600 bash -c "until curl -s localhost:$((8000+i))/health > /dev/null 2>&1; do sleep 5; done"
|
||||
echo " inst_$i healthy"
|
||||
done
|
||||
for bp in $(seq 8998 9005); do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query > /dev/null 2>&1; do sleep 2; done"
|
||||
done
|
||||
echo " All ready"
|
||||
}
|
||||
|
||||
run_exp() {
|
||||
local tag=$1 mode=$2 reqs=$3 sess=$4
|
||||
local outdir=outputs/$tag
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " $tag (mode=$mode, reqs=$reqs, sess=$sess)"
|
||||
echo " $(date)"
|
||||
echo "================================================================"
|
||||
cleanup
|
||||
launch_7c_1ps "$outdir"
|
||||
|
||||
$VENV/python scripts/cache_aware_proxy.py \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 \
|
||||
http://127.0.0.1:8003 http://127.0.0.1:8004 http://127.0.0.1:8005 http://127.0.0.1:8006 \
|
||||
--ps-instances http://127.0.0.1:8007 --ps-bootstrap-ports 9005 \
|
||||
--bootstrap-ports 8998,8999,9000,9001,9002,9003,9004 \
|
||||
--offload-mode "$mode" --port 9090 > "$outdir/proxy.log" 2>&1 &
|
||||
sleep 3
|
||||
|
||||
$VENV/python -m replayer --trace "$TRACE" \
|
||||
--output "$outdir/metrics.jsonl" --endpoint http://localhost:9090 --model "$MODEL" \
|
||||
--time-scale 20 --max-inflight-sessions "$sess" --request-limit "$reqs" -v 2>&1 | tail -5
|
||||
|
||||
curl -sf http://localhost:9090/breakdown > "$outdir/breakdown.json" 2>/dev/null || true
|
||||
curl -sf http://localhost:9090/stats > "$outdir/stats.json" 2>/dev/null || true
|
||||
echo " Done: $tag"
|
||||
}
|
||||
|
||||
run_exp ps_cost cost 200 7
|
||||
run_exp ps_highload cost 1000 7
|
||||
cleanup
|
||||
|
||||
echo ""
|
||||
echo "=== ALL REMAINING DONE $(date) ==="
|
||||
156
scripts/legacy/run_v2_offload.sh
Executable file
156
scripts/legacy/run_v2_offload.sh
Executable file
@@ -0,0 +1,156 @@
|
||||
#!/bin/bash
|
||||
# V2: P2P KV offload — C_s (session-sticky, cached) prefills, D (least-loaded) decodes
|
||||
# 8 combined instances (all kv_both), no dedicated PS
|
||||
set -euo pipefail
|
||||
cd /home/admin/cpfs/wjh/agentic-kv
|
||||
source .venv/bin/activate
|
||||
|
||||
MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
OUTDIR=outputs/v2_offload
|
||||
|
||||
cleanup() {
|
||||
for p in $(ps aux | grep -E 'vllm serve|cache_aware_proxy' | grep -v grep | awk '{print $2}' 2>/dev/null); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 3
|
||||
for p in $(fuser /dev/nvidia* 2>/dev/null | tr ' ' '\n' | sort -u | grep -v '^$' || true); do kill -9 "$p" 2>/dev/null || true; done
|
||||
sleep 5
|
||||
}
|
||||
|
||||
cleanup
|
||||
mkdir -p "$OUTDIR"
|
||||
|
||||
echo "=== Verifying GPUs free ==="
|
||||
nvidia-smi --query-gpu=index,memory.used --format=csv,noheader
|
||||
|
||||
# ---- Launch 8 Combined instances (GPUs 0-7, all kv_both) ----
|
||||
echo "=== Launching 8 Combined instances ==="
|
||||
for i in $(seq 0 7); do
|
||||
echo "Starting C instance $i on GPU $i, port $((8000+i)), bootstrap $((8998+i))"
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT=$((8998+i)) MASTER_PORT=$((29500+i)) CUDA_VISIBLE_DEVICES=$i \
|
||||
.venv/bin/vllm serve "$MODEL" --host 0.0.0.0 --port $((8000+i)) --tensor-parallel-size 1 \
|
||||
--trust-remote-code --enable-prefix-caching --enforce-eager \
|
||||
--dtype auto --gpu-memory-utilization 0.9 --max-model-len 200000 \
|
||||
--kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both"}' \
|
||||
> "$OUTDIR/vllm_$i.log" 2>&1 &
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ---- Wait for all instances healthy ----
|
||||
echo "=== Waiting for all instances to be healthy ==="
|
||||
for port in $(seq 8000 8007); do
|
||||
echo -n " Waiting for port $port..."
|
||||
timeout 600 bash -c "until curl -sf http://127.0.0.1:$port/health > /dev/null 2>&1; do sleep 5; done"
|
||||
echo " OK"
|
||||
done
|
||||
|
||||
# Wait for bootstrap ports
|
||||
for bp in $(seq 8998 9005); do
|
||||
timeout 120 bash -c "until curl -s localhost:$bp/query > /dev/null 2>&1; do sleep 2; done"
|
||||
done
|
||||
echo "=== All instances healthy ==="
|
||||
sleep 5
|
||||
|
||||
# ---- Launch Proxy with V2 offload ----
|
||||
echo "=== Launching cache_aware_proxy with V2 offload ==="
|
||||
.venv/bin/python scripts/cache_aware_proxy.py \
|
||||
--combined http://127.0.0.1:8000 http://127.0.0.1:8001 http://127.0.0.1:8002 \
|
||||
http://127.0.0.1:8003 http://127.0.0.1:8004 http://127.0.0.1:8005 \
|
||||
http://127.0.0.1:8006 http://127.0.0.1:8007 \
|
||||
--bootstrap-ports 8998,8999,9000,9001,9002,9003,9004,9005 \
|
||||
--offload --port 9090 \
|
||||
> "$OUTDIR/proxy.log" 2>&1 &
|
||||
PROXY_PID=$!
|
||||
echo "Proxy PID: $PROXY_PID"
|
||||
|
||||
# Wait for proxy ready
|
||||
echo -n " Waiting for proxy..."
|
||||
until curl -sf "http://127.0.0.1:9090/stats" > /dev/null 2>&1; do
|
||||
sleep 2
|
||||
echo -n "."
|
||||
done
|
||||
echo " OK"
|
||||
|
||||
# ---- Run benchmark ----
|
||||
echo "=== Running benchmark: 200 req, time_scale=20, max-inflight-sessions=8 ==="
|
||||
.venv/bin/python -m replayer --trace traces/sampled_1000req_seed42.jsonl \
|
||||
--output "$OUTDIR/metrics.jsonl" \
|
||||
--endpoint http://localhost:9090 --model "$MODEL" \
|
||||
--time-scale 20 --max-inflight-sessions 8 --request-limit 200 -v
|
||||
|
||||
# ---- Save proxy data BEFORE cleanup ----
|
||||
echo "=== Saving proxy breakdown and stats ==="
|
||||
curl -sf "http://127.0.0.1:9090/breakdown" > "$OUTDIR/breakdown.json" 2>/dev/null || true
|
||||
curl -sf "http://127.0.0.1:9090/stats" > "$OUTDIR/stats.json" 2>/dev/null || true
|
||||
|
||||
# ---- Quick analysis ----
|
||||
echo "=== Quick metrics summary ==="
|
||||
.venv/bin/python -c "
|
||||
import json
|
||||
from collections import Counter
|
||||
|
||||
rows = [json.loads(l) for l in open('$OUTDIR/metrics.jsonl')]
|
||||
ok = [r for r in rows if not r.get('error')]
|
||||
fail = [r for r in rows if r.get('error')]
|
||||
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
|
||||
ttfts = sorted([r['ttft_s'] for r in ok if r.get('ttft_s')])
|
||||
tpots = sorted([r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e2es = sorted([r['latency_s'] for r in ok])
|
||||
|
||||
print(f'OK={len(ok)}/{len(rows)} TTFT50={p(ttfts,.5):.3f} TTFT90={p(ttfts,.9):.3f} TPOT90={p(tpots,.9):.4f} E2E50={p(e2es,.5):.3f}')
|
||||
|
||||
# Per-class breakdown
|
||||
for lo,hi,cl in [(0,5000,'WARM'),(5000,20000,'MED'),(20000,200000,'HEAVY')]:
|
||||
sub = [r for r in ok if lo <= r['input_length'] < hi and r.get('ttft_s')]
|
||||
if sub:
|
||||
t = sorted([r['ttft_s'] for r in sub])
|
||||
tp = sorted([r['tpot_s'] for r in sub if r.get('tpot_s') and r['tpot_s']>0])
|
||||
e = sorted([r['latency_s'] for r in sub])
|
||||
print(f' {cl:6s} n={len(sub):3d} TTFT50={p(t,.5):.3f} TTFT90={p(t,.9):.3f} TPOT90={p(tp,.9):.4f} E2E50={p(e,.5):.3f}')
|
||||
|
||||
# Route distribution from breakdown
|
||||
try:
|
||||
bd = json.load(open('$OUTDIR/breakdown.json'))
|
||||
rc = Counter(b.get('route_class','') for b in bd)
|
||||
print(f'\nRoute class distribution:')
|
||||
for cls, cnt in sorted(rc.items()):
|
||||
print(f' {cls}: {cnt}')
|
||||
|
||||
# Offload timing
|
||||
offloaded = [b for b in bd if b.get('route_class') == 'HEAVY_OFFLOAD']
|
||||
if offloaded:
|
||||
pf = [b['t_prefill_done']-b['t_prefill_sent'] for b in offloaded if b.get('t_prefill_done') and b.get('t_prefill_sent')]
|
||||
if pf:
|
||||
pf.sort()
|
||||
print(f'\nHEAVY_OFFLOAD prefill time: n={len(pf)} p50={p(pf,.5):.2f}s p90={p(pf,.9):.2f}s')
|
||||
|
||||
# Offload reasons for HEAVY
|
||||
heavy = [b for b in bd if b.get('route_class','').startswith('HEAVY')]
|
||||
reasons = Counter(b.get('offload_reason','') for b in heavy)
|
||||
if reasons:
|
||||
print(f'HEAVY offload reasons: {dict(reasons)}')
|
||||
|
||||
# Per-instance P and D counts
|
||||
p_counts = Counter(b.get('p_inst','') for b in bd if b.get('p_inst'))
|
||||
d_counts = Counter(b.get('d_inst','') for b in bd if b.get('d_inst'))
|
||||
if p_counts:
|
||||
print(f'\nP-instance distribution: {dict(p_counts)}')
|
||||
if d_counts:
|
||||
print(f'D-instance distribution: {dict(d_counts)}')
|
||||
|
||||
except Exception as e:
|
||||
print(f'Breakdown analysis error: {e}')
|
||||
|
||||
if fail:
|
||||
print(f'\nFailed requests ({len(fail)}):')
|
||||
for r in fail[:5]:
|
||||
print(f' input={r[\"input_length\"]} error={r[\"error\"][:80]}')
|
||||
|
||||
print()
|
||||
print('=== Baselines for comparison ===')
|
||||
print('Phase0A (7C kv_both): OK=198/200 TTFT50=1.073 TPOT90=0.0738 E2E50=5.096')
|
||||
print('Baseline (8C plain): OK=198/200 TTFT50=1.075 TPOT90=0.0761 E2E50=5.075')
|
||||
print('PS V1 flexD: OK=172/186 TTFT50=0.978 TPOT90=0.0758 E2E50=5.623')
|
||||
"
|
||||
|
||||
cleanup
|
||||
echo "=== DONE $(date) ==="
|
||||
Reference in New Issue
Block a user