Gahow Wang
547611e022
D2: run_benchmark.sh and run_experiments.sh still pass --time-scale and --max-inflight-sessions to the replayer, but those flags were removed when the project moved to trace-driven dispatch. The scripts cannot run as-is. D3: ~25 ad-hoc analyze_* / compare_* / profile_* / final_* scripts and a handful of single-experiment run_*.sh point at /home/admin/cpfs paths, deleted output directories, or a sampled trace file that no longer exists. Keep them in scripts/legacy/ for historical reference; the scripts that remain in scripts/ (analyze_trace, analyze_breakdown, analyze_cache_hit, analyze_eviction, compare_results, compute_roofline, sample_trace, analyze_agentic_patterns, simulate_cache_policies, plus launch_*.sh, gpu_monitor.sh, bench.sh) cover the current workflow. Adds scripts/legacy/README.md to document the archival policy.
57 lines
2.1 KiB
Bash
Executable File
57 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# A/B comparison: linear (session-sticky) vs lmetric (OSDI'26, no affinity).
|
|
# Wraps bench.sh for guaranteed fresh state between experiments.
|
|
#
|
|
# Usage:
|
|
# bash scripts/run_lmetric_ab.sh # defaults: 200 req
|
|
# bash scripts/run_lmetric_ab.sh --requests 1000 # override request count
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
VENV="${VENV_PATH:-$PROJECT_DIR/.venv/bin}"
|
|
TS=$(date +%Y%m%d_%H%M%S)
|
|
|
|
echo "================================================================"
|
|
echo " A/B: Linear vs LMetric routing policy"
|
|
echo " $(date)"
|
|
echo "================================================================"
|
|
|
|
echo ""
|
|
echo "=== Experiment 1/2: Linear (session-sticky) ==="
|
|
bash "$SCRIPT_DIR/bench.sh" --tag "ab_linear_${TS}" --mode baseline --policy linear "$@"
|
|
|
|
echo ""
|
|
echo "=== Experiment 2/2: LMetric (no affinity) ==="
|
|
bash "$SCRIPT_DIR/bench.sh" --tag "ab_lmetric_${TS}" --mode baseline --policy lmetric "$@"
|
|
|
|
echo ""
|
|
echo "================================================================"
|
|
echo " Results comparison"
|
|
echo "================================================================"
|
|
"$VENV/python" -c "
|
|
import json
|
|
|
|
def summarize(path):
|
|
rows = [json.loads(l) for l in open(path)]
|
|
ok = [r for r in rows if not r.get('error')]
|
|
p = lambda v,q: sorted(v)[min(int(q*len(v)),len(v)-1)] if v else 0
|
|
ttfts = [r['ttft_s'] for r in ok if r.get('ttft_s')]
|
|
tpots = [r['tpot_s'] for r in ok if r.get('tpot_s') and r['tpot_s']>0]
|
|
e2es = [r['latency_s'] for r in ok]
|
|
return len(ok), len(rows), p(ttfts,.5), p(ttfts,.9), p(tpots,.9), p(e2es,.5)
|
|
|
|
lin = summarize('$PROJECT_DIR/outputs/ab_linear_${TS}/metrics.jsonl')
|
|
lm = summarize('$PROJECT_DIR/outputs/ab_lmetric_${TS}/metrics.jsonl')
|
|
|
|
fmt = ' %-20s OK=%3d/%3d TTFT50=%7.3f TTFT90=%7.3f TPOT90=%6.4f E2E50=%7.3f'
|
|
print(fmt % ('Linear', *lin))
|
|
print(fmt % ('LMetric', *lm))
|
|
print()
|
|
for name, i in [('TTFT50',2),('TTFT90',3),('TPOT90',4),('E2E50',5)]:
|
|
d = (lm[i] - lin[i]) / lin[i] * 100 if lin[i] else 0
|
|
print(' %s delta: %+.1f%%' % (name, d))
|
|
"
|
|
|
|
echo ""
|
|
echo "Done at $(date)"
|