Add --offload-mode switch for ablation (direct_read vs cached_prefill)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-24 11:24:15 +08:00
parent cdf83493ab
commit bf76273778
2 changed files with 29 additions and 10 deletions

View File

@@ -37,6 +37,7 @@ OVERLOAD_FACTOR_ARG=""
MAX_BATCHED_TOKENS=""
MAX_OFFLOAD_INFLIGHT=""
CACHE_GATE_RATIO=""
OFFLOAD_MODE=""
# Parse args
while [[ $# -gt 0 ]]; do
@@ -53,6 +54,7 @@ while [[ $# -gt 0 ]]; do
--max-batched-tokens) MAX_BATCHED_TOKENS="$2"; shift 2 ;;
--max-offload-inflight) MAX_OFFLOAD_INFLIGHT="$2"; shift 2 ;;
--cache-gate-ratio) CACHE_GATE_RATIO="$2"; shift 2 ;;
--offload-mode) OFFLOAD_MODE="$2"; shift 2 ;;
*) echo "Unknown: $1"; exit 1 ;;
esac
done
@@ -218,6 +220,9 @@ launch_proxy() {
if [ -n "$CACHE_GATE_RATIO" ]; then
extra_args="$extra_args --cache-gate-ratio $CACHE_GATE_RATIO"
fi
if [ -n "$OFFLOAD_MODE" ]; then
extra_args="$extra_args --offload-mode $OFFLOAD_MODE"
fi
if [ "$MODE" = "elastic" ]; then
local bp_list=""
for i in $(seq 0 $((N_INSTANCES - 1))); do