Add --offload-mode switch for ablation (direct_read vs cached_prefill)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -37,6 +37,7 @@ OVERLOAD_FACTOR_ARG=""
|
||||
MAX_BATCHED_TOKENS=""
|
||||
MAX_OFFLOAD_INFLIGHT=""
|
||||
CACHE_GATE_RATIO=""
|
||||
OFFLOAD_MODE=""
|
||||
|
||||
# Parse args
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -53,6 +54,7 @@ while [[ $# -gt 0 ]]; do
|
||||
--max-batched-tokens) MAX_BATCHED_TOKENS="$2"; shift 2 ;;
|
||||
--max-offload-inflight) MAX_OFFLOAD_INFLIGHT="$2"; shift 2 ;;
|
||||
--cache-gate-ratio) CACHE_GATE_RATIO="$2"; shift 2 ;;
|
||||
--offload-mode) OFFLOAD_MODE="$2"; shift 2 ;;
|
||||
*) echo "Unknown: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
@@ -218,6 +220,9 @@ launch_proxy() {
|
||||
if [ -n "$CACHE_GATE_RATIO" ]; then
|
||||
extra_args="$extra_args --cache-gate-ratio $CACHE_GATE_RATIO"
|
||||
fi
|
||||
if [ -n "$OFFLOAD_MODE" ]; then
|
||||
extra_args="$extra_args --offload-mode $OFFLOAD_MODE"
|
||||
fi
|
||||
if [ "$MODE" = "elastic" ]; then
|
||||
local bp_list=""
|
||||
for i in $(seq 0 $((N_INSTANCES - 1))); do
|
||||
|
||||
Reference in New Issue
Block a user