Extend serving rotary cache for code context

2026-07-24 01:14:15 +08:00
parent 766f09d3ed
commit da480a8761
2 changed files with 4 additions and 0 deletions
--- a/runs/frontier-code-trace-v0/run_max_model_len_smoke.sh
+++ b/runs/frontier-code-trace-v0/run_max_model_len_smoke.sh
@@ -80,6 +80,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
  --served-model-name "${SERVED_MODEL}" \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.92 \
+  --hf-overrides "{\"max_position_embeddings\":${MAX_MODEL_LEN}}" \
  --max-model-len "${MAX_MODEL_LEN}" \
  --max-num-batched-tokens 8192 \
  --max-num-seqs 16 \