gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extend serving rotary cache for code context

Browse Source
This commit is contained in:
Gahow Wang
2026-07-24 01:14:15 +08:00
parent 766f09d3ed
commit da480a8761
2 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
runs/frontier-code-trace-v0/run_max_model_len_smoke.sh
View File

@@ -80,6 +80,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
--served-model-name "${SERVED_MODEL}" \ --served-model-name "${SERVED_MODEL}" \
--tensor-parallel-size 4 \ --tensor-parallel-size 4 \
--gpu-memory-utilization 0.92 \ --gpu-memory-utilization 0.92 \
--hf-overrides "{\"max_position_embeddings\":${MAX_MODEL_LEN}}" \
--max-model-len "${MAX_MODEL_LEN}" \ --max-model-len "${MAX_MODEL_LEN}" \
--max-num-batched-tokens 8192 \ --max-num-batched-tokens 8192 \
--max-num-seqs 16 \ --max-num-seqs 16 \

3
runs/frontier-s3-real-v0/run_full_real.sh
View File

@@ -152,13 +152,16 @@ PY
export TOKENIZERS_PARALLELISM=false VLLM_USE_V1=1 TORCH_CUDA_ARCH_LIST=9.0 PREFIX_CACHING=true export TOKENIZERS_PARALLELISM=false VLLM_USE_V1=1 TORCH_CUDA_ARCH_LIST=9.0 PREFIX_CACHING=true
export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 FLASHINFER_WORKSPACE_BASE export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 FLASHINFER_WORKSPACE_BASE
HF_OVERRIDE_ARGS=()
if [[ "${ALLOW_LONG_CONTEXT_SERVER}" == "true" ]]; then if [[ "${ALLOW_LONG_CONTEXT_SERVER}" == "true" ]]; then
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
HF_OVERRIDE_ARGS=(--hf-overrides "{\"max_position_embeddings\":${MAX_MODEL_LEN}}")
fi fi
ulimit -n 65536 ulimit -n 65536
setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
--host 127.0.0.1 --port "${PORT}" --served-model-name "${SERVED_MODEL}" \ --host 127.0.0.1 --port "${PORT}" --served-model-name "${SERVED_MODEL}" \
--tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \ --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \
"${HF_OVERRIDE_ARGS[@]}" \
--max-model-len "${MAX_MODEL_LEN}" --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ --max-model-len "${MAX_MODEL_LEN}" --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
--enable-prefix-caching --block-size 16 --enable-chunked-prefill --no-enable-log-requests \ --enable-prefix-caching --block-size 16 --enable-chunked-prefill --no-enable-log-requests \
--enable-logging-iteration-details \ --enable-logging-iteration-details \