Add vLLM v0.18.1 source tree with KV transfer abort fix
third_party/vllm/ now tracked in git for direct patch management.
Based on vLLM v0.18.1 release with one patch applied:
vllm/v1/core/sched/scheduler.py:
Replace fatal assert with graceful skip when KV transfer callback
arrives for an already-aborted request during PD disaggregated serving.
Future vLLM modifications should be made directly in third_party/vllm/
and committed normally. The patches/ directory is kept as documentation
of what changed from upstream.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
24
third_party/vllm/examples/online_serving/sagemaker-entrypoint.sh
vendored
Normal file
24
third_party/vllm/examples/online_serving/sagemaker-entrypoint.sh
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Define the prefix for environment variables to look for
|
||||
PREFIX="SM_VLLM_"
|
||||
ARG_PREFIX="--"
|
||||
|
||||
# Initialize an array for storing the arguments
|
||||
# port 8080 required by sagemaker, https://docs.aws.amazon.com/sagemaker/latest/dg/your-algorithms-inference-code.html#your-algorithms-inference-code-container-response
|
||||
ARGS=(--port 8080)
|
||||
|
||||
# Loop through all environment variables
|
||||
while IFS='=' read -r key value; do
|
||||
# Remove the prefix from the key, convert to lowercase, and replace underscores with dashes
|
||||
arg_name=$(echo "${key#"${PREFIX}"}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
|
||||
|
||||
# Add the argument name and value to the ARGS array
|
||||
ARGS+=("${ARG_PREFIX}${arg_name}")
|
||||
if [ -n "$value" ]; then
|
||||
ARGS+=("$value")
|
||||
fi
|
||||
done < <(env | grep "^${PREFIX}")
|
||||
|
||||
# Pass the collected arguments to the main entrypoint
|
||||
exec standard-supervisor vllm serve "${ARGS[@]}"
|
||||
Reference in New Issue
Block a user