Gahow Wang
445e491123
third_party/vllm/ now tracked in git for direct patch management.
Based on vLLM v0.18.1 release with one patch applied:
vllm/v1/core/sched/scheduler.py:
Replace fatal assert with graceful skip when KV transfer callback
arrives for an already-aborted request during PD disaggregated serving.
Future vLLM modifications should be made directly in third_party/vllm/
and committed normally. The patches/ directory is kept as documentation
of what changed from upstream.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
41 lines
989 B
Python
41 lines
989 B
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import pytest
|
|
from typing_extensions import LiteralString
|
|
|
|
from ..utils import compare_two_settings, create_new_process_for_each_test
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"PP_SIZE, MODEL_NAME",
|
|
[
|
|
(2, "JackFram/llama-160m"),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"ATTN_BACKEND",
|
|
[
|
|
"FLASH_ATTN",
|
|
],
|
|
)
|
|
@create_new_process_for_each_test()
|
|
def test_pp_cudagraph(
|
|
PP_SIZE: int,
|
|
MODEL_NAME: str,
|
|
ATTN_BACKEND: LiteralString,
|
|
):
|
|
cudagraph_args = [
|
|
# use half precision for speed and memory savings in CI environment
|
|
"--dtype",
|
|
"float16",
|
|
"--pipeline-parallel-size",
|
|
str(PP_SIZE),
|
|
"--distributed-executor-backend",
|
|
"mp",
|
|
f"--attention-backend={ATTN_BACKEND}",
|
|
]
|
|
|
|
eager_args = cudagraph_args + ["--enforce-eager"]
|
|
|
|
compare_two_settings(MODEL_NAME, eager_args, cudagraph_args)
|