Add vLLM v0.18.1 source tree with KV transfer abort fix
third_party/vllm/ now tracked in git for direct patch management.
Based on vLLM v0.18.1 release with one patch applied:
vllm/v1/core/sched/scheduler.py:
Replace fatal assert with graceful skip when KV transfer callback
arrives for an already-aborted request during PD disaggregated serving.
Future vLLM modifications should be made directly in third_party/vllm/
and committed normally. The patches/ directory is kept as documentation
of what changed from upstream.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
312
third_party/vllm/tests/lora/test_qwenvl.py
vendored
Normal file
312
third_party/vllm/tests/lora/test_qwenvl.py
vendored
Normal file
@@ -0,0 +1,312 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
|
||||
from packaging.version import Version
|
||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||
|
||||
import vllm
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
model_path: str
|
||||
lora_path: str
|
||||
max_num_seqs: int = 2
|
||||
max_loras: int = 2
|
||||
max_lora_rank: int = 32
|
||||
enable_tower_connector_lora: bool = False
|
||||
max_model_len: int = 8192
|
||||
gpu_memory_utilization: float = 0.85
|
||||
mm_processor_kwargs: dict[str, object] | None = None
|
||||
mm_processor_cache_gb: float = 4
|
||||
|
||||
def __post_init__(self):
|
||||
if self.mm_processor_kwargs is None:
|
||||
# There is a bug in transformers v4 where size is ignored by
|
||||
# `Qwen2VLProcessor.__call__`
|
||||
if Version(TRANSFORMERS_VERSION) < Version("5.2.0"):
|
||||
self.mm_processor_kwargs = {
|
||||
"min_pixels": 28 * 28,
|
||||
"max_pixels": 1280 * 28 * 28,
|
||||
}
|
||||
else:
|
||||
self.mm_processor_kwargs = {
|
||||
"size": {
|
||||
"shortest_edge": 28 * 28,
|
||||
"longest_edge": 1280 * 28 * 28,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class Qwen2VLTester:
|
||||
"""Test helper for Qwen2 VL models with LoRA"""
|
||||
|
||||
PROMPT_TEMPLATE = (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
|
||||
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
|
||||
"What is in the image?<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
def __init__(self, config: TestConfig):
|
||||
self.config = config
|
||||
self.llm = self._initialize_llm()
|
||||
|
||||
def _initialize_llm(self) -> vllm.LLM:
|
||||
"""Initialize the LLM with given configuration"""
|
||||
return vllm.LLM(
|
||||
model=self.config.model_path,
|
||||
max_num_seqs=self.config.max_num_seqs,
|
||||
enable_lora=True,
|
||||
max_loras=self.config.max_loras,
|
||||
max_lora_rank=self.config.max_lora_rank,
|
||||
enable_tower_connector_lora=self.config.enable_tower_connector_lora,
|
||||
trust_remote_code=True,
|
||||
gpu_memory_utilization=self.config.gpu_memory_utilization,
|
||||
mm_processor_kwargs=self.config.mm_processor_kwargs,
|
||||
mm_processor_cache_gb=self.config.mm_processor_cache_gb,
|
||||
max_model_len=self.config.max_model_len,
|
||||
)
|
||||
|
||||
def run_test(
|
||||
self,
|
||||
images: list[ImageAsset],
|
||||
expected_outputs: list[str],
|
||||
lora_id: int | None = None,
|
||||
lora_name: str | None = None,
|
||||
temperature: float = 0,
|
||||
max_tokens: int = 5,
|
||||
):
|
||||
sampling_params = vllm.SamplingParams(
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
inputs = [
|
||||
{
|
||||
"prompt": self.PROMPT_TEMPLATE,
|
||||
"multi_modal_data": {"image": asset.pil_image},
|
||||
}
|
||||
for asset in images
|
||||
]
|
||||
|
||||
lora_request = LoRARequest(
|
||||
lora_name if lora_name else str(lora_id), lora_id, self.config.lora_path
|
||||
)
|
||||
outputs = self.llm.generate(inputs, sampling_params, lora_request=lora_request)
|
||||
generated_texts = [output.outputs[0].text.strip() for output in outputs]
|
||||
# Validate outputs
|
||||
for generated, expected in zip(generated_texts, expected_outputs):
|
||||
assert expected.startswith(generated), (
|
||||
f"Generated text {generated} doesn't match expected pattern {expected}"
|
||||
)
|
||||
|
||||
def run_beam_search_test(
|
||||
self,
|
||||
images: list[ImageAsset],
|
||||
expected_outputs: list[list[str]],
|
||||
lora_id: int | None = None,
|
||||
temperature: float = 0,
|
||||
beam_width: int = 2,
|
||||
max_tokens: int = 5,
|
||||
):
|
||||
beam_search_params = BeamSearchParams(
|
||||
beam_width=beam_width, max_tokens=max_tokens, temperature=temperature
|
||||
)
|
||||
|
||||
inputs = [
|
||||
{
|
||||
"prompt": self.PROMPT_TEMPLATE,
|
||||
"multi_modal_data": {"image": asset.pil_image},
|
||||
}
|
||||
for asset in images
|
||||
]
|
||||
|
||||
lora_request = LoRARequest(str(lora_id), lora_id, self.config.lora_path)
|
||||
outputs = self.llm.beam_search(
|
||||
inputs, beam_search_params, lora_request=lora_request
|
||||
)
|
||||
|
||||
for output_obj, expected_texts in zip(outputs, expected_outputs):
|
||||
output_texts = [seq.text for seq in output_obj.sequences]
|
||||
|
||||
for output_text, expected_text in zip(output_texts, expected_texts):
|
||||
# NOTE beam search .text contains the whole text including inputs
|
||||
assert output_text.endswith(expected_text), (
|
||||
f"Generated {output_text} does not match expected {expected_text}"
|
||||
)
|
||||
|
||||
|
||||
TEST_IMAGES = [
|
||||
ImageAsset("stop_sign"),
|
||||
ImageAsset("cherry_blossom"),
|
||||
]
|
||||
|
||||
EXPECTED_OUTPUTS = [
|
||||
"A red stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
|
||||
"A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501
|
||||
]
|
||||
|
||||
EXPECTED_OUTPUTS_LANGUAGE = [
|
||||
"A stop sign is shown in an Asian city, with buildings and a car in the "
|
||||
"background.",
|
||||
"The Tokyo Skytree can be seen behind the pink blossoms of the cherry trees.",
|
||||
]
|
||||
|
||||
EXPECTED_OUTPUTS_VISION = [
|
||||
"A stop sign in front of oriental buildings.",
|
||||
"A tree with pink flowers in front of it and a blue sky behind the flowers.",
|
||||
]
|
||||
|
||||
EXPECTED_OUTPUTS_VISION_NO_CONNECTOR = [
|
||||
"A stop sign is located on the street of a Chinese neighborhood.",
|
||||
"A closeup shot of the Tokyo Skytree with pink flowers in the foreground.",
|
||||
]
|
||||
|
||||
EXPECTED_BEAM_SEARCH_OUTPUTS = [
|
||||
[
|
||||
"A majestic skyscraper stands",
|
||||
"A majestic tower stands tall",
|
||||
],
|
||||
]
|
||||
|
||||
QWEN2VL_MODEL_PATH = "Qwen/Qwen2-VL-2B-Instruct"
|
||||
QWEN25VL_MODEL_PATH = "Qwen/Qwen2.5-VL-3B-Instruct"
|
||||
QWEN3VL_MODEL_PATH = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
|
||||
|
||||
def test_qwen2vl_lora(qwen2vl_lora_files):
|
||||
"""Test Qwen 2.0 VL model with LoRA"""
|
||||
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
|
||||
tester = Qwen2VLTester(config)
|
||||
|
||||
# Test with different LoRA IDs
|
||||
for lora_id in [1, 2]:
|
||||
tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
|
||||
|
||||
|
||||
def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
|
||||
"""Test Qwen 2.0 VL model with LoRA through beam search."""
|
||||
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
|
||||
tester = Qwen2VLTester(config)
|
||||
|
||||
# Test with different LoRA IDs
|
||||
for lora_id in [1, 2]:
|
||||
# NOTE currently, we only test cherry blossom since stop sign
|
||||
# output is slightly different for v1; - the root cause is likely
|
||||
# independent of the intent of this test, which is to ensure beam
|
||||
# search passes through lora through correctly.
|
||||
tester.run_beam_search_test(
|
||||
[ImageAsset("cherry_blossom")],
|
||||
expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
|
||||
lora_id=lora_id,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen25vl_lora(qwen25vl_lora_files):
|
||||
"""Test Qwen 2.5 VL model with LoRA"""
|
||||
config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files)
|
||||
tester = Qwen2VLTester(config)
|
||||
|
||||
# Test with different LoRA IDs
|
||||
for lora_id in [1, 2]:
|
||||
tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
|
||||
|
||||
|
||||
def test_qwen25vl_vision_lora(qwen25vl_vision_lora_files):
|
||||
config = TestConfig(
|
||||
model_path=QWEN25VL_MODEL_PATH,
|
||||
lora_path=qwen25vl_vision_lora_files,
|
||||
# Currently, tower_connector_lora is incompatible with
|
||||
# the multi-modal processor cache.
|
||||
# TODO: Remove this restriction
|
||||
mm_processor_cache_gb=0,
|
||||
enable_tower_connector_lora=True,
|
||||
)
|
||||
tester = Qwen2VLTester(config)
|
||||
for lora_id in [1, 2]:
|
||||
tester.run_test(
|
||||
TEST_IMAGES,
|
||||
expected_outputs=EXPECTED_OUTPUTS,
|
||||
lora_id=lora_id,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
|
||||
config = TestConfig(
|
||||
model_path=QWEN3VL_MODEL_PATH,
|
||||
lora_path=qwen3vl_vision_lora_files,
|
||||
# Currently, tower_connector_lora is incompatible with
|
||||
# the multi-modal processor cache.
|
||||
# TODO: Remove this restriction
|
||||
mm_processor_cache_gb=0,
|
||||
enable_tower_connector_lora=True,
|
||||
)
|
||||
tester = Qwen2VLTester(config)
|
||||
for lora_id in [1, 2]:
|
||||
tester.run_test(
|
||||
TEST_IMAGES,
|
||||
expected_outputs=EXPECTED_OUTPUTS,
|
||||
lora_id=lora_id,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen2vl_multiple_lora_types(
|
||||
qwen2vl_language_lora_files,
|
||||
qwen2vl_vision_tower_connector_lora_files,
|
||||
qwen2vl_vision_tower_lora_files,
|
||||
):
|
||||
"""
|
||||
Test multiple LoRA adapter types (language, vision tower + connector,
|
||||
vision tower only) using the same LLM instance to verify mm_encoder_cache
|
||||
behavior with different LoRA requests.
|
||||
|
||||
By reusing the same LLM instance across different LoRA requests, we ensure that
|
||||
the multimodal encoder cache correctly manages state transitions between
|
||||
language-only and vision-enabled LoRA adapters.
|
||||
"""
|
||||
config = TestConfig(
|
||||
model_path=QWEN2VL_MODEL_PATH,
|
||||
# We'll override the lora_path for each specific test, but need to provide
|
||||
# an initial path for initialization
|
||||
lora_path=qwen2vl_language_lora_files,
|
||||
# Currently, tower_connector_lora is incompatible with
|
||||
# the multi-modal processor cache.
|
||||
# TODO: Remove this restriction
|
||||
mm_processor_cache_gb=0,
|
||||
enable_tower_connector_lora=True,
|
||||
)
|
||||
tester = Qwen2VLTester(config)
|
||||
|
||||
# Test 1: Language-only LoRA adapter
|
||||
tester.config.lora_path = qwen2vl_language_lora_files
|
||||
for lora_id in [1, 2]:
|
||||
tester.run_test(
|
||||
TEST_IMAGES,
|
||||
expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
|
||||
lora_id=lora_id,
|
||||
lora_name="language_only",
|
||||
)
|
||||
|
||||
# Test 2: Vision tower + connector LoRA adapter
|
||||
tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
|
||||
for lora_id in [3, 4]:
|
||||
tester.run_test(
|
||||
TEST_IMAGES,
|
||||
expected_outputs=EXPECTED_OUTPUTS_VISION,
|
||||
lora_id=lora_id,
|
||||
lora_name="vision_tower_connector",
|
||||
)
|
||||
|
||||
# Test 3: Vision tower only LoRA adapter (no connector)
|
||||
tester.config.lora_path = qwen2vl_vision_tower_lora_files
|
||||
for lora_id in [5, 6]:
|
||||
tester.run_test(
|
||||
TEST_IMAGES,
|
||||
expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
|
||||
lora_id=lora_id,
|
||||
lora_name="vision_tower",
|
||||
)
|
||||
Reference in New Issue
Block a user