Gahow Wang
445e491123
third_party/vllm/ now tracked in git for direct patch management.
Based on vLLM v0.18.1 release with one patch applied:
vllm/v1/core/sched/scheduler.py:
Replace fatal assert with graceful skip when KV transfer callback
arrives for an already-aborted request during PD disaggregated serving.
Future vLLM modifications should be made directly in third_party/vllm/
and committed normally. The patches/ directory is kept as documentation
of what changed from upstream.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
37 lines
1.2 KiB
Python
37 lines
1.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import uuid
|
|
|
|
import torch
|
|
|
|
MASK_64_BITS = (1 << 64) - 1
|
|
|
|
|
|
def random_uuid() -> str:
|
|
return f"{uuid.uuid4().int & MASK_64_BITS:016x}" # 16 hex chars
|
|
|
|
|
|
def length_from_prompt_token_ids_or_embeds(
|
|
prompt_token_ids: list[int] | torch.Tensor | None,
|
|
prompt_embeds: torch.Tensor | None,
|
|
) -> int:
|
|
"""Calculate the request length (in number of tokens) give either
|
|
prompt_token_ids or prompt_embeds.
|
|
"""
|
|
prompt_token_len = None if prompt_token_ids is None else len(prompt_token_ids)
|
|
prompt_embeds_len = None if prompt_embeds is None else len(prompt_embeds)
|
|
|
|
if prompt_token_len is None:
|
|
if prompt_embeds_len is None:
|
|
raise ValueError("Neither prompt_token_ids nor prompt_embeds were defined.")
|
|
return prompt_embeds_len
|
|
else:
|
|
if prompt_embeds_len is not None and prompt_embeds_len != prompt_token_len:
|
|
raise ValueError(
|
|
"Prompt token ids and prompt embeds had different lengths"
|
|
f" prompt_token_ids={prompt_token_len}"
|
|
f" prompt_embeds={prompt_embeds_len}"
|
|
)
|
|
return prompt_token_len
|