Gahow Wang
445e491123
third_party/vllm/ now tracked in git for direct patch management.
Based on vLLM v0.18.1 release with one patch applied:
vllm/v1/core/sched/scheduler.py:
Replace fatal assert with graceful skip when KV transfer callback
arrives for an already-aborted request during PD disaggregated serving.
Future vLLM modifications should be made directly in third_party/vllm/
and committed normally. The patches/ directory is kept as documentation
of what changed from upstream.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
97 lines
4.3 KiB
Django/Jinja
97 lines
4.3 KiB
Django/Jinja
{% if not add_generation_prompt is defined %}
|
||
{% set add_generation_prompt = false %}
|
||
{% endif %}
|
||
|
||
{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}
|
||
|
||
{%- for message in messages %}
|
||
{%- if message['role'] == 'system' %}
|
||
{%- if ns.is_first_sp %}
|
||
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
|
||
{% set ns.is_first_sp = false %}
|
||
{%- else %}
|
||
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
|
||
{%- endif %}
|
||
{%- endif %}
|
||
{%- endfor %}
|
||
|
||
{{ bos_token }}
|
||
{{ ns.system_prompt }}
|
||
{%- if tools %}
|
||
{{"\n\n# Tools\n\nYou may call one or more functions to assist with the user query." }}
|
||
{%- for tool in tools %}
|
||
{{- "\n" }}
|
||
{{- tool | tojson }}
|
||
{%- endfor %}
|
||
{{"\n</tools>\n\n"}}
|
||
|
||
{{"For function call returns, you should first print <｜tool▁calls▁begin｜>"}}
|
||
|
||
{{"For each function call, you should return object like:\n" }}
|
||
{{"<｜tool▁call▁begin｜>function<｜tool▁sep｜><function_name>\n```json\n<function_arguments_in_json_format>\n```<｜tool▁call▁end｜>"}}
|
||
|
||
{{"At the end of function call returns, you should print <｜tool▁calls▁end｜><｜end▁of▁sentence｜>"}}
|
||
{%- endif %}
|
||
|
||
{%- for message in messages %}
|
||
{%- if message['role'] == 'user' %}
|
||
{%- set ns.is_tool = false -%}
|
||
{%- set ns.is_first = false -%}
|
||
{%- set ns.is_last_user = true -%}
|
||
{{'<｜User｜>' + message['content'] + '<｜Assistant｜>'}}
|
||
{%- endif %}
|
||
|
||
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
|
||
{%- set ns.is_last_user = false -%}
|
||
{%- if ns.is_tool %}
|
||
{{'<｜tool▁outputs▁end｜>'}}
|
||
{%- endif %}
|
||
{%- set ns.is_first = false %}
|
||
{%- set ns.is_tool = false -%}
|
||
{%- set ns.is_output_first = true %}
|
||
|
||
{%- for tool in message['tool_calls'] %}
|
||
{%- if not ns.is_first %}
|
||
{%- if message['content'] is none %}
|
||
{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}}
|
||
{%- else %}
|
||
{{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}}
|
||
{%- endif %}
|
||
{%- set ns.is_first = true -%}
|
||
{%- else %}
|
||
{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}}
|
||
{%- endif %}
|
||
{%- endfor %}
|
||
{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
|
||
{%- endif %}
|
||
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}
|
||
{%- set ns.is_last_user = false -%}
|
||
{%- if ns.is_tool %}
|
||
{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}
|
||
{%- set ns.is_tool = false -%}
|
||
{%- else %}
|
||
{% set content = message['content'] %}
|
||
{{content + '<｜end▁of▁sentence｜>'}}
|
||
{%- endif %}
|
||
{%- endif %}
|
||
|
||
{%- if message['role'] == 'tool' %}
|
||
{%- set ns.is_last_user = false -%}
|
||
{%- set ns.is_tool = true -%}
|
||
{%- if ns.is_output_first %}
|
||
{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
|
||
{%- set ns.is_output_first = false %}
|
||
{%- else %}
|
||
{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
|
||
{%- endif %}
|
||
{%- endif %}
|
||
{%- endfor -%}
|
||
|
||
{% if ns.is_tool %}
|
||
{{'<｜tool▁outputs▁end｜>'}}
|
||
{% endif %}
|
||
|
||
{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
|
||
{{'<｜Assistant｜>'}}
|
||
{% endif %}
|