60 lines
1.7 KiB
Python
60 lines
1.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Run Frontier's linear profiler against the vLLM 0.20 RoPE API.
|
|
|
|
Frontier passes the pre-v0.20 ``get_rope`` arguments separately, while vLLM
|
|
0.20 carries the same values in ``rope_parameters``. Keep both repositories
|
|
unchanged and adapt only that call boundary for the profiling experiment.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
from typing import Any
|
|
|
|
import torch
|
|
from vllm.model_executor.layers.rotary_embedding import get_rope as vllm_get_rope
|
|
|
|
from frontier.profiling.common.layers import rotary_embedding as frontier_rope
|
|
|
|
|
|
def _vllm020_get_rope_adapter(
|
|
*,
|
|
head_size: int,
|
|
rotary_dim: int,
|
|
max_position: int,
|
|
base: int | float,
|
|
is_neox_style: bool,
|
|
rope_scaling: dict[str, Any] | None,
|
|
dtype: torch.dtype | None = None,
|
|
) -> Any:
|
|
rope_parameters = dict(rope_scaling or {})
|
|
rope_parameters["rope_theta"] = base
|
|
rope_parameters["rope_dim"] = rotary_dim
|
|
return vllm_get_rope(
|
|
head_size=head_size,
|
|
max_position=max_position,
|
|
is_neox_style=is_neox_style,
|
|
rope_parameters=rope_parameters,
|
|
dtype=dtype,
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
parameters = inspect.signature(vllm_get_rope).parameters
|
|
if "rope_parameters" not in parameters or "rotary_dim" in parameters:
|
|
raise RuntimeError(
|
|
"Expected the vLLM 0.20 get_rope API with rope_parameters; "
|
|
f"found {inspect.signature(vllm_get_rope)}"
|
|
)
|
|
|
|
frontier_rope._VLLM_GET_ROPE = _vllm020_get_rope_adapter
|
|
frontier_rope._VLLM_GET_ROPE_IMPORT_ERROR = None
|
|
|
|
from frontier.profiling.linear_op.main import main as frontier_main
|
|
|
|
frontier_main()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|