chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/e2e_test/fixtures/__init__.py

@@ -0,0 +1,51 @@
+"""Fixtures for E2E tests.
+
+This package contains modular pytest fixtures split by responsibility:
+- hooks.py: Pytest collection hooks and marker registration
+- pool.py: Model pool fixtures (session-scoped worker management)
+- setup_backend.py: Backend setup fixtures (class/function-scoped)
+- markers.py: Helper utilities for marker extraction
+
+Legacy modules (to be removed during e2e_response_api migration):
+- ports.py: Use infra.get_open_port() instead
+- router_manager.py: Use infra.Gateway instead
+"""
+
+# Pytest hooks (imported by conftest.py via pytest_plugins)
+from .hooks import (
+    get_pool_requirements,
+    is_parallel_execution,
+    pytest_collection_finish,
+    pytest_collection_modifyitems,
+    pytest_configure,
+    pytest_runtest_setup,
+    validate_gpu_requirements,
+)
+
+# Marker helpers
+from .markers import get_marker_kwargs, get_marker_value
+
+# Fixtures (imported by conftest.py)
+from .pool import model_base_url, model_client, model_pool
+from .setup_backend import backend_router, setup_backend
+
+__all__ = [
+    # Hooks
+    "pytest_collection_modifyitems",
+    "pytest_collection_finish",
+    "pytest_configure",
+    "pytest_runtest_setup",
+    "get_pool_requirements",
+    "validate_gpu_requirements",
+    "is_parallel_execution",
+    # Pool fixtures
+    "model_pool",
+    "model_client",
+    "model_base_url",
+    # Backend fixtures
+    "setup_backend",
+    "backend_router",
+    # Marker helpers
+    "get_marker_value",
+    "get_marker_kwargs",
+]

third_party/sglang/sgl-model-gateway/e2e_test/fixtures/hooks.py

@@ -0,0 +1,430 @@
+"""Pytest hooks for E2E test collection and validation.
+
+This module handles:
+- Test collection: Scanning markers to determine required workers
+- GPU validation: Ensuring sufficient GPUs for test requirements
+- Marker registration: Defining custom pytest markers
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from infra import ConnectionMode, WorkerIdentity, WorkerType
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Test collection state
+# ---------------------------------------------------------------------------
+
+# Track max worker counts: (model_id, mode, worker_type) -> max_count
+_worker_counts: dict[tuple["ConnectionMode", "WorkerType"], int] = {}
+
+# Track first-seen order to preserve test collection order
+_first_seen_order: list[tuple[str, "ConnectionMode", "WorkerType"]] = []
+
+# Track max GPU requirement for any single test (for validation)
+_max_test_gpu_requirement: int = 0
+_max_test_name: str = ""
+
+_needs_default_model: bool = False
+
+
+def reset_collection_state() -> None:
+    """Reset collection state (useful for testing)."""
+    global _worker_counts, _first_seen_order
+    global _max_test_gpu_requirement, _max_test_name, _needs_default_model
+    _worker_counts = {}
+    _first_seen_order = []
+    _max_test_gpu_requirement = 0
+    _max_test_name = ""
+    _needs_default_model = False
+
+
+def get_worker_counts() -> dict:
+    """Get the worker counts dictionary."""
+    return _worker_counts
+
+
+def get_first_seen_order() -> list:
+    """Get the first-seen order list."""
+    return _first_seen_order
+
+
+def get_max_gpu_requirement() -> tuple[int, str]:
+    """Get the max GPU requirement and test name."""
+    return _max_test_gpu_requirement, _max_test_name
+
+
+def needs_default_model() -> bool:
+    """Check if any test needs the default model."""
+    return _needs_default_model
+
+
+# ---------------------------------------------------------------------------
+# Test collection hook
+# ---------------------------------------------------------------------------
+
+
+def pytest_collection_modifyitems(
+    session: pytest.Session,
+    config: pytest.Config,
+    items: list[pytest.Item],
+) -> None:
+    """Scan collected tests to determine required workers.
+
+    This runs after test collection but before tests execute.
+    It extracts worker requirements from markers in test collection order,
+    tracking the max count needed for each (model, mode, worker_type) combination.
+    """
+    global _worker_counts, _first_seen_order, _needs_default_model
+    global _max_test_gpu_requirement, _max_test_name
+
+    from infra import (
+        DEFAULT_MODEL,
+        LOG_SEPARATOR_WIDTH,
+        MODEL_SPECS,
+        PARAM_MODEL,
+        PARAM_SETUP_BACKEND,
+        ConnectionMode,
+        WorkerType,
+    )
+
+    def track_worker(
+        model_id: str, mode: ConnectionMode, worker_type: WorkerType, count: int
+    ) -> None:
+        """Track a worker requirement, updating max count if needed."""
+        key = (model_id, mode, worker_type)
+        if key not in _worker_counts:
+            _first_seen_order.append(key)
+            _worker_counts[key] = count
+        else:
+            _worker_counts[key] = max(_worker_counts[key], count)
+
+    def calculate_test_gpus(
+        model_id: str, prefill: int, decode: int, regular: int
+    ) -> int:
+        """Calculate GPU requirement for a single test."""
+        if model_id not in MODEL_SPECS:
+            return 0
+        tp = MODEL_SPECS[model_id].get("tp", 1)
+        return tp * (prefill + decode + regular)
+
+    for item in items:
+        # Extract model from marker or use default
+        # First check the class directly (handles inheritance correctly)
+        model_id = None
+        if hasattr(item, "cls") and item.cls is not None:
+            for marker in (
+                item.cls.pytestmark if hasattr(item.cls, "pytestmark") else []
+            ):
+                if marker.name == PARAM_MODEL and marker.args:
+                    model_id = marker.args[0]
+                    break
+        # Fall back to get_closest_marker for method-level markers
+        if model_id is None:
+            model_marker = item.get_closest_marker(PARAM_MODEL)
+            model_id = (
+                model_marker.args[0] if model_marker and model_marker.args else None
+            )
+
+        # Check parametrize for model
+        if model_id is None:
+            for marker in item.iter_markers("parametrize"):
+                if marker.args and len(marker.args) >= 2:
+                    param_name = marker.args[0]
+                    if param_name == PARAM_MODEL or PARAM_MODEL in param_name:
+                        param_values = marker.args[1]
+                        if isinstance(param_values, (list, tuple)) and param_values:
+                            model_id = param_values[0]
+                        break
+
+        # Extract backends from parametrize
+        backends: list[str] = []
+        for marker in item.iter_markers("parametrize"):
+            if marker.args and len(marker.args) >= 2:
+                param_name = marker.args[0]
+                param_values = marker.args[1]
+                if param_name == PARAM_SETUP_BACKEND:
+                    if isinstance(param_values, (list, tuple)):
+                        backends.extend(param_values)
+
+        # Check for workers marker
+        workers_marker = item.get_closest_marker("workers")
+        prefill_count = 0
+        decode_count = 0
+        regular_count = 1
+        if workers_marker:
+            prefill_count = workers_marker.kwargs.get("prefill") or 0
+            decode_count = workers_marker.kwargs.get("decode") or 0
+            regular_count = workers_marker.kwargs.get("count") or 1
+
+        # Track if this test needs default model
+        is_e2e = item.get_closest_marker("e2e") is not None
+        if model_id is None and is_e2e:
+            _needs_default_model = True
+            model_id = DEFAULT_MODEL
+
+        # Track worker requirements
+        test_gpus = 0
+        if model_id and backends:
+            for backend in backends:
+                if backend == "pd":
+                    mode = ConnectionMode.HTTP
+                    p_count = prefill_count if prefill_count > 0 else 1
+                    d_count = decode_count if decode_count > 0 else 1
+                    track_worker(model_id, mode, WorkerType.PREFILL, p_count)
+                    track_worker(model_id, mode, WorkerType.DECODE, d_count)
+                    test_gpus = max(
+                        test_gpus, calculate_test_gpus(model_id, p_count, d_count, 0)
+                    )
+                else:
+                    try:
+                        mode = ConnectionMode(backend)
+                    except ValueError:
+                        continue
+
+                    if prefill_count > 0 or decode_count > 0:
+                        track_worker(model_id, mode, WorkerType.PREFILL, prefill_count)
+                        track_worker(model_id, mode, WorkerType.DECODE, decode_count)
+                        test_gpus = max(
+                            test_gpus,
+                            calculate_test_gpus(
+                                model_id, prefill_count, decode_count, 0
+                            ),
+                        )
+                    else:
+                        track_worker(model_id, mode, WorkerType.REGULAR, regular_count)
+                        test_gpus = max(
+                            test_gpus,
+                            calculate_test_gpus(model_id, 0, 0, regular_count),
+                        )
+
+        elif model_id and is_e2e:
+            track_worker(model_id, ConnectionMode.HTTP, WorkerType.REGULAR, 1)
+            test_gpus = calculate_test_gpus(model_id, 0, 0, 1)
+
+        if test_gpus > _max_test_gpu_requirement:
+            _max_test_gpu_requirement = test_gpus
+            _max_test_name = item.nodeid
+
+    # Log results
+    if _worker_counts:
+        summary = []
+        for key in _first_seen_order:
+            model_id, mode, worker_type = key
+            count = _worker_counts[key]
+            if worker_type == WorkerType.REGULAR:
+                summary.append(f"{model_id}:{mode.value}x{count}")
+            else:
+                summary.append(f"{model_id}:{mode.value}:{worker_type.value}x{count}")
+        logger.info("Scanned worker requirements (in test order): %s", summary)
+        logger.info(
+            "Max GPU requirement for single test: %d (%s)",
+            _max_test_gpu_requirement,
+            _max_test_name,
+        )
+    else:
+        logger.info("Scanned worker requirements: (none)")
+
+
+# ---------------------------------------------------------------------------
+# Pool requirements
+# ---------------------------------------------------------------------------
+
+
+def get_pool_requirements() -> list["WorkerIdentity"]:
+    """Build pool requirements from scanned test markers.
+
+    Returns:
+        List of WorkerIdentity objects to pre-launch.
+    """
+    from infra import DEFAULT_MODEL, ConnectionMode, WorkerIdentity, WorkerType
+
+    # Track which models have PD workers as their first requirement
+    models_with_pd_first: set[str] = set()
+    first_worker_type_per_model: dict[str, WorkerType] = {}
+
+    for model_id, mode, worker_type in _first_seen_order:
+        if model_id not in first_worker_type_per_model:
+            first_worker_type_per_model[model_id] = worker_type
+            if worker_type in (WorkerType.PREFILL, WorkerType.DECODE):
+                models_with_pd_first.add(model_id)
+                logger.info(
+                    "Model %s has PD test first - skipping regular worker pre-launch",
+                    model_id,
+                )
+
+    # Generate individual WorkerIdentity objects in first-seen order
+    requirements: list[WorkerIdentity] = []
+    for model_id, mode, worker_type in _first_seen_order:
+        if model_id in models_with_pd_first and worker_type == WorkerType.REGULAR:
+            continue
+
+        count = _worker_counts.get((model_id, mode, worker_type), 1)
+        for i in range(count):
+            requirements.append(WorkerIdentity(model_id, mode, worker_type, i))
+
+    if not requirements:
+        requirements.append(WorkerIdentity(DEFAULT_MODEL, ConnectionMode.HTTP))
+
+    return requirements
+
+
+# ---------------------------------------------------------------------------
+# GPU validation
+# ---------------------------------------------------------------------------
+
+
+def _count_gpus_without_cuda() -> int:
+    """Count available GPUs without initializing CUDA.
+
+    Uses nvidia-smi to avoid CUDA initialization, which is critical for
+    pytest-parallel compatibility. CUDA cannot be re-initialized after a fork.
+    """
+    import subprocess
+
+    try:
+        result = subprocess.run(
+            ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode == 0:
+            return len([line for line in result.stdout.strip().split("\n") if line])
+    except (subprocess.SubprocessError, FileNotFoundError, OSError):
+        pass
+    return 0
+
+
+def validate_gpu_requirements() -> tuple[int, int]:
+    """Check if there are enough GPUs for any single test.
+
+    Uses nvidia-smi instead of torch.cuda to avoid CUDA initialization,
+    which would break pytest-parallel (CUDA cannot be re-initialized after fork).
+
+    Returns:
+        Tuple of (max_required_gpus, available_gpus).
+    """
+    available_gpus = _count_gpus_without_cuda()
+    return _max_test_gpu_requirement, available_gpus
+
+
+def pytest_collection_finish(session: pytest.Session) -> None:
+    """Validate GPU requirements after test collection."""
+    from infra import ENV_SKIP_MODEL_POOL, LOG_SEPARATOR_WIDTH
+
+    if not _worker_counts:
+        return
+
+    if os.environ.get(ENV_SKIP_MODEL_POOL, "").lower() in ("1", "true", "yes"):
+        return
+
+    max_required, available_gpus = validate_gpu_requirements()
+
+    if max_required > available_gpus:
+        sep = "=" * LOG_SEPARATOR_WIDTH
+        raise pytest.UsageError(
+            f"\n{sep}\n"
+            f"GPU REQUIREMENTS EXCEEDED\n"
+            f"{sep}\n"
+            f"Test '{_max_test_name}' requires {max_required} GPUs\n"
+            f"Available: {available_gpus} GPUs\n"
+            f"\nOptions:\n"
+            f"  1. Run tests that fit: pytest -k 'not {_max_test_name.split('::')[0]}'\n"
+            f"  2. Reduce workers: @pytest.mark.workers(prefill=1, decode=1)\n"
+            f"  3. Skip GPU tests: SKIP_MODEL_POOL=1 pytest\n"
+            f"{sep}"
+        )
+
+    logger.info(
+        "GPU validation passed: max %d required (by %s), %d available",
+        max_required,
+        _max_test_name,
+        available_gpus,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Marker registration
+# ---------------------------------------------------------------------------
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Register custom markers."""
+    config.addinivalue_line(
+        "markers",
+        "model(name): mark test to use a specific model from MODEL_SPECS",
+    )
+    config.addinivalue_line(
+        "markers",
+        "backend(name): mark test to use a specific backend (grpc, http, openai, etc.)",
+    )
+    config.addinivalue_line(
+        "markers",
+        "workers(count=1, prefill=None, decode=None): "
+        "worker configuration - use count for regular workers, "
+        "or prefill/decode for PD disaggregation mode",
+    )
+    config.addinivalue_line(
+        "markers",
+        "gateway(policy='round_robin', timeout=None, extra_args=None): "
+        "gateway/router configuration",
+    )
+    config.addinivalue_line(
+        "markers",
+        "e2e: mark test as an end-to-end test requiring GPU workers",
+    )
+    config.addinivalue_line(
+        "markers",
+        "slow: mark test as slow-running",
+    )
+    config.addinivalue_line(
+        "markers",
+        "thread_unsafe: mark test as incompatible with parallel thread execution",
+    )
+    config.addinivalue_line(
+        "markers",
+        "storage(backend): mark test to use a specific history storage backend "
+        "(memory, oracle). Default is memory.",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Parallel execution support
+# ---------------------------------------------------------------------------
+
+
+def is_parallel_execution(config: pytest.Config) -> bool:
+    """Check if tests are running in parallel mode (pytest-parallel).
+
+    Returns True if --tests-per-worker > 1, indicating concurrent thread execution.
+    """
+    # pytest-parallel adds the 'tests_per_worker' option
+    tests_per_worker = getattr(config.option, "tests_per_worker", None)
+    if tests_per_worker is None:
+        return False
+
+    if tests_per_worker == "auto":
+        return True
+
+    try:
+        return int(tests_per_worker) > 1
+    except (ValueError, TypeError):
+        return False
+
+
+def pytest_runtest_setup(item: pytest.Item) -> None:
+    """Skip thread_unsafe tests when running in parallel mode."""
+    if is_parallel_execution(item.config):
+        marker = item.get_closest_marker("thread_unsafe")
+        if marker:
+            reason = marker.kwargs.get("reason", "Test is not thread-safe")
+            pytest.skip(f"Skipping in parallel mode: {reason}")

third_party/sglang/sgl-model-gateway/e2e_test/fixtures/markers.py

@@ -0,0 +1,57 @@
+"""Marker helper utilities for E2E tests.
+
+This module provides helper functions for extracting values from pytest markers.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+
+def get_marker_value(
+    request: pytest.FixtureRequest,
+    marker_name: str,
+    arg_index: int = 0,
+    default: Any = None,
+) -> Any:
+    """Get a value from a pytest marker.
+
+    Args:
+        request: The pytest fixture request.
+        marker_name: Name of the marker to look for.
+        arg_index: Index of positional argument to extract.
+        default: Default value if marker not found.
+
+    Returns:
+        The marker argument value or default.
+    """
+    marker = request.node.get_closest_marker(marker_name)
+    if marker is None:
+        return default
+    if marker.args and len(marker.args) > arg_index:
+        return marker.args[arg_index]
+    return default
+
+
+def get_marker_kwargs(
+    request: pytest.FixtureRequest,
+    marker_name: str,
+    defaults: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Get keyword arguments from a pytest marker.
+
+    Args:
+        request: The pytest fixture request.
+        marker_name: Name of the marker to look for.
+        defaults: Default values if marker not found or missing kwargs.
+
+    Returns:
+        Dict of keyword arguments merged with defaults.
+    """
+    result = dict(defaults) if defaults else {}
+    marker = request.node.get_closest_marker(marker_name)
+    if marker is not None:
+        result.update(marker.kwargs)
+    return result

third_party/sglang/sgl-model-gateway/e2e_test/fixtures/pool.py

@@ -0,0 +1,242 @@
+"""Model pool fixtures for E2E tests.
+
+This module provides session-scoped fixtures for managing SGLang worker processes.
+Workers are expensive to start (~30-60s each), so they're kept running across tests.
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+import threading
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from infra import ModelPool
+
+from .hooks import get_pool_requirements
+
+logger = logging.getLogger(__name__)
+
+# Global model pool instance with thread-safe initialization
+_model_pool: "ModelPool | None" = None
+_model_pool_lock = threading.Lock()
+_shutdown_registered = False
+
+
+def _shutdown_model_pool() -> None:
+    """Shutdown the global model pool at process exit.
+
+    This is registered with atexit to ensure cleanup happens after all tests
+    complete, which is important for pytest-parallel where multiple threads
+    share the session-scoped fixture.
+    """
+    global _model_pool
+    if _model_pool is not None:
+        logger.info("Shutting down model pool at process exit")
+        _model_pool.shutdown()
+        _model_pool = None
+
+
+@pytest.fixture(scope="session")
+def model_pool(request: pytest.FixtureRequest) -> "ModelPool":
+    """Session-scoped fixture that manages SGLang worker processes.
+
+    Workers (sglang.launch_server) are expensive to start (~30-60s each due to
+    model loading). This fixture starts them ONCE per session and keeps them
+    running across all tests. The setup_backend fixture then launches cheap
+    routers (~1-2s) pointing to these workers.
+
+    Startup behavior:
+    - Scans test markers to determine required workers (model, mode, type, count)
+    - Launches workers in test collection order
+    - Waits for all workers to become healthy before returning
+
+    Test requirements are auto-detected from:
+    - @pytest.mark.parametrize("setup_backend", ["grpc", "http", "pd"])
+    - @pytest.mark.model("model-name")
+    - @pytest.mark.workers(count=N) for regular workers
+    - @pytest.mark.workers(prefill=N, decode=N) for PD workers
+
+    Environment variable overrides:
+    - E2E_MODELS: Comma-separated model IDs (e.g., "llama-8b,qwen-7b")
+    - E2E_BACKENDS: Comma-separated backends (e.g., "grpc,http")
+    - SKIP_MODEL_POOL: Set to "1" to skip worker startup
+    """
+    global _model_pool
+
+    from infra import (
+        DEFAULT_MODEL,
+        ENV_BACKENDS,
+        ENV_MODELS,
+        ENV_SKIP_MODEL_POOL,
+        ENV_STARTUP_TIMEOUT,
+        LOCAL_MODES,
+        MODEL_SPECS,
+        ConnectionMode,
+        GPUAllocator,
+        ModelPool,
+        WorkerIdentity,
+        WorkerType,
+    )
+
+    # Thread-safe initialization: use lock to ensure only one thread creates the pool
+    # This is critical for pytest-parallel which runs tests as concurrent threads
+    with _model_pool_lock:
+        if _model_pool is not None:
+            return _model_pool
+
+        # Check if we should skip model startup
+        if os.environ.get(ENV_SKIP_MODEL_POOL, "").lower() in ("1", "true", "yes"):
+            logger.info("%s is set, skipping model pool startup", ENV_SKIP_MODEL_POOL)
+            _model_pool = ModelPool(GPUAllocator(gpus=[]))
+            return _model_pool
+
+        # Determine requirements from scanned tests or env vars
+        models_env = os.environ.get(ENV_MODELS, "")
+        backends_env = os.environ.get(ENV_BACKENDS, "")
+
+        if models_env or backends_env:
+            # Use env var overrides
+            models = (
+                {m.strip() for m in models_env.split(",") if m.strip()}
+                if models_env
+                else {DEFAULT_MODEL}
+            )
+
+            # Parse backend strings to ConnectionMode enums
+            backend_modes: set[ConnectionMode] = set()
+            if backends_env:
+                for b in backends_env.split(","):
+                    b = b.strip()
+                    if b:
+                        try:
+                            mode = ConnectionMode(b)
+                            if mode in LOCAL_MODES:
+                                backend_modes.add(mode)
+                        except ValueError:
+                            logger.warning("Unknown backend '%s', skipping", b)
+
+            # Default to HTTP if no valid backends
+            if not backend_modes:
+                backend_modes = {ConnectionMode.HTTP}
+
+            # Create WorkerIdentity objects (regular workers only from env vars)
+            requirements = [
+                WorkerIdentity(m, b, WorkerType.REGULAR, 0)
+                for m in models
+                for b in backend_modes
+            ]
+            logger.info(
+                "Using env var requirements: %s", [str(r) for r in requirements]
+            )
+        else:
+            # Use scanned requirements from test markers
+            requirements = get_pool_requirements()
+            logger.info(
+                "Using scanned requirements: %s", [str(r) for r in requirements]
+            )
+
+        # Filter to valid models
+        requirements = [r for r in requirements if r.model_id in MODEL_SPECS]
+
+        if not requirements:
+            logger.warning("No valid requirements, model pool will be empty")
+            _model_pool = ModelPool(GPUAllocator(gpus=[]))
+            return _model_pool
+
+        # Create and start the pool
+        allocator = GPUAllocator()
+        _model_pool = ModelPool(allocator)
+
+        startup_timeout = int(os.environ.get(ENV_STARTUP_TIMEOUT, "300"))
+        _model_pool.startup(
+            requirements=requirements,
+            startup_timeout=startup_timeout,
+        )
+
+        # Log final GPU allocation summary
+        logger.info(_model_pool.allocator.summary())
+
+        # Register cleanup with atexit instead of request.addfinalizer
+        # This is critical for pytest-parallel where multiple threads share
+        # the session-scoped fixture - addfinalizer can fire too early
+        global _shutdown_registered
+        if not _shutdown_registered:
+            atexit.register(_shutdown_model_pool)
+            _shutdown_registered = True
+
+        return _model_pool
+
+
+@pytest.fixture
+def model_client(request: pytest.FixtureRequest, model_pool: "ModelPool"):
+    """Get OpenAI client for the model specified by @pytest.mark.model().
+
+    Usage:
+        @pytest.mark.model("llama-8b")
+        def test_chat(model_client):
+            response = model_client.chat.completions.create(...)
+    """
+    import openai
+    from infra import PARAM_MODEL
+
+    marker = request.node.get_closest_marker(PARAM_MODEL)
+    if marker is None:
+        pytest.fail(
+            f"Test must be marked with @pytest.mark.{PARAM_MODEL}('model-id') "
+            "to use model_client fixture"
+        )
+
+    model_id = marker.args[0]
+
+    try:
+        # get() auto-acquires the returned instance
+        instance = model_pool.get(model_id)
+    except KeyError:
+        pytest.skip(f"Model {model_id} not available in model pool")
+
+    client = openai.OpenAI(
+        base_url=f"{instance.base_url}/v1",
+        api_key="not-used",
+    )
+
+    yield client
+
+    # Release reference to allow eviction
+    instance.release()
+
+
+@pytest.fixture
+def model_base_url(request: pytest.FixtureRequest, model_pool: "ModelPool") -> str:
+    """Get the base URL for the model specified by @pytest.mark.model().
+
+    Usage:
+        @pytest.mark.model("llama-8b")
+        def test_direct_http(model_base_url):
+            response = httpx.get(f"{model_base_url}/health")
+    """
+    from infra import PARAM_MODEL
+
+    marker = request.node.get_closest_marker(PARAM_MODEL)
+    if marker is None:
+        pytest.fail(
+            f"Test must be marked with @pytest.mark.{PARAM_MODEL}('model-id') "
+            "to use model_base_url fixture"
+        )
+
+    model_id = marker.args[0]
+
+    try:
+        # get() auto-acquires the returned instance
+        instance = model_pool.get(model_id)
+    except KeyError:
+        pytest.skip(f"Model {model_id} not available in model pool")
+
+    yield instance.base_url
+
+    # Release reference to allow eviction
+    instance.release()

third_party/sglang/sgl-model-gateway/e2e_test/fixtures/ports.py

@@ -0,0 +1,14 @@
+"""Legacy port utilities.
+
+DEPRECATED: This module will be removed during e2e_response_api migration.
+Use infra.get_open_port() instead.
+"""
+
+import socket
+
+
+def find_free_port() -> int:
+    """Return an available TCP port on localhost."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]

third_party/sglang/sgl-model-gateway/e2e_test/fixtures/setup_backend.py

@@ -0,0 +1,449 @@
+"""Backend setup fixtures for E2E tests.
+
+This module provides fixtures for launching gateways/routers for different backends.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from infra import ModelPool
+
+from .markers import get_marker_kwargs, get_marker_value
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="class")
+def setup_backend(request: pytest.FixtureRequest, model_pool: "ModelPool"):
+    """Class-scoped fixture that launches a router for each test class.
+
+    Routers are cheap to start (~1-2s) compared to workers (~30-60s), so we
+    launch a fresh router per test class for isolation while reusing the
+    expensive workers from model_pool.
+
+    Backend types:
+    - "http", "grpc": Gets existing worker from model_pool, launches router
+    - "pd": Launches prefill/decode workers via model_pool, launches PD router
+    - "openai", "xai", etc.: Launches cloud router (no local workers)
+
+    Configuration via markers:
+    - @pytest.mark.model("model-id"): Override default model
+    - @pytest.mark.workers(count=1): Number of regular workers behind router
+    - @pytest.mark.workers(prefill=1, decode=1): PD worker configuration
+    - @pytest.mark.gateway(policy="round_robin", timeout=60): Gateway configuration
+
+    Returns:
+        Tuple of (backend_name, model_path, openai_client, gateway)
+
+    Usage:
+        @pytest.mark.parametrize("setup_backend", ["http"], indirect=True)
+        class TestBasic:
+            def test_chat(self, setup_backend):
+                backend, model, client, gateway = setup_backend
+    """
+    import openai
+    from infra import (
+        DEFAULT_MODEL,
+        DEFAULT_ROUTER_TIMEOUT,
+        ENV_MODEL,
+        ENV_SKIP_BACKEND_SETUP,
+        LOCAL_MODES,
+        ConnectionMode,
+        Gateway,
+        WorkerIdentity,
+        WorkerType,
+    )
+
+    backend_name = request.param
+
+    # Skip if requested
+    if os.environ.get(ENV_SKIP_BACKEND_SETUP, "").lower() in ("1", "true", "yes"):
+        pytest.skip(f"{ENV_SKIP_BACKEND_SETUP} is set")
+
+    # Get model from marker or env var or default
+    model_id = get_marker_value(request, "model")
+    if model_id is None:
+        model_id = os.environ.get(ENV_MODEL, DEFAULT_MODEL)
+
+    # Get worker configuration from marker
+    workers_config = get_marker_kwargs(
+        request, "workers", defaults={"count": 1, "prefill": None, "decode": None}
+    )
+
+    # Get gateway configuration from marker
+    gateway_config = get_marker_kwargs(
+        request,
+        "gateway",
+        defaults={
+            "policy": "round_robin",
+            "timeout": DEFAULT_ROUTER_TIMEOUT,
+            "extra_args": None,
+        },
+    )
+
+    # PD disaggregation backend
+    if backend_name == "pd":
+        yield from _setup_pd_backend(
+            request, model_pool, model_id, workers_config, gateway_config
+        )
+        return
+
+    # Check if this is a local backend (grpc, http)
+    try:
+        connection_mode = ConnectionMode(backend_name)
+        is_local = connection_mode in LOCAL_MODES
+    except ValueError:
+        is_local = False
+        connection_mode = None
+
+    # Local backends: use worker from pool + launch gateway
+    if is_local:
+        yield from _setup_local_backend(
+            request,
+            model_pool,
+            backend_name,
+            model_id,
+            connection_mode,
+            workers_config,
+            gateway_config,
+        )
+        return
+
+    # Get storage backend from marker (default: memory)
+    storage_backend = get_marker_value(request, "storage", default="memory")
+
+    # Cloud backends: launch cloud router
+    yield from _setup_cloud_backend(backend_name, storage_backend, gateway_config)
+
+
+def _setup_pd_backend(
+    request: pytest.FixtureRequest,
+    model_pool: "ModelPool",
+    model_id: str,
+    workers_config: dict,
+    gateway_config: dict,
+):
+    """Setup PD disaggregation backend."""
+    import openai
+    from infra import ConnectionMode, Gateway, WorkerIdentity, WorkerType
+
+    logger.info("Setting up PD backend for model %s", model_id)
+
+    # Get PD configuration from workers marker
+    num_prefill = workers_config.get("prefill") or 1
+    num_decode = workers_config.get("decode") or 1
+    logger.info("PD config: %d prefill, %d decode workers", num_prefill, num_decode)
+
+    # Try to use pre-launched PD workers, or launch additional ones if needed
+    # get_workers_by_type auto-acquires all returned workers
+    existing_prefills = model_pool.get_workers_by_type(model_id, WorkerType.PREFILL)
+    existing_decodes = model_pool.get_workers_by_type(model_id, WorkerType.DECODE)
+
+    # Calculate how many more we need
+    missing_prefill = max(0, num_prefill - len(existing_prefills))
+    missing_decode = max(0, num_decode - len(existing_decodes))
+
+    if missing_prefill == 0 and missing_decode == 0:
+        prefills = existing_prefills[:num_prefill]
+        decodes = existing_decodes[:num_decode]
+        # Release excess workers we won't use
+        for w in existing_prefills[num_prefill:]:
+            w.release()
+        for w in existing_decodes[num_decode:]:
+            w.release()
+        logger.info(
+            "Using pre-launched PD workers: %d prefill, %d decode",
+            len(prefills),
+            len(decodes),
+        )
+    else:
+        # Build WorkerIdentity list for missing workers
+        workers_to_launch: list[WorkerIdentity] = []
+        for i in range(missing_prefill):
+            workers_to_launch.append(
+                WorkerIdentity(
+                    model_id,
+                    ConnectionMode.HTTP,
+                    WorkerType.PREFILL,
+                    len(existing_prefills) + i,
+                )
+            )
+        for i in range(missing_decode):
+            workers_to_launch.append(
+                WorkerIdentity(
+                    model_id,
+                    ConnectionMode.HTTP,
+                    WorkerType.DECODE,
+                    len(existing_decodes) + i,
+                )
+            )
+
+        logger.info(
+            "Have %d/%d prefill, %d/%d decode. Launching %d more workers",
+            len(existing_prefills),
+            num_prefill,
+            len(existing_decodes),
+            num_decode,
+            len(workers_to_launch),
+        )
+        new_instances = model_pool.launch_workers(
+            workers_to_launch, startup_timeout=300
+        )
+
+        if not new_instances:
+            # Release any existing workers we acquired
+            for w in existing_prefills + existing_decodes:
+                w.release()
+            pytest.fail(
+                f"Failed to launch PD workers: needed {len(workers_to_launch)} workers "
+                f"but could not allocate GPUs (all in use or timeout)"
+            )
+
+        # Acquire newly launched instances (launch_workers doesn't auto-acquire)
+        for inst in new_instances:
+            inst.acquire()
+
+        new_prefills = [w for w in new_instances if w.worker_type == WorkerType.PREFILL]
+        new_decodes = [w for w in new_instances if w.worker_type == WorkerType.DECODE]
+        prefills = existing_prefills + new_prefills
+        decodes = existing_decodes + new_decodes
+
+    # All workers in prefills and decodes are now acquired
+
+    if not prefills or not decodes:
+        # This shouldn't happen but guard against it
+        for w in prefills + decodes:
+            w.release()
+        pytest.fail(
+            f"PD setup incomplete: have {len(prefills)} prefill, {len(decodes)} decode "
+            f"(need {num_prefill} prefill, {num_decode} decode)"
+        )
+
+    model_path = prefills[0].model_path
+
+    # Launch PD gateway
+    gateway = Gateway()
+    gateway.start(
+        prefill_workers=prefills,
+        decode_workers=decodes,
+        policy=gateway_config["policy"],
+        timeout=gateway_config["timeout"],
+        extra_args=gateway_config["extra_args"],
+    )
+
+    client = openai.OpenAI(
+        base_url=f"{gateway.base_url}/v1",
+        api_key="not-used",
+    )
+
+    logger.info(
+        "Setup PD backend: model=%s, %d prefill + %d decode workers, "
+        "gateway=%s, policy=%s",
+        model_id,
+        len(prefills),
+        len(decodes),
+        gateway.base_url,
+        gateway_config["policy"],
+    )
+
+    try:
+        yield "pd", model_path, client, gateway
+    finally:
+        logger.info("Tearing down PD gateway")
+        gateway.shutdown()
+        # Release references to allow eviction
+        for worker in prefills + decodes:
+            worker.release()
+
+
+def _setup_local_backend(
+    request: pytest.FixtureRequest,
+    model_pool: "ModelPool",
+    backend_name: str,
+    model_id: str,
+    connection_mode,
+    workers_config: dict,
+    gateway_config: dict,
+):
+    """Setup local backend (grpc, http)."""
+    import openai
+    from infra import Gateway, WorkerIdentity, WorkerType
+
+    num_workers = workers_config.get("count") or 1
+    instances: list = []  # Track instances for reference counting
+
+    try:
+        if num_workers > 1:
+            # get_workers_by_type auto-acquires all returned workers
+            all_existing = model_pool.get_workers_by_type(model_id, WorkerType.REGULAR)
+            existing_for_mode = [w for w in all_existing if w.mode == connection_mode]
+
+            # Release workers we won't use (wrong mode)
+            for w in all_existing:
+                if w not in existing_for_mode:
+                    w.release()
+
+            if len(existing_for_mode) >= num_workers:
+                instances = existing_for_mode[:num_workers]
+                # Release excess workers we won't use
+                for w in existing_for_mode[num_workers:]:
+                    w.release()
+            else:
+                missing = num_workers - len(existing_for_mode)
+                workers_to_launch = [
+                    WorkerIdentity(
+                        model_id,
+                        connection_mode,
+                        WorkerType.REGULAR,
+                        len(existing_for_mode) + i,
+                    )
+                    for i in range(missing)
+                ]
+                new_instances = model_pool.launch_workers(
+                    workers_to_launch, startup_timeout=300
+                )
+                # Acquire newly launched instances
+                for inst in new_instances:
+                    inst.acquire()
+                instances = existing_for_mode + new_instances
+
+            if not instances:
+                pytest.fail(f"Failed to get {num_workers} workers for {model_id}")
+            worker_urls = [inst.worker_url for inst in instances]
+            model_path = instances[0].model_path
+        else:
+            # get() auto-acquires the returned instance
+            instance = model_pool.get(model_id, connection_mode)
+            instances = [instance]
+            worker_urls = [instance.worker_url]
+            model_path = instance.model_path
+    except RuntimeError as e:
+        pytest.fail(str(e))
+
+    # Launch gateway
+    gateway = Gateway()
+    gateway.start(
+        worker_urls=worker_urls,
+        model_path=model_path,
+        policy=gateway_config["policy"],
+        timeout=gateway_config["timeout"],
+        extra_args=gateway_config["extra_args"],
+    )
+
+    client = openai.OpenAI(
+        base_url=f"{gateway.base_url}/v1",
+        api_key="not-used",
+    )
+
+    logger.info(
+        "Setup %s backend: model=%s, workers=%d, gateway=%s, policy=%s",
+        backend_name,
+        model_id,
+        num_workers,
+        gateway.base_url,
+        gateway_config["policy"],
+    )
+
+    try:
+        yield backend_name, model_path, client, gateway
+    finally:
+        logger.info("Tearing down gateway for %s backend", backend_name)
+        gateway.shutdown()
+        # Release references to allow eviction
+        for inst in instances:
+            inst.release()
+
+
+def _setup_cloud_backend(
+    backend_name: str,
+    storage_backend: str = "memory",
+    gateway_config: dict | None = None,
+):
+    """Setup cloud backend (openai, xai, etc.).
+
+    Args:
+        backend_name: Cloud backend name (openai, xai).
+        storage_backend: History storage backend (memory, oracle).
+        gateway_config: Gateway configuration from marker.
+    """
+    import openai
+    from infra import THIRD_PARTY_MODELS, launch_cloud_gateway
+
+    if backend_name not in THIRD_PARTY_MODELS:
+        pytest.fail(f"Unknown cloud runtime: {backend_name}")
+
+    cfg = THIRD_PARTY_MODELS[backend_name]
+    api_key_env = cfg.get("api_key_env")
+
+    if api_key_env and not os.environ.get(api_key_env):
+        pytest.skip(f"{api_key_env} not set, skipping {backend_name} tests")
+
+    extra_args = gateway_config.get("extra_args") if gateway_config else None
+
+    logger.info(
+        "Launching cloud backend: %s with storage=%s", backend_name, storage_backend
+    )
+    gateway = launch_cloud_gateway(
+        backend_name,
+        history_backend=storage_backend,
+        extra_args=extra_args,
+    )
+
+    api_key = os.environ.get(api_key_env) if api_key_env else "not-used"
+    client = openai.OpenAI(
+        base_url=f"{gateway.base_url}/v1",
+        api_key=api_key,
+    )
+
+    try:
+        yield backend_name, cfg["model"], client, gateway
+    finally:
+        logger.info("Tearing down cloud backend: %s", backend_name)
+        gateway.shutdown()
+
+
+@pytest.fixture
+def backend_router(request: pytest.FixtureRequest, model_pool: "ModelPool"):
+    """Function-scoped fixture for launching a fresh router per test.
+
+    This launches a new Gateway for each test, pointing to workers from the pool.
+    Use for tests that need isolated router state.
+
+    Usage:
+        @pytest.mark.parametrize("backend_router", ["grpc", "http"], indirect=True)
+        def test_router_state(backend_router):
+            gateway = backend_router
+    """
+    from infra import DEFAULT_MODEL, ENV_MODEL, ConnectionMode, Gateway
+
+    backend_name = request.param
+    model_id = os.environ.get(ENV_MODEL, DEFAULT_MODEL)
+
+    connection_mode = ConnectionMode(backend_name)
+
+    try:
+        # get() auto-acquires the returned instance
+        instance = model_pool.get(model_id, connection_mode)
+    except KeyError:
+        pytest.skip(f"Model {model_id}:{backend_name} not available in pool")
+    except RuntimeError as e:
+        pytest.fail(str(e))
+
+    gateway = Gateway()
+    gateway.start(
+        worker_urls=[instance.worker_url],
+        model_path=instance.model_path,
+    )
+
+    try:
+        yield gateway
+    finally:
+        gateway.shutdown()
+        # Release reference to allow eviction
+        instance.release()