chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/e2e_test/chat_completions/test_enable_thinking.py

@@ -0,0 +1,168 @@
+"""Enable Thinking E2E Tests.
+
+Tests for chat completions with enable_thinking feature (Qwen3 reasoning).
+
+Source: Migrated from e2e_grpc/features/test_enable_thinking.py
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+
+import pytest
+import requests
+
+logger = logging.getLogger(__name__)
+
+# API key is not validated by the gateway, but required for OpenAI-compatible headers
+API_KEY = "not-used"
+
+
+# =============================================================================
+# Enable Thinking Tests (Qwen 30B)
+# =============================================================================
+
+
+@pytest.mark.model("qwen-30b")
+@pytest.mark.gateway(
+    extra_args=["--reasoning-parser", "qwen3", "--history-backend", "memory"]
+)
+@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
+class TestEnableThinking:
+    """Tests for enable_thinking feature with Qwen3 reasoning parser."""
+
+    def test_chat_completion_with_reasoning(self, setup_backend):
+        """Test non-streaming with enable_thinking=True, reasoning_content should not be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = requests.post(
+            f"{gateway.base_url}/v1/chat/completions",
+            headers={"Authorization": f"Bearer {API_KEY}"},
+            json={
+                "model": model,
+                "messages": [{"role": "user", "content": "Hello"}],
+                "temperature": 0,
+                "separate_reasoning": True,
+                "chat_template_kwargs": {"enable_thinking": True},
+            },
+        )
+
+        assert response.status_code == 200, f"Failed with: {response.text}"
+        data = response.json()
+
+        assert "choices" in data
+        assert len(data["choices"]) > 0
+        assert "message" in data["choices"][0]
+        assert "reasoning_content" in data["choices"][0]["message"]
+        assert data["choices"][0]["message"]["reasoning_content"] is not None
+
+    def test_chat_completion_without_reasoning(self, setup_backend):
+        """Test non-streaming with enable_thinking=False, reasoning_content should be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = requests.post(
+            f"{gateway.base_url}/v1/chat/completions",
+            headers={"Authorization": f"Bearer {API_KEY}"},
+            json={
+                "model": model,
+                "messages": [{"role": "user", "content": "Hello"}],
+                "temperature": 0,
+                "separate_reasoning": True,
+                "chat_template_kwargs": {"enable_thinking": False},
+            },
+        )
+
+        assert response.status_code == 200, f"Failed with: {response.text}"
+        data = response.json()
+
+        assert "choices" in data
+        assert len(data["choices"]) > 0
+        assert "message" in data["choices"][0]
+
+        if "reasoning_content" in data["choices"][0]["message"]:
+            assert data["choices"][0]["message"]["reasoning_content"] is None
+
+    def test_stream_chat_completion_with_reasoning(self, setup_backend):
+        """Test streaming with enable_thinking=True, reasoning_content should not be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = requests.post(
+            f"{gateway.base_url}/v1/chat/completions",
+            headers={"Authorization": f"Bearer {API_KEY}"},
+            json={
+                "model": model,
+                "messages": [{"role": "user", "content": "Hello"}],
+                "temperature": 0,
+                "separate_reasoning": True,
+                "stream": True,
+                "chat_template_kwargs": {"enable_thinking": True},
+            },
+            stream=True,
+        )
+
+        assert response.status_code == 200, f"Failed with: {response.text}"
+
+        has_reasoning = False
+        has_content = False
+
+        for line in response.iter_lines():
+            if line:
+                line = line.decode("utf-8")
+                if line.startswith("data:") and not line.startswith("data: [DONE]"):
+                    data = json.loads(line[6:])
+                    if "choices" in data and len(data["choices"]) > 0:
+                        delta = data["choices"][0].get("delta", {})
+
+                        if "reasoning_content" in delta and delta["reasoning_content"]:
+                            has_reasoning = True
+
+                        if "content" in delta and delta["content"]:
+                            has_content = True
+
+        assert (
+            has_reasoning
+        ), "The reasoning content is not included in the stream response"
+        assert has_content, "The stream response does not contain normal content"
+
+    def test_stream_chat_completion_without_reasoning(self, setup_backend):
+        """Test streaming with enable_thinking=False, reasoning_content should be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = requests.post(
+            f"{gateway.base_url}/v1/chat/completions",
+            headers={"Authorization": f"Bearer {API_KEY}"},
+            json={
+                "model": model,
+                "messages": [{"role": "user", "content": "Hello"}],
+                "temperature": 0,
+                "separate_reasoning": True,
+                "stream": True,
+                "chat_template_kwargs": {"enable_thinking": False},
+            },
+            stream=True,
+        )
+
+        assert response.status_code == 200, f"Failed with: {response.text}"
+
+        has_reasoning = False
+        has_content = False
+
+        for line in response.iter_lines():
+            if line:
+                line = line.decode("utf-8")
+                if line.startswith("data:") and not line.startswith("data: [DONE]"):
+                    data = json.loads(line[6:])
+                    if "choices" in data and len(data["choices"]) > 0:
+                        delta = data["choices"][0].get("delta", {})
+
+                        if "reasoning_content" in delta and delta["reasoning_content"]:
+                            has_reasoning = True
+
+                        if "content" in delta and delta["content"]:
+                            has_content = True
+
+        assert (
+            not has_reasoning
+        ), "The reasoning content should not be included in the stream response"
+        assert has_content, "The stream response does not contain normal content"

third_party/sglang/sgl-model-gateway/e2e_test/chat_completions/test_openai_server.py

@@ -0,0 +1,316 @@
+"""Chat Completions API E2E Tests - OpenAI Server Compatibility.
+
+Tests for OpenAI-compatible chat completions API through the gateway.
+
+Source: Migrated from e2e_grpc/basic/test_openai_server.py
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Chat Completion Tests (Llama 8B)
+# =============================================================================
+
+
+@pytest.mark.model("llama-8b")
+@pytest.mark.gateway(extra_args=["--history-backend", "memory"])
+@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
+class TestChatCompletion:
+    """Tests for OpenAI-compatible chat completions API."""
+
+    @pytest.mark.parametrize("logprobs", [None, 5])
+    @pytest.mark.parametrize("parallel_sample_num", [1, 2])
+    def test_chat_completion(self, setup_backend, logprobs, parallel_sample_num):
+        """Test non-streaming chat completion with logprobs and parallel sampling."""
+        _, model, client, gateway = setup_backend
+        self._run_chat_completion(client, model, logprobs, parallel_sample_num)
+
+    @pytest.mark.parametrize("logprobs", [None, 5])
+    @pytest.mark.parametrize("parallel_sample_num", [1, 2])
+    def test_chat_completion_stream(self, setup_backend, logprobs, parallel_sample_num):
+        """Test streaming chat completion with logprobs and parallel sampling."""
+        _, model, client, gateway = setup_backend
+        self._run_chat_completion_stream(client, model, logprobs, parallel_sample_num)
+
+    def test_regex(self, setup_backend):
+        """Test structured output with regex constraint."""
+        _, model, client, gateway = setup_backend
+
+        regex = (
+            r"""\{\n"""
+            + r"""   "name": "[\w]+",\n"""
+            + r"""   "population": [\d]+\n"""
+            + r"""\}"""
+        )
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful AI assistant"},
+                {"role": "user", "content": "Introduce the capital of France."},
+            ],
+            temperature=0,
+            max_tokens=128,
+            extra_body={"regex": regex},
+        )
+        text = response.choices[0].message.content
+
+        try:
+            js_obj = json.loads(text)
+        except (TypeError, json.decoder.JSONDecodeError):
+            raise
+        assert isinstance(js_obj["name"], str)
+        assert isinstance(js_obj["population"], int)
+
+    def test_penalty(self, setup_backend):
+        """Test frequency penalty parameter."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful AI assistant"},
+                {"role": "user", "content": "Introduce the capital of France."},
+            ],
+            temperature=0,
+            max_tokens=32,
+            frequency_penalty=1.0,
+        )
+        text = response.choices[0].message.content
+        assert isinstance(text, str)
+
+    def test_response_prefill(self, setup_backend):
+        """Test assistant message prefill with continue_final_message."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful AI assistant"},
+                {
+                    "role": "user",
+                    "content": """
+Extract the name, size, price, and color from this product description as a JSON object:
+
+<description>
+The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices.
+</description>
+""",
+                },
+                {
+                    "role": "assistant",
+                    "content": "{\n",
+                },
+            ],
+            temperature=0,
+            extra_body={"continue_final_message": True},
+        )
+
+        assert (
+            response.choices[0]
+            .message.content.strip()
+            .startswith('"name": "SmartHome Mini",')
+        )
+
+    def test_model_list(self, setup_backend):
+        """Test listing available models."""
+        _, model, client, gateway = setup_backend
+
+        models = list(client.models.list().data)
+        assert len(models) == 1
+
+    @pytest.mark.skip(
+        reason="Skipping retrieve model test as it is not supported by the router"
+    )
+    def test_retrieve_model(self, setup_backend):
+        """Test retrieving a specific model."""
+        import openai
+
+        _, model, client, gateway = setup_backend
+
+        retrieved_model = client.models.retrieve(model)
+        assert retrieved_model.id == model
+        assert retrieved_model.root == model
+
+        with pytest.raises(openai.NotFoundError):
+            client.models.retrieve("non-existent-model")
+
+    # -------------------------------------------------------------------------
+    # Helper methods
+    # -------------------------------------------------------------------------
+
+    def _run_chat_completion(self, client, model, logprobs, parallel_sample_num):
+        """Run a non-streaming chat completion and verify response."""
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful AI assistant"},
+                {
+                    "role": "user",
+                    "content": "What is the capital of France? Answer in a few words.",
+                },
+            ],
+            temperature=0,
+            logprobs=logprobs is not None and logprobs > 0,
+            top_logprobs=logprobs,
+            n=parallel_sample_num,
+        )
+
+        if logprobs:
+            assert isinstance(
+                response.choices[0].logprobs.content[0].top_logprobs[0].token, str
+            )
+
+            ret_num_top_logprobs = len(
+                response.choices[0].logprobs.content[0].top_logprobs
+            )
+            assert (
+                ret_num_top_logprobs == logprobs
+            ), f"{ret_num_top_logprobs} vs {logprobs}"
+
+        assert len(response.choices) == parallel_sample_num
+        assert response.choices[0].message.role == "assistant"
+        assert isinstance(response.choices[0].message.content, str)
+        assert response.id
+        assert response.created
+        assert response.usage.prompt_tokens > 0
+        assert response.usage.completion_tokens > 0
+        assert response.usage.total_tokens > 0
+
+    def _run_chat_completion_stream(
+        self, client, model, logprobs, parallel_sample_num=1
+    ):
+        """Run a streaming chat completion and verify response chunks."""
+        generator = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful AI assistant"},
+                {"role": "user", "content": "What is the capital of France?"},
+            ],
+            temperature=0,
+            logprobs=logprobs is not None and logprobs > 0,
+            top_logprobs=logprobs,
+            stream=True,
+            stream_options={"include_usage": True},
+            n=parallel_sample_num,
+        )
+
+        is_firsts = {}
+        is_finished = {}
+        finish_reason_counts = {}
+        for response in generator:
+            usage = response.usage
+            if usage is not None:
+                assert usage.prompt_tokens > 0, "usage.prompt_tokens was zero"
+                assert usage.completion_tokens > 0, "usage.completion_tokens was zero"
+                assert usage.total_tokens > 0, "usage.total_tokens was zero"
+                continue
+
+            index = response.choices[0].index
+            finish_reason = response.choices[0].finish_reason
+            if finish_reason is not None:
+                is_finished[index] = True
+                finish_reason_counts[index] = finish_reason_counts.get(index, 0) + 1
+
+            data = response.choices[0].delta
+
+            if is_firsts.get(index, True):
+                assert (
+                    data.role == "assistant"
+                ), "data.role was not 'assistant' for first chunk"
+                is_firsts[index] = False
+                continue
+
+            if logprobs and not is_finished.get(index, False):
+                assert response.choices[0].logprobs, "logprobs was not returned"
+                assert isinstance(
+                    response.choices[0].logprobs.content[0].top_logprobs[0].token, str
+                ), "top_logprobs token was not a string"
+                assert isinstance(
+                    response.choices[0].logprobs.content[0].top_logprobs, list
+                ), "top_logprobs was not a list"
+                ret_num_top_logprobs = len(
+                    response.choices[0].logprobs.content[0].top_logprobs
+                )
+                assert (
+                    ret_num_top_logprobs == logprobs
+                ), f"{ret_num_top_logprobs} vs {logprobs}"
+
+            assert (
+                isinstance(data.content, str)
+                or isinstance(data.reasoning_content, str)
+                or (isinstance(data.tool_calls, list) and len(data.tool_calls) > 0)
+                or response.choices[0].finish_reason
+            )
+            assert response.id
+            assert response.created
+
+        for index in range(parallel_sample_num):
+            assert not is_firsts.get(
+                index, True
+            ), f"index {index} is not found in the response"
+
+        for index in range(parallel_sample_num):
+            assert (
+                index in finish_reason_counts
+            ), f"No finish_reason found for index {index}"
+            assert finish_reason_counts[index] == 1, (
+                f"Expected 1 finish_reason chunk for index {index}, "
+                f"got {finish_reason_counts[index]}"
+            )
+
+
+# =============================================================================
+# Chat Completion Tests (GPT-OSS)
+#
+# NOTE: Some tests are skipped because they don't work with OSS models:
+# - test_regex: OSS models don't support regex constraints
+# - test_penalty: OSS models don't support frequency_penalty
+# - test_response_prefill: OSS models don't support continue_final_message
+# =============================================================================
+
+
+@pytest.mark.model("gpt-oss")
+@pytest.mark.gateway(
+    extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]
+)
+class TestChatCompletionGptOss(TestChatCompletion):
+    """Tests for chat completions API with GPT-OSS model.
+
+    Inherits from TestChatCompletion and overrides tests that don't work
+    with OSS models.
+    """
+
+    @pytest.mark.parametrize("logprobs", [None])  # No logprobs for OSS
+    @pytest.mark.parametrize("parallel_sample_num", [1, 2])
+    def test_chat_completion(self, setup_backend, logprobs, parallel_sample_num):
+        """Test non-streaming chat completion with parallel sampling (no logprobs)."""
+        super().test_chat_completion(setup_backend, logprobs, parallel_sample_num)
+
+    @pytest.mark.parametrize("logprobs", [None])  # No logprobs for OSS
+    @pytest.mark.parametrize("parallel_sample_num", [1, 2])
+    def test_chat_completion_stream(self, setup_backend, logprobs, parallel_sample_num):
+        """Test streaming chat completion with parallel sampling (no logprobs)."""
+        super().test_chat_completion_stream(
+            setup_backend, logprobs, parallel_sample_num
+        )
+
+    @pytest.mark.skip(reason="OSS models don't support regex constraints")
+    def test_regex(self, setup_backend):
+        pass
+
+    @pytest.mark.skip(reason="OSS models don't support frequency_penalty")
+    def test_penalty(self, setup_backend):
+        pass
+
+    @pytest.mark.skip(reason="OSS models don't support continue_final_message")
+    def test_response_prefill(self, setup_backend):
+        pass

third_party/sglang/sgl-model-gateway/e2e_test/chat_completions/test_reasoning_content.py

@@ -0,0 +1,165 @@
+"""Reasoning Content E2E Tests.
+
+Tests for chat completions with reasoning content (DeepSeek R1 reasoning parser).
+
+Source: Migrated from e2e_grpc/features/test_reasoning_content.py
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Reasoning Content API Tests (DeepSeek 7B)
+# =============================================================================
+
+
+@pytest.mark.model("deepseek-7b")
+@pytest.mark.gateway(
+    extra_args=["--reasoning-parser", "deepseek_r1", "--history-backend", "memory"]
+)
+@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
+class TestReasoningContentAPI:
+    """Tests for reasoning content API with DeepSeek R1 reasoning parser."""
+
+    def test_streaming_separate_reasoning_false(self, setup_backend):
+        """Test streaming with separate_reasoning=False, reasoning_content should be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": "What is 1+3?",
+                }
+            ],
+            max_tokens=100,
+            stream=True,
+            extra_body={"separate_reasoning": False},
+        )
+
+        reasoning_content = ""
+        content = ""
+        for chunk in response:
+            if chunk.choices[0].delta.content:
+                content += chunk.choices[0].delta.content
+            elif chunk.choices[0].delta.reasoning_content:
+                reasoning_content += chunk.choices[0].delta.reasoning_content
+
+        assert len(reasoning_content) == 0
+        assert len(content) > 0
+
+    def test_streaming_separate_reasoning_true(self, setup_backend):
+        """Test streaming with separate_reasoning=True, reasoning_content should not be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": "What is 1+3?",
+                }
+            ],
+            max_tokens=100,
+            stream=True,
+            extra_body={"separate_reasoning": True},
+        )
+
+        reasoning_content = ""
+        content = ""
+        for chunk in response:
+            if chunk.choices[0].delta.content:
+                content += chunk.choices[0].delta.content
+            elif chunk.choices[0].delta.reasoning_content:
+                reasoning_content += chunk.choices[0].delta.reasoning_content
+
+        assert len(reasoning_content) > 0
+        assert len(content) > 0
+
+    def test_streaming_separate_reasoning_true_stream_reasoning_false(
+        self, setup_backend
+    ):
+        """Test streaming with separate_reasoning=True and stream_reasoning=False."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": "What is 1+3?",
+                }
+            ],
+            max_tokens=100,
+            stream=True,
+            extra_body={"separate_reasoning": True, "stream_reasoning": False},
+        )
+
+        reasoning_content = ""
+        content = ""
+        first_chunk = False
+        for chunk in response:
+            if chunk.choices[0].delta.reasoning_content:
+                reasoning_content = chunk.choices[0].delta.reasoning_content
+                first_chunk = True
+            if chunk.choices[0].delta.content:
+                content += chunk.choices[0].delta.content
+                if not first_chunk:
+                    reasoning_content = chunk.choices[0].delta.reasoning_content
+                first_chunk = True
+            if not first_chunk:
+                assert (
+                    not chunk.choices[0].delta.reasoning_content
+                    or len(chunk.choices[0].delta.reasoning_content) == 0
+                )
+
+        assert len(reasoning_content) > 0
+        assert len(content) > 0
+
+    def test_nonstreaming_separate_reasoning_false(self, setup_backend):
+        """Test non-streaming with separate_reasoning=False, reasoning_content should be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": "What is 1+3?",
+                }
+            ],
+            max_tokens=100,
+            extra_body={"separate_reasoning": False},
+        )
+
+        assert (
+            not response.choices[0].message.reasoning_content
+            or len(response.choices[0].message.reasoning_content) == 0
+        )
+        assert len(response.choices[0].message.content) > 0
+
+    def test_nonstreaming_separate_reasoning_true(self, setup_backend):
+        """Test non-streaming with separate_reasoning=True, reasoning_content should not be empty."""
+        _, model, client, gateway = setup_backend
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": "What is 1+3?",
+                }
+            ],
+            max_tokens=100,
+            extra_body={"separate_reasoning": True},
+        )
+
+        assert len(response.choices[0].message.reasoning_content) > 0
+        assert len(response.choices[0].message.content) > 0

third_party/sglang/sgl-model-gateway/e2e_test/chat_completions/test_validation.py

@@ -0,0 +1,167 @@
+"""Validation E2E Tests.
+
+Tests for validation features like ignore_eos and large token handling.
+
+Source: Migrated from e2e_grpc/validation/test_openai_server_ignore_eos.py
+        and e2e_grpc/validation/test_large_max_new_tokens.py
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+# Lazy load tokenizer to avoid import errors if transformers not installed
+_tokenizer_cache: dict = {}
+_tokenizer_lock = threading.Lock()
+
+
+def get_tokenizer(model_path: str):
+    """Get tokenizer for a model, with caching."""
+    if model_path not in _tokenizer_cache:
+        with _tokenizer_lock:
+            # Re-check after acquiring the lock to handle race conditions
+            if model_path not in _tokenizer_cache:
+                from transformers import AutoTokenizer
+
+                _tokenizer_cache[model_path] = AutoTokenizer.from_pretrained(model_path)
+    return _tokenizer_cache[model_path]
+
+
+# =============================================================================
+# Ignore EOS Tests (Llama 8B)
+# =============================================================================
+
+
+@pytest.mark.model("llama-8b")
+@pytest.mark.gateway(extra_args=["--history-backend", "memory"])
+@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
+class TestIgnoreEOS:
+    """Tests for ignore_eos feature."""
+
+    def test_ignore_eos(self, setup_backend):
+        """Test that ignore_eos=True allows generation to continue beyond EOS token.
+
+        When ignore_eos=True, the model should generate until max_tokens is reached,
+        even if it encounters an EOS token.
+        """
+        _, model, client, _ = setup_backend
+
+        tokenizer = get_tokenizer(model)
+        max_tokens = 200
+
+        # Request without ignore_eos (default behavior - stops at EOS)
+        response_default = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "Count from 1 to 20."},
+            ],
+            temperature=0,
+            max_tokens=max_tokens,
+            extra_body={"ignore_eos": False},
+        )
+
+        # Request with ignore_eos=True (continues past EOS until max_tokens)
+        response_ignore_eos = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "Count from 1 to 20."},
+            ],
+            temperature=0,
+            max_tokens=max_tokens,
+            extra_body={"ignore_eos": True},
+        )
+
+        default_tokens = len(
+            tokenizer.encode(response_default.choices[0].message.content)
+        )
+        ignore_eos_tokens = len(
+            tokenizer.encode(response_ignore_eos.choices[0].message.content)
+        )
+
+        # Check if ignore_eos resulted in more tokens or exactly max_tokens
+        # The ignore_eos response should either:
+        # 1. Have more tokens than the default response (if default stopped at EOS before max_tokens)
+        # 2. Have exactly max_tokens (if it reached the max_tokens limit)
+        assert (
+            ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens
+        ), f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}"
+
+        assert response_ignore_eos.choices[0].finish_reason == "length", (
+            f"Expected finish_reason='length' for ignore_eos=True, "
+            f"got {response_ignore_eos.choices[0].finish_reason}"
+        )
+
+
+# =============================================================================
+# Large Max New Tokens Tests (Llama 8B)
+#
+# NOTE: This test verifies concurrent request handling with large token limits.
+# The original test monitored server logs to verify concurrency, which is not
+# possible with the pool-based infrastructure. This simplified version verifies
+# that concurrent requests complete successfully.
+# =============================================================================
+
+
+@pytest.mark.model("llama-8b")
+@pytest.mark.gateway(extra_args=["--history-backend", "memory"])
+@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
+class TestLargeMaxNewTokens:
+    """Tests for handling large max_new_tokens with concurrent requests."""
+
+    def test_concurrent_chat_completions(self, setup_backend):
+        """Test that multiple concurrent requests with large token generation complete.
+
+        This test sends multiple requests that ask for long outputs concurrently
+        to verify the server can handle concurrent long-running requests.
+        """
+        _, model, client, _ = setup_backend
+
+        num_requests = 4
+
+        def run_chat_completion():
+            response = client.chat.completions.create(
+                model=model,
+                messages=[
+                    {"role": "system", "content": "You are a helpful AI assistant"},
+                    {
+                        "role": "user",
+                        "content": "Please repeat the word 'hello' for 100 times.",
+                    },
+                ],
+                temperature=0,
+                max_tokens=256,  # Reasonable limit for concurrent test
+            )
+            return response
+
+        # Send concurrent requests
+        start_time = time.time()
+        futures = []
+        with ThreadPoolExecutor(max_workers=num_requests) as executor:
+            for _ in range(num_requests):
+                futures.append(executor.submit(run_chat_completion))
+
+            # Wait for all to complete and collect results
+            responses = [f.result() for f in futures]
+
+        elapsed = time.time() - start_time
+        logger.info("Completed %d concurrent requests in %.2fs", num_requests, elapsed)
+
+        # Verify all requests completed successfully
+        assert len(responses) == num_requests
+        for i, response in enumerate(responses):
+            assert response.choices[
+                0
+            ].message.content, f"Request {i} returned empty content"
+            assert response.choices[0].finish_reason in ("stop", "length"), (
+                f"Request {i} had unexpected finish_reason: "
+                f"{response.choices[0].finish_reason}"
+            )