From 99ba2733b7e046361e8cf5450eaafe7c9bb39a65 Mon Sep 17 00:00:00 2001 From: lonexreb Date: Fri, 10 Jul 2026 11:25:58 -0500 Subject: [PATCH 1/5] [None][fix] validate sampling params in trtllm-serve request models Invalid sampling parameters (e.g. top_k=-1, top_p=2.0, negative temperature) reached the backend and could crash the server with an unhandled error. Add pydantic Field bounds mirroring SamplingParams' own constraints (top_k>=0, 0<=top_p<=1, temperature>=0) to CompletionRequest, ChatCompletionRequest and ResponsesRequest so bad input is rejected at the API boundary (HTTP 422) instead of crashing. No currently-valid request is newly rejected. Fixes #6329 Signed-off-by: lonexreb --- tensorrt_llm/serve/openai_protocol.py | 16 ++-- .../apps/test_openai_protocol_validation.py | 80 +++++++++++++++++++ 2 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 tests/unittest/llmapi/apps/test_openai_protocol_validation.py diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 7c2ab0812bf2..f45aa3c7e7f0 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -654,15 +654,15 @@ class CompletionRequest(OpenAIBaseModel): stream: Optional[bool] = False stream_options: Optional[StreamOptions] = None suffix: Optional[str] = None - temperature: Optional[float] = None - top_p: Optional[float] = None + temperature: Optional[float] = Field(default=None, ge=0.0) + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) user: Optional[str] = None lora_request: Optional[LoRARequest] = None prompt_ignore_length: Optional[int] = 0 # doc: begin-completion-sampling-params use_beam_search: bool = False - top_k: int = 0 + top_k: int = Field(default=0, ge=0) top_p_min: float = 0.0 min_p: float = 0.0 repetition_penalty: float = 1.0 @@ -1092,8 +1092,8 @@ class ChatCompletionRequest(OpenAIBaseModel): stop: Optional[Union[str, List[str]]] = Field(default_factory=list) stream: Optional[bool] = False stream_options: Optional[StreamOptions] = None - temperature: Optional[float] = None - top_p: Optional[float] = None + temperature: Optional[float] = Field(default=None, ge=0.0) + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) tools: Optional[List[ChatCompletionToolsParam]] = None tool_choice: Optional[Union[Literal["none", "auto", "required"], ChatCompletionNamedToolChoiceParam]] = "none" @@ -1121,7 +1121,7 @@ class ChatCompletionRequest(OpenAIBaseModel): # doc: begin-chat-completion-sampling-params best_of: Optional[int] = None use_beam_search: bool = False - top_k: int = 0 + top_k: int = Field(default=0, ge=0) top_p_min: float = 0.0 min_p: float = 0.0 repetition_penalty: float = 1.0 @@ -1623,12 +1623,12 @@ def _materialize_lazy_item_content(cls, value): "priority"] = "auto" store: Optional[bool] = True stream: Optional[bool] = False - temperature: Optional[float] = None + temperature: Optional[float] = Field(default=None, ge=0.0) text: Optional[ResponseTextConfig] = None tool_choice: ToolChoice = "auto" tools: list[Tool] = Field(default_factory=list) top_logprobs: Optional[int] = 0 - top_p: Optional[float] = None + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) truncation: Optional[Literal["auto", "disabled"]] = "disabled" user: Optional[str] = None diff --git a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py new file mode 100644 index 000000000000..b07d957865b6 --- /dev/null +++ b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Boundary validation for trtllm-serve request models (issue #6329). + +Invalid sampling parameters must be rejected at the API boundary with a +pydantic ValidationError (surfaced to clients as HTTP 422) instead of crashing +the server downstream. +""" + +import pytest +from pydantic import ValidationError + +from tensorrt_llm.serve.openai_protocol import ( + ChatCompletionRequest, + CompletionRequest, + ResponsesRequest, +) + + +def _completion(**kw): + return CompletionRequest(model="m", prompt="hi", **kw) + + +def _chat(**kw): + return ChatCompletionRequest(model="m", messages=[{"role": "user", "content": "hi"}], **kw) + + +def _responses(**kw): + return ResponsesRequest(model="m", input="hi", **kw) + + +# top_k only exists on completion / chat requests. +BUILDERS_WITH_TOP_K = [_completion, _chat] +ALL_BUILDERS = [_completion, _chat, _responses] + + +@pytest.mark.parametrize("build", ALL_BUILDERS) +def test_valid_sampling_params_accepted(build): + # Boundary-valid values must pass unchanged. + req = build(temperature=0.0, top_p=1.0) + assert req.temperature == 0.0 + assert req.top_p == 1.0 + + +@pytest.mark.parametrize("build", ALL_BUILDERS) +def test_negative_temperature_rejected(build): + with pytest.raises(ValidationError): + build(temperature=-0.1) + + +@pytest.mark.parametrize("build", ALL_BUILDERS) +@pytest.mark.parametrize("bad_top_p", [-0.1, 1.1, 2.0]) +def test_out_of_range_top_p_rejected(build, bad_top_p): + with pytest.raises(ValidationError): + build(top_p=bad_top_p) + + +@pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) +def test_negative_top_k_rejected(build): + # top_k = -1 is vLLM's default; TRT-LLM's backend requires top_k >= 0, so + # reject it cleanly at the boundary rather than crashing. + with pytest.raises(ValidationError): + build(top_k=-1) + + +@pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) +def test_top_k_zero_accepted(build): + assert build(top_k=0).top_k == 0 From 9d15c4abd038a004bf1045c260593546d79f44e1 Mon Sep 17 00:00:00 2001 From: lonexreb Date: Fri, 10 Jul 2026 11:46:35 -0500 Subject: [PATCH 2/5] [None][doc] address review: describe constrained serve sampling fields Add description= (with allowed ranges) to temperature, top_p and top_k on the completion/chat/responses request models so the OpenAPI schema and generated clients expose the validation contract. Signed-off-by: lonexreb --- tensorrt_llm/serve/openai_protocol.py | 28 +++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index f45aa3c7e7f0..278011f2071a 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -654,15 +654,23 @@ class CompletionRequest(OpenAIBaseModel): stream: Optional[bool] = False stream_options: Optional[StreamOptions] = None suffix: Optional[str] = None - temperature: Optional[float] = Field(default=None, ge=0.0) - top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) + temperature: Optional[float] = Field( + default=None, ge=0.0, description="Sampling temperature; must be >= 0.") + top_p: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description="Nucleus sampling probability; must be in [0, 1].") user: Optional[str] = None lora_request: Optional[LoRARequest] = None prompt_ignore_length: Optional[int] = 0 # doc: begin-completion-sampling-params use_beam_search: bool = False - top_k: int = Field(default=0, ge=0) + top_k: int = Field( + default=0, + ge=0, + description="Top-k sampling; must be >= 0 (0 disables top-k).") top_p_min: float = 0.0 min_p: float = 0.0 repetition_penalty: float = 1.0 @@ -1092,8 +1100,13 @@ class ChatCompletionRequest(OpenAIBaseModel): stop: Optional[Union[str, List[str]]] = Field(default_factory=list) stream: Optional[bool] = False stream_options: Optional[StreamOptions] = None - temperature: Optional[float] = Field(default=None, ge=0.0) - top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) + temperature: Optional[float] = Field( + default=None, ge=0.0, description="Sampling temperature; must be >= 0.") + top_p: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description="Nucleus sampling probability; must be in [0, 1].") tools: Optional[List[ChatCompletionToolsParam]] = None tool_choice: Optional[Union[Literal["none", "auto", "required"], ChatCompletionNamedToolChoiceParam]] = "none" @@ -1121,7 +1134,10 @@ class ChatCompletionRequest(OpenAIBaseModel): # doc: begin-chat-completion-sampling-params best_of: Optional[int] = None use_beam_search: bool = False - top_k: int = Field(default=0, ge=0) + top_k: int = Field( + default=0, + ge=0, + description="Top-k sampling; must be >= 0 (0 disables top-k).") top_p_min: float = 0.0 min_p: float = 0.0 repetition_penalty: float = 1.0 From a74bbe66db83918d11ae21a8c6754e01aa45ee66 Mon Sep 17 00:00:00 2001 From: lonexreb Date: Sat, 24 Oct 2026 13:13:57 -0700 Subject: [PATCH 3/5] [None][fix] address review: register test in l0_cpu, correct 400 claim, bound min_p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Register test_openai_protocol_validation.py in l0_cpu.yml so CI runs it. - Correct the test docstring: these values were already rejected with a 400 by SamplingParams._validate; the pydantic bounds give an earlier, field-specific error (also surfaced as 400 via the server's RequestValidationError handler) and schema-visible bounds — not a crash fix. - Bound min_p to [0, 1] on CompletionRequest and ChatCompletionRequest for consistency, with tests. - Add description= to the constrained ResponsesRequest fields. Signed-off-by: lonexreb --- tensorrt_llm/serve/openai_protocol.py | 23 +++++++++++++++---- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../apps/test_openai_protocol_validation.py | 22 ++++++++++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 278011f2071a..4650cdb71892 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -672,7 +672,12 @@ class CompletionRequest(OpenAIBaseModel): ge=0, description="Top-k sampling; must be >= 0 (0 disables top-k).") top_p_min: float = 0.0 - min_p: float = 0.0 + min_p: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Minimum token probability scaled by the top token's " + "probability; must be in [0, 1] (0 disables min-p).") repetition_penalty: float = 1.0 # Unset by default so the engine picks its own beam-search defaults, as it # does for requests coming through the Python API. Sending concrete values @@ -1139,7 +1144,12 @@ class ChatCompletionRequest(OpenAIBaseModel): ge=0, description="Top-k sampling; must be >= 0 (0 disables top-k).") top_p_min: float = 0.0 - min_p: float = 0.0 + min_p: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Minimum token probability scaled by the top token's " + "probability; must be in [0, 1] (0 disables min-p).") repetition_penalty: float = 1.0 # Unset by default so the engine picks its own beam-search defaults, as it # does for requests coming through the Python API. Sending concrete values @@ -1639,12 +1649,17 @@ def _materialize_lazy_item_content(cls, value): "priority"] = "auto" store: Optional[bool] = True stream: Optional[bool] = False - temperature: Optional[float] = Field(default=None, ge=0.0) + temperature: Optional[float] = Field( + default=None, ge=0.0, description="Sampling temperature; must be >= 0.") text: Optional[ResponseTextConfig] = None tool_choice: ToolChoice = "auto" tools: list[Tool] = Field(default_factory=list) top_logprobs: Optional[int] = 0 - top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) + top_p: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description="Nucleus sampling probability; must be in [0, 1].") truncation: Optional[Literal["auto", "disabled"]] = "disabled" user: Optional[str] = None diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index a42393e3d02a..d4af717212d1 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -190,6 +190,7 @@ l0_cpu: - unittest/llmapi/apps/test_responses_usage.py - unittest/llmapi/apps/test_serving_extensions.py - unittest/llmapi/apps/test_tool_parsers.py + - unittest/llmapi/apps/test_openai_protocol_validation.py - unittest/llmapi/apps/test_web_search_shared.py - unittest/llmapi/test_bench_async.py - unittest/llmapi/test_config_overrides.py diff --git a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py index b07d957865b6..2109f2b65c96 100644 --- a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py +++ b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py @@ -14,9 +14,11 @@ # limitations under the License. """Boundary validation for trtllm-serve request models (issue #6329). -Invalid sampling parameters must be rejected at the API boundary with a -pydantic ValidationError (surfaced to clients as HTTP 422) instead of crashing -the server downstream. +Invalid sampling parameters are rejected at the pydantic boundary with a +field-specific ValidationError (surfaced to clients as HTTP 400 by the +server's RequestValidationError handler). SamplingParams._validate already +rejects these values later with a 400; the benefit here is the earlier, +field-named error and schema-visible bounds, not a crash fix. """ import pytest @@ -34,7 +36,12 @@ def _completion(**kw): def _chat(**kw): - return ChatCompletionRequest(model="m", messages=[{"role": "user", "content": "hi"}], **kw) + return ChatCompletionRequest(model="m", + messages=[{ + "role": "user", + "content": "hi" + }], + **kw) def _responses(**kw): @@ -67,6 +74,13 @@ def test_out_of_range_top_p_rejected(build, bad_top_p): build(top_p=bad_top_p) +@pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) +@pytest.mark.parametrize("bad_min_p", [-0.1, 1.1]) +def test_out_of_range_min_p_rejected(build, bad_min_p): + with pytest.raises(ValidationError): + build(min_p=bad_min_p) + + @pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) def test_negative_top_k_rejected(build): # top_k = -1 is vLLM's default; TRT-LLM's backend requires top_k >= 0, so From 20fcc50b5eb3b496cdb3e55284debf9e7bec814d Mon Sep 17 00:00:00 2001 From: lonexreb Date: Sat, 24 Oct 2026 14:27:36 -0700 Subject: [PATCH 4/5] [None][test] apply ruff-format and assert field location in protocol validation test Collapse the _chat helper onto one line as ruff-format requires (fixes the pre-commit failure). Rejection tests now assert the failing field's location so they prove the field-specific error the PR claims; add an inclusive-boundary acceptance test for min_p. Signed-off-by: lonexreb --- .../apps/test_openai_protocol_validation.py | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py index 2109f2b65c96..f342b46a081e 100644 --- a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py +++ b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py @@ -36,18 +36,20 @@ def _completion(**kw): def _chat(**kw): - return ChatCompletionRequest(model="m", - messages=[{ - "role": "user", - "content": "hi" - }], - **kw) + return ChatCompletionRequest(model="m", messages=[{"role": "user", "content": "hi"}], **kw) def _responses(**kw): return ResponsesRequest(model="m", input="hi", **kw) +def _rejected_field(build, **kw): + """Return the set of top-level field names that failed validation.""" + with pytest.raises(ValidationError) as exc_info: + build(**kw) + return {err["loc"][0] for err in exc_info.value.errors()} + + # top_k only exists on completion / chat requests. BUILDERS_WITH_TOP_K = [_completion, _chat] ALL_BUILDERS = [_completion, _chat, _responses] @@ -63,30 +65,32 @@ def test_valid_sampling_params_accepted(build): @pytest.mark.parametrize("build", ALL_BUILDERS) def test_negative_temperature_rejected(build): - with pytest.raises(ValidationError): - build(temperature=-0.1) + assert "temperature" in _rejected_field(build, temperature=-0.1) @pytest.mark.parametrize("build", ALL_BUILDERS) @pytest.mark.parametrize("bad_top_p", [-0.1, 1.1, 2.0]) def test_out_of_range_top_p_rejected(build, bad_top_p): - with pytest.raises(ValidationError): - build(top_p=bad_top_p) + assert "top_p" in _rejected_field(build, top_p=bad_top_p) @pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) @pytest.mark.parametrize("bad_min_p", [-0.1, 1.1]) def test_out_of_range_min_p_rejected(build, bad_min_p): - with pytest.raises(ValidationError): - build(min_p=bad_min_p) + assert "min_p" in _rejected_field(build, min_p=bad_min_p) @pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) def test_negative_top_k_rejected(build): # top_k = -1 is vLLM's default; TRT-LLM's backend requires top_k >= 0, so # reject it cleanly at the boundary rather than crashing. - with pytest.raises(ValidationError): - build(top_k=-1) + assert "top_k" in _rejected_field(build, top_k=-1) + + +@pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) +@pytest.mark.parametrize("edge_min_p", [0.0, 1.0]) +def test_min_p_boundaries_accepted(build, edge_min_p): + assert build(min_p=edge_min_p).min_p == edge_min_p @pytest.mark.parametrize("build", BUILDERS_WITH_TOP_K) From 67121c560939b645bc33176a15ee03ac48bc9e76 Mon Sep 17 00:00:00 2001 From: lonexreb Date: Wed, 28 Oct 2026 01:02:00 -0700 Subject: [PATCH 5/5] [None][test] mark request-model validation tests cpu_only The CPU stage runs pytest -m cpu_only and its conftest skips files without the marker, so this file was listed in l0_cpu.yml but never collected. Signed-off-by: lonexreb --- tests/unittest/llmapi/apps/test_openai_protocol_validation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py index f342b46a081e..146292f58914 100644 --- a/tests/unittest/llmapi/apps/test_openai_protocol_validation.py +++ b/tests/unittest/llmapi/apps/test_openai_protocol_validation.py @@ -30,6 +30,9 @@ ResponsesRequest, ) +# The CPU CI stage runs pytest with -m cpu_only and skips files without this marker. +pytestmark = pytest.mark.cpu_only + def _completion(**kw): return CompletionRequest(model="m", prompt="hi", **kw)