From 490ad743bf81b922445c0ebf2691149d34cbc5a4 Mon Sep 17 00:00:00 2001 From: daixu Date: Tue, 20 Oct 2026 12:05:05 +0800 Subject: [PATCH 1/2] [Fix] Preserve tool-result image ownership Keep Anthropic tool images with their tool_call_id and preserve surrounding user content order. Align GLM-5.3 multimodal tool results before both prompt rendering and media collection, and support Responses image outputs with grouped parallel calls. Add protocol, HTTP/SSE, and shared preprocessing regressions and update English/Chinese documentation. --- docs/en/llm/api_server_anthropic.md | 1 + docs/en/llm/api_server_responses.md | 27 ++- docs/zh_cn/llm/api_server_anthropic.md | 1 + docs/zh_cn/llm/api_server_responses.md | 27 ++- lmdeploy/model.py | 42 ++++ lmdeploy/serve/anthropic/adapter.py | 44 ++-- lmdeploy/serve/openai/responses/request.py | 27 +-- lmdeploy/serve/processors/multimodal.py | 1 + .../anthropic/test_adapter_conversion.py | 9 +- .../serve/anthropic/test_endpoints.py | 36 ++++ .../serve/openai/responses/test_images.py | 58 +++++- tests/test_lmdeploy/serve/test_tool_images.py | 189 ++++++++++++++++++ 12 files changed, 410 insertions(+), 52 deletions(-) create mode 100644 tests/test_lmdeploy/serve/test_tool_images.py diff --git a/docs/en/llm/api_server_anthropic.md b/docs/en/llm/api_server_anthropic.md index d8f58b4e58..560cdf7758 100644 --- a/docs/en/llm/api_server_anthropic.md +++ b/docs/en/llm/api_server_anthropic.md @@ -18,6 +18,7 @@ For Anthropic `POST` endpoints, include: ## Notes and Current Limits - `POST /v1/messages` supports text output, reasoning output (`thinking` blocks), and tool-use output (`tool_use` blocks). +- With a vision-capable model, `tool_result.content` can contain text and image blocks. These blocks remain in the same tool result, retaining `tool_use_id` and their internal order. Ordinary user content before, between, or after tool results retains its position. - Tool use requires launching API server with a configured tool parser (`--tool-call-parser ...`). - Reasoning block extraction depends on parser configuration (same parser stack used by OpenAI-compatible chat endpoint). - `count_tokens` is tokenizer/chat-template based and is intended for practical estimation. diff --git a/docs/en/llm/api_server_responses.md b/docs/en/llm/api_server_responses.md index ee834e030d..adb49b7ef2 100644 --- a/docs/en/llm/api_server_responses.md +++ b/docs/en/llm/api_server_responses.md @@ -16,11 +16,12 @@ For `POST /v1/responses`, include: ## Notes and Current Limits -- `POST /v1/responses` supports text and user-message image inputs with a vision-capable model. +- `POST /v1/responses` supports text, user-message images, and tool-result images with a vision-capable model. - `input` may be a string or a list of Responses input items. - User content arrays can mix `input_text` and `input_image` blocks. Images use `image_url` with an HTTP(S) URL or a base64 image data URL. Multiple images and their order are preserved in both streaming and non-streaming requests. - Image `detail` and the `min_dynamic_patch` / `max_dynamic_patch` extensions are forwarded like Chat image parameters; their effect depends on the model's image processor. -- Image `file_id` references, images in system/developer/assistant messages, and images in `function_call_output` are not supported. +- `function_call_output.output` can also contain an array of `input_text` and `input_image` blocks. Images stay associated with the result's `call_id`. Consecutive `function_call` items are treated as one parallel tool-call group. +- Image `file_id` references and images in system/developer/assistant messages are not supported. - `instructions`, `developer`, and `system` messages are merged into a single leading system message for chat-template compatibility. - Function tools are converted to LMDeploy's OpenAI-compatible tool format. Tool calling requires launching API server with a configured tool parser (`--tool-call-parser ...`). - `parallel_tool_calls` defaults to `true`. When it is `false`, LMDeploy follows vLLM-style compatibility and returns only the first parsed function call. @@ -122,3 +123,25 @@ When `stream=true`, the endpoint returns `text/event-stream` events such as: ## Codex Integration May refer to [codex](../integration/codex.md). + +## Example: tool results with images + +Replay the tool call and return its image using the same `call_id`. For parallel calls, place the consecutive `function_call` items before their results; results may arrive in a different order. Each result preserves its internal text/image order. + +```json +{ + "model": "your-vision-model", + "input": [ + {"role": "user", "content": "Describe the screenshot."}, + {"type": "function_call", "call_id": "call_screenshot", "name": "screenshot", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call_screenshot", + "output": [ + {"type": "input_text", "text": "Screenshot result:"}, + {"type": "input_image", "image_url": "https://example.com/screenshot.png"} + ] + } + ] +} +``` diff --git a/docs/zh_cn/llm/api_server_anthropic.md b/docs/zh_cn/llm/api_server_anthropic.md index 88a8bc247d..7962901930 100644 --- a/docs/zh_cn/llm/api_server_anthropic.md +++ b/docs/zh_cn/llm/api_server_anthropic.md @@ -18,6 +18,7 @@ LMDeploy 提供了轻量级的 Anthropic 兼容接口，方便接入 Anthropic ## 说明与当前限制 - `POST /v1/messages` 支持文本输出、推理输出（`thinking` block）和工具调用输出（`tool_use` block）。 +- 使用视觉模型时，`tool_result.content` 可以包含 text 和 image 块。这些块保留在同一工具结果中，并保留 `tool_use_id` 和内部顺序；工具结果前、后及结果之间的普通 user 内容也保留原有位置。 - 工具调用需要在启动 API Server 时配置工具解析器（`--tool-call-parser ...`）。 - 推理内容 block 的提取依赖解析器配置（与 OpenAI 兼容 chat 接口使用同一套解析器）。 - `count_tokens` 基于 tokenizer 和 chat template 计算，适合用于实用估算。 diff --git a/docs/zh_cn/llm/api_server_responses.md b/docs/zh_cn/llm/api_server_responses.md index 09630d758b..6f47b7aaa9 100644 --- a/docs/zh_cn/llm/api_server_responses.md +++ b/docs/zh_cn/llm/api_server_responses.md @@ -16,11 +16,12 @@ LMDeploy 提供了轻量级的 OpenAI Responses 兼容接口，方便接入使 ## 说明与当前限制 -- `POST /v1/responses` 支持文本输入，以及视觉模型的 user 消息图片输入。 +- `POST /v1/responses` 支持文本输入，以及视觉模型的 user 消息图片和工具结果图片输入。 - `input` 可以是字符串，也可以是 Responses input item 列表。 - user 的 content 数组可以混合 `input_text` 和 `input_image`。图片通过 `image_url` 指定 HTTP(S) URL 或 base64 图片 data URL；流式和非流式请求均支持多图，并保留图文顺序。 - 图片的 `detail` 和 `min_dynamic_patch` / `max_dynamic_patch` 扩展参数按 Chat 图片参数方式传递，实际效果取决于模型的图片处理器。 -- 暂不支持图片 `file_id` 引用、system/developer/assistant 消息中的图片，以及 `function_call_output` 中的图片。 +- `function_call_output.output` 也可以包含 `input_text` 和 `input_image` 数组。图片保留所属结果的 `call_id`，连续的 `function_call` 条目会转换为同一组并行工具调用。 +- 暂不支持图片 `file_id` 引用，以及 system/developer/assistant 消息中的图片。 - `instructions`、`developer` 和 `system` 消息会合并为一个位于最前面的 system message，以兼容 chat template。 - function tools 会转换为 LMDeploy 的 OpenAI 兼容 tool 格式。工具调用需要在启动 API Server 时配置工具解析器（`--tool-call-parser ...`）。 - `parallel_tool_calls` 默认为 `true`。当设置为 `false` 时，LMDeploy 会保持与 vLLM 类似的兼容行为，仅返回解析到的第一个 function call。 @@ -137,3 +138,25 @@ stream_idle_timeout_ms = 300000 ``` `model` 的值必须与 LMDeploy 暴露的模型名完全一致。 + +## 示例：包含图片的工具结果 + +重放工具调用，并使用相同的 `call_id` 返回图片。并行调用时，将连续的 `function_call` 条目放在结果之前；结果可以按不同顺序返回，每个结果内部的图文顺序都会保留。 + +```json +{ + "model": "your-vision-model", + "input": [ + {"role": "user", "content": "Describe the screenshot."}, + {"type": "function_call", "call_id": "call_screenshot", "name": "screenshot", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call_screenshot", + "output": [ + {"type": "input_text", "text": "Screenshot result:"}, + {"type": "input_image", "image_url": "https://example.com/screenshot.png"} + ] + } + ] +} +``` diff --git a/lmdeploy/model.py b/lmdeploy/model.py index 284f266539..ae9128ef1a 100644 --- a/lmdeploy/model.py +++ b/lmdeploy/model.py @@ -114,6 +114,10 @@ def from_json(cls, file_or_string): class BaseChatTemplate: """Base Chat template.""" + def prepare_multimodal_messages(self, messages: list[dict]) -> list[dict]: + """Prepare message order before rendering and collecting media.""" + return messages + def __init__(self, system='', meta_instruction='', @@ -788,11 +792,49 @@ def __init__(self, model_path: str = '', trust_remote_code: bool = False, **kwar self.stop_words.append(self.tokenizer.eot_token) arch, _ = get_model_arch(model_path) self.is_gpt_oss = arch == 'GptOssForCausalLM' + self.is_glm5_next = arch == 'Glm5NextForConditionalGeneration' if self.is_gpt_oss: self.stop_words.append('<|call|>') except Exception as e: raise ValueError(f'Try apply_chat_template failed: {e}') + def prepare_multimodal_messages(self, messages: list[dict]) -> list[dict]: + """Match GLM-5.3 tool rendering order before collecting media. + + Its template renders an unambiguous, contiguous result group in the preceding assistant's tool-call order. Both + the prompt and media collector must see that order. Other templates keep their original message order. + """ + if not self.is_glm5_next: + return messages + ordered = list(messages) + start = 0 + while start < len(ordered): + if ordered[start]['role'] != 'tool': + start += 1 + continue + end = start + 1 + while end < len(ordered) and ordered[end]['role'] == 'tool': + end += 1 + group = ordered[start:end] + has_media = any(isinstance(message.get('content'), list) + and any(isinstance(part, dict) and part.get('type') in ('image', 'video', 'audio') + for part in message['content']) for message in group) + previous = ordered[start - 1] if start else {} + calls = (previous.get('tool_calls') or []) if previous.get('role') == 'assistant' else [] + # Match the template's tool_call_id/id fallback. Ambiguous groups + # are rendered in input order, so leave their media in that order. + call_ids = [call.get('tool_call_id') or call.get('id') for call in calls] + result_ids = [message.get('tool_call_id') or message.get('id') for message in group] + valid_ids = all(isinstance(identifier, str) and identifier for identifier in call_ids + result_ids) + if (has_media and call_ids and valid_ids + and len(set(call_ids)) == len(call_ids) and len(set(result_ids)) == len(result_ids) + and set(result_ids).issubset(call_ids)): + positions = {identifier: index for index, identifier in enumerate(call_ids)} + ordered[start:end] = sorted(group, key=lambda message: positions[message.get('tool_call_id') + or message.get('id')]) + start = end + return ordered + def get_prompt(self, prompt, **kwargs): messages = [{'role': 'user', 'content': prompt}] return self.messages2prompt(messages, **kwargs) diff --git a/lmdeploy/serve/anthropic/adapter.py b/lmdeploy/serve/anthropic/adapter.py index 189cb5664f..5af2212176 100644 --- a/lmdeploy/serve/anthropic/adapter.py +++ b/lmdeploy/serve/anthropic/adapter.py @@ -251,37 +251,28 @@ def _merge_system_messages(messages: list[_MessageT]) -> list[_MessageT]: def _convert_user_tool_result(block: ContentBlockParam | dict[str, Any]) -> list[dict[str, Any]]: - tool_text = '' - tool_image_urls: list[str] = [] content = _block_get(block, 'content') - + tool_content: str | list[dict[str, Any]] = '' if isinstance(content, str): - tool_text = content + tool_content = content elif isinstance(content, list): - text_parts: list[str] = [] + parts: list[dict[str, Any]] = [] for item in content: if _block_get(item, 'type') == 'text': - text_parts.append(_block_get(item, 'text', '')) + parts.append(dict(type='text', text=_block_get(item, 'text', ''))) elif _block_get(item, 'type') == 'image': url = _convert_image_source_to_url(_block_get(item, 'source', {})) if url: - tool_image_urls.append(url) - tool_text = '\n'.join(text_parts) - - messages = [ - dict( - role='tool', - tool_call_id=_block_get(block, 'tool_use_id') or _block_get(block, 'id') or '', - content=tool_text or '', - ) - ] - if tool_image_urls: - messages.append( - dict( - role='user', - content=[dict(type='image_url', image_url=dict(url=url)) for url in tool_image_urls], - )) - return messages + parts.append(dict(type='image_url', image_url=dict(url=url))) + # A separate user image message loses its tool_call_id and interrupts + # the contiguous tool-result group used by chat templates. + if any(part['type'] == 'image_url' for part in parts): + tool_content = parts + else: + tool_content = '\n'.join(part['text'] for part in parts) + return [dict(role='tool', + tool_call_id=_block_get(block, 'tool_use_id') or _block_get(block, 'id') or '', + content=tool_content)] def to_openai_messages( @@ -340,6 +331,13 @@ def to_openai_messages( if block_type == 'tool_result': if message.role == 'user': + # Ordinary user content is a boundary between tool groups. + if content_parts: + content = (content_parts[0]['text'] + if len(content_parts) == 1 and content_parts[0]['type'] == 'text' + else content_parts) + openai_messages.append(dict(role='user', content=content)) + content_parts = [] openai_messages.extend(_convert_user_tool_result(block)) else: result_text = _text_from_block_content( diff --git a/lmdeploy/serve/openai/responses/request.py b/lmdeploy/serve/openai/responses/request.py index 9b74d4dc1d..7b0d6cbde1 100644 --- a/lmdeploy/serve/openai/responses/request.py +++ b/lmdeploy/serve/openai/responses/request.py @@ -278,12 +278,15 @@ def messages_from_input(request: ResponsesRequest) -> list[dict[str, Any]]: messages.append(dict(role='user', content=request.input)) return ([dict(role='system', content='\n\n'.join(system_parts))] if system_parts else []) + messages + previous_item_type = None for idx, item in enumerate(request.input): item = _as_dict(item) if not item: raise ValueError(f'Unsupported Responses input item at index {idx}.') item_type = item.get('type', 'message') + adjacent_call = previous_item_type == 'function_call' + previous_item_type = item_type if item_type == 'function_call': call_id = item.get('call_id') or item.get('id') name = item.get('name') @@ -292,21 +295,13 @@ def messages_from_input(request: ResponsesRequest) -> list[dict[str, Any]]: arguments = item.get('arguments') or '' if not isinstance(arguments, str): raise ValueError(f'Unsupported `arguments` in function_call item at index {idx}. Expected string.') - messages.append( - dict( - role='assistant', - content=None, - tool_calls=[ - dict( - id=call_id, - type='function', - function=dict( - name=name, - arguments=arguments, - ), - ) - ], - )) + call = dict(id=call_id, type='function', function=dict(name=name, arguments=arguments)) + # Adjacent calls belong to one parallel invocation. Keep all IDs + # visible to templates associating the following tool results. + if adjacent_call: + messages[-1]['tool_calls'].append(call) + else: + messages.append(dict(role='assistant', content=None, tool_calls=[call])) continue if item_type == 'function_call_output': call_id = item.get('call_id') @@ -316,7 +311,7 @@ def messages_from_input(request: ResponsesRequest) -> list[dict[str, Any]]: dict( role='tool', tool_call_id=call_id, - content=_content_from_input(item.get('output', ''), f'input[{idx}].output'), + content=_content_from_input(item.get('output', ''), f'input[{idx}].output', allow_images=True), )) continue if item_type != 'message': diff --git a/lmdeploy/serve/processors/multimodal.py b/lmdeploy/serve/processors/multimodal.py index 3623150077..66db813bbc 100644 --- a/lmdeploy/serve/processors/multimodal.py +++ b/lmdeploy/serve/processors/multimodal.py @@ -426,6 +426,7 @@ async def _get_multimodal_prompt_input(self, messages = await self.async_parse_multimodal_item(messages, media_io_kwargs, allowed_media_domains=self.allowed_media_domains) + messages = chat_template.prepare_multimodal_messages(messages) if self.backend == 'turbomind': if self.vl_encoder._uses_new_preprocess: diff --git a/tests/test_lmdeploy/serve/anthropic/test_adapter_conversion.py b/tests/test_lmdeploy/serve/anthropic/test_adapter_conversion.py index cb9bb74494..b81f56018a 100644 --- a/tests/test_lmdeploy/serve/anthropic/test_adapter_conversion.py +++ b/tests/test_lmdeploy/serve/anthropic/test_adapter_conversion.py @@ -266,7 +266,7 @@ def test_to_openai_messages_maps_user_tool_result_to_tool_role_message(): } -def test_to_openai_messages_maps_user_tool_result_images_to_user_message(): +def test_to_openai_messages_keeps_tool_result_images_with_tool_id(): request = _make_request( messages=[{ 'role': 'user', @@ -293,11 +293,10 @@ def test_to_openai_messages_maps_user_tool_result_images_to_user_message(): { 'role': 'tool', 'tool_call_id': 'toolu_123', - 'content': 'see image', - }, - { - 'role': 'user', 'content': [{ + 'type': 'text', + 'text': 'see image', + }, { 'type': 'image_url', 'image_url': { 'url': 'data:image/jpeg;base64,xyz789', diff --git a/tests/test_lmdeploy/serve/anthropic/test_endpoints.py b/tests/test_lmdeploy/serve/anthropic/test_endpoints.py index 01eb5f02a7..d07261e663 100644 --- a/tests/test_lmdeploy/serve/anthropic/test_endpoints.py +++ b/tests/test_lmdeploy/serve/anthropic/test_endpoints.py @@ -1150,3 +1150,39 @@ def test_messages_rejects_logprobs_when_engine_logprobs_mode_disabled(): response = _post_messages(client, messages=[{'role': 'user', 'content': 'Hi'}], return_logprob=True) assert response.status_code == 400 assert 'return_logprob' in response.json()['error']['message'] + + +@pytest.mark.parametrize('stream', [False, True]) +def test_http_tool_result_images_keep_their_ids_and_content_order(stream): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + context = _FakeServerContext() + app = FastAPI() + app.include_router(create_anthropic_router(context)) + messages = [ + dict(role='user', content='Read both screenshots.'), + dict(role='assistant', content=[dict(type='tool_use', id=identifier, name='screenshot', input={}) + for identifier in ('first', 'second')]), + dict(role='user', content=[dict(type='tool_result', tool_use_id=identifier, content=[ + dict(type='text', text='before'), + dict(type='image', source=dict(type='url', url=f'https://example.com/{identifier}.png')), + dict(type='text', text='after'), + ]) for identifier in ('second', 'first')]), + ] + with TestClient(app) as client: + response = client.post('/v1/messages', headers=ANTHROPIC_HEADERS, + json=_messages_payload(messages=messages, stream=stream)) + assert response.status_code == 200 + if stream: + assert 'event: message_stop' in response.text + else: + assert response.json()['stop_reason'] == 'end_turn' + forwarded = context.async_engine.preprocess_calls[0][0][0] + assert [m['role'] for m in forwarded] == ['user', 'assistant', 'tool', 'tool'] + for message, identifier in zip(forwarded[2:], ('second', 'first')): + assert message['tool_call_id'] == identifier + assert message['content'] == [dict(type='text', text='before'), + dict(type='image_url', image_url=dict(url=f'https://example.com/{identifier}.png')), + dict(type='text', text='after')] + assert len(context.session_mgr.removed) == 1 diff --git a/tests/test_lmdeploy/serve/openai/responses/test_images.py b/tests/test_lmdeploy/serve/openai/responses/test_images.py index 48dee3887a..18a989b442 100644 --- a/tests/test_lmdeploy/serve/openai/responses/test_images.py +++ b/tests/test_lmdeploy/serve/openai/responses/test_images.py @@ -104,12 +104,14 @@ def test_images_outside_user_messages_remain_unsupported(role): messages_from_input(request) -def test_tool_result_images_remain_unsupported(): +def test_tool_result_images_preserve_call_id_and_image_parameters(): + image = dict(_image(), detail='high', max_dynamic_patch=4) request = ResponsesRequest(model='fake-model', input=[ - dict(type='function_call_output', call_id='call_1', output=[_image()]), + dict(type='function_call_output', call_id='call_1', output=[image]), ]) - with pytest.raises(ValueError, match=r'input_image.*input\[0\].output\[0\]'): - messages_from_input(request) + assert messages_from_input(request) == [dict( + role='tool', tool_call_id='call_1', content=[dict( + type='image_url', image_url=dict(url=image['image_url'], detail='high', max_dynamic_patch=4))])] @pytest.mark.parametrize('part, message', [ @@ -178,3 +180,51 @@ async def preprocess(prompt, session, **kwargs): assert [part['data'].getpixel((0, 0)) for part in content[1:]] == [(255, 0, 0), (0, 0, 255)] assert context.async_engine.preprocess_kwargs['do_preprocess'] is True assert context.async_engine.session_mgr.removed == context.sessions + + +@pytest.mark.parametrize('stream', [False, True]) +def test_http_tool_images_preserve_parallel_calls_and_result_ownership(responses_endpoint, stream): + _, context = responses_endpoint + original_preprocess = context.async_engine.preprocess + decoded = [] + + async def preprocess(prompt, session, **kwargs): + decoded.extend(await MultimodalProcessor.async_parse_multimodal_item(prompt)) + return await original_preprocess(prompt, session, **kwargs) + + context.async_engine.preprocess = preprocess + app = FastAPI() + app.include_router(create_responses_router(context)) + items = [dict(role='user', content='Read both tool images.')] + items += [dict(type='function_call', call_id=call_id, name='screenshot', arguments='{}') + for call_id in ('red', 'blue')] + items += [dict(type='function_call_output', call_id=color, output=[_image(color)]) + for color in ('blue', 'red')] + with TestClient(app) as client: + response = client.post('/v1/responses', json=dict(model='fake-model', input=items, stream=stream)) + assert response.status_code == 200 + if stream: + assert 'event: response.completed' in response.text + assert 'event: response.failed' not in response.text + else: + assert response.json()['status'] == 'completed' + assert [call['id'] for call in decoded[1]['tool_calls']] == ['red', 'blue'] + assert [(m['tool_call_id'], m['content'][0]['data'].getpixel((0, 0))) for m in decoded[2:]] == [ + ('blue', (0, 0, 255)), ('red', (255, 0, 0)), + ] + assert context.async_engine.session_mgr.removed == context.sessions + + +@pytest.mark.parametrize('stream', [False, True]) +def test_invalid_tool_image_returns_400_before_streaming(responses_endpoint, stream): + _, context = responses_endpoint + app = FastAPI() + app.include_router(create_responses_router(context)) + with TestClient(app) as client: + response = client.post('/v1/responses', json=dict(model='fake-model', stream=stream, input=[ + dict(type='function_call_output', call_id='call_1', output=[dict(type='input_image', file_id='file_1')]), + ])) + assert response.status_code == 400 + assert 'input[0].output[0]' in response.json()['error']['message'] + assert context.async_engine.prompt is None + assert context.sessions == [] diff --git a/tests/test_lmdeploy/serve/test_tool_images.py b/tests/test_lmdeploy/serve/test_tool_images.py new file mode 100644 index 0000000000..657c048c6c --- /dev/null +++ b/tests/test_lmdeploy/serve/test_tool_images.py @@ -0,0 +1,189 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import asyncio +import base64 +import copy +import itertools +from io import BytesIO +from types import SimpleNamespace + +import pytest +from PIL import Image + +from lmdeploy.model import BaseChatTemplate, HFChatTemplate +from lmdeploy.serve.anthropic.adapter import to_openai_messages +from lmdeploy.serve.anthropic.protocol import MessagesRequest +from lmdeploy.serve.openai.protocol import ChatCompletionRequest +from lmdeploy.serve.openai.responses import ResponsesRequest +from lmdeploy.serve.openai.responses.request import messages_from_input +from lmdeploy.serve.processors.multimodal import MultimodalProcessor +from lmdeploy.vl.model.base import VisionModel + +CALL_IDS = ['call_z', 'call_a', 'call_m'] +COLORS = ['red', 'blue', 'lime'] +PIXELS = [(255, 0, 0), (0, 0, 255), (0, 255, 0)] + + +def _image(color): + buf = BytesIO() + Image.new('RGB', (28, 28), color).save(buf, format='PNG') + return 'data:image/png;base64,' + base64.b64encode(buf.getvalue()).decode() + + +def _messages(protocol, order, two_images=False): + calls = [dict(id=call_id, type='function', function=dict(name='screenshot', arguments='{}')) + for call_id in CALL_IDS] + contents = [] + for color in COLORS: + parts = [dict(type='text', text='before'), dict(type='image_url', image_url=dict(url=_image(color))), + dict(type='text', text='after')] + if two_images: + parts += [dict(type='image_url', image_url=dict(url=_image('white')))] + contents.append(parts) + messages = [dict(role='user', content='Read the screenshots.'), + dict(role='assistant', content=None, tool_calls=calls)] + messages += [dict(role='tool', tool_call_id=CALL_IDS[i], content=contents[i]) for i in order] + if protocol == 'chat': + return ChatCompletionRequest(model='fake-model', messages=messages).messages + if protocol == 'messages': + uses = [dict(type='tool_use', id=call_id, name='screenshot', input={}) for call_id in CALL_IDS] + results = [] + for i in order: + parts = [dict(type='image', source=dict(type='url', url=p['image_url']['url'])) + if p['type'] == 'image_url' else p for p in contents[i]] + results.append(dict(type='tool_result', tool_use_id=CALL_IDS[i], content=parts)) + request = MessagesRequest(model='fake-model', max_tokens=32, messages=[messages[0], + dict(role='assistant', content=uses), dict(role='user', content=results)]) + return to_openai_messages(request) + items = [messages[0]] + items += [dict(type='function_call', call_id=call_id, name='screenshot', arguments='{}') for call_id in CALL_IDS] + for i in order: + parts = [dict(type='input_image', image_url=p['image_url']['url']) if p['type'] == 'image_url' + else dict(type='input_text', text=p['text']) for p in contents[i]] + items.append(dict(type='function_call_output', call_id=CALL_IDS[i], output=parts)) + return messages_from_input(ResponsesRequest(model='fake-model', input=items)) + + +def _template(glm=True): + template = HFChatTemplate.__new__(HFChatTemplate) + template.is_glm5_next = glm + return template + + +@pytest.mark.parametrize('protocol', ['chat', 'messages', 'responses']) +@pytest.mark.parametrize('order', list(itertools.permutations(range(3)))) +@pytest.mark.parametrize('two_images', [False, True]) +def test_parallel_tool_images_keep_ownership_through_shared_processor(protocol, order, two_images): + messages = _messages(protocol, order, two_images) + before = copy.deepcopy(messages) + assert [m['tool_call_id'] for m in messages[2:]] == [CALL_IDS[i] for i in order] + assert [call['id'] for call in messages[1]['tool_calls']] == CALL_IDS + # Exercise the actual shared path used before both prompt rendering and + # media collection, with real decoding and no model/GPU allocation. + seen = {} + + def get_input_prompt(messages, **kwargs): + seen['rendered'] = messages + return 'prompt' + + async def preprocess(messages, input_prompt, **kwargs): + assert messages is seen['rendered'] + assert input_prompt == 'prompt' + seen['collected'] = VisionModel.collect_multimodal_items(messages) + return messages + + encoder = SimpleNamespace(_uses_new_preprocess=True, model=SimpleNamespace(get_input_prompt=get_input_prompt), + preprocess=preprocess) + processor = MultimodalProcessor(None, _template(), vl_encoder=encoder, backend='pytorch') + result = asyncio.run(processor.get_prompt_input(messages, do_preprocess=True)) + assert [m['tool_call_id'] for m in result[2:]] == CALL_IDS + pixels = [data.getpixel((0, 0)) for _, data, _ in seen['collected']] + expected = [p for color in PIXELS for p in ([color, (255, 255, 255)] if two_images else [color])] + assert pixels == expected + for message in result[2:]: + assert [part['type'] for part in message['content']][:3] == ['text', 'image', 'text'] + assert message['content'][0]['text'] == 'before' + assert message['content'][2]['text'] == 'after' + assert messages == before + + +def _parsed(): + return asyncio.run(MultimodalProcessor.async_parse_multimodal_item(_messages('chat', (2, 1, 0)))) + + +@pytest.mark.parametrize('kind', ['missing_result', 'duplicate_result', 'unknown_result', + 'missing_call', 'duplicate_call', 'no_calls']) +def test_ambiguous_ids_keep_template_fallback_order(kind): + messages = _parsed() + if kind == 'missing_result': + messages[2].pop('tool_call_id') + elif kind == 'duplicate_result': + messages[2]['tool_call_id'] = messages[3]['tool_call_id'] + elif kind == 'unknown_result': + messages[2]['tool_call_id'] = 'unknown' + elif kind == 'missing_call': + messages[1]['tool_calls'][0].pop('id') + elif kind == 'duplicate_call': + messages[1]['tool_calls'][0]['id'] = messages[1]['tool_calls'][1]['id'] + else: + messages[1].pop('tool_calls') + ordered = _template().prepare_multimodal_messages(messages) + assert all(left is right for left, right in zip(ordered, messages)) + + +def test_partial_group_id_fallback_and_mixed_text_result(): + messages = _parsed() + messages.pop(3) # A subset of the calls has returned. + messages[2]['id'] = messages[2].pop('tool_call_id') + messages[2]['content'] = 'plain text result' + ordered = _template().prepare_multimodal_messages(messages) + assert ordered[2] is messages[3] + assert ordered[3] is messages[2] + + +def test_no_sort_for_other_templates_or_text_only_results(): + messages = _parsed() + assert _template(glm=False).prepare_multimodal_messages(messages) is messages + assert BaseChatTemplate().prepare_multimodal_messages(messages) is messages + for message in messages[2:]: + message['content'] = 'text only' + ordered = _template().prepare_multimodal_messages(messages) + assert all(left is right for left, right in zip(ordered, messages)) + + +def test_user_boundaries_and_separate_tool_rounds(): + first = _parsed() + second = _parsed() + messages = first + [dict(role='user', content='next round')] + second + ordered = _template().prepare_multimodal_messages(messages) + assert [m['tool_call_id'] for m in ordered if m['role'] == 'tool'] == CALL_IDS * 2 + split = first[:2] + [dict(role='user', content='boundary')] + first[2:] + ordered = _template().prepare_multimodal_messages(split) + assert all(left is right for left, right in zip(ordered, split)) + + +def test_messages_preserves_user_content_around_tool_results(): + result = dict(type='tool_result', tool_use_id='call_z', content=[ + dict(type='text', text='inside before'), dict(type='image', source=dict(type='url', url=_image('red'))), + dict(type='text', text='inside after'), + ]) + request = MessagesRequest(model='fake-model', max_tokens=32, messages=[dict(role='user', content=[ + dict(type='text', text='before'), result, dict(type='text', text='between'), result, + dict(type='image', source=dict(type='url', url=_image('blue'))), dict(type='text', text='after'), + ])]) + messages = to_openai_messages(request) + assert [m['role'] for m in messages] == ['user', 'tool', 'user', 'tool', 'user'] + assert messages[0]['content'] == 'before' + assert messages[2]['content'] == 'between' + assert [p['type'] for p in messages[1]['content']] == ['text', 'image_url', 'text'] + assert [p['type'] for p in messages[4]['content']] == ['image_url', 'text'] + assert messages[4]['content'][1]['text'] == 'after' + + +def test_responses_does_not_merge_calls_across_rounds_or_messages(): + call = dict(type='function_call', call_id='call_z', name='screenshot', arguments='{}') + output = dict(type='function_call_output', call_id='call_z', output='text') + request = ResponsesRequest(model='fake-model', input=[call, output, call, + dict(role='user', content='boundary'), call]) + messages = messages_from_input(request) + assert [m['role'] for m in messages] == ['assistant', 'tool', 'assistant', 'user', 'assistant'] + assert all(len(m['tool_calls']) == 1 for m in messages if m['role'] == 'assistant') From c56b75165fe17e6b5eef05cc3cf5494027a4e931 Mon Sep 17 00:00:00 2001 From: daixu Date: Tue, 20 Oct 2026 12:13:31 +0800 Subject: [PATCH 2/2] [Fix] Complete tool-image HTTP regression coverage Add streaming and non-streaming Chat HTTP tests with real image decoding and shared tool-result ordering. Clarify that the model-specific fix targets GLM-5.3-Flash and distinguish three-protocol preprocessing tests from HTTP endpoint tests. --- lmdeploy/model.py | 3 +- .../chat_completions/test_chat_tool_images.py | 86 +++++++++++++++++++ tests/test_lmdeploy/serve/test_tool_images.py | 5 ++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 tests/test_lmdeploy/serve/openai/chat_completions/test_chat_tool_images.py diff --git a/lmdeploy/model.py b/lmdeploy/model.py index ae9128ef1a..ff1b4a9997 100644 --- a/lmdeploy/model.py +++ b/lmdeploy/model.py @@ -792,6 +792,7 @@ def __init__(self, model_path: str = '', trust_remote_code: bool = False, **kwar self.stop_words.append(self.tokenizer.eot_token) arch, _ = get_model_arch(model_path) self.is_gpt_oss = arch == 'GptOssForCausalLM' + # GLM-5.3-Flash's multimodal architecture, not the text-only GLM-5.3. self.is_glm5_next = arch == 'Glm5NextForConditionalGeneration' if self.is_gpt_oss: self.stop_words.append('<|call|>') @@ -799,7 +800,7 @@ def __init__(self, model_path: str = '', trust_remote_code: bool = False, **kwar raise ValueError(f'Try apply_chat_template failed: {e}') def prepare_multimodal_messages(self, messages: list[dict]) -> list[dict]: - """Match GLM-5.3 tool rendering order before collecting media. + """Match GLM-5.3-Flash tool rendering order before collecting media. Its template renders an unambiguous, contiguous result group in the preceding assistant's tool-call order. Both the prompt and media collector must see that order. Other templates keep their original message order. diff --git a/tests/test_lmdeploy/serve/openai/chat_completions/test_chat_tool_images.py b/tests/test_lmdeploy/serve/openai/chat_completions/test_chat_tool_images.py new file mode 100644 index 0000000000..96029de04d --- /dev/null +++ b/tests/test_lmdeploy/serve/openai/chat_completions/test_chat_tool_images.py @@ -0,0 +1,86 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""HTTP regressions for tool images; generation is simulated, media decoding is +real.""" +import base64 +from io import BytesIO +from types import SimpleNamespace + +import pytest +from fastapi import APIRouter, FastAPI +from fastapi.testclient import TestClient +from PIL import Image + +from lmdeploy.model import HFChatTemplate +from lmdeploy.serve.openai.chat_completions import register +from lmdeploy.serve.processors.multimodal import MultimodalProcessor +from lmdeploy.vl.model.base import VisionModel + + +def _image(color): + buffer = BytesIO() + Image.new('RGB', (28, 28), color).save(buffer, format='PNG') + data = base64.b64encode(buffer.getvalue()).decode() + return dict(type='image_url', image_url=dict(url=f'data:image/png;base64,{data}')) + + +@pytest.mark.parametrize('stream', [False, True]) +def test_http_tool_images_reach_shared_preprocessing_with_correct_ownership(chat_endpoint, stream): + _, context = chat_endpoint + seen = {} + template = HFChatTemplate.__new__(HFChatTemplate) + template.is_glm5_next = True + + def get_input_prompt(messages, **kwargs): + seen['rendered'] = messages + return 'prompt' + + async def preprocess_images(messages, input_prompt, **kwargs): + assert messages is seen['rendered'] + seen['media'] = VisionModel.collect_multimodal_items(messages) + return messages + + encoder = SimpleNamespace(_uses_new_preprocess=True, model=SimpleNamespace(get_input_prompt=get_input_prompt), + preprocess=preprocess_images) + processor = MultimodalProcessor(None, template, vl_encoder=encoder, backend='pytorch') + original_preprocess = context.async_engine.preprocess + + async def preprocess(prompt, session, **kwargs): + seen['received'] = prompt + await processor.get_prompt_input(prompt, do_preprocess=kwargs['do_preprocess']) + return await original_preprocess(prompt, session, **kwargs) + + context.async_engine.preprocess = preprocess + messages = [dict(role='user', content='Read the tool screenshots.'), dict(role='assistant', content=None, + tool_calls=[dict(id=identifier, type='function', function=dict(name='screenshot', arguments='{}')) + for identifier in ('first', 'second')])] + messages += [dict(role='tool', tool_call_id=identifier, content=[ + dict(type='text', text='before'), _image(color), dict(type='text', text='between'), _image('white'), + dict(type='text', text='after'), + ]) for identifier, color in [('second', 'blue'), ('first', 'red')]] + app = FastAPI() + router = APIRouter() + register(router, context) + app.include_router(router) + with TestClient(app) as client: + response = client.post('/v1/chat/completions', json=dict(model='fake-model', messages=messages, stream=stream)) + + assert response.status_code == 200 + if stream: + assert response.headers['content-type'].startswith('text/event-stream') + assert response.text.rstrip().endswith('data: [DONE]') + assert 'choice-1' in response.text + else: + assert response.json()['choices'][0]['message']['content'] == 'choice-1' + assert response.json()['choices'][0]['finish_reason'] == 'stop' + # Request validation/forwarding keeps IDs, content, and arrival order; + # GLM-5.3-Flash preprocessing then aligns both media and prompt order. + assert seen['received'] == messages + assert [m['tool_call_id'] for m in seen['rendered'][2:]] == ['first', 'second'] + for message in seen['rendered'][2:]: + assert [p['type'] for p in message['content']] == ['text', 'image', 'text', 'image', 'text'] + assert [p['text'] for p in message['content'] if p['type'] == 'text'] == ['before', 'between', 'after'] + assert [image.getpixel((0, 0)) for _, image, _ in seen['media']] == [ + (255, 0, 0), (255, 255, 255), (0, 0, 255), (255, 255, 255), + ] + assert len(context.session_manager.removed) == 1 + assert context.session_manager.sessions == {} diff --git a/tests/test_lmdeploy/serve/test_tool_images.py b/tests/test_lmdeploy/serve/test_tool_images.py index 657c048c6c..b0c059d7e4 100644 --- a/tests/test_lmdeploy/serve/test_tool_images.py +++ b/tests/test_lmdeploy/serve/test_tool_images.py @@ -1,4 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. +"""Three-protocol conversion and shared preprocessing tests, without HTTP or +generation. + +HTTP/SSE tool-image tests live in each protocol's endpoint test directory. +""" import asyncio import base64 import copy