From 826195f5d6f934e88e89615b99c53e0893b2fe76 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Tue, 28 Jul 2026 22:12:13 +0200 Subject: [PATCH] feat(openai): surface Gemini thinking + request thought summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenAI-compatible path only carried the Gemini thought_signature; it never surfaced the model's reasoning. Two additions, mirroring the C# SDK change (askui/csharp-sdk#13): - Parse reasoning into a thinking block: read `reasoning_content` / `reasoning` from the response message's extra fields, and split inline `…` out of `content` (how Gemini via Vertex's OpenAI endpoint returns its thought summary). Either becomes a BetaThinkingBlock ahead of the spoken text. - Request the summary on the AskUI Gemini path via `extra_body.google.thinking_config.include_thoughts`. Vertex rejects `reasoning_effort` and a custom `thinking_config` together ("found both", 400), so only the latter is sent; caller-supplied options win. Co-Authored-By: Claude Fable 5 --- .../model_providers/askui_vlm_provider.py | 22 +++++ src/askui/models/openai/messages_api.py | 49 ++++++++++- .../test_askui_vlm_provider.py | 50 ++++++++++- tests/unit/models/openai/test_messages_api.py | 86 +++++++++++++++++++ 4 files changed, 205 insertions(+), 2 deletions(-) diff --git a/src/askui/model_providers/askui_vlm_provider.py b/src/askui/model_providers/askui_vlm_provider.py index 4c92cfcc..e1843d3e 100644 --- a/src/askui/model_providers/askui_vlm_provider.py +++ b/src/askui/model_providers/askui_vlm_provider.py @@ -65,6 +65,26 @@ def _infer_backend(model_id: str) -> _Backend: raise ValueError(error_msg) +def _with_google_thinking( + provider_options: dict[str, Any] | None, +) -> dict[str, Any]: + """Ask Gemini (Vertex's OpenAI-compatible endpoint) for its thought summary. + + Sets ``extra_body.google.thinking_config.include_thoughts`` so the summary + comes back (inline ```` tags / ``reasoning_content``, surfaced as a + thinking block). Vertex rejects ``reasoning_effort`` and a custom + ``thinking_config`` in the same request ("found both", 400), so only the + latter is sent. Caller-supplied options win — an existing ``extra_body`` is + left untouched. + """ + options = dict(provider_options) if provider_options else {} + options.setdefault( + "extra_body", + {"google": {"thinking_config": {"include_thoughts": True}}}, + ) + return options + + class AskUIVlmProvider(VlmProvider): """VLM provider that routes requests through AskUI's hosted model proxies. @@ -228,6 +248,8 @@ def create_message( ) -> MessageParam: if system is not None: system = self.augment_system_prompt(system) + if self._backend is _Backend.GOOGLE: + provider_options = _with_google_thinking(provider_options) result: MessageParam = self._messages_api.create_message( messages=messages, model_id=self._model_id_value, diff --git a/src/askui/models/openai/messages_api.py b/src/askui/models/openai/messages_api.py index 83d8c4d4..d7e54aa4 100644 --- a/src/askui/models/openai/messages_api.py +++ b/src/askui/models/openai/messages_api.py @@ -2,6 +2,7 @@ import json import logging +import re from collections.abc import Callable from typing import Any @@ -273,6 +274,37 @@ def _parse_tool_calls( ) +_THINK_TAG_PATTERN = re.compile(r"(.*?)", re.DOTALL | re.IGNORECASE) + + +def _split_inline_thinking(content: str) -> tuple[str | None, str | None]: + """Split ``…`` reasoning out of an inline content string. + + Gemini via Vertex's OpenAI-compatible endpoint returns its thought summary + inline in ``content`` wrapped in ```` tags rather than in a separate + ``reasoning_content`` field. Returns ``(thinking, text)`` — either may be + ``None`` (no tags found → ``(None, content)``; nothing but tags → text is + ``None``). + """ + matches = _THINK_TAG_PATTERN.findall(content) + if not matches: + return None, content + thinking = "\n".join(m.strip() for m in matches) + text = _THINK_TAG_PATTERN.sub("", content).strip() + return (thinking or None), (text or None) + + +def _extract_reasoning(message: ChatCompletionMessage) -> str | None: + """Read a model's reasoning summary from the non-standard response fields. + + OpenAI-compatible reasoning models return it in ``reasoning_content`` + (Vertex/DeepSeek) or ``reasoning`` (OpenRouter); both land in ``model_extra``. + """ + extra = message.model_extra or {} + raw = extra.get("reasoning_content") or extra.get("reasoning") + return raw if isinstance(raw, str) and raw else None + + def _from_openai_response(response: ChatCompletion) -> MessageParam: """Convert an OpenAI ``ChatCompletion`` to an internal `MessageParam`.""" choice = response.choices[0] @@ -281,8 +313,23 @@ def _from_openai_response(response: ChatCompletion) -> MessageParam: content_blocks: list[ContentBlockParam] = [] + # Reasoning can arrive either in a dedicated field (`reasoning_content` / + # `reasoning`) or inline in `content` wrapped in `` tags (Gemini via + # the Vertex OpenAI-compatible endpoint). Surface either as a thinking block + # ahead of the spoken text so it reads as the model's private reasoning. + field_reasoning = _extract_reasoning(message) + inline_reasoning: str | None = None + answer_text: str | None = None if message.content: - content_blocks.append(TextBlockParam(text=message.content)) + inline_reasoning, answer_text = _split_inline_thinking(message.content) + + reasoning = field_reasoning or inline_reasoning + if reasoning: + content_blocks.append( + BetaThinkingBlock(signature="", thinking=reasoning, type="thinking") + ) + if answer_text: + content_blocks.append(TextBlockParam(text=answer_text)) _parse_tool_calls(message, content_blocks) diff --git a/tests/unit/model_providers/test_askui_vlm_provider.py b/tests/unit/model_providers/test_askui_vlm_provider.py index a3052632..8265609a 100644 --- a/tests/unit/model_providers/test_askui_vlm_provider.py +++ b/tests/unit/model_providers/test_askui_vlm_provider.py @@ -6,7 +6,10 @@ from anthropic import Anthropic from openai import OpenAI -from askui.model_providers.askui_vlm_provider import AskUIVlmProvider +from askui.model_providers.askui_vlm_provider import ( + AskUIVlmProvider, + _with_google_thinking, +) from askui.models.anthropic.messages_api import AnthropicMessagesApi from askui.models.askui.inference_api_settings import AskUiInferenceApiSettings from askui.models.openai.messages_api import OpenAIMessagesApi @@ -149,3 +152,48 @@ def test_gemini_uses_scaled_coordinate_space( assert provider.coordinate_space == ScaledCoordinateSpace( width=1000, height=1000 ) + + +class TestWithGoogleThinking: + def test_injects_include_thoughts_when_absent(self) -> None: + options = _with_google_thinking(None) + assert options == { + "extra_body": {"google": {"thinking_config": {"include_thoughts": True}}} + } + + def test_preserves_other_provider_options(self) -> None: + options = _with_google_thinking({"temperature": 0.2}) + assert options["temperature"] == 0.2 + assert options["extra_body"]["google"]["thinking_config"]["include_thoughts"] + + def test_caller_extra_body_wins(self) -> None: + caller = {"extra_body": {"google": {"foo": "bar"}}} + options = _with_google_thinking(caller) + assert options["extra_body"] == {"google": {"foo": "bar"}} + + +class TestAskUIVlmProviderThinkingRequest: + def test_gemini_requests_thoughts( + self, askui_settings: AskUiInferenceApiSettings + ) -> None: + provider = AskUIVlmProvider( + askui_settings=askui_settings, + model_id="gemini-2.5-pro", + ) + stub = MagicMock() + provider.__dict__["_messages_api"] = stub + provider.create_message(messages=[]) + forwarded = stub.create_message.call_args.kwargs["provider_options"] + assert forwarded["extra_body"]["google"]["thinking_config"]["include_thoughts"] + + def test_claude_does_not_request_thoughts( + self, askui_settings: AskUiInferenceApiSettings + ) -> None: + provider = AskUIVlmProvider( + askui_settings=askui_settings, + model_id="claude-sonnet-4-6", + ) + stub = MagicMock() + provider.__dict__["_messages_api"] = stub + provider.create_message(messages=[]) + assert stub.create_message.call_args.kwargs["provider_options"] is None diff --git a/tests/unit/models/openai/test_messages_api.py b/tests/unit/models/openai/test_messages_api.py index b3ebfeb2..d21be9e4 100644 --- a/tests/unit/models/openai/test_messages_api.py +++ b/tests/unit/models/openai/test_messages_api.py @@ -67,6 +67,28 @@ def _make_completion( ) +def _completion_with_extra( + content: str | None, + extra: dict[str, object], +) -> ChatCompletion: + """Build a completion whose message carries non-standard fields. + + Reasoning models return their summary in extra fields (``reasoning_content`` + / ``reasoning``) that land in ``ChatCompletionMessage.model_extra``. + """ + message = ChatCompletionMessage.model_validate( + {"role": "assistant", "content": content, **extra} + ) + return ChatCompletion( + id="chatcmpl-test", + choices=[Choice(finish_reason="stop", index=0, message=message)], + created=1234567890, + model="gemini-2.5-pro", + object="chat.completion", + usage=None, + ) + + class TestMapFinishReason: def test_stop_maps_to_end_turn(self) -> None: assert _map_finish_reason("stop") == "end_turn" @@ -457,6 +479,70 @@ def test_usage_captured(self) -> None: assert result.usage.output_tokens == 100 +class TestFromOpenaiResponseThinking: + def test_inline_think_tags_become_thinking_block(self) -> None: + completion = _make_completion( + content="I should click the button.Clicking now." + ) + result = _from_openai_response(completion) + assert isinstance(result.content, list) + assert len(result.content) == 2 + thinking, text = result.content + assert isinstance(thinking, BetaThinkingBlock) + assert thinking.thinking == "I should click the button." + assert isinstance(text, TextBlockParam) + assert text.text == "Clicking now." + + def test_multiple_think_tags_joined(self) -> None: + completion = _make_completion( + content="firstanswersecond" + ) + result = _from_openai_response(completion) + assert isinstance(result.content, list) + thinking = result.content[0] + assert isinstance(thinking, BetaThinkingBlock) + assert thinking.thinking == "first\nsecond" + text = result.content[1] + assert isinstance(text, TextBlockParam) + assert text.text == "answer" + + def test_reasoning_content_field_becomes_thinking_block(self) -> None: + completion = _completion_with_extra( + content="The answer.", + extra={"reasoning_content": "Deliberating..."}, + ) + result = _from_openai_response(completion) + assert isinstance(result.content, list) + assert len(result.content) == 2 + assert isinstance(result.content[0], BetaThinkingBlock) + assert result.content[0].thinking == "Deliberating..." + assert isinstance(result.content[1], TextBlockParam) + assert result.content[1].text == "The answer." + + def test_reasoning_field_alias_becomes_thinking_block(self) -> None: + completion = _completion_with_extra( + content="Done.", + extra={"reasoning": "OpenRouter style."}, + ) + result = _from_openai_response(completion) + assert isinstance(result.content, list) + assert isinstance(result.content[0], BetaThinkingBlock) + assert result.content[0].thinking == "OpenRouter style." + + def test_thinking_only_response_has_no_text_block(self) -> None: + completion = _make_completion(content="only thinking") + result = _from_openai_response(completion) + assert isinstance(result.content, list) + assert len(result.content) == 1 + assert isinstance(result.content[0], BetaThinkingBlock) + assert result.content[0].thinking == "only thinking" + + def test_plain_text_without_thinking_unaffected(self) -> None: + completion = _make_completion(content="Just a plain answer.") + result = _from_openai_response(completion) + assert result.content == "Just a plain answer." + + class TestOpenAIMessagesApi: def test_create_message_delegates_to_client(self) -> None: mock_client = MagicMock()