Move to Claude 5.5 models

.changeset/claude-5-5-models.md

@@ -0,0 +1,5 @@
+---
+"apollo": minor
+---
+
+Use Claude Opus 5.5, Sonnet 5.5 and Haiku 5.5 across all services

.changeset/model-refusal-error.md

@@ -0,0 +1,5 @@
+---
+"apollo": patch
+---
+
+Return a `MODEL_REFUSAL` error when the AI service declines a request, instead of an empty-output error

services/doc_agent_chat/agent.py

@@ -6,7 +6,7 @@
 from doc_agent_chat.prompt import build_system_prompt
 from doc_agent_chat.tools import TOOL_DEFINITIONS, search_documents, format_search_results_as_documents
 from doc_agent_chat.config_loader import ConfigLoader
-from models import resolve_model
+from models import resolve_model, NO_THINKING_SONNET
 
 logger = create_logger("agent")
 
@@ -63,7 +63,8 @@ def run(
                 max_tokens=self.max_tokens,
                 system=system_prompt,
                 messages=messages,
-                tools=TOOL_DEFINITIONS
+                tools=TOOL_DEFINITIONS,
+                thinking=NO_THINKING_SONNET,
             )
 
             if hasattr(response, "usage"):

services/global_chat/config.yaml

@@ -8,7 +8,7 @@ router:
 # Planner configuration (complex orchestration)
 planner:
   model: "claude-opus"
-  max_tokens: 24576
+  max_tokens: 32768
   max_tool_calls: 20
   max_pause_continuations: 5
   web_search: # Server-side web search/fetch for planner.

services/global_chat/planner.py

@@ -18,7 +18,7 @@
 sys.path.append(str(Path(__file__).parent.parent))
 
 from langfuse import observe
-from util import create_logger, ApolloError, sum_usage, mask_secrets, format_attachments
+from util import create_logger, ApolloError, sum_usage, mask_secrets, format_attachments, refusal_error
 from streaming_util import (
     StreamManager,
     STATUS_REVIEWING_WORKFLOW,
@@ -39,7 +39,6 @@
 # max_tool_calls, so this is a backstop against a raised budget: at the budget
 # it could only fire on the wrap-up round, clearing what that round summarises.
 _CONTEXT_MANAGEMENT = {
-    "betas": ["context-management-2025-06-27"],
     "context_management": {
         "edits": [
             {
@@ -53,7 +52,13 @@
     },
 }
 
-# API puts a breakpoint on the last block of each request, 
+# The model writes its notes between tool calls into thinking blocks, which
+# come back empty unless display is "updates". Those notes are the narration.
+_THINKING = {"type": "adaptive", "display": "updates"}
+
+_BETAS = ["context-management-2025-06-27", "thinking-display-updates-2026-08-18"]
+
+# API puts a breakpoint on the last block of each request,
 # so each tool-loop round reads the rounds before it from cache.
 _HISTORY_CACHE = {"cache_control": {"type": "ephemeral"}}
 
@@ -251,7 +256,7 @@ def __init__(
 
         planner_config = config_loader.config.get("planner", {})
         self.model = resolve_model(planner_config.get("model", "claude-opus"))
-        self.max_tokens = planner_config.get("max_tokens", 24576)
+        self.max_tokens = planner_config.get("max_tokens", 32768)
         self.max_tool_calls = planner_config.get("max_tool_calls", 20)
         self.max_pause_continuations = planner_config.get("max_pause_continuations", 5)
 
@@ -391,6 +396,10 @@ def run(
 
                     logger.info(f"Claude API call {tool_call_count + 1}: stop_reason={response.stop_reason}")
 
+                    refusal = refusal_error(response)
+                    if refusal:
+                        raise refusal
+
                     # Text from every round is part of the answer the user saw
                     # (tool rounds may narrate before calling tools).
                     round_text = self._extract_text(response)
@@ -599,10 +608,10 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice
         rounds close the open text block), so the client can weave text and
         status events with its own formatting.
 
-        Adaptive thinking is enabled for better reasoning but thinking content
-        is not streamed to the client — it exposes internal details like tool
-        names and agent architecture. User-facing progress comes from the
-        task-specific status messages sent before each tool execution.
+        Reasoning is never streamed — it exposes internal details like tool
+        names and agent architecture — and under display "updates" it comes
+        back empty. The only thinking text is the model's progress notes
+        between tool calls, which stream as narration.
         """
         choice = {"tool_choice": tool_choice} if tool_choice else {}
 
@@ -614,15 +623,19 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice
                 system=system_prompt,
                 messages=messages,
                 tools=self.tools,
-                thinking={"type": "adaptive"},
+                thinking=_THINKING,
                 output_config={"effort": "medium"},
+                betas=_BETAS,
                 **choice,
                 **_CONTEXT_MANAGEMENT,
                 **_HISTORY_CACHE,
             ) as stream_obj:
                 for event in stream_obj:
-                    if event.type == "content_block_delta" and event.delta.type == "text_delta":
-                        stream_manager.send_text(event.delta.text)
+                    if event.type == "content_block_delta":
+                        if event.delta.type == "text_delta":
+                            stream_manager.send_text(event.delta.text)
+                        elif event.delta.type == "thinking_delta" and event.delta.thinking:
+                            stream_manager.send_text(event.delta.thinking)
                     elif event.type == "content_block_start":
                         if event.content_block.type == "server_tool_use":
                             self._send_spinner(stream_manager, STATUS_SEARCHING_WEB)
@@ -641,8 +654,9 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice
                 system=system_prompt,
                 messages=messages,
                 tools=self.tools,
-                thinking={"type": "adaptive"},
+                thinking=_THINKING,
                 output_config={"effort": "medium"},
+                betas=_BETAS,
                 **choice,
                 # Per-request timeout (same values as the SDK default):
                 # required for non-streaming calls with max_tokens > ~21k,
@@ -1048,8 +1062,12 @@ def _format_display_name(name: str) -> str:
         return name.replace("-", " ").replace("_", " ").title()
 
     def _extract_text(self, response):
-        """Extract text from response content, concatenated as it was streamed."""
-        return "".join(block.text for block in response.content if block.type == "text")
+        """Extract text and progress notes, concatenated as they were streamed."""
+        return "".join(
+            block.text if block.type == "text" else block.thinking
+            for block in response.content
+            if block.type == "text" or (block.type == "thinking" and block.thinking)
+        )
 
     @staticmethod
     def _count_server_tool_uses(response) -> dict:

services/global_chat/router.py

@@ -21,7 +21,7 @@
 from util import create_logger, ApolloError, sum_usage, attachments_to_context
 from streaming_util import StreamManager
 from global_chat.config_loader import ConfigLoader
-from models import resolve_model
+from models import resolve_model, NO_THINKING_HAIKU
 from yaml_utils import get_step_name_from_page, get_page_view, find_job_in_yaml, stitch_job_code, workflow_has_job_code
 
 logger = create_logger(__name__)
@@ -184,6 +184,9 @@ def _make_routing_decision(
             messages=[
                 {"role": "user", "content": routing_message}
             ],
+            # Routing is meant to be fast: a request that needs reasoning is
+            # one to send to the planner, not to think about here.
+            thinking=NO_THINKING_HAIKU,
             output_config={"format": {"type": "json_schema", "schema": routing_schema}}
         )
 
@@ -194,7 +197,7 @@ def _make_routing_decision(
             "cache_read_input_tokens": getattr(response.usage, "cache_read_input_tokens", 0),
         }
 
-        response_text = response.content[0].text if response.content else "{}"
+        response_text = "".join(block.text for block in response.content if block.type == "text") or "{}"
 
         try:
             decision_data = json.loads(response_text)

services/global_chat/tests/unit/test_planner.py

@@ -611,6 +611,48 @@ def test_model_api_error_is_not_reported_as_a_tool_failure() -> None:
     )
 
 
+class FakeThinking:
+    type = "thinking"
+    signature = "sig"
+
+    def __init__(self, thinking: str) -> None:
+        self.thinking = thinking
+
+
+def test_a_refusal_is_reported_as_one_not_as_an_empty_answer() -> None:
+    """A decline is a 200 with no text, which otherwise surfaces as our own
+    EMPTY_OUTPUT failure."""
+    planner = make_planner()
+    planner.model = "test-model"
+    planner.max_tool_calls = 5
+    refused = FakeResponse("refusal", [])
+    refused.stop_details = type("StopDetails", (), {"category": "cyber"})()
+
+    with (
+        patch.object(planner, "_call_api", return_value=refused),
+        patch.object(planner, "_build_system_prompt", return_value="sys"),
+        pytest.raises(ApolloError) as raised,
+    ):
+        planner.run(content="x", workflow_yaml=None, page=None, history=[], stream=False)
+
+    assert raised.value.code == 422
+    assert raised.value.type == "MODEL_REFUSAL"
+    assert raised.value.details == {"category": "cyber"}
+
+
+def test_progress_notes_count_as_narration_and_hidden_reasoning_does_not() -> None:
+    """Notes written before a tool call arrive as thinking blocks; reasoning
+    arrives as thinking blocks too, but empty."""
+    response = FakeResponse("tool_use", [
+        FakeThinking(""),
+        FakeThinking("Checking the fetch step first."),
+        FakeText(" Then the mapping."),
+        FakeToolUse("inspect_job_code", {"job_keys": ["a"]}, "tu_1"),
+    ])
+
+    assert make_planner()._extract_text(response) == "Checking the fetch step first. Then the mapping."
+
+
 def _raise_from_planner(error: Exception) -> ApolloError:
     planner = make_planner()
     planner.model = "test-model"

services/job_chat/job_chat.py

@@ -19,7 +19,7 @@
 import sentry_sdk
 from langfuse import observe, propagate_attributes, get_client as get_langfuse_client
 from langfuse_util import should_track, build_tags, build_generation_diff, mask_secrets
-from util import ApolloError, create_logger, AdaptorSpecifier, add_page_prefix, APOLLO_VERSION
+from util import ApolloError, create_logger, AdaptorSpecifier, add_page_prefix, refusal_error, APOLLO_VERSION
 from yaml_utils import INSPECT_JOB_CODE_TOOL, inspect_job_code
 from .prompt import build_prompt, build_error_correction_prompt
 from .old_prompt import build_old_prompt
@@ -192,7 +192,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Payload":
 @dataclass
 class ChatConfig:
     model: str = _MODEL
-    max_tokens: int = 24576
+    max_tokens: int = 32768
     api_key: Optional[str] = None
 
 
@@ -378,6 +378,11 @@ def generate(
                     if hasattr(message, "usage"):
                         usage_events.append(message.usage.model_dump())
 
+                    refusal = refusal_error(message)
+                    if refusal:
+                        stream_manager.end_stream()
+                        raise refusal
+
                     for content_block in message.content:
                         if getattr(content_block, "type", None) == "text" and content_block.text:
                             text_parts.append(content_block.text)

services/job_chat/retrieve_docs.py

@@ -14,7 +14,7 @@
     UnprocessableEntityError,
 )
 from langfuse import observe
-from models import resolve_model
+from models import resolve_model, NO_THINKING_SONNET
 from search_docsite.search_docsite import resolve_backend
 from util import ApolloError, create_logger
 
@@ -225,7 +225,8 @@ def call_llm(model, system_prompt, user_prompt, client, output_schema=None):
             model=model,
             max_tokens=1024,
             system=system_prompt,
-            messages=messages
+            messages=messages,
+            thinking=NO_THINKING_SONNET,
         )
         if output_schema:
             kwargs["output_config"] = {
@@ -238,7 +239,7 @@ def call_llm(model, system_prompt, user_prompt, client, output_schema=None):
         if not message.content or len(message.content) == 0:
             raise ApolloError(500, "Empty response from AI service", type="EMPTY_LLM_RESPONSE")
 
-        response_text = message.content[0].text
+        response_text = "".join(block.text for block in message.content if block.type == "text")
         if not response_text:
             raise ApolloError(500, "Empty text in AI service response", type="EMPTY_LLM_RESPONSE")
 

services/job_chat/tests/unit/test_retrieve_docs.py

@@ -44,7 +44,7 @@ def test_generate_queries_raises_apollo_error_on_invalid_json():
 
 def test_call_llm_returns_text_and_usage_on_success():
     message = MagicMock()
-    message.content = [MagicMock(text="hello")]
+    message.content = [MagicMock(type="text", text="hello")]
     message.usage.model_dump.return_value = {"input_tokens": 5, "output_tokens": 2}
     client = MagicMock()
     client.messages.create.return_value = message

services/models.py

@@ -4,15 +4,21 @@
 """
 
 CLAUDE_MODELS: dict[str, str] = {
-    "claude-opus":   "claude-opus-4-8",
-    "claude-sonnet": "claude-sonnet-4-6",
-    "claude-haiku":  "claude-haiku-4-5-20251001",
+    "claude-opus":   "claude-opus-5-5",
+    "claude-sonnet": "claude-sonnet-5-5",
+    "claude-haiku":  "claude-haiku-5-5",
 }
 
 CLAUDE_OPUS:   str = CLAUDE_MODELS["claude-opus"]
 CLAUDE_SONNET: str = CLAUDE_MODELS["claude-sonnet"]
 CLAUDE_HAIKU:  str = CLAUDE_MODELS["claude-haiku"]
 
+# These models think when `thinking` is omitted, where the 4.x ones didn't.
+# Calls that never thought pass these to keep it off. Each is model-specific:
+# Sonnet 5.5 rejects "disabled", and only Sonnet 5.5 accepts "between_tools".
+NO_THINKING_HAIKU:  dict = {"type": "disabled"}
+NO_THINKING_SONNET: dict = {"type": "between_tools"}
+
 
 def resolve_model(alias: str) -> str:
     """Resolve a model alias to its full ID. Passes through unknown strings unchanged."""

services/status/status.py

@@ -5,7 +5,7 @@
 from anthropic import Anthropic
 import psycopg2
 from util import create_logger
-from models import CLAUDE_HAIKU
+from models import CLAUDE_HAIKU, NO_THINKING_HAIKU
 
 logger = create_logger("status")
 
@@ -29,6 +29,7 @@ def test_anthropic_key(api_key):
         client.messages.create(
             model=CLAUDE_HAIKU,
             max_tokens=1,
+            thinking=NO_THINKING_HAIKU,
             messages=[{"role": "user", "content": "test"}]
         )
         return True

services/testing/judge.py

@@ -34,7 +34,7 @@
 
 import yaml
 from anthropic import Anthropic
-from models import CLAUDE_SONNET
+from models import CLAUDE_SONNET, NO_THINKING_SONNET
 from testing.judges import load_judge
 
 DEFAULT_MODEL = CLAUDE_SONNET
@@ -417,13 +417,14 @@ def evaluate(
     response = client.messages.create(
         model=model,
         max_tokens=16384,
+        thinking=NO_THINKING_SONNET,
         system=system_prompt,
         messages=[
             {"role": "user", "content": user_prompt},
         ],
     )
 
-    raw_text = response.content[0].text
+    raw_text = "".join(block.text for block in response.content if block.type == "text")
     usage = {
         "input_tokens": response.usage.input_tokens,
         "output_tokens": response.usage.output_tokens,

services/util.py

@@ -397,6 +397,24 @@ def check_attachment_size(attachments: list[dict] | None) -> None:
     )
 
 
+def refusal_error(message: Any) -> ApolloError | None:  # noqa: ANN401
+    """The typed error for a safety decline, or None if the model answered.
+
+    A decline is an HTTP 200 with stop_reason "refusal", so nothing upstream
+    raises; left alone it reads as an empty or unparseable answer and gets
+    retried or reported as our failure.
+    """
+    if getattr(message, "stop_reason", None) != "refusal":
+        return None
+    category = getattr(getattr(message, "stop_details", None), "category", None)
+    return ApolloError(
+        422,
+        "The AI service declined to answer this request. Try rephrasing it.",
+        type="MODEL_REFUSAL",
+        details={"category": category},
+    )
+
+
 # Payload attachment type -> the job_chat context field that already renders it.
 # job_chat has had typed <run_logs>/<input>/<output> blocks all along; attachments
 # arriving from global_chat reuse them rather than adding a second channel.