Support for embedding models via Ollama and OpenAI-like endpoints

AGENTS.md

@@ -118,7 +118,7 @@ Invocation (the plain `docker compose` command uses `docker-compose.yml`):
 - **Extensions**: Installed under `config/extensions/`. Each extension can include its own `requirements.txt` (auto-installed by `setup.py`). Enabled/disabled via `extensions.enabled` setting.
 
 ## Versioning and Migrations
-- The current version is stored in the `VERSION` file (currently `1.53`). **Do not edit this file casually.**
+- The current version is stored in the `VERSION` file. **Do not edit this file casually.**
 - `config/.current-version` tracks the last-migrated version for the running instance.
 - `helper-scripts/migrate.py` compares `VERSION` to `.current-version` and runs the appropriate `migrate-X.XX-X.XX.py` scripts from `helper-scripts/migrate/` in sequence.
 - Migration scripts handle database schema changes, data transformations, and config updates between versions.

VERSION

@@ -1,4 +1,4 @@
-1.57
+1.58
 
 This file should not be modified. It is used by 4CAT to determine whether it
 needs to run migration scripts to e.g. update the database structure to a more

common/assets/llms.json

@@ -1,4 +1,25 @@
 [
+  {
+    "model": "gpt-5.6-sol",
+    "name": "[OpenAI] GPT-5.6 Sol",
+    "model_card": "https://developers.openai.com/api/docs/models/gpt-5.6-sol",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.6-terra",
+    "name": "[OpenAI] GPT-5.6 Terra",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.6-terra",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.6-luna",
+    "name": "[OpenAI] GPT-5.6 Luna",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.6-luna",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.5",
+    "name": "[OpenAI] GPT-5.5",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.5",
+    "provider": "openai"
+  },
   {
     "model": "gpt-5.5",
     "name": "[OpenAI] GPT-5.5",
@@ -48,44 +69,32 @@
     "provider": "google"
   },
   {
-    "model": "gemini-3.5-flash",
-    "name": "[Google] Gemini 3.5 Flash",
-    "model_card": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash",
+    "model": "gemini-3.8-flash",
+    "name": "[Google] Gemini 3.8 Flash",
+    "model_card": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash",
     "provider": "google"
   },
   {
-    "model": "gemini-3-flash-preview",
-    "name": "[Google] Gemini 3 Flash",
-    "model_card": "https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-flash",
+    "model": "gemini-3.5-flash-lite-preview",
+    "name": "[Google] Gemini 3.5 Flash Lite",
+    "model_card": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-5-flash-lite",
     "provider": "google"
   },
   {
-    "model": "gemini-3.1-flash-lite-preview",
-    "name": "[Google] Gemini 3.1 Flash Lite",
-    "provider": "google",
-    "model_card": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-flash-lite"
-  },
-  {
-    "model": "claude-fable-5",
-    "name": "[Anthropic] Claude Fable 5",
+    "model": "claude-fable-5-1",
+    "name": "[Anthropic] Claude Fable 5.1",
     "model_card": "https://www.anthropic.com/claude/fable",
     "provider": "anthropic"
   },
   {
-    "model": "claude-opus-4-8",
-    "name": "[Anthropic] Claude Opus 4.8",
-    "model_card": "https://www.anthropic.com/claude/opus",
-    "provider": "anthropic"
-  },
-  {
-    "model": "claude-opus-4-6",
-    "name": "[Anthropic] Claude Opus 4.6",
+    "model": "claude-opus-5",
+    "name": "[Anthropic] Claude Opus 5",
     "model_card": "https://www.anthropic.com/claude/opus",
     "provider": "anthropic"
   },
   {
-    "model": "claude-sonnet-4-6",
-    "name": "[Anthropic] Claude Sonnet 4.6 (latest)",
+    "model": "claude-sonnet-5",
+    "name": "[Anthropic] Claude Sonnet 5 (latest)",
     "model_card": "https://www.anthropic.com/claude/sonnet",
     "provider": "anthropic"
   },

common/lib/config_definition.py

@@ -636,13 +636,13 @@
                 "type": UserInput.OPTION_TEXT,
                 "help": "Authentication Header",
                 "default": "",
-                "tooltip": "The HTTP header used to authenticate with the server (e.g. 'X-API-KEY', 'Authorization'). Passed with the Authentication Key as value.",
+                "tooltip": "The HTTP header used to authenticate with the server (e.g. 'X-API-KEY', 'Authorization'). Passed with the Authentication Key as value. For a bearer token you can write either 'Authorization' with 'Bearer <key>' as the value, or 'Authorization: Bearer' with just the key.",
             },
             "auth_key": {
                 "type": UserInput.OPTION_TEXT,
                 "default": "",
                 "help": "Authentication Key",
-                "tooltip": "The API key to access the LLM server, if required.",
+                "tooltip": "The API key to access the LLM server, if required. 'Bearer <key>' for bearer tokens.",
             },
         }
     },

common/lib/embeddings.py

@@ -0,0 +1,121 @@
+"""
+Helpers for processors that need all vectors of an embeddings dataset at once.
+Mostly to reduce memory errors
+
+"""
+import numpy as np
+
+from common.lib.exceptions import ProcessorInterruptedException
+
+# How often to report progress while reading, in items
+STATUS_EVERY = 1000
+
+
+def get_vector(record: dict) -> list:
+    """
+    Get the embedding of a record as a list of numbers.
+
+    :param dict record:  Embedding record
+    :return list:  The vector, or an empty list if the record has none
+    """
+    vector = record.get("embedding")
+    if isinstance(vector, str):
+        vector = [float(value) for value in vector.split() if value]
+    return vector or []
+
+
+def read_embedding_matrix(dataset, limit: int, processor=None, accept=None, keep=None) -> tuple:
+    """
+    Read up to `limit` embeddings into one float32 array.
+
+    The array is allocated once, at its full size, instead of collecting rows
+    and stacking them, which would hold every vector twice for a moment.
+
+    :param dataset:  Embeddings dataset
+    :param int limit:  Most embeddings to read
+    :param processor:  Processor reading it, for progress and interruption
+    :param accept:  Optional `function(record) -> bool`; records it rejects
+      are skipped
+    :param keep:  Optional `function(record) -> anything`, to keep a small
+      summary of each record read (never the vector itself)
+    :return tuple:  `(matrix, positions, kept)`: one row per embedding read;
+      the position of each among the items `iterate_items()` yields, to find
+      them again in a second pass; and what `keep` returned for each (empty
+      without `keep`)
+    :raises ValueError:  If the embeddings are not all the same size
+    """
+    matrix = None
+    positions, kept = [], []
+
+    for position, item in enumerate(dataset.iterate_items(processor, get_annotations=False)):
+        if processor is not None and processor.interrupted:
+            raise ProcessorInterruptedException("Interrupted while reading embeddings")
+
+        if len(positions) >= limit:
+            break
+
+        record = item.original
+
+        if accept is not None and not accept(record):
+            continue
+
+        vector = get_vector(record)
+        if not vector:
+            continue
+
+        if matrix is None:
+            matrix = np.empty((limit, len(vector)), dtype=np.float32)
+        elif len(vector) != matrix.shape[1]:
+            # a ragged matrix cannot be compared; this means the dataset mixes
+            # models, which should not happen but is worth saying out loud
+            raise ValueError("The embeddings are not all the same size.")
+
+        matrix[len(positions)] = vector
+        positions.append(position)
+        if keep is not None:
+            kept.append(keep(record))
+
+        if processor is not None and len(positions) % STATUS_EVERY == 0:
+            processor.dataset.update_status(f"Read {len(positions):,} embeddings")
+
+    if matrix is None:
+        return np.empty((0, 0), dtype=np.float32), [], []
+
+    return matrix[:len(positions)], positions, kept
+
+
+def find_embedding(dataset, match, processor=None):
+    """
+    Find the first record that matches, and its vector.
+
+    :param dataset:  Embeddings dataset
+    :param match:  `function(record) -> bool`
+    :param processor:  Processor reading it, to stop when it is interrupted
+    :return tuple:  `(vector as float32 array, record)`, or `(None, None)`
+    """
+    for item in dataset.iterate_items(processor, get_annotations=False):
+        if processor is not None and processor.interrupted:
+            raise ProcessorInterruptedException("Interrupted while reading embeddings")
+
+        record = item.original
+        if match(record):
+            vector = get_vector(record)
+            if vector:
+                return np.asarray(vector, dtype=np.float32), record
+    return None, None
+
+
+def normalise_in_place(matrix):
+    """
+    Scale each row to length 1, without making a copy.
+
+    Embeddings are compared by angle (cosine similarity), not length. On unit
+    vectors, Euclidean distances and dot products follow the angles.
+
+    :param matrix:  2D float array, one embedding per row; changed in place
+    :return:  The same array
+    """
+    norms = np.linalg.norm(matrix, axis=1, keepdims=True)
+    norms[norms == 0] = 1
+    matrix /= norms
+    return matrix

common/lib/exceptions.py

@@ -162,3 +162,9 @@ class MediaSignatureException(FourcatException):
     Raise in media processors when the media cannot be read
     """
     pass
+
+class LLMServerException(FourcatException):
+    """
+    Raise when an LLM server request fails or returns something unusable
+    """
+    pass

common/lib/llm/clients/ollama_client.py

@@ -3,6 +3,7 @@
 """
 import requests
 
+from common.lib.exceptions import LLMServerException
 from common.lib.llm.llm_client import LLMServerClient
 
 
@@ -46,7 +47,7 @@ def parse_supported_media_types(self, meta: dict) -> list[str]:
         **Primary path**: reads `meta["capabilities"]`:
         - `"completion"` → `"text"`
         - `"vision"`     → `"image"`
-        - `"embedding"`  → `"embedding"`
+        - `"embedding"`  → `"text"` (an embedder still consumes text)
 
         **Fallback path** (used when capabilities are absent or only yield `"text"`):
         inspects GGUF `model_info` / `details` for vision signals and adds
@@ -57,16 +58,16 @@ def parse_supported_media_types(self, meta: dict) -> list[str]:
           Returns `[]` when `meta` is `None` (unknown — callers should
           include the model, not block it).
         """
-        if meta is None or not meta.get("metadata"):
+        if meta is None or not (meta.get("metadata") or meta.get("capabilities")):
             return []
 
-        capabilities = meta["metadata"].get("capabilities", [])
+        capabilities = self._get_capabilities(meta)
         media_types: list[str] = []
 
         _cap_map = {
             "completion": "text",
+            "embedding": "text",
             "vision": "image",
-            "embedding": "embedding",
         }
         for cap in capabilities:
             mapped = _cap_map.get(cap)
@@ -88,6 +89,106 @@ def parse_supported_media_types(self, meta: dict) -> list[str]:
 
         return media_types
 
+    @staticmethod
+    def _get_capabilities(meta: dict) -> list[str]:
+        """
+        Read a model's capability list.
+
+        Ollama reports capabilities in two places: nested under `metadata` for
+        the `/api/show` response `list_models()` attaches, and inline on the
+        `/api/tags` entry itself. Prefer the former and fall back to the latter,
+        so a model whose `/api/show` request failed is still classified.
+
+        :param dict meta:  Model metadata, or `None`.
+        :returns list[str]:  Ollama capability strings, or `[]` if unknown.
+        """
+        if not meta:
+            return []
+
+        metadata = meta.get("metadata") or {}
+        return metadata.get("capabilities") or meta.get("capabilities") or []
+
+    def parse_supported_tasks(self, meta: dict) -> list[str]:
+        """
+        Derive the tasks a model supports from its Ollama capabilities.
+
+        :param dict meta:  `/api/show` response dict, or `None`.
+        :returns list[str]:  Supported tasks - `"generate"`, `"embed"`, or both.
+        """
+        capabilities = self._get_capabilities(meta)
+
+        tasks = []
+        if "completion" in capabilities:
+            tasks.append("generate")
+        if "embedding" in capabilities:
+            tasks.append("embed")
+
+        # An unreachable or partial /api/show leaves nothing to go on. Assume
+        # generation, matching the base class, rather than hiding the model
+        # from every processor at once.
+        return tasks if tasks else ["generate"]
+
+    def embed(self, model_id: str, inputs: list, media: list | None = None, timeout: int = 300,
+              text_as_instruction: bool = False) -> list[list[float]]:
+        """
+        Embed inputs via Ollama's `/api/embed` endpoint.
+
+        The endpoint accepts a list and returns one vector per item, so a caller
+        that batches its inputs gets one round-trip per batch rather than per
+        item. NOTE: Ollama currently only supports text embeddings.
+
+        todo: revise later, since Ollama notes that OpenAI-like embedding endpoints are coming soon,
+         https://ollama.com/blog/embedding-models
+
+        :param str model_id:  Ollama model name.
+        :param list inputs:  Strings to embed.
+        :param list media:  Not supported; passing any raises.
+        :param int timeout:  Request timeout in seconds.
+        :param bool text_as_instruction:  Ignored; only relevant with media,
+          which Ollama does not support.
+        :returns list[list[float]]:  One vector per input, in input order.
+        :raises LLMServerException:  If media is passed, if the server errors,
+          or if it returns a number of vectors that does not match the number of
+          inputs (which would silently misalign vectors with their items).
+        """
+        if media:
+            raise LLMServerException(
+                "Ollama does not support non-text inputs yet. Use a "
+                "server and model that supports multimodal embeddings, like vLLM.")
+
+        if not inputs:
+            return []
+
+        try:
+            response = self._session.post(
+                f"{self.base_url}/api/embed",
+                headers=self._headers,
+                json={"model": model_id, "input": inputs},
+                timeout=timeout,
+            )
+        except requests.RequestException as e:
+            raise LLMServerException(f"Could not reach Ollama server at {self.base_url}: {e}")
+
+        if response.status_code != 200:
+            raise LLMServerException(
+                f"Ollama server returned status {response.status_code} while embedding with model "
+                f"'{model_id}': {response.text}")
+
+        try:
+            embeddings = response.json().get("embeddings")
+        except ValueError as e:
+            raise LLMServerException(f"Ollama server returned invalid JSON while embedding: {e}")
+
+        if not embeddings:
+            raise LLMServerException(f"Ollama server returned no embeddings for model '{model_id}'")
+
+        if len(embeddings) != len(inputs):
+            raise LLMServerException(
+                f"Ollama server returned {len(embeddings)} embeddings for {len(inputs)} inputs; cannot map "
+                f"vectors back to items")
+
+        return embeddings
+
     def format_display_name(self, meta: dict) -> str:
         """
         Build a human-readable display name for a model.