hotfix: allow saving long summaries and descriptions on prod

api/migrations/0054_rebuild_long_text_search_indexes.py

@@ -0,0 +1,71 @@
+"""One-time rebuild of search indexes that still have a single-token text copy.
+
+Long text fields (use case and collaborative summaries, descriptions, bios) had
+a ``raw`` sub-field with the ``keyword`` analyzer, which turns the whole value
+into one token. Elasticsearch refuses tokens over 32,766 bytes, so a long
+summary failed to index: saving a published item returned an error (#266) and
+every deploy failed at ``search_index --populate``. The documents no longer
+define those sub-fields, but an existing index keeps its old mapping.
+
+For each search index whose live mapping still has such a sub-field, drop it
+and create it again empty with the current mapping. The deploy's release step
+runs migrations first and then populates every index from the database, so
+search is refilled right after. Nothing in the database changes.
+
+Does nothing when an index is missing or already clean, and never fails the
+migration: problems are only logged.
+"""
+
+import structlog
+from django.conf import settings
+from django.db import migrations
+
+logger = structlog.get_logger(__name__)
+
+
+def _has_single_token_text(properties):
+    for field in properties.values():
+        for sub in field.get("fields", {}).values():
+            if sub.get("type") == "text" and sub.get("analyzer") == "keyword":
+                return True
+        if _has_single_token_text(field.get("properties", {})):
+            return True
+    return False
+
+
+def rebuild_long_text_indexes(apps, schema_editor):
+    try:
+        from django_elasticsearch_dsl.registries import registry
+        from elasticsearch_dsl.connections import connections
+
+        es = connections.get_connection().options(request_timeout=30)
+        indices = {index._name: index for index in registry.get_indices()}
+        for name in set(settings.ELASTICSEARCH_INDEX_NAMES.values()):
+            if not es.indices.exists(index=name):
+                continue
+            mappings = es.indices.get_mapping(index=name)
+            if not any(
+                _has_single_token_text(body.get("mappings", {}).get("properties", {}))
+                for body in mappings.values()
+            ):
+                continue
+            for concrete in mappings:
+                es.indices.delete(index=concrete)
+            # Create it again at once so a write from the running server cannot
+            # make Elasticsearch create it with guessed field types.
+            if name in indices:
+                indices[name].create()
+            logger.info("long_text_search_index_rebuilt", index=name)
+    except Exception as exc:  # never block a deploy on search problems
+        logger.error("long_text_search_index_rebuild_failed", error=str(exc))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0053_alter_collaborative_summary"),
+    ]
+
+    operations = [
+        migrations.RunPython(rebuild_long_text_indexes, migrations.RunPython.noop),
+    ]

search/documents/aimodel_document.py

@@ -34,12 +34,8 @@ class AIModelDocument(Document):
         fields=title_subfields(),
     )
 
-    description = fields.TextField(
-        analyzer=html_strip,
-        fields={
-            "raw": fields.TextField(analyzer="keyword"),
-        },
-    )
+    # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit
+    description = fields.TextField(analyzer=html_strip)
 
     version = fields.KeywordField()
     provider = fields.KeywordField()

search/documents/collaborative_document.py

@@ -61,12 +61,8 @@ class CollaborativeDocument(Document):
         fields=title_subfields(),
     )
 
-    summary = fields.TextField(
-        analyzer=html_strip,
-        fields={
-            "raw": fields.TextField(analyzer="keyword"),
-        },
-    )
+    # No keyword-analyzed sub-field: summaries can exceed Lucene's 32,766-byte token limit
+    summary = fields.TextField(analyzer=html_strip)
 
     logo = fields.TextField(analyzer=ngram_analyser)
     cover_image = fields.TextField(analyzer=ngram_analyser)

search/documents/dataset_document.py

@@ -47,12 +47,8 @@ class DatasetDocument(Document):
         fields=title_subfields(),
     )
 
-    description = fields.TextField(
-        analyzer=html_strip,
-        fields={
-            "raw": fields.TextField(analyzer="keyword"),
-        },
-    )
+    # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit
+    description = fields.TextField(analyzer=html_strip)
 
     status = fields.KeywordField()
 

search/documents/publication_document.py

@@ -33,10 +33,8 @@ class PublicationDocument(Document):
         analyzer=ngram_analyser,
         fields=title_subfields(),
     )
-    description = fields.TextField(
-        analyzer=html_strip,
-        fields={"raw": fields.TextField(analyzer="keyword")},
-    )
+    # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit
+    description = fields.TextField(analyzer=html_strip)
     status = fields.KeywordField()
 
     # Resource Type facet (name).

search/documents/publisher_document.py

@@ -27,12 +27,8 @@ class PublisherDocument(Document):
         },
     )
 
-    description = fields.TextField(
-        analyzer=html_strip,
-        fields={
-            "raw": fields.TextField(analyzer="keyword"),
-        },
-    )
+    # No keyword-analyzed sub-field: user bios can exceed Lucene's 32,766-byte token limit
+    description = fields.TextField(analyzer=html_strip)
 
     publisher_type = fields.KeywordField()  # 'organization' or 'user'
 
@@ -47,12 +43,8 @@ class PublisherDocument(Document):
     location = fields.TextField(analyzer=ngram_analyser)
 
     # User specific fields
-    bio = fields.TextField(
-        analyzer=html_strip,
-        fields={
-            "raw": fields.TextField(analyzer="keyword"),
-        },
-    )
+    # No keyword-analyzed sub-field: bios can exceed Lucene's 32,766-byte token limit
+    bio = fields.TextField(analyzer=html_strip)
     profile_picture = fields.TextField(analyzer=ngram_analyser)
     username = fields.KeywordField()
     email = fields.KeywordField()

search/documents/usecase_document.py

@@ -45,12 +45,8 @@ class UseCaseDocument(Document):
         fields=title_subfields(),
     )
 
-    summary = fields.TextField(
-        analyzer=html_strip,
-        fields={
-            "raw": fields.TextField(analyzer="keyword"),
-        },
-    )
+    # No keyword-analyzed sub-field: summaries can exceed Lucene's 32,766-byte token limit
+    summary = fields.TextField(analyzer=html_strip)
 
     logo = fields.TextField(analyzer=ngram_analyser)
 

tests/test_rebuild_long_text_indexes.py

@@ -0,0 +1,73 @@
+"""Migration 0054 rebuilds only search indexes that still have a single-token copy."""
+
+import importlib
+from unittest import mock
+
+from django.test import override_settings
+
+migration = importlib.import_module("api.migrations.0054_rebuild_long_text_search_indexes")
+
+OLD = {
+    "properties": {
+        "summary": {
+            "type": "text",
+            "fields": {"raw": {"type": "text", "analyzer": "keyword"}},
+        }
+    }
+}
+CLEAN = {
+    "properties": {
+        "summary": {"type": "text", "analyzer": "html_strip"},
+        "title": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
+    }
+}
+NAMES = {"a": "usecase", "b": "dataset", "c": "event"}
+
+
+def _run(mappings):
+    es = mock.MagicMock()
+    es.options.return_value = es
+    es.indices.exists.side_effect = lambda index: index in mappings
+    es.indices.get_mapping.side_effect = lambda index: {index: {"mappings": mappings[index]}}
+    indices = {name: mock.MagicMock(_name=name) for name in NAMES.values()}
+    with (
+        override_settings(ELASTICSEARCH_INDEX_NAMES=NAMES),
+        mock.patch("elasticsearch_dsl.connections.connections.get_connection", return_value=es),
+        mock.patch(
+            "django_elasticsearch_dsl.registries.registry.get_indices",
+            return_value=list(indices.values()),
+        ),
+    ):
+        migration.rebuild_long_text_indexes(None, None)
+    return es, indices
+
+
+def test_old_index_is_dropped_and_created_again():
+    es, indices = _run({"usecase": OLD, "dataset": CLEAN})
+
+    es.indices.delete.assert_called_once_with(index="usecase")
+    indices["usecase"].create.assert_called_once()
+    indices["dataset"].create.assert_not_called()
+
+
+def test_nested_single_token_copy_is_found():
+    nested = {"properties": {"datasets": {"type": "nested", "properties": OLD["properties"]}}}
+
+    es, _ = _run({"dataset": nested})
+
+    es.indices.delete.assert_called_once_with(index="dataset")
+
+
+def test_clean_or_missing_indexes_are_left_alone():
+    es, indices = _run({"usecase": CLEAN})
+
+    es.indices.delete.assert_not_called()
+    assert not any(index.create.called for index in indices.values())
+
+
+def test_search_errors_never_fail_the_migration():
+    with mock.patch(
+        "elasticsearch_dsl.connections.connections.get_connection",
+        side_effect=ConnectionError("down"),
+    ):
+        migration.rebuild_long_text_indexes(None, None)

tests/test_search_long_text_mapping.py

@@ -0,0 +1,63 @@
+"""Regression test for long text fields failing to index.
+
+Use case and collaborative summaries are rich-text JSON documents of up to
+200,000 characters, and AI model descriptions, publication descriptions and user
+bios have no length limit either. Their ``raw`` sub-fields used the ``keyword``
+analyzer, which emits the whole value as one token, and Lucene rejects any token
+over 32,766 bytes. Elasticsearch then refused the whole document with an
+"immense term" error, so the record never reached search.
+
+Nothing queries these sub-fields, so they are gone. The fields themselves stay
+indexed: search results return them and text queries match on them. The publisher
+``description`` is included because the user index fills it from ``bio``.
+"""
+
+import pytest
+from django.core.cache import cache
+
+from api.views.search_usecase import SearchUseCase
+from search.documents.aimodel_document import AIModelDocument
+from search.documents.collaborative_document import CollaborativeDocument
+from search.documents.dataset_document import DatasetDocument
+from search.documents.publication_document import PublicationDocument
+from search.documents.publisher_document import (
+    OrganizationPublisherDocument,
+    UserPublisherDocument,
+)
+from search.documents.usecase_document import UseCaseDocument
+
+LONG_TEXT_FIELDS = [
+    (UseCaseDocument, "summary"),
+    (CollaborativeDocument, "summary"),
+    (DatasetDocument, "description"),
+    (AIModelDocument, "description"),
+    (PublicationDocument, "description"),
+    (OrganizationPublisherDocument, "description"),
+    (OrganizationPublisherDocument, "bio"),
+    (UserPublisherDocument, "description"),
+    (UserPublisherDocument, "bio"),
+]
+
+
+@pytest.mark.parametrize("document,field", LONG_TEXT_FIELDS)
+def test_long_text_field_has_no_single_token_subfield(document, field):
+    mapping = document._doc_type.mapping.to_dict()["properties"][field]
+
+    assert "raw" not in mapping.get("fields", {})
+
+
+@pytest.mark.parametrize("document,field", LONG_TEXT_FIELDS)
+def test_long_text_field_stays_searchable(document, field):
+    mapping = document._doc_type.mapping.to_dict()["properties"][field]
+
+    assert mapping["type"] == "text"
+    assert mapping["analyzer"] == "html_strip"
+
+
+@pytest.mark.django_db
+def test_usecase_search_matches_on_summary():
+    cache.delete("usecase_search_metadata_config")
+
+    searchable_fields, _ = SearchUseCase().get_searchable_and_aggregations()
+
+    assert "summary" in searchable_fields