From cbdc87ccaead9c16a8db53c790d7f8cfeb9669ed Mon Sep 17 00:00:00 2001 From: Anant Jain <86653304+anantjain341@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:37:55 +0530 Subject: [PATCH] hotfix: allow saving long summaries and descriptions (#283) Use case and collaborative summaries can be up to 200,000 characters, but their search mapping kept a keyword-analyzed raw copy that turns the whole value into one token, and Elasticsearch rejects tokens over 32,766 bytes. Saving a published item with a long summary returned an error. Removes those raw sub-fields from the long text fields of all search documents (same change as #278 and #281 on dev) and adds migration 0054, which drops and recreates any live index still carrying one; the deploy then refills it. Fixes #266 --- .../0054_rebuild_long_text_search_indexes.py | 71 ++++++++++++++++++ search/documents/aimodel_document.py | 8 +- search/documents/collaborative_document.py | 8 +- search/documents/dataset_document.py | 8 +- search/documents/publication_document.py | 6 +- search/documents/publisher_document.py | 16 +--- search/documents/usecase_document.py | 8 +- tests/test_rebuild_long_text_indexes.py | 73 +++++++++++++++++++ tests/test_search_long_text_mapping.py | 63 ++++++++++++++++ 9 files changed, 221 insertions(+), 40 deletions(-) create mode 100644 api/migrations/0054_rebuild_long_text_search_indexes.py create mode 100644 tests/test_rebuild_long_text_indexes.py create mode 100644 tests/test_search_long_text_mapping.py diff --git a/api/migrations/0054_rebuild_long_text_search_indexes.py b/api/migrations/0054_rebuild_long_text_search_indexes.py new file mode 100644 index 00000000..c2627ecc --- /dev/null +++ b/api/migrations/0054_rebuild_long_text_search_indexes.py @@ -0,0 +1,71 @@ +"""One-time rebuild of search indexes that still have a single-token text copy. + +Long text fields (use case and collaborative summaries, descriptions, bios) had +a ``raw`` sub-field with the ``keyword`` analyzer, which turns the whole value +into one token. Elasticsearch refuses tokens over 32,766 bytes, so a long +summary failed to index: saving a published item returned an error (#266) and +every deploy failed at ``search_index --populate``. The documents no longer +define those sub-fields, but an existing index keeps its old mapping. + +For each search index whose live mapping still has such a sub-field, drop it +and create it again empty with the current mapping. The deploy's release step +runs migrations first and then populates every index from the database, so +search is refilled right after. Nothing in the database changes. + +Does nothing when an index is missing or already clean, and never fails the +migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def _has_single_token_text(properties): + for field in properties.values(): + for sub in field.get("fields", {}).values(): + if sub.get("type") == "text" and sub.get("analyzer") == "keyword": + return True + if _has_single_token_text(field.get("properties", {})): + return True + return False + + +def rebuild_long_text_indexes(apps, schema_editor): + try: + from django_elasticsearch_dsl.registries import registry + from elasticsearch_dsl.connections import connections + + es = connections.get_connection().options(request_timeout=30) + indices = {index._name: index for index in registry.get_indices()} + for name in set(settings.ELASTICSEARCH_INDEX_NAMES.values()): + if not es.indices.exists(index=name): + continue + mappings = es.indices.get_mapping(index=name) + if not any( + _has_single_token_text(body.get("mappings", {}).get("properties", {})) + for body in mappings.values() + ): + continue + for concrete in mappings: + es.indices.delete(index=concrete) + # Create it again at once so a write from the running server cannot + # make Elasticsearch create it with guessed field types. + if name in indices: + indices[name].create() + logger.info("long_text_search_index_rebuilt", index=name) + except Exception as exc: # never block a deploy on search problems + logger.error("long_text_search_index_rebuild_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_collaborative_summary"), + ] + + operations = [ + migrations.RunPython(rebuild_long_text_indexes, migrations.RunPython.noop), + ] diff --git a/search/documents/aimodel_document.py b/search/documents/aimodel_document.py index 6cd6a1a7..6c89bc2b 100644 --- a/search/documents/aimodel_document.py +++ b/search/documents/aimodel_document.py @@ -34,12 +34,8 @@ class AIModelDocument(Document): fields=title_subfields(), ) - description = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) version = fields.KeywordField() provider = fields.KeywordField() diff --git a/search/documents/collaborative_document.py b/search/documents/collaborative_document.py index ef1ec70f..0fcedeed 100644 --- a/search/documents/collaborative_document.py +++ b/search/documents/collaborative_document.py @@ -61,12 +61,8 @@ class CollaborativeDocument(Document): fields=title_subfields(), ) - summary = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: summaries can exceed Lucene's 32,766-byte token limit + summary = fields.TextField(analyzer=html_strip) logo = fields.TextField(analyzer=ngram_analyser) cover_image = fields.TextField(analyzer=ngram_analyser) diff --git a/search/documents/dataset_document.py b/search/documents/dataset_document.py index 757920c2..03fd2a38 100644 --- a/search/documents/dataset_document.py +++ b/search/documents/dataset_document.py @@ -47,12 +47,8 @@ class DatasetDocument(Document): fields=title_subfields(), ) - description = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) status = fields.KeywordField() diff --git a/search/documents/publication_document.py b/search/documents/publication_document.py index 3c1e6e7f..5b5e04a8 100644 --- a/search/documents/publication_document.py +++ b/search/documents/publication_document.py @@ -33,10 +33,8 @@ class PublicationDocument(Document): analyzer=ngram_analyser, fields=title_subfields(), ) - description = fields.TextField( - analyzer=html_strip, - fields={"raw": fields.TextField(analyzer="keyword")}, - ) + # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) status = fields.KeywordField() # Resource Type facet (name). diff --git a/search/documents/publisher_document.py b/search/documents/publisher_document.py index b63744d1..7273ad82 100644 --- a/search/documents/publisher_document.py +++ b/search/documents/publisher_document.py @@ -27,12 +27,8 @@ class PublisherDocument(Document): }, ) - description = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: user bios can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) publisher_type = fields.KeywordField() # 'organization' or 'user' @@ -47,12 +43,8 @@ class PublisherDocument(Document): location = fields.TextField(analyzer=ngram_analyser) # User specific fields - bio = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: bios can exceed Lucene's 32,766-byte token limit + bio = fields.TextField(analyzer=html_strip) profile_picture = fields.TextField(analyzer=ngram_analyser) username = fields.KeywordField() email = fields.KeywordField() diff --git a/search/documents/usecase_document.py b/search/documents/usecase_document.py index b2b9e023..4b2b1ea4 100644 --- a/search/documents/usecase_document.py +++ b/search/documents/usecase_document.py @@ -45,12 +45,8 @@ class UseCaseDocument(Document): fields=title_subfields(), ) - summary = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: summaries can exceed Lucene's 32,766-byte token limit + summary = fields.TextField(analyzer=html_strip) logo = fields.TextField(analyzer=ngram_analyser) diff --git a/tests/test_rebuild_long_text_indexes.py b/tests/test_rebuild_long_text_indexes.py new file mode 100644 index 00000000..1c35d6ad --- /dev/null +++ b/tests/test_rebuild_long_text_indexes.py @@ -0,0 +1,73 @@ +"""Migration 0054 rebuilds only search indexes that still have a single-token copy.""" + +import importlib +from unittest import mock + +from django.test import override_settings + +migration = importlib.import_module("api.migrations.0054_rebuild_long_text_search_indexes") + +OLD = { + "properties": { + "summary": { + "type": "text", + "fields": {"raw": {"type": "text", "analyzer": "keyword"}}, + } + } +} +CLEAN = { + "properties": { + "summary": {"type": "text", "analyzer": "html_strip"}, + "title": {"type": "text", "fields": {"raw": {"type": "keyword"}}}, + } +} +NAMES = {"a": "usecase", "b": "dataset", "c": "event"} + + +def _run(mappings): + es = mock.MagicMock() + es.options.return_value = es + es.indices.exists.side_effect = lambda index: index in mappings + es.indices.get_mapping.side_effect = lambda index: {index: {"mappings": mappings[index]}} + indices = {name: mock.MagicMock(_name=name) for name in NAMES.values()} + with ( + override_settings(ELASTICSEARCH_INDEX_NAMES=NAMES), + mock.patch("elasticsearch_dsl.connections.connections.get_connection", return_value=es), + mock.patch( + "django_elasticsearch_dsl.registries.registry.get_indices", + return_value=list(indices.values()), + ), + ): + migration.rebuild_long_text_indexes(None, None) + return es, indices + + +def test_old_index_is_dropped_and_created_again(): + es, indices = _run({"usecase": OLD, "dataset": CLEAN}) + + es.indices.delete.assert_called_once_with(index="usecase") + indices["usecase"].create.assert_called_once() + indices["dataset"].create.assert_not_called() + + +def test_nested_single_token_copy_is_found(): + nested = {"properties": {"datasets": {"type": "nested", "properties": OLD["properties"]}}} + + es, _ = _run({"dataset": nested}) + + es.indices.delete.assert_called_once_with(index="dataset") + + +def test_clean_or_missing_indexes_are_left_alone(): + es, indices = _run({"usecase": CLEAN}) + + es.indices.delete.assert_not_called() + assert not any(index.create.called for index in indices.values()) + + +def test_search_errors_never_fail_the_migration(): + with mock.patch( + "elasticsearch_dsl.connections.connections.get_connection", + side_effect=ConnectionError("down"), + ): + migration.rebuild_long_text_indexes(None, None) diff --git a/tests/test_search_long_text_mapping.py b/tests/test_search_long_text_mapping.py new file mode 100644 index 00000000..28d52f9d --- /dev/null +++ b/tests/test_search_long_text_mapping.py @@ -0,0 +1,63 @@ +"""Regression test for long text fields failing to index. + +Use case and collaborative summaries are rich-text JSON documents of up to +200,000 characters, and AI model descriptions, publication descriptions and user +bios have no length limit either. Their ``raw`` sub-fields used the ``keyword`` +analyzer, which emits the whole value as one token, and Lucene rejects any token +over 32,766 bytes. Elasticsearch then refused the whole document with an +"immense term" error, so the record never reached search. + +Nothing queries these sub-fields, so they are gone. The fields themselves stay +indexed: search results return them and text queries match on them. The publisher +``description`` is included because the user index fills it from ``bio``. +""" + +import pytest +from django.core.cache import cache + +from api.views.search_usecase import SearchUseCase +from search.documents.aimodel_document import AIModelDocument +from search.documents.collaborative_document import CollaborativeDocument +from search.documents.dataset_document import DatasetDocument +from search.documents.publication_document import PublicationDocument +from search.documents.publisher_document import ( + OrganizationPublisherDocument, + UserPublisherDocument, +) +from search.documents.usecase_document import UseCaseDocument + +LONG_TEXT_FIELDS = [ + (UseCaseDocument, "summary"), + (CollaborativeDocument, "summary"), + (DatasetDocument, "description"), + (AIModelDocument, "description"), + (PublicationDocument, "description"), + (OrganizationPublisherDocument, "description"), + (OrganizationPublisherDocument, "bio"), + (UserPublisherDocument, "description"), + (UserPublisherDocument, "bio"), +] + + +@pytest.mark.parametrize("document,field", LONG_TEXT_FIELDS) +def test_long_text_field_has_no_single_token_subfield(document, field): + mapping = document._doc_type.mapping.to_dict()["properties"][field] + + assert "raw" not in mapping.get("fields", {}) + + +@pytest.mark.parametrize("document,field", LONG_TEXT_FIELDS) +def test_long_text_field_stays_searchable(document, field): + mapping = document._doc_type.mapping.to_dict()["properties"][field] + + assert mapping["type"] == "text" + assert mapping["analyzer"] == "html_strip" + + +@pytest.mark.django_db +def test_usecase_search_matches_on_summary(): + cache.delete("usecase_search_metadata_config") + + searchable_fields, _ = SearchUseCase().get_searchable_and_aggregations() + + assert "summary" in searchable_fields