diff --git a/src/services/docling_service.py b/src/services/docling_service.py index a96fca248..0d3b3022d 100644 --- a/src/services/docling_service.py +++ b/src/services/docling_service.py @@ -1,6 +1,7 @@ import asyncio import json import platform +import re from dataclasses import dataclass from enum import StrEnum from pathlib import Path @@ -232,6 +233,96 @@ def _format_docling_error(payload: dict[str, Any]) -> str: return result +# Keeps the "incorrect password" substring that task_service._infer_failure_metadata +# maps to the user-facing password-protected message. +PASSWORD_PROTECTED_PDF_DETAIL = "PDF is password-protected (Incorrect password)" + +# The trailer / xref-stream dictionary holding /Encrypt is never compressed, so a +# byte scan finds it without a PDF library. Applied only to the active trailer +# dictionary (_active_trailer_dict): /Encrypt elsewhere in the file, such as page +# text or an attached PDF, does not make the document encrypted. Matches an +# indirect reference (/Encrypt 5 0 R) or a direct dictionary (/Encrypt <<). +_PDF_ENCRYPT_RE = re.compile(rb"/Encrypt\s*(?:\d+\s+\d+\s+R|<<)") +_STARTXREF_RE = re.compile(rb"startxref\s+(\d+)") +# What a startxref offset may point at: a classic `xref` table (group 1) or the +# header of an indirect object that should be the xref stream. +_XREF_SECTION_START_RE = re.compile(rb"\s*(?:(xref)|\d+\s+\d+\s+obj\b)") +# The object's dictionary ends where its stream data or the object itself ends. +_OBJ_DICT_END_RE = re.compile(rb"stream|endobj") +_XREF_TYPE_RE = re.compile(rb"/Type\s*/XRef\b") + + +def is_docling_load_failure(message: str) -> bool: + """True when docling reports that its backend could not open the document.""" + return "could not load document" in message.lower() + + +def _active_trailer_dict(content: bytes) -> bytes | None: + """Return the trailer or xref-stream dictionary that the final ``startxref`` names. + + PDF encryption is declared only there. Incremental updates repeat /Encrypt in + the newest trailer, and a linearized file's final ``startxref`` points at the + first-page section, which carries the full trailer. Offsets are tried both from + the start of the file and from the ``%PDF-`` header, because some producers + write junk before the header. An object at the offset counts only if it is an + xref stream (``/Type /XRef``), so a stale offset in a damaged file cannot pick + up an unrelated dictionary. Returns ``None`` when the content is not a PDF or + the trailer cannot be located. + + Searches ``content`` in place and slices out only the dictionary, because a + linearized file's offset is near the start and a tail copy would duplicate + almost the whole upload. + """ + header = content.find(b"%PDF-", 0, 1024) + if header < 0: + return None + matches = list(_STARTXREF_RE.finditer(content, max(0, len(content) - 1024))) + if not matches: + logger.debug("No startxref found; not treating the PDF as encrypted") + return None + offset = int(matches[-1].group(1)) + for start in dict.fromkeys((offset, offset + header)): + section = _XREF_SECTION_START_RE.match(content, start) + if section is None: + continue + if section.group(1): + # Classic table: the dictionary follows the `trailer` keyword. + trailer = content.find(b"trailer", section.end()) + if trailer < 0: + logger.debug("Cross-reference table has no trailer", offset=start) + return None + end = content.find(b"startxref", trailer) + return content[trailer : end if end >= 0 else None] + # Xref stream: the dictionary precedes `stream`; the data is excluded. + dict_end = _OBJ_DICT_END_RE.search(content, section.end()) + xref_dict = content[section.end() : dict_end.start() if dict_end else None] + if _XREF_TYPE_RE.search(xref_dict): + return xref_dict + logger.debug("startxref offset does not point at a cross-reference section", offset=offset) + return None + + +def is_encrypted_pdf(content: bytes) -> bool: + """True when ``content`` is a PDF whose active trailer declares an encryption dictionary.""" + trailer = _active_trailer_dict(content) + return trailer is not None and _PDF_ENCRYPT_RE.search(trailer) is not None + + +def password_protected_pdf_error(message: str, content: bytes) -> str | None: + """Return the password-protected detail for a docling load failure on an encrypted PDF. + + docling 2.108 pre-opened PDFs with pypdfium2, whose error read "Incorrect + password". Later versions load through docling-parse first, which reports only + "Failed to load document with key ...", so the reason has to be recovered from + the bytes. This is checked only after docling failed to load the file: a PDF + with just an owner password also declares /Encrypt but converts fine. A file + whose trailer cannot be located keeps docling's own message. + """ + if is_docling_load_failure(message) and is_encrypted_pdf(content): + return PASSWORD_PROTECTED_PDF_DETAIL + return None + + def _extract_document_json_content(payload: dict[str, Any], *, task_id: str) -> dict[str, Any]: """Return ``document.json_content`` from a ``/v1/result`` payload. @@ -851,8 +942,8 @@ async def convert_file( ocr=ocr, picture_descriptions=picture_descriptions, ) - return await self.get_docling_result_async( - task_id, user_id=user_id, auth_header=auth_header + return await self._get_result_for_content( + task_id, file_bytes, user_id=user_id, auth_header=auth_header ) async def convert_bytes( @@ -868,6 +959,30 @@ async def convert_bytes( task_id = await self.upload_to_docling_direct_async( filename, content, user_id=user_id, auth_header=auth_header ) - return await self.get_docling_result_async( - task_id, user_id=user_id, auth_header=auth_header + return await self._get_result_for_content( + task_id, content, user_id=user_id, auth_header=auth_header ) + + async def _get_result_for_content( + self, + task_id: str, + content: bytes, + *, + user_id: str | None, + auth_header: str | None, + ) -> dict[str, Any]: + """Fetch the conversion result, naming a password-protected PDF on load failure.""" + try: + return await self.get_docling_result_async( + task_id, user_id=user_id, auth_header=auth_header + ) + except DoclingServeError as exc: + detail = password_protected_pdf_error(str(exc), content) + if detail is None: + raise + logger.warning( + "Docling could not load an encrypted PDF", + task_id=task_id, + docling_error=str(exc), + ) + raise DoclingServeError(f"Docling processing failed: {detail}") from exc diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py index 47af817d0..5c60eb581 100644 --- a/src/services/langflow_file_service.py +++ b/src/services/langflow_file_service.py @@ -1176,9 +1176,18 @@ async def upload_and_ingest_file( "elapsed_seconds": round(poll_result.elapsed_seconds, 2), }, ) + from services.docling_service import password_protected_pdf_error + + detail = poll_result.detail + if ( + poll_result.outcome == PollOutcome.FAILED + and detail + and isinstance(content, bytes) + ): + detail = password_protected_pdf_error(detail, content) or detail raise Exception( f"Docling conversion did not complete ({poll_result.outcome.value}): " - f"{poll_result.detail or 'no detail provided'}" + f"{detail or 'no detail provided'}" ) if file_task is not None: diff --git a/src/services/task_service.py b/src/services/task_service.py index 11b361039..f4de56d35 100644 --- a/src/services/task_service.py +++ b/src/services/task_service.py @@ -1359,7 +1359,9 @@ def _infer_failure_metadata(self, file_task: FileTask) -> dict | None: if opensearch_meta: return opensearch_meta - if "incorrect password" in error.lower(): # for password protected pdf cases + # Password-protected PDFs. Newer docling no longer names the reason, so the + # backend recovers it from the file bytes (docling_service.password_protected_pdf_error). + if "incorrect password" in error.lower(): return { "component": "docling", "failure_phase": "parsing", diff --git a/tests/unit/test_docling_service.py b/tests/unit/test_docling_service.py index 945954572..85ba5cab0 100644 --- a/tests/unit/test_docling_service.py +++ b/tests/unit/test_docling_service.py @@ -8,7 +8,14 @@ import httpx import pytest -from services.docling_service import DoclingPayloadTooLarge, DoclingServeError, DoclingService +from services.docling_service import ( + PASSWORD_PROTECTED_PDF_DETAIL, + DoclingPayloadTooLarge, + DoclingServeError, + DoclingService, + is_encrypted_pdf, + password_protected_pdf_error, +) from utils.upload_limits import UploadTooLarge # Shape of a /v1/result body for a password-protected PDF: the task ran to @@ -166,6 +173,196 @@ async def test_poll_result_surfaces_conversion_failure_from_result_payload( assert "missing document.json_content" not in message +# ── Password-protected PDFs on newer docling ─────────────────────── + +# docling >= 2.141 loads through docling-parse first, whose load failure no longer +# names the reason (seen on CPD, tracker #99817). +_DOCLING_PARSE_LOAD_FAILURE = ( + "Docling processing failed: docling-parse could not load document 8fc525: " + "Failed to load document with key key=8fc525" +) +_XREF_TABLE = b"xref\n0 1\n0000000000 65535 f \n" + + +def _pdf(body: bytes, trailer: bytes, *, junk: bytes = b"") -> bytes: + """Minimal classic-xref PDF; with ``junk``, offsets are relative to the header.""" + head = b"%PDF-1.7\n" + body + return junk + head + _XREF_TABLE + trailer + b"\nstartxref\n%d\n%%%%EOF\n" % len(head) + + +def _append_revision(pdf: bytes, body: bytes, trailer: bytes) -> bytes: + """Append an incremental update whose trailer becomes the active one.""" + head = pdf + body + return head + _XREF_TABLE + trailer + b"\nstartxref\n%d\n%%%%EOF\n" % len(head) + + +def _xref_stream_pdf(entries: bytes, data: bytes) -> bytes: + """Minimal PDF whose active cross-reference section is an xref stream.""" + head = b"%PDF-1.5\n" + xref = b"9 0 obj\n<< /Type /XRef %s /Length %d >>\nstream\n%s\nendstream\nendobj\n" % ( + entries, + len(data), + data, + ) + return head + xref + b"startxref\n%d\n%%%%EOF\n" % len(head) + + +def _stale_startxref_pdf(objects: bytes) -> bytes: + """PDF whose ``startxref`` names the first of ``objects`` rather than an xref section.""" + head = b"%PDF-1.5\n" + return head + objects + b"startxref\n%d\n%%%%EOF\n" % len(head) + + +_CATALOG = b"1 0 obj\n<< /Type /Catalog >>\nendobj\n" +_ENCRYPTED_PDF = _pdf(_CATALOG, b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R /Size 6 >>") +_PLAIN_PDF = _pdf(_CATALOG, b"trailer\n<< /Root 1 0 R >>") +# Unencrypted, but a content stream shows the literal text "/Encrypt 1 0 R". +_PLAIN_PDF_WITH_ENCRYPT_TEXT = _pdf( + _CATALOG + b"4 0 obj\n<< /Length 25 >>\nstream\nBT (/Encrypt 1 0 R) Tj ET\nendstream\nendobj\n", + b"trailer\n<< /Root 1 0 R >>", +) + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + pytest.param(_ENCRYPTED_PDF, True, id="classic-trailer-encrypted"), + pytest.param( + _xref_stream_pdf(b"/Encrypt << /Filter /Standard >>", b"abc"), + True, + id="xref-stream-encrypted", + ), + pytest.param( + _append_revision( + _ENCRYPTED_PDF, + b"2 0 obj\n<< /Type /Page >>\nendobj\n", + b"trailer\n<< /Root 1 0 R /Prev %d /Encrypt 5 0 R /Size 6 >>" + % _ENCRYPTED_PDF.index(b"xref"), + ), + True, + id="incremental-update-encrypted", + ), + pytest.param( + _pdf(_CATALOG, b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R >>", junk=b"JUNK\n"), + True, + id="junk-before-header", + ), + pytest.param(_PLAIN_PDF, False, id="classic-trailer-plain"), + pytest.param(_PLAIN_PDF_WITH_ENCRYPT_TEXT, False, id="encrypt-in-content-stream"), + pytest.param( + _xref_stream_pdf(b"", b"/Encrypt 1 0 R"), False, id="encrypt-in-xref-stream-data" + ), + pytest.param( + b"%PDF-1.7\n" + _CATALOG + b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R >>\n", + False, + id="truncated-no-startxref", + ), + pytest.param( + b"%PDF-1.7\ntrailer\n<< /Encrypt 5 0 R >>\nstartxref\n3\n%%EOF\n", + False, + id="startxref-points-at-garbage", + ), + pytest.param( + _stale_startxref_pdf(b"3 0 obj\n<< /Type /Filespec /Encrypt 5 0 R >>\nendobj\n"), + False, + id="startxref-points-at-non-xref-object", + ), + pytest.param( + # The non-xref dictionary ends at `endobj`; the later xref stream's + # /Type /XRef and /Encrypt must not be read as belonging to it. + _stale_startxref_pdf( + b"3 0 obj\n<< /Type /Filespec >>\nendobj\n" + b"9 0 obj\n<< /Type /XRef /Encrypt 5 0 R /Length 3 >>\nstream\nabc\n" + b"endstream\nendobj\n" + ), + False, + id="startxref-points-at-object-before-xref-stream", + ), + pytest.param( + # Offsets are relative to the header, as in the classic junk case. + b"JUNK\n" + _xref_stream_pdf(b"/Encrypt 5 0 R", b"abc"), + True, + id="xref-stream-junk-before-header", + ), + pytest.param(b"not a pdf /Encrypt 5 0 R startxref 0", False, id="not-a-pdf"), + ], +) +def test_is_encrypted_pdf(content, expected): + assert is_encrypted_pdf(content) is expected + + +def test_password_protected_pdf_error_needs_load_failure_and_encryption(): + assert ( + password_protected_pdf_error(_DOCLING_PARSE_LOAD_FAILURE, _ENCRYPTED_PDF) + == PASSWORD_PROTECTED_PDF_DETAIL + ) + # A corrupt but unencrypted PDF keeps docling's own message. + assert password_protected_pdf_error(_DOCLING_PARSE_LOAD_FAILURE, _PLAIN_PDF) is None + # An owner-password-only PDF that failed for another reason is not relabelled. + assert ( + password_protected_pdf_error("Docling processing failed: OCR crashed", _ENCRYPTED_PDF) + is None + ) + + +@pytest.mark.asyncio +async def test_convert_bytes_names_password_protected_pdf(docling_service): + with ( + patch.object( + docling_service, "upload_to_docling_direct_async", AsyncMock(return_value="t1") + ), + patch.object( + docling_service, + "get_docling_result_async", + AsyncMock(side_effect=DoclingServeError(_DOCLING_PARSE_LOAD_FAILURE)), + ), + pytest.raises(DoclingServeError) as exc_info, + ): + await docling_service.convert_bytes(_ENCRYPTED_PDF, "secret.pdf") + + assert str(exc_info.value) == f"Docling processing failed: {PASSWORD_PROTECTED_PDF_DETAIL}" + assert str(exc_info.value.__cause__) == _DOCLING_PARSE_LOAD_FAILURE + + +@pytest.mark.asyncio +async def test_convert_bytes_keeps_docling_error_when_encrypt_appears_outside_trailer( + docling_service, +): + """Literal /Encrypt text in page content must not relabel an unrelated load failure.""" + with ( + patch.object( + docling_service, "upload_to_docling_direct_async", AsyncMock(return_value="t1") + ), + patch.object( + docling_service, + "get_docling_result_async", + AsyncMock(side_effect=DoclingServeError(_DOCLING_PARSE_LOAD_FAILURE)), + ), + pytest.raises(DoclingServeError) as exc_info, + ): + await docling_service.convert_bytes(_PLAIN_PDF_WITH_ENCRYPT_TEXT, "about-pdf.pdf") + + assert str(exc_info.value) == _DOCLING_PARSE_LOAD_FAILURE + + +@pytest.mark.asyncio +async def test_convert_file_keeps_docling_error_for_unencrypted_pdf(docling_service, tmp_path): + pdf = tmp_path / "broken.pdf" + pdf.write_bytes(_PLAIN_PDF) + with ( + patch.object( + docling_service, "upload_to_docling_direct_async", AsyncMock(return_value="t1") + ), + patch.object( + docling_service, + "get_docling_result_async", + AsyncMock(side_effect=DoclingServeError(_DOCLING_PARSE_LOAD_FAILURE)), + ), + pytest.raises(DoclingServeError, match="Failed to load document with key"), + ): + await docling_service.convert_file(str(pdf)) + + @pytest.mark.asyncio async def test_poll_result_http_error(docling_service, mock_httpx_client): """Propagates HTTP errors during polling as DoclingServeError.""" diff --git a/tests/unit/test_langflow_file_service_two_phase.py b/tests/unit/test_langflow_file_service_two_phase.py index ac1f543ae..4105e53e8 100644 --- a/tests/unit/test_langflow_file_service_two_phase.py +++ b/tests/unit/test_langflow_file_service_two_phase.py @@ -113,6 +113,58 @@ async def test_langflow_not_invoked_on_docling_failure( assert file_task.phase == IngestionPhase.DOCLING +@pytest.mark.asyncio +async def test_docling_load_failure_on_encrypted_pdf_names_password_protection( + langflow_service, mock_polling_service, file_task +): + """Newer docling drops the PDFium reason; the bytes still show the PDF is encrypted.""" + head = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n" + pdf = ( + head + + b"xref\n0 1\n0000000000 65535 f \ntrailer\n<< /Root 1 0 R /Encrypt 5 0 R >>\n" + + b"startxref\n%d\n%%%%EOF\n" % len(head) + ) + encrypted = ("secret.pdf", pdf, "application/pdf") + mock_polling_service.poll_until_ready.return_value = DoclingPollResult( + outcome=PollOutcome.FAILED, + detail=( + "Docling result unavailable after SUCCESS status: Docling processing failed: " + "docling-parse could not load document 8fc525: " + "Failed to load document with key key=8fc525" + ), + ) + + with pytest.raises(Exception) as exc_info: + await langflow_service.upload_and_ingest_file( + file_tuple=encrypted, + docling_polling_service=mock_polling_service, + file_task=file_task, + ) + + message = str(exc_info.value) + assert message.startswith("Docling conversion did not complete (failed): ") + assert "Incorrect password" in message + assert "with key" not in message + assert langflow_service.run_ingestion_flow.call_count == 0 + + +@pytest.mark.asyncio +async def test_docling_load_failure_on_unencrypted_pdf_keeps_docling_detail( + langflow_service, mock_polling_service, file_tuple, file_task +): + detail = "docling-parse could not load document 8fc525: Failed to load document with key k" + mock_polling_service.poll_until_ready.return_value = DoclingPollResult( + outcome=PollOutcome.FAILED, detail=detail + ) + + with pytest.raises(Exception, match="Failed to load document with key k"): + await langflow_service.upload_and_ingest_file( + file_tuple=file_tuple, + docling_polling_service=mock_polling_service, + file_task=file_task, + ) + + @pytest.mark.asyncio async def test_langflow_not_invoked_on_docling_expired( langflow_service, mock_polling_service, file_tuple, file_task diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.py index 32c5a5444..e21ee6742 100644 --- a/tests/unit/test_task_service_get_task_status2.py +++ b/tests/unit/test_task_service_get_task_status2.py @@ -14,6 +14,7 @@ import pytest from models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask +from services.docling_service import PASSWORD_PROTECTED_PDF_DETAIL from services.task_service import TaskService, _opensearch_failure_metadata # --------------------------------------------------------------------------- @@ -203,6 +204,31 @@ def test_password_protected_pdf_non_langflow_path_maps_to_clean_message(self, ta assert "docling-parse" not in msg.lower() assert "cbfc231c" not in msg + @pytest.mark.parametrize( + "error", + [ + # Langflow path (upload_and_ingest_file) + f"Docling conversion did not complete (failed): {PASSWORD_PROTECTED_PDF_DETAIL}", + # Non-Langflow path (DoclingService.convert_file / convert_bytes) + f"Docling processing failed: {PASSWORD_PROTECTED_PDF_DETAIL}", + ], + ) + def test_backend_detected_password_protected_pdf_maps_to_clean_message( + self, task_service, error + ): + """docling >= 2.141 no longer names the reason; the backend's remap must still match.""" + ft = _make_file_task( + phase=IngestionPhase.DOCLING, + docling_status=DoclingPhaseStatus.FAILED, + error=error, + ) + meta = task_service._infer_failure_metadata(ft) + assert meta is not None + assert meta["actionable_by"] == "USER_ACTIONABLE" + assert meta["component"] == "docling" + assert meta["failure_phase"] == "parsing" + assert meta["user_facing_message"].startswith("This PDF is password-protected.") + def test_langflow_empty_content_not_retryable(self, task_service): ft = _make_file_task( phase=IngestionPhase.LANGFLOW,