Fix broken diagrams in /docs/1.16.0 and /docs/1.17.0-SNAPSHOT; fail the build on them

.gitignore

@@ -12,3 +12,6 @@ site-src/.hugo_build.lock
 
 # Generated commit message for docs/VERSION updates (scripts/generate_docs.py)
 /DOCS-LOG-MESSAGE
+
+# Python bytecode from running the scripts/ unit tests
+__pycache__/

README.md

@@ -74,6 +74,14 @@ backend. To refresh it:
    Make sure the clone's working tree is clean (`git status`) -- the commit hash
    is recorded so the copied doc can be traced back to the exact source commit.
 
+   Install Graphviz (`dot`), plus `vl2vg` and `vg2svg` (`npm install -g vega-cli
+   vega-lite`), first. Without them the build still succeeds but leaves
+   diagrams out: PlantUML draws a "Cannot find Graphviz" error image in place of
+   each of its diagrams. `fineract-site docs` refuses such a page, and
+   `fineract-site build` fails on any already in `docs/`
+   (`scripts/check_doc_diagrams.py`). After installing Graphviz, delete
+   `fineract-doc/build` before rebuilding, as generated diagrams are cached.
+
 2. Run the `docs` command in the site tool image, mounting both repos:
 
    ```bash
@@ -91,7 +99,8 @@ This copies `fineract-doc/build/docs/html/en/index.html` from the backend clone
 into `docs/VERSION/index.html`, then rewrites the Google Fonts and Font Awesome
 CDN links to point at the site's local `css/` stylesheets (per Apache project
 website requirements). Versions ending in `-SNAPSHOT` are overwritten; released
-versions are not.
+versions are not, unless you pass `--replace-release` to republish one from the
+same release tag (e.g. to fix a broken build).
 
 It also suggests a commit message. Use it like so:
 

scripts/check_doc_diagrams.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+"""Fail on diagrams that PlantUML rendered as an error image.
+
+When Graphviz is missing, `./gradlew asciidoctor` still succeeds: PlantUML
+draws "Cannot find Graphviz" in place of each diagram, and the docs embed that
+image as base64. That is how docs/1.16.0 and docs/1.17.0-SNAPSHOT were first
+published. This decodes every embedded diagram: SVGs are searched for
+PlantUML's error text, and PlantUML PNGs, whose metadata is the same whether or
+not they rendered, for the green PlantUML draws that text in.
+"""
+
+import argparse
+import base64
+import binascii
+import re
+import struct
+import sys
+import zlib
+from pathlib import Path
+from typing import List, Optional
+
+EMBEDDED_IMAGE = re.compile(
+    r"""src=["']data:image/(svg\+xml|png);base64,([A-Za-z0-9+/=]+)["']"""
+)
+
+# Text PlantUML draws into its error images.
+ERROR_MARKERS = (
+    "Cannot find Graphviz",
+    "Dot Executable:",
+    "Syntax Error?",
+)
+
+PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
+PLANTUML_PNG_TAG = b"Generated by http://plantuml.com"
+# The colour of PlantUML's error text (#33FF02). No working diagram in
+# docs/1.12.1 to docs/1.15.0 uses it.
+ERROR_GREEN = b"\x33\xff\x02"
+PNG_ERROR_MARKER = "PlantUML error image (PNG)"
+
+
+def png_chunks(png: bytes):
+    """Yield (type, data) for each chunk of a PNG."""
+    pos = len(PNG_SIGNATURE)
+    while pos + 8 <= len(png):
+        length, kind = struct.unpack(">I4s", png[pos : pos + 8])
+        yield kind, png[pos + 8 : pos + 8 + length]
+        pos += 12 + length
+
+
+def paeth(left: int, up: int, up_left: int) -> int:
+    p = left + up - up_left
+    pa, pb, pc = abs(p - left), abs(p - up), abs(p - up_left)
+    if pa <= pb and pa <= pc:
+        return left
+    if pb <= pc:
+        return up
+    return up_left
+
+
+def unfilter_row(kind: int, row: bytearray, previous: bytearray, bpp: int) -> None:
+    """Undo a PNG row filter in place (https://www.w3.org/TR/png/#9Filters)."""
+    for x in range(len(row)):
+        left = row[x - bpp] if x >= bpp else 0
+        up = previous[x]
+        if kind == 1:
+            predictor = left
+        elif kind == 2:
+            predictor = up
+        elif kind == 3:
+            predictor = (left + up) // 2
+        elif kind == 4:
+            predictor = paeth(left, up, previous[x - bpp] if x >= bpp else 0)
+        else:
+            return
+        row[x] = (row[x] + predictor) & 0xFF
+
+
+def png_rows(png: bytes) -> Optional[List[bytes]]:
+    """RGB pixel rows of an 8-bit, non-interlaced RGB or RGBA PNG, else None."""
+    if not png.startswith(PNG_SIGNATURE):
+        return None
+    chunks = list(png_chunks(png))
+    header = next((data for kind, data in chunks if kind == b"IHDR"), None)
+    if header is None:
+        return None
+    width, height, bit_depth, colour_type, _, _, interlace = struct.unpack(">IIBBBBB", header)
+    if bit_depth != 8 or colour_type not in (2, 6) or interlace != 0:
+        return None
+    bpp = 3 if colour_type == 2 else 4
+    stride = width * bpp
+    raw = zlib.decompress(b"".join(data for kind, data in chunks if kind == b"IDAT"))
+    rows, previous = [], bytearray(stride)
+    for y in range(height):
+        start = y * (stride + 1)
+        row = bytearray(raw[start + 1 : start + 1 + stride])
+        unfilter_row(raw[start], row, previous, bpp)
+        # Drop the alpha channel so every row is plain RGB triples.
+        rows.append(bytes(row) if bpp == 3 else bytes(b for i, b in enumerate(row) if i % 4 != 3))
+        previous = row
+    return rows
+
+
+def is_plantuml_error_png(png: bytes) -> bool:
+    # Only PlantUML's own PNGs are decoded; the large screenshots are skipped.
+    if PLANTUML_PNG_TAG not in png:
+        return False
+    rows = png_rows(png)
+    if rows is None:
+        return False
+    return any(
+        row[i : i + 3] == ERROR_GREEN for row in rows for i in range(0, len(row), 3)
+    )
+
+
+def broken_diagrams(html: str) -> List[str]:
+    """Return the error marker found in each broken embedded diagram, in page order."""
+    found = []
+    for match in EMBEDDED_IMAGE.finditer(html):
+        try:
+            image = base64.b64decode(match.group(2))
+        except binascii.Error:
+            continue
+        if match.group(1) == "png":
+            if is_plantuml_error_png(image):
+                found.append(PNG_ERROR_MARKER)
+            continue
+        svg = image.decode("utf-8", errors="ignore")
+        for marker in ERROR_MARKERS:
+            if marker in svg:
+                found.append(marker)
+                break
+    return found
+
+
+# Broken diagrams already published before this check existed, per version.
+# A version is only allowed exactly this many; any other count fails.
+KNOWN_BROKEN = {
+    # The batch-jobs "Fault-tolerance scenarios" diagram (a PNG), broken since
+    # the 1.13.0 release. 1.13.0 is EOL, so it is not being rebuilt.
+    "1.13.0": 1,
+}
+
+
+def check_docs(docs_root: Path, known_broken=None) -> List[str]:
+    known_broken = KNOWN_BROKEN if known_broken is None else known_broken
+    errors = []
+    for page in sorted(docs_root.glob("*/index.html")):
+        markers = broken_diagrams(page.read_text(encoding="utf-8", errors="ignore"))
+        if len(markers) == known_broken.get(page.parent.name, 0):
+            continue
+        for marker in markers:
+            errors.append(f"{page}: diagram rendered as a PlantUML error ('{marker}')")
+        if not markers:
+            errors.append(f"{page}: no broken diagrams left; remove it from KNOWN_BROKEN")
+    return errors
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "--docs-root",
+        default="docs",
+        help="Directory holding VERSION/index.html pages (default: docs).",
+    )
+    args = parser.parse_args()
+
+    errors = check_docs(Path(args.docs_root))
+    if errors:
+        print("Broken diagrams found. Install Graphviz (`dot`) and rebuild the docs:")
+        for error in errors:
+            print(f"  - {error}")
+        return 1
+    print("Docs diagram check passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/generate_docs.py

@@ -31,6 +31,8 @@
 import sys
 from pathlib import Path
 
+from check_doc_diagrams import broken_diagrams
+
 REPO_ROOT = Path(__file__).resolve().parent.parent
 DEFAULT_BACKEND_DIR = "/fineract"
 DOC_HTML_RELATIVE_PATH = Path("fineract-doc/build/docs/html/en/index.html")
@@ -64,6 +66,12 @@ def parse_args():
         help="Path to the apache/fineract clone "
         f"(default: $FINERACT_REPO_DIR or {DEFAULT_BACKEND_DIR})",
     )
+    parser.add_argument(
+        "--replace-release",
+        action="store_true",
+        help="Overwrite docs for a released version, e.g. to republish a broken "
+        "build from the same release tag",
+    )
     args = parser.parse_args()
     if not args.version:
         parser.error("a version is required: pass --version or set FINERACT_DOC_VERSION")
@@ -132,6 +140,14 @@ def main():
             "'./gradlew asciidoctor' in the backend clone."
         )
 
+    broken = broken_diagrams(source_html.read_text(encoding="utf-8", errors="ignore"))
+    if broken:
+        sys.exit(
+            f"{source_html} has {len(broken)} diagram(s) rendered as a PlantUML error "
+            f"('{broken[0]}'). Install Graphviz (`dot`), then rebuild with "
+            "'./gradlew asciidoctor' after deleting fineract-doc/build."
+        )
+
     require_clean_git_status(backend_dir)
     commit_hash = get_commit_hash(backend_dir)
     print(f"Backend commit: {commit_hash}")
@@ -140,10 +156,11 @@ def main():
     dest_html = dest_dir / "index.html"
     is_snapshot = args.version.endswith("-SNAPSHOT")
 
-    if dest_html.exists() and not is_snapshot:
+    if dest_html.exists() and not is_snapshot and not args.replace_release:
         sys.exit(
             f"{dest_html} already exists and {args.version} is not a -SNAPSHOT "
-            "version. Refusing to overwrite a released doc."
+            "version. Refusing to overwrite a released doc (pass --replace-release "
+            "to republish it from the same release tag)."
         )
 
     dest_dir.mkdir(parents=True, exist_ok=True)

scripts/site-tool.sh

@@ -9,7 +9,7 @@ Commands:
   build  Build site into /src/.build/site and run checks
   check  Run internal link check against /src/.build/site
   docs   Copy generated Fineract API docs into /src/docs/VERSION
-  test   Run unit tests for whimsy checks
+  test   Run unit tests for whimsy checks and docs tooling
   serve  Run hugo server on port 1313
   shell  Open an interactive shell
 EOF
@@ -46,11 +46,13 @@ run_whimsy_checks() {
 
 run_checks() {
   python3 "${REPO_ROOT}/scripts/check_internal_links.py" --site-root "${REPO_ROOT}/.build/site"
+  python3 "${REPO_ROOT}/scripts/check_doc_diagrams.py" --docs-root "${REPO_ROOT}/docs"
   run_whimsy_checks
 }
 
 run_tests() {
   ruby "${REPO_ROOT}/scripts/test_run_whimsy_checks.rb"
+  python3 -m unittest discover -s "${REPO_ROOT}/scripts" -p 'test_*.py'
 }
 
 generate_docs() {