diff --git a/.gitignore b/.gitignore index cd9ceee..598a2f5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ site-src/.hugo_build.lock # Generated commit message for docs/VERSION updates (scripts/generate_docs.py) /DOCS-LOG-MESSAGE + +# Python bytecode from running the scripts/ unit tests +__pycache__/ diff --git a/README.md b/README.md index 0f24691..ec416e1 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,14 @@ backend. To refresh it: Make sure the clone's working tree is clean (`git status`) -- the commit hash is recorded so the copied doc can be traced back to the exact source commit. + Install Graphviz (`dot`), plus `vl2vg` and `vg2svg` (`npm install -g vega-cli + vega-lite`), first. Without them the build still succeeds but leaves + diagrams out: PlantUML draws a "Cannot find Graphviz" error image in place of + each of its diagrams. `fineract-site docs` refuses such a page, and + `fineract-site build` fails on any already in `docs/` + (`scripts/check_doc_diagrams.py`). After installing Graphviz, delete + `fineract-doc/build` before rebuilding, as generated diagrams are cached. + 2. Run the `docs` command in the site tool image, mounting both repos: ```bash @@ -91,7 +99,8 @@ This copies `fineract-doc/build/docs/html/en/index.html` from the backend clone into `docs/VERSION/index.html`, then rewrites the Google Fonts and Font Awesome CDN links to point at the site's local `css/` stylesheets (per Apache project website requirements). Versions ending in `-SNAPSHOT` are overwritten; released -versions are not. +versions are not, unless you pass `--replace-release` to republish one from the +same release tag (e.g. to fix a broken build). It also suggests a commit message. Use it like so: diff --git a/docs/1.16.0/index.html b/docs/1.16.0/index.html index b3c005c..94c9393 100644 --- a/docs/1.16.0/index.html +++ b/docs/1.16.0/index.html @@ -927,7 +927,7 @@

Preface

Version: 1.16.0

-

Built on: Sat Oct 24 21:56:23 IST 2026

+

Built on: Sun Oct 25 13:19:31 IST 2026

Note the version in the URL path:

@@ -2715,7 +2715,7 @@
-Diagram +Diagram
@@ -46929,7 +46929,7 @@

Core Componen

-Diagram +Diagram
Figure 7. Command Processing Architecture Level 1 - Core
@@ -46947,7 +46947,7 @@

Dispatcher

-Diagram +Diagram
Figure 8. Command Processing Architecture Level 2 - Dispatcher
@@ -47033,7 +47033,7 @@

Command Handl
-Diagram +Diagram
Figure 9. Command Processing Architecture Level 2 - Handlers
@@ -47089,7 +47089,7 @@

Command Hooks
-Diagram +Diagram
Figure 10. Command Processing Architecture Level 2 - Hooks
@@ -47173,7 +47173,7 @@

Command Store
-Diagram +Diagram
Figure 11. Command Processing Architecture Level 2 - Store
@@ -66052,7 +66052,7 @@

Data Model

-Diagram +Diagram
Figure 14. Entity-Relationship — action tables reused by Disable / Enable
@@ -79035,7 +79035,7 @@

Release Proce
-Diagram +Diagram
Figure 19. Release Process Diagram
@@ -81227,7 +81227,7 @@

App diff --git a/docs/1.17.0-SNAPSHOT/index.html b/docs/1.17.0-SNAPSHOT/index.html index 9973239..63378f6 100644 --- a/docs/1.17.0-SNAPSHOT/index.html +++ b/docs/1.17.0-SNAPSHOT/index.html @@ -928,7 +928,7 @@

Preface

Version: 1.17.0-SNAPSHOT

-

Built on: Sat Oct 24 22:03:54 IST 2026

+

Built on: Sun Oct 25 13:21:51 IST 2026

Note the version in the URL path:

@@ -2716,7 +2716,7 @@
-Diagram +Diagram
@@ -46945,7 +46945,7 @@

Core Componen

-Diagram +Diagram
Figure 7. Command Processing Architecture Level 1 - Core
@@ -46963,7 +46963,7 @@

Dispatcher

-Diagram +Diagram
Figure 8. Command Processing Architecture Level 2 - Dispatcher
@@ -47049,7 +47049,7 @@

Command Handl
-Diagram +Diagram
Figure 9. Command Processing Architecture Level 2 - Handlers
@@ -47105,7 +47105,7 @@

Command Hooks
-Diagram +Diagram
Figure 10. Command Processing Architecture Level 2 - Hooks
@@ -47189,7 +47189,7 @@

Command Store
-Diagram +Diagram
Figure 11. Command Processing Architecture Level 2 - Store
@@ -66068,7 +66068,7 @@

Data Model

-Diagram +Diagram
Figure 14. Entity-Relationship — action tables reused by Disable / Enable
@@ -79291,7 +79291,7 @@

Release Proce
-Diagram +Diagram
Figure 19. Release Process Diagram
@@ -81484,7 +81484,7 @@

App diff --git a/scripts/check_doc_diagrams.py b/scripts/check_doc_diagrams.py new file mode 100644 index 0000000..0f67af8 --- /dev/null +++ b/scripts/check_doc_diagrams.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Fail on diagrams that PlantUML rendered as an error image. + +When Graphviz is missing, `./gradlew asciidoctor` still succeeds: PlantUML +draws "Cannot find Graphviz" in place of each diagram, and the docs embed that +image as base64. That is how docs/1.16.0 and docs/1.17.0-SNAPSHOT were first +published. This decodes every embedded diagram: SVGs are searched for +PlantUML's error text, and PlantUML PNGs, whose metadata is the same whether or +not they rendered, for the green PlantUML draws that text in. +""" + +import argparse +import base64 +import binascii +import re +import struct +import sys +import zlib +from pathlib import Path +from typing import List, Optional + +EMBEDDED_IMAGE = re.compile( + r"""src=["']data:image/(svg\+xml|png);base64,([A-Za-z0-9+/=]+)["']""" +) + +# Text PlantUML draws into its error images. +ERROR_MARKERS = ( + "Cannot find Graphviz", + "Dot Executable:", + "Syntax Error?", +) + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +PLANTUML_PNG_TAG = b"Generated by http://plantuml.com" +# The colour of PlantUML's error text (#33FF02). No working diagram in +# docs/1.12.1 to docs/1.15.0 uses it. +ERROR_GREEN = b"\x33\xff\x02" +PNG_ERROR_MARKER = "PlantUML error image (PNG)" + + +def png_chunks(png: bytes): + """Yield (type, data) for each chunk of a PNG.""" + pos = len(PNG_SIGNATURE) + while pos + 8 <= len(png): + length, kind = struct.unpack(">I4s", png[pos : pos + 8]) + yield kind, png[pos + 8 : pos + 8 + length] + pos += 12 + length + + +def paeth(left: int, up: int, up_left: int) -> int: + p = left + up - up_left + pa, pb, pc = abs(p - left), abs(p - up), abs(p - up_left) + if pa <= pb and pa <= pc: + return left + if pb <= pc: + return up + return up_left + + +def unfilter_row(kind: int, row: bytearray, previous: bytearray, bpp: int) -> None: + """Undo a PNG row filter in place (https://www.w3.org/TR/png/#9Filters).""" + for x in range(len(row)): + left = row[x - bpp] if x >= bpp else 0 + up = previous[x] + if kind == 1: + predictor = left + elif kind == 2: + predictor = up + elif kind == 3: + predictor = (left + up) // 2 + elif kind == 4: + predictor = paeth(left, up, previous[x - bpp] if x >= bpp else 0) + else: + return + row[x] = (row[x] + predictor) & 0xFF + + +def png_rows(png: bytes) -> Optional[List[bytes]]: + """RGB pixel rows of an 8-bit, non-interlaced RGB or RGBA PNG, else None.""" + if not png.startswith(PNG_SIGNATURE): + return None + chunks = list(png_chunks(png)) + header = next((data for kind, data in chunks if kind == b"IHDR"), None) + if header is None: + return None + width, height, bit_depth, colour_type, _, _, interlace = struct.unpack(">IIBBBBB", header) + if bit_depth != 8 or colour_type not in (2, 6) or interlace != 0: + return None + bpp = 3 if colour_type == 2 else 4 + stride = width * bpp + raw = zlib.decompress(b"".join(data for kind, data in chunks if kind == b"IDAT")) + rows, previous = [], bytearray(stride) + for y in range(height): + start = y * (stride + 1) + row = bytearray(raw[start + 1 : start + 1 + stride]) + unfilter_row(raw[start], row, previous, bpp) + # Drop the alpha channel so every row is plain RGB triples. + rows.append(bytes(row) if bpp == 3 else bytes(b for i, b in enumerate(row) if i % 4 != 3)) + previous = row + return rows + + +def is_plantuml_error_png(png: bytes) -> bool: + # Only PlantUML's own PNGs are decoded; the large screenshots are skipped. + if PLANTUML_PNG_TAG not in png: + return False + rows = png_rows(png) + if rows is None: + return False + return any( + row[i : i + 3] == ERROR_GREEN for row in rows for i in range(0, len(row), 3) + ) + + +def broken_diagrams(html: str) -> List[str]: + """Return the error marker found in each broken embedded diagram, in page order.""" + found = [] + for match in EMBEDDED_IMAGE.finditer(html): + try: + image = base64.b64decode(match.group(2)) + except binascii.Error: + continue + if match.group(1) == "png": + if is_plantuml_error_png(image): + found.append(PNG_ERROR_MARKER) + continue + svg = image.decode("utf-8", errors="ignore") + for marker in ERROR_MARKERS: + if marker in svg: + found.append(marker) + break + return found + + +# Broken diagrams already published before this check existed, per version. +# A version is only allowed exactly this many; any other count fails. +KNOWN_BROKEN = { + # The batch-jobs "Fault-tolerance scenarios" diagram (a PNG), broken since + # the 1.13.0 release. 1.13.0 is EOL, so it is not being rebuilt. + "1.13.0": 1, +} + + +def check_docs(docs_root: Path, known_broken=None) -> List[str]: + known_broken = KNOWN_BROKEN if known_broken is None else known_broken + errors = [] + for page in sorted(docs_root.glob("*/index.html")): + markers = broken_diagrams(page.read_text(encoding="utf-8", errors="ignore")) + if len(markers) == known_broken.get(page.parent.name, 0): + continue + for marker in markers: + errors.append(f"{page}: diagram rendered as a PlantUML error ('{marker}')") + if not markers: + errors.append(f"{page}: no broken diagrams left; remove it from KNOWN_BROKEN") + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--docs-root", + default="docs", + help="Directory holding VERSION/index.html pages (default: docs).", + ) + args = parser.parse_args() + + errors = check_docs(Path(args.docs_root)) + if errors: + print("Broken diagrams found. Install Graphviz (`dot`) and rebuild the docs:") + for error in errors: + print(f" - {error}") + return 1 + print("Docs diagram check passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py index 817744f..b2a72a5 100644 --- a/scripts/generate_docs.py +++ b/scripts/generate_docs.py @@ -31,6 +31,8 @@ import sys from pathlib import Path +from check_doc_diagrams import broken_diagrams + REPO_ROOT = Path(__file__).resolve().parent.parent DEFAULT_BACKEND_DIR = "/fineract" DOC_HTML_RELATIVE_PATH = Path("fineract-doc/build/docs/html/en/index.html") @@ -64,6 +66,12 @@ def parse_args(): help="Path to the apache/fineract clone " f"(default: $FINERACT_REPO_DIR or {DEFAULT_BACKEND_DIR})", ) + parser.add_argument( + "--replace-release", + action="store_true", + help="Overwrite docs for a released version, e.g. to republish a broken " + "build from the same release tag", + ) args = parser.parse_args() if not args.version: parser.error("a version is required: pass --version or set FINERACT_DOC_VERSION") @@ -132,6 +140,14 @@ def main(): "'./gradlew asciidoctor' in the backend clone." ) + broken = broken_diagrams(source_html.read_text(encoding="utf-8", errors="ignore")) + if broken: + sys.exit( + f"{source_html} has {len(broken)} diagram(s) rendered as a PlantUML error " + f"('{broken[0]}'). Install Graphviz (`dot`), then rebuild with " + "'./gradlew asciidoctor' after deleting fineract-doc/build." + ) + require_clean_git_status(backend_dir) commit_hash = get_commit_hash(backend_dir) print(f"Backend commit: {commit_hash}") @@ -140,10 +156,11 @@ def main(): dest_html = dest_dir / "index.html" is_snapshot = args.version.endswith("-SNAPSHOT") - if dest_html.exists() and not is_snapshot: + if dest_html.exists() and not is_snapshot and not args.replace_release: sys.exit( f"{dest_html} already exists and {args.version} is not a -SNAPSHOT " - "version. Refusing to overwrite a released doc." + "version. Refusing to overwrite a released doc (pass --replace-release " + "to republish it from the same release tag)." ) dest_dir.mkdir(parents=True, exist_ok=True) diff --git a/scripts/site-tool.sh b/scripts/site-tool.sh index f821c33..7860e7f 100644 --- a/scripts/site-tool.sh +++ b/scripts/site-tool.sh @@ -9,7 +9,7 @@ Commands: build Build site into /src/.build/site and run checks check Run internal link check against /src/.build/site docs Copy generated Fineract API docs into /src/docs/VERSION - test Run unit tests for whimsy checks + test Run unit tests for whimsy checks and docs tooling serve Run hugo server on port 1313 shell Open an interactive shell EOF @@ -46,11 +46,13 @@ run_whimsy_checks() { run_checks() { python3 "${REPO_ROOT}/scripts/check_internal_links.py" --site-root "${REPO_ROOT}/.build/site" + python3 "${REPO_ROOT}/scripts/check_doc_diagrams.py" --docs-root "${REPO_ROOT}/docs" run_whimsy_checks } run_tests() { ruby "${REPO_ROOT}/scripts/test_run_whimsy_checks.rb" + python3 -m unittest discover -s "${REPO_ROOT}/scripts" -p 'test_*.py' } generate_docs() { diff --git a/scripts/test_check_doc_diagrams.py b/scripts/test_check_doc_diagrams.py new file mode 100644 index 0000000..004e1d8 --- /dev/null +++ b/scripts/test_check_doc_diagrams.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Tests for check_doc_diagrams.py.""" + +import base64 +import struct +import tempfile +import unittest +import zlib +from pathlib import Path + +import check_doc_diagrams + +TESTDATA = Path(__file__).resolve().parent / "testdata" +# The batch-jobs diagram as first published in docs/1.16.0, and as rebuilt. +ERROR_PNG = (TESTDATA / "plantuml-graphviz-error.png").read_bytes() +GOOD_PNG = (TESTDATA / "plantuml-batch-jobs.png").read_bytes() + +# The text of one of the broken diagrams first published in docs/1.16.0. +GRAPHVIZ_ERROR_SVG = ( + "" + "Command Processing Architecture (Level 1) - Core Components" + "Dot Executable: /opt/local/bin/dot" + "File does not exist" + "Cannot find Graphviz. You should try" + "@startumltestdot@enduml" + "" +) +GOOD_SVG = ( + "" + "CommandHandler" +) + + +def img(svg): + encoded = base64.b64encode(svg.encode("utf-8")).decode("ascii") + return f'diagram' + + +def png_img(png): + return f'' + + +def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def make_png(pixels, filter_type=0, plantuml=True, alpha=False): + """Encode rows of RGB tuples as an 8-bit PNG using one row filter throughout.""" + bpp = 4 if alpha else 3 + raw, previous = b"", bytes(len(pixels[0]) * bpp) + for row_pixels in pixels: + row = b"".join(bytes(p) + (b"\xff" if alpha else b"") for p in row_pixels) + encoded = bytearray(row) + for x in range(len(row)): + left = row[x - bpp] if x >= bpp else 0 + up = previous[x] + up_left = previous[x - bpp] if x >= bpp else 0 + predictor = (0, left, up, (left + up) // 2, check_doc_diagrams.paeth(left, up, up_left))[filter_type] + encoded[x] = (row[x] - predictor) & 0xFF + raw += bytes([filter_type]) + bytes(encoded) + previous = row + header = struct.pack(">IIBBBBB", len(pixels[0]), len(pixels), 8, 6 if alpha else 2, 0, 0, 0) + tag = chunk(b"tEXt", b"copyleft\0" + check_doc_diagrams.PLANTUML_PNG_TAG) if plantuml else b"" + return ( + check_doc_diagrams.PNG_SIGNATURE + + chunk(b"IHDR", header) + + tag + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + + +WHITE, BLACK, GREEN = (255, 255, 255), (0, 0, 0), (0x33, 0xFF, 0x02) +WITH_GREEN = [[WHITE, BLACK, WHITE], [BLACK, GREEN, WHITE], [WHITE, WHITE, BLACK]] +WITHOUT_GREEN = [[WHITE, BLACK, WHITE], [BLACK, (0x33, 0xFF, 0x03), WHITE], [WHITE, WHITE, BLACK]] + + +class PlantumlPngTest(unittest.TestCase): + def test_published_graphviz_error_png_is_detected(self): + self.assertTrue(check_doc_diagrams.is_plantuml_error_png(ERROR_PNG)) + + def test_rebuilt_png_passes(self): + self.assertFalse(check_doc_diagrams.is_plantuml_error_png(GOOD_PNG)) + + def test_every_row_filter_is_decoded(self): + for filter_type in range(5): + for alpha in (False, True): + with self.subTest(filter_type=filter_type, alpha=alpha): + self.assertTrue( + check_doc_diagrams.is_plantuml_error_png(make_png(WITH_GREEN, filter_type, alpha=alpha)) + ) + self.assertFalse( + check_doc_diagrams.is_plantuml_error_png(make_png(WITHOUT_GREEN, filter_type, alpha=alpha)) + ) + + def test_png_not_made_by_plantuml_is_not_decoded(self): + self.assertFalse(check_doc_diagrams.is_plantuml_error_png(make_png(WITH_GREEN, plantuml=False))) + + +class BrokenDiagramsTest(unittest.TestCase): + def test_graphviz_error_image_is_reported(self): + html = f"

{img(GOOD_SVG)}

{img(GRAPHVIZ_ERROR_SVG)}

" + self.assertEqual(check_doc_diagrams.broken_diagrams(html), ["Cannot find Graphviz"]) + + def test_good_diagrams_pass(self): + self.assertEqual(check_doc_diagrams.broken_diagrams(img(GOOD_SVG) * 3), []) + + def test_error_text_outside_an_image_is_ignored(self): + # The docs may quote PlantUML's messages in prose; only images count. + html = "

If you see Cannot find Graphviz, install it.

" + self.assertEqual(check_doc_diagrams.broken_diagrams(html), []) + + def test_broken_png_is_reported_in_page_order(self): + html = png_img(GOOD_PNG) + png_img(ERROR_PNG) + img(GRAPHVIZ_ERROR_SVG) + self.assertEqual( + check_doc_diagrams.broken_diagrams(html), + [check_doc_diagrams.PNG_ERROR_MARKER, "Cannot find Graphviz"], + ) + + def test_truncated_png_and_invalid_base64_are_skipped(self): + html = '' + self.assertEqual(check_doc_diagrams.broken_diagrams(html), []) + + +class CheckDocsTest(unittest.TestCase): + def test_reports_each_broken_diagram_per_version(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for version, body in ( + ("1.16.0", img(GRAPHVIZ_ERROR_SVG) * 2), + ("1.15.0", img(GOOD_SVG)), + ): + (root / version).mkdir() + (root / version / "index.html").write_text(body, encoding="utf-8") + + errors = check_doc_diagrams.check_docs(root, known_broken={}) + + self.assertEqual(len(errors), 2) + self.assertTrue(all("1.16.0" in error for error in errors)) + + def write_page(self, root, version, body): + (root / version).mkdir() + (root / version / "index.html").write_text(body, encoding="utf-8") + + def test_known_broken_count_is_allowed(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_page(Path(tmp), "1.13.0", png_img(ERROR_PNG)) + self.assertEqual(check_doc_diagrams.check_docs(Path(tmp), {"1.13.0": 1}), []) + + def test_new_breakage_in_a_known_version_fails(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_page(Path(tmp), "1.13.0", png_img(ERROR_PNG) + img(GRAPHVIZ_ERROR_SVG)) + self.assertEqual(len(check_doc_diagrams.check_docs(Path(tmp), {"1.13.0": 1})), 2) + + def test_fixed_known_version_asks_for_the_entry_to_go(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_page(Path(tmp), "1.13.0", png_img(GOOD_PNG)) + errors = check_doc_diagrams.check_docs(Path(tmp), {"1.13.0": 1}) + self.assertEqual(len(errors), 1) + self.assertIn("remove it from KNOWN_BROKEN", errors[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/testdata/plantuml-batch-jobs.png b/scripts/testdata/plantuml-batch-jobs.png new file mode 100644 index 0000000..c538fdb Binary files /dev/null and b/scripts/testdata/plantuml-batch-jobs.png differ diff --git a/scripts/testdata/plantuml-graphviz-error.png b/scripts/testdata/plantuml-graphviz-error.png new file mode 100644 index 0000000..8d9444c Binary files /dev/null and b/scripts/testdata/plantuml-graphviz-error.png differ