diff --git a/.github/docs.json b/.github/docs.json new file mode 100644 index 00000000..0a88e18e --- /dev/null +++ b/.github/docs.json @@ -0,0 +1,59 @@ +{ + "schema": 1, + "site": { + "title": "ConnectBot SSH Library", + "base_url": "https://sshlib.connectbot.org" + }, + "repository": { + "slug": "connectbot/cbssh" + }, + "tags": { + "minimum": "0.5.0" + }, + "build": { + "project": "sshlib", + "gradle_overlay": true, + "arguments": [ + "--dependency-verification", + "strict" + ], + "overlays": [ + ".github/scripts/templates", + "gradle/verification-metadata.xml" + ] + }, + "api": { + "sshlib": "SSH" + }, + "ci": { + "jobs": [ + { + "names": [ + "Build and test (17)", + "Build and test (21)", + "Build and test (25)" + ], + "fallback_steps": [ + "Build with Gradle", + { + "name": "Check TLA+ models", + "conclusions": [ + "success", + "skipped" + ], + "optional": true + }, + { + "name": "Verify release tag", + "conclusions": [ + "success", + "skipped" + ], + "optional": true + } + ] + }, + "Workflow scripts" + ] + } +} diff --git a/.github/scripts/docs_site.py b/.github/scripts/docs_site.py deleted file mode 100644 index 2083c93e..00000000 --- a/.github/scripts/docs_site.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Render the version homepage and sitemaps for a complete documentation tree.""" - -from html import escape -from urllib.parse import quote -import xml.etree.ElementTree as ET - - -INDEX_HTML = """ - - - - ConnectBot SSH Library — Documentation - - - - - - - - - - - -
- -
-
-
-

ConnectBot SSH Library

-
-
-

A Kotlin and Java SSH client library with coroutine APIs, modern cryptography, SFTP, and port forwarding.

-
-
-

Documentation versions

-
-${VERSION_ROWS}
-
-
-
- -
-
- Generated by - Dokka -
Copyright Kenny Root
-
-
-
-
-
-
-
- - -\n""" - - -def generate_site(output, versions, base_url): - """Versions are supplied in display order: main, then descending releases.""" - available = [name for name in versions if (output / name / "index.html").is_file()] - sitemap_index = ET.Element("sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9") - rows = [] - for name in available: - sitemap = ET.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9") - for page in sorted((output / name).rglob("*.html")): - if page.name == "navigation.html": - continue - entry = ET.SubElement(sitemap, "url") - ET.SubElement(entry, "loc").text = base_url + "/" + quote(page.relative_to(output).as_posix(), safe="/") - ET.ElementTree(sitemap).write(output / name / "sitemap.xml", encoding="utf-8", xml_declaration=True) - entry = ET.SubElement(sitemap_index, "sitemap") - ET.SubElement(entry, "loc").text = f"{base_url}/{quote(name, safe='')}/sitemap.xml" - label = 'main snapshot' if name == "main" else escape(name) - href = escape(f"{base_url}/{quote(name, safe='')}/index.html", quote=True) - rows.append('
' - '
' - f'{label}
\n') - if "main" not in available: - return False - index = INDEX_HTML.replace("${BASE_URL}", escape(base_url, quote=True)).replace("${VERSION_ROWS}", "".join(rows)) - (output / "index.html").write_text(index, encoding="utf-8") - ET.ElementTree(sitemap_index).write(output / "sitemap_index.xml", encoding="utf-8", xml_declaration=True) - return True diff --git a/.github/scripts/publish-docs.py b/.github/scripts/publish-docs.py deleted file mode 100644 index 61ae0cdb..00000000 --- a/.github/scripts/publish-docs.py +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env python3 -"""Reconcile release documentation, then save a complete GitHub Pages archive. - -The archive is a generated Git tree: committing it does not use or change the -source checkout's index. Only this script's --publish path pushes gh-pages. -""" - -import argparse -from dataclasses import asdict, dataclass, field -from enum import Enum -import hashlib -import json -import os -from pathlib import Path -import posixpath -import re -import shutil -import subprocess -import tarfile -import tempfile -from urllib.parse import unquote, urlsplit, urlunsplit - -from docs_site import generate_site - - -BASE_URL = "https://sshlib.connectbot.org" -POLICY = 1 # Bump if the public-only filtering contract changes. -TAG = re.compile(r"v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)" - r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?" - r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\Z") - - -class BuildStatus(str, Enum): - SUCCESS = "success" - FAILED = "failed" - SKIPPED = "skipped" - - -@dataclass(frozen=True) -class DocumentationConfiguration: - commit: str - config: str - policy: int = POLICY - - @classmethod - def from_dict(cls, value): - # Missing policy means the artifact predates explicit visibility filtering. - return cls(value["commit"], value["config"], value.get("policy", 0)) - - -@dataclass(frozen=True) -class BuildAttempt: - configuration: DocumentationConfiguration - status: BuildStatus - - @classmethod - def from_dict(cls, value): - configuration = DocumentationConfiguration( - value.get("commit", ""), value.get("config", ""), value.get("policy", 0)) - return cls(configuration, BuildStatus(value["status"])) - - def to_dict(self): - return {**asdict(self.configuration), - "status": self.status.value} - - -@dataclass -class VersionDocumentation: - published: DocumentationConfiguration | None = None - attempt: BuildAttempt | None = None - - @classmethod - def from_dict(cls, value): - return cls( - DocumentationConfiguration.from_dict(value["published"]) if value.get("published") else None, - BuildAttempt.from_dict(value["attempt"]) if value.get("attempt") else None, - ) - - def to_dict(self): - result = {} - if self.published: - result["published"] = asdict(self.published) - if self.attempt: - result["attempt"] = self.attempt.to_dict() - return result - - def copy_verified_archive(self, source, destination): - if (self.published and self.published.policy == POLICY - and (source / "index.html").is_file() - and (source / "index.md").is_file()): - shutil.copytree(source, destination) - return self.published - return None - - def should_build(self, name, desired, retry_failed): - if (name != "main" and self.attempt - and self.attempt.status is BuildStatus.FAILED and not retry_failed): - return False - return self.published != desired - - def message(self, name, attempted=False): - if attempted: - message = f"{name}: {self.attempt.status.value}" - elif self.attempt and self.attempt.status is BuildStatus.FAILED: - message = f"{name}: previous failure; not retried" - else: - message = f"{name}: reused" - if self.attempt and self.attempt.status is BuildStatus.FAILED and self.published: - message += " (serving previous successful build)" - return message - - -@dataclass -class DocumentationManifest: - versions: dict[str, VersionDocumentation] = field(default_factory=dict) - - @classmethod - def read(cls, path): - if not path.exists(): - return cls() - document = json.loads(path.read_text()) - if document.get("schema") != 1: - raise RuntimeError("Unknown archive manifest schema") - return cls({name: VersionDocumentation.from_dict(value) - for name, value in document.get("versions", {}).items()}) - - def write(self, path): - document = {"schema": 1, - "versions": {name: version.to_dict() - for name, version in self.versions.items()}} - path.write_text(json.dumps(document, indent=2) + "\n") - - -def release_key(tag): - match = TAG.fullmatch(tag) - if not match: - return None - core = tuple(map(int, match.group(1, 2, 3))) - pre = match.group(4) - if core < (0, 5, 0) or (core == (0, 5, 0) and pre): - return None - if pre and any(part.isdigit() and len(part) > 1 and part[0] == '0' - for part in pre.split('.')): - return None - identifiers = tuple((0, int(part)) if part.isdigit() else (1, part) - for part in pre.split('.')) if pre else () - return core, pre is None, identifiers, tag - - -def git(repo, *args, data=None): - return subprocess.check_output(["git", "-C", str(repo), *args], input=data) - - -def discover(repo): - tags = git(repo, "tag", "--list").decode().splitlines() - tags = sorted((tag for tag in tags if release_key(tag)), - key=release_key, reverse=True) - return {name: git(repo, "rev-parse", f"{ref}^{{commit}}").decode().strip() - for name, ref in [("main", "HEAD")] + [(tag, f"refs/tags/{tag}") for tag in tags]} - - -def ci_passed(commit): - """Require successful library verification across the latest push CI matrix.""" - workflow = "/repos/connectbot/cbssh/actions/workflows/ci.yml/runs" - runs = json.loads(subprocess.check_output([ - "gh", "api", "--method", "GET", workflow, - "-f", f"head_sha={commit}", "-f", "event=push", "-f", "per_page=1", - ]))["workflow_runs"] - if not runs or runs[0]["head_sha"] != commit: - return False - # Inspect the actual verification steps, excluding Maven upload and analysis. - pages = json.loads(subprocess.check_output([ - "gh", "api", "--paginate", "--slurp", - f"/repos/connectbot/cbssh/actions/runs/{runs[0]['id']}/jobs?per_page=100", - ])) - jobs = [job for page in pages for job in page["jobs"]] - expected = {"Build and test (17)", "Build and test (21)", "Build and test (25)"} - builds = [job for job in jobs if job["name"] in expected] - if {job["name"] for job in builds} != expected or len(builds) != len(expected): - return False - # Publishing and analysis happen inside build jobs; admit their build steps. - for job in builds: - if job["conclusion"] == "success": - continue - steps = {step["name"]: step["conclusion"] for step in job.get("steps", [])} - if steps.get("Build with Gradle") != "success": - return False - for required in ("Check TLA+ models", "Verify release tag"): - if required in steps and steps[required] not in ("success", "skipped"): - return False - checks = [job for job in jobs if job["name"] == "Workflow scripts"] - return len(checks) == 1 and checks[0]["conclusion"] == "success" - - -def load_archive(repo, archive): - result = subprocess.run(["git", "-C", str(repo), "ls-remote", "--exit-code", - "origin", "refs/heads/gh-pages"], capture_output=True) - if result.returncode == 2: - return None, DocumentationManifest() - if result.returncode: - raise RuntimeError(result.stderr.decode()) - git(repo, "fetch", "--no-tags", "origin", "refs/heads/gh-pages") - parent = git(repo, "rev-parse", "FETCH_HEAD").decode().strip() - with tempfile.TemporaryFile() as packed: - subprocess.run(["git", "-C", str(repo), "archive", parent], stdout=packed, check=True) - packed.seek(0) - with tarfile.open(fileobj=packed) as contents: - contents.extractall(archive, filter="data") - return parent, DocumentationManifest.read(archive / "manifest.json") - - -def build_version(repo, scripts, commit, name, destination, log): - with tempfile.TemporaryDirectory(prefix="sshlib-docs-build-") as temporary: - checkout = Path(temporary) / "source" - with log.open("w") as output: - subprocess.run(["git", "clone", "--shared", "--no-checkout", str(repo), str(checkout)], - stdout=output, stderr=subprocess.STDOUT, check=True) - subprocess.run(["git", "-C", str(checkout), "checkout", "--detach", commit], - stdout=output, stderr=subprocess.STDOUT, check=True) - # Overlay only documentation configuration; keep tagged source intact. - docs = checkout / ".github/scripts" - docs.mkdir(parents=True, exist_ok=True) - shutil.copytree(scripts / "templates", docs / "templates", dirs_exist_ok=True) - # Use the publisher's reviewed trust metadata for newly enabled formats. - shutil.copy2(repo / "gradle/verification-metadata.xml", - checkout / "gradle/verification-metadata.xml") - build_file = checkout / "sshlib/build.gradle.kts" - build = build_file.read_text().replace( - "alias(libs.plugins.dokka) apply false", "alias(libs.plugins.dokka)") - current = (repo / "sshlib/build.gradle.kts").read_text() - config = current.split("// BEGIN SITE DOCUMENTATION\n", 1)[1].split("// END SITE DOCUMENTATION", 1)[0] - imports = "\n".join(line for line in current.splitlines() - if line.startswith("import org.jetbrains.dokka.")) - if "// BEGIN SITE DOCUMENTATION" in build: - before, _, remainder = build.partition("// BEGIN SITE DOCUMENTATION") - _, marker, after = remainder.partition("// END SITE DOCUMENTATION") - if not marker: - raise RuntimeError("Unterminated documentation configuration") - build = before + after - build = "\n".join(line for line in build.splitlines() - if not line.startswith("import org.jetbrains.dokka.")) - build = imports + "\n" + build + "\n// BEGIN SITE DOCUMENTATION\n" + config + "// END SITE DOCUMENTATION\n" - build_file.write_text(build) - subprocess.run([ - "./gradlew", ":sshlib:dokkaGeneratePublicationHtml", ":sshlib:dokkaGenerateMarkdown", - "--dependency-verification", "strict", - "--no-configuration-cache", "--no-daemon", "--console=plain", - f"-PdocsSourceCommit={commit}", f"-PdocsVersion={name}", - ], cwd=checkout, stdout=output, stderr=subprocess.STDOUT, check=True, timeout=1800) - generated = checkout / "sshlib/build/dokka" - if not (generated / "html/index.html").is_file() or not (generated / "markdown/index.md").is_file(): - raise RuntimeError("Dokka did not produce both HTML and Markdown entrypoints") - assemble_publication(generated / "html", generated / "markdown", destination) - - -def assemble_publication(html, markdown, destination): - """Align GFM's extra module-name directory with aggregated HTML paths.""" - mapping = {} - for source in markdown.rglob("*.md"): - relative = source.relative_to(markdown) - parts = relative.parts - # GFM adds its module name between the Gradle module and package paths. - target = relative if (html / relative.with_suffix(".html")).is_file() else ( - Path(parts[0], *parts[2:]) if len(parts) >= 3 else relative) - if target in mapping.values(): - raise RuntimeError(f"Duplicate Markdown page: {target}") - mapping[relative.as_posix()] = target - for page in html.rglob("*.html"): - if page.name != "navigation.html" and page.relative_to(html).with_suffix(".md") not in mapping.values(): - raise RuntimeError(f"HTML page has no Markdown counterpart: {page}") - shutil.copytree(html, destination) - for original, target in mapping.items(): - def rebase(match): - url = urlsplit(match.group(1)) - if url.scheme or url.netloc or not url.path.endswith(".md"): - return match.group(0) - resolved = posixpath.normpath(posixpath.join(posixpath.dirname(original), unquote(url.path))) - if resolved not in mapping: - raise RuntimeError(f"Broken Markdown link in {original}: {url.path}") - path = os.path.relpath(mapping[resolved], target.parent).replace(os.sep, "/") - return "](" + urlunsplit(("", "", path, url.query, url.fragment)) + ")" - content = re.sub(r"\]\(([^)\n]+)\)", rebase, (markdown / original).read_text()) - path = destination / target - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content) - - -def reconcile(repo, scripts, archive, output, logs, previous, versions, config, - retry_failed=False, builder=build_version, admission=None): - records = {} - messages = [] - for name, commit in versions.items(): - old = previous.versions.get(name, VersionDocumentation()) - record = VersionDocumentation(attempt=old.attempt) - record.published = old.copy_verified_archive(archive / name, output / name) - desired = DocumentationConfiguration(commit, config) - files = git(repo, "ls-tree", "--name-only", commit, "sshlib/build.gradle.kts").decode() - build_file = git(repo, "show", f"{commit}:sshlib/build.gradle.kts").decode() if files.strip() else "" - if admission is not None and not admission(commit): - messages.append(f"{name}: pending (library CI has not succeeded)") - elif not re.search(r"\bdokka\b", build_file): - record.attempt = BuildAttempt(desired, BuildStatus.SKIPPED) - messages.append(f"{name}: skipped (no Dokka configuration)") - elif record.should_build(name, desired, retry_failed): - print(f"Building documentation for {name} ({commit})", flush=True) - status = BuildStatus.FAILED - # Build separately so failures cannot damage the previous successful tree. - with tempfile.TemporaryDirectory(prefix="sshlib-docs-result-") as temporary: - result = Path(temporary) / "result" - log = logs / f"{name}.log" - try: - builder(repo, scripts, commit, name, result, log) - if any(path.is_symlink() for path in result.rglob("*")): - raise RuntimeError("Dokka output contains symlinks") - except (subprocess.SubprocessError, OSError, RuntimeError) as error: - with log.open("a") as failed_log: - failed_log.write(f"\nPublisher: {error}\n") - else: - # Assembly errors are fatal: never publish a partially copied tree. - if (output / name).exists(): - shutil.rmtree(output / name) - shutil.copytree(result, output / name) - record.published = desired - status = BuildStatus.SUCCESS - record.attempt = BuildAttempt(desired, status) - messages.append(record.message(name, attempted=True)) - else: - messages.append(record.message(name)) - records[name] = record - return DocumentationManifest(records), messages - - -def write_tree(repo, directory): - """Store generated files without touching the developer's Git index.""" - entries = [] - for path in sorted(directory.iterdir()): - if path.is_symlink(): - raise RuntimeError(f"Archive cannot contain symlinks: {path}") - if path.is_dir(): - mode, kind, oid = "040000", "tree", write_tree(repo, path) - else: - mode, kind = "100644", "blob" - oid = git(repo, "hash-object", "-w", str(path)).decode().strip() - entries.append(f"{mode} {kind} {oid}\t{path.name}\0".encode()) - return git(repo, "mktree", "-z", data=b"".join(entries)).decode().strip() - - -def save_archive(repo, output, parent): - tree = write_tree(repo, output) - if parent and git(repo, "rev-parse", f"{parent}^{{tree}}").decode().strip() == tree: - return - args = ["-c", "user.name=github-actions[bot]", "-c", - "user.email=41898282+github-actions[bot]@users.noreply.github.com", - "commit-tree", tree] - if parent: - args.extend(["-p", parent]) - commit = git(repo, *args, data=b"Update versioned public API documentation\n").decode().strip() - # An ordinary fast-forward push detects external writers instead of overwriting them. - git(repo, "push", "origin", f"{commit}:refs/heads/gh-pages") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--logs", type=Path, required=True) - parser.add_argument("--retry-failed", choices=["true", "false"], default="false") - parser.add_argument("--require-ci", action="store_true", help="Require successful CI for each source commit") - parser.add_argument("--publish", action="store_true", help="Push the generated archive to gh-pages") - args = parser.parse_args() - if args.publish and not args.require_ci: - parser.error("--publish requires --require-ci") - scripts = Path(__file__).resolve().parent - repo = scripts.parent.parent - output = args.output.resolve() - output.mkdir(parents=True, exist_ok=False) - args.logs.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha256() - for path in [Path(__file__), repo / "sshlib/build.gradle.kts", repo / "gradle/verification-metadata.xml", - *sorted((scripts / "templates").rglob("*.ftl"))]: - digest.update(path.read_bytes()) - versions = discover(repo) - with tempfile.TemporaryDirectory(prefix="sshlib-docs-archive-") as temporary: - archive = Path(temporary) - parent, previous = load_archive(repo, archive) - manifest, messages = reconcile(repo, scripts, archive, output, args.logs, previous, - versions, digest.hexdigest(), args.retry_failed == "true", - admission=ci_passed if args.require_ci else None) - manifest.write(output / "manifest.json") - (output / ".nojekyll").touch() - ready = generate_site(output, versions, BASE_URL) - failures = sum(record.published is None or - (record.attempt is not None and record.attempt.status is BuildStatus.FAILED) - or record.published != DocumentationConfiguration(versions[name], digest.hexdigest()) - for name, record in manifest.versions.items()) - summary = "## Documentation versions\n\n" + "\n".join(f"- {message}" for message in messages) + "\n" - if not ready: - summary += "\nNo verified main documentation is available; the live site will be left unchanged.\n" - print(summary) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as stream: - stream.write(summary) - if args.publish: - save_archive(repo, output, parent) - if os.environ.get("GITHUB_OUTPUT"): - with open(os.environ["GITHUB_OUTPUT"], "a") as stream: - stream.write(f"ready={str(ready).lower()}\nfailures={failures}\n") - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/tests/test_publish_docs.py b/.github/scripts/tests/test_publish_docs.py deleted file mode 100644 index aad3b37e..00000000 --- a/.github/scripts/tests/test_publish_docs.py +++ /dev/null @@ -1,324 +0,0 @@ -import importlib.util -import json -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest -from unittest.mock import patch -import xml.etree.ElementTree as ET - - -SCRIPTS = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(SCRIPTS)) -spec = importlib.util.spec_from_file_location("publisher", SCRIPTS / "publish-docs.py") -publisher = importlib.util.module_from_spec(spec) -spec.loader.exec_module(publisher) - - -def publication(path, text="public API"): - path.mkdir(parents=True) - (path / "index.html").write_text(text) - (path / "index.md").write_text(text) - - -class PublisherTests(unittest.TestCase): - def setUp(self): - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.root = Path(self.temporary.name) - self.archive = self.root / "archive" - self.output = self.root / "output" - self.logs = self.root / "logs" - for path in (self.archive, self.output, self.logs): - path.mkdir() - self.calls = [] - - def builder(self, repo, scripts, commit, name, output, log): - self.calls.append(name) - publication(output, commit) - - def run_reconcile(self, previous=None, versions=None, builder=None, retry=False, config="config"): - with patch.object(publisher, "git", return_value=b"plugins { alias(libs.plugins.dokka) }"): - return publisher.reconcile( - self.root, SCRIPTS, self.archive, self.output, self.logs, - previous or publisher.DocumentationManifest(), - versions or {"main": "new-main", "0.1.0": "release"}, - config, retry, builder or self.builder) - - def previous(self, name, commit="release", config="config", status="success"): - publication(self.archive / name, "previous content") - published = publisher.DocumentationConfiguration(commit, config) - attempt = publisher.BuildAttempt(published, publisher.BuildStatus(status)) - return publisher.DocumentationManifest({name: publisher.VersionDocumentation(published, attempt)}) - - def test_supported_tags_and_semver_order(self): - tags = ["0.4.2", "v0.3.0", "v0.5.0-rc.1", "main", "test/0.5.0", "0.5.0-01"] - self.assertTrue(all(publisher.release_key(tag) is None for tag in tags)) - supported = ["v0.5.1", "0.10.0", "0.5.1-rc.2", "0.5.1-rc.10", "v0.5.0"] - self.assertEqual(sorted(supported, key=publisher.release_key, reverse=True), - ["0.10.0", "v0.5.1", "0.5.1-rc.10", "0.5.1-rc.2", "v0.5.0"]) - self.assertIsNotNone(publisher.release_key("v0.5.1-rc.1")) - - def test_ci_admission_blocks_new_content_and_preserves_archive(self): - previous = self.previous("main", "old-main") - with patch.object(publisher, "git", return_value=b"plugins { dokka }"): - records, messages = publisher.reconcile( - self.root, SCRIPTS, self.archive, self.output, self.logs, - previous, {"main": "new-main", "v0.1.0": "release"}, - "config", builder=self.builder, admission=lambda commit: False) - self.assertEqual(self.calls, []) - self.assertEqual((self.output / "main/index.html").read_text(), "previous content") - self.assertIsNone(records.versions["v0.1.0"].published) - self.assertTrue(all("pending" in message for message in messages)) - - def ci_jobs(self): - return [{"name": f"Build and test ({jdk})", "conclusion": "success"} for jdk in (17, 21, 25)] + [ - {"name": "Workflow scripts", "conclusion": "success"}] - - def query_ci(self, jobs, commit="source"): - responses = [ - json.dumps({"workflow_runs": [{"id": 42, "head_sha": commit, "conclusion": "failure"}]}).encode(), - json.dumps([{"jobs": jobs}]).encode(), - ] - with patch.object(publisher.subprocess, "check_output", side_effect=responses): - return publisher.ci_passed("source") - - def test_ci_admission_uses_build_steps_despite_maven_failure(self): - jobs = self.ci_jobs() - jobs[0].update(conclusion="failure", steps=[ - {"name": "Build with Gradle", "conclusion": "success"}, - {"name": "Check TLA+ models", "conclusion": "success"}, - {"name": "Verify release tag", "conclusion": "success"}, - {"name": "Upload release to Sonatype", "conclusion": "failure"}, - ]) - self.assertTrue(self.query_ci(jobs)) - - def test_ci_admission_rejects_missing_failed_and_different_commit(self): - for runs in ([], [{"id": 42, "head_sha": "other"}]): - with patch.object(publisher.subprocess, "check_output", - return_value=json.dumps({"workflow_runs": runs}).encode()): - self.assertFalse(publisher.ci_passed("source")) - for conclusion in (None, "failure", "skipped", "cancelled"): - jobs = self.ci_jobs() - jobs[0]["conclusion"] = conclusion - self.assertFalse(self.query_ci(jobs)) - self.assertFalse(self.query_ci(self.ci_jobs()[1:])) - self.assertFalse(self.query_ci(self.ci_jobs()[:-1])) - - def test_failed_model_tag_or_workflow_check_blocks_admission(self): - for name in ("Check TLA+ models", "Verify release tag"): - jobs = self.ci_jobs() - jobs[0].update(conclusion="failure", steps=[ - {"name": "Build with Gradle", "conclusion": "success"}, - {"name": name, "conclusion": "failure"}, - ]) - self.assertFalse(self.query_ci(jobs)) - jobs = self.ci_jobs() - jobs[-1]["conclusion"] = "failure" - self.assertFalse(self.query_ci(jobs)) - - def test_initial_backfill_and_no_unverified_content(self): - publication(self.archive / "0.1.0", "unverified") - records, _ = self.run_reconcile() - self.assertEqual(self.calls, ["main", "0.1.0"]) - self.assertEqual((self.output / "0.1.0/index.html").read_text(), "release") - self.assertEqual(records.versions["0.1.0"].published.policy, publisher.POLICY) - - def test_new_tag_preserves_existing_release(self): - previous = self.previous("0.1.0") - self.run_reconcile(previous, {"main": "main", "0.1.0": "release", "0.2.0": "new"}) - self.assertEqual(self.calls, ["main", "0.2.0"]) - self.assertEqual((self.output / "0.1.0/index.html").read_text(), "previous content") - - def test_failed_tag_not_retried_even_after_source_or_config_change(self): - previous = publisher.DocumentationManifest({ - "0.1.0": publisher.VersionDocumentation( - attempt=publisher.BuildAttempt( - publisher.DocumentationConfiguration("old", "config"), - publisher.BuildStatus.FAILED)), - }) - records, messages = self.run_reconcile(previous, config="changed") - self.assertEqual(self.calls, ["main"]) - self.assertIsNone(records.versions["0.1.0"].published) - self.assertIn("not retried", messages[-1]) - self.assertFalse(messages[-1].split(": ", 1)[-1].startswith("failed")) - - def test_manual_retry(self): - previous = publisher.DocumentationManifest({ - "0.1.0": publisher.VersionDocumentation( - attempt=publisher.BuildAttempt( - publisher.DocumentationConfiguration("old", "config"), - publisher.BuildStatus.FAILED)), - }) - records, _ = self.run_reconcile(previous, retry=True) - self.assertIs(records.versions["0.1.0"].attempt.status, publisher.BuildStatus.SUCCESS) - self.assertIn("0.1.0", self.calls) - - def test_failed_main_keeps_previous_and_retries_next_run(self): - previous = self.previous("main", "old-main") - - def fail(*args): - raise RuntimeError("build failure") - - records, _ = self.run_reconcile(previous, {"main": "new-main"}, fail) - self.assertEqual((self.output / "main/index.html").read_text(), "previous content") - main = records.versions["main"] - self.assertIs(main.attempt.status, publisher.BuildStatus.FAILED) - self.assertTrue(main.should_build( - "main", publisher.DocumentationConfiguration("new-main", "config"), False)) - - def test_configuration_change_rebuilds_successful_release(self): - self.run_reconcile(self.previous("0.1.0"), config="new-config") - self.assertIn("0.1.0", self.calls) - - def test_missing_dokka_is_skipped(self): - with patch.object(publisher, "git", return_value=b"plugins { kotlin() }"): - records, _ = publisher.reconcile( - self.root, SCRIPTS, self.archive, self.output, self.logs, - publisher.DocumentationManifest(), {"0.1.0": "release"}, - "config", builder=self.builder) - self.assertIs(records.versions["0.1.0"].attempt.status, publisher.BuildStatus.SKIPPED) - self.assertEqual(self.calls, []) - - def test_failed_initial_main_has_no_deployable_entrypoint(self): - def fail(*args): - raise RuntimeError("build failure") - - self.run_reconcile(builder=fail) - self.assertFalse((self.output / "main/index.html").exists()) - - def test_absent_library_is_skipped(self): - with patch.object(publisher, "git", return_value=b""): - records, _ = publisher.reconcile( - self.root, SCRIPTS, self.archive, self.output, self.logs, - publisher.DocumentationManifest(), {"0.1.0": "release"}, - "config", builder=self.builder) - self.assertIs(records.versions["0.1.0"].attempt.status, publisher.BuildStatus.SKIPPED) - - def test_old_visibility_policy_is_not_served_after_failure(self): - previous = self.previous("0.1.0") - previous.versions["0.1.0"].published = publisher.DocumentationConfiguration( - "release", "config", policy=0) - - def fail(*args): - raise RuntimeError("build failure") - - records, _ = self.run_reconcile(previous, {"0.1.0": "release"}, fail) - self.assertIsNone(records.versions["0.1.0"].published) - self.assertFalse((self.output / "0.1.0").exists()) - - def test_manifest_without_policy_is_unverified(self): - path = self.archive / "manifest.json" - path.write_text(json.dumps({ - "schema": 1, - "versions": {"0.1.0": {"published": { - "commit": "release", "config": "config", - }}}, - })) - manifest = publisher.DocumentationManifest.read(path) - self.assertEqual(manifest.versions["0.1.0"].published.policy, 0) - - def test_manifest_round_trip_uses_configuration_classes(self): - manifest = self.previous("0.1.0") - path = self.archive / "manifest.json" - manifest.write(path) - self.assertEqual(publisher.DocumentationManifest.read(path), manifest) - - def test_site_without_main_is_not_deployable(self): - publication(self.output / "0.1.0") - self.assertFalse(publisher.generate_site(self.output, ["main", "0.1.0"], publisher.BASE_URL)) - self.assertFalse((self.output / "index.html").exists()) - - def test_indexes_and_sitemaps(self): - for name in ("main", "0.1.0", "v0.2.0", "0.10.0"): - publication(self.output / name) - (self.output / name / "a space.html").write_text("API") - publisher.generate_site(self.output, ["main", "0.10.0", "v0.2.0", "0.1.0"], publisher.BASE_URL) - index = (self.output / "index.html").read_text() - links = [index.index(f'{publisher.BASE_URL}/{name}/index.html') - for name in ("main", "0.10.0", "v0.2.0", "0.1.0")] - self.assertEqual(links, sorted(links)) - self.assertIn("a%20space.html", (self.output / "main/sitemap.xml").read_text()) - self.assertEqual(len(ET.parse(self.output / "sitemap_index.xml").getroot()), 4) - - def test_sitemap_encodes_and_escapes_urls(self): - publication(self.output / "main") - publication(self.output / "0.2.0+build") - (self.output / "main/a&b.html").write_text("API") - (self.output / "main/navigation.html").write_text("navigation") - publisher.generate_site(self.output, ["main", "0.2.0+build", "0.3.0"], publisher.BASE_URL) - self.assertIn("a%26b.html", (self.output / "main/sitemap.xml").read_text()) - self.assertNotIn("navigation.html", (self.output / "main/sitemap.xml").read_text()) - self.assertIn("0.2.0%2Bbuild", (self.output / "index.html").read_text()) - self.assertNotIn("0.3.0", (self.output / "index.html").read_text()) - - def test_archive_round_trip_and_unchanged_tree(self): - remote = self.root / "remote.git" - repo = self.root / "repo" - subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) - subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) - publisher.git(repo, "remote", "add", "origin", str(remote)) - parent, manifest = publisher.load_archive(repo, self.archive) - self.assertIsNone(parent) - self.assertEqual(manifest, publisher.DocumentationManifest()) - publication(self.output / "main") - (self.output / "manifest.json").write_text(json.dumps({"schema": 1, "versions": {"main": {}}})) - publisher.save_archive(repo, self.output, None) - parent, manifest = publisher.load_archive(repo, self.archive) - self.assertIn("main", manifest.versions) - self.assertEqual((self.archive / "main/index.md").read_text(), "public API") - publisher.save_archive(repo, self.output, parent) - self.assertEqual(publisher.git(remote, "rev-parse", "gh-pages").decode().strip(), parent) - self.assertFalse((repo / ".git/index").exists()) - - def test_aggregated_markdown_paths_and_cross_module_links_match_html(self): - html = self.root / "html" - markdown = self.root / "markdown" - for relative in ("index.html", "core/index.html", "core/pkg/type.html", - "ktor/index.html", "ktor/pkg/connect.html"): - page = html / relative - page.parent.mkdir(parents=True, exist_ok=True) - page.write_text("public API") - pages = { - "index.md": "[core](core/index.md)", - "core/index.md": "[type](-core/pkg/type.md)", - "core/-core/pkg/type.md": "[core](../../index.md)", - "ktor/index.md": "[connect](-ktor/pkg/connect.md)", - "ktor/-ktor/pkg/connect.md": "[type](../../../core/-core/pkg/type.md#member)", - } - for relative, content in pages.items(): - page = markdown / relative - page.parent.mkdir(parents=True, exist_ok=True) - page.write_text(content) - result = self.root / "combined" - publisher.assemble_publication(html, markdown, result) - self.assertEqual((result / "core/index.md").read_text(), "[type](pkg/type.md)") - self.assertEqual((result / "core/pkg/type.md").read_text(), "[core](../index.md)") - self.assertEqual((result / "ktor/pkg/connect.md").read_text(), - "[type](../../core/pkg/type.md#member)") - self.assertTrue(all(page.with_suffix(".md").is_file() for page in result.rglob("*.html"))) - - def test_single_module_markdown_keeps_matching_html_paths(self): - html, markdown = self.root / "html", self.root / "markdown" - for root, extension in ((html, ".html"), (markdown, ".md")): - (root / "sshlib/pkg").mkdir(parents=True) - (root / f"index{extension}").write_text("API") - (root / f"sshlib/pkg/index{extension}").write_text("API") - (markdown / "index.md").write_text("[package](sshlib/pkg/index.md)") - publisher.assemble_publication(html, markdown, self.root / "combined") - self.assertEqual((self.root / "combined/index.md").read_text(), - "[package](sshlib/pkg/index.md)") - self.assertTrue((self.root / "combined/sshlib/pkg/index.md").exists()) - - def test_missing_markdown_counterpart_prevents_assembly(self): - html, markdown = self.root / "html", self.root / "markdown" - html.mkdir() - markdown.mkdir() - (html / "index.html").write_text("API") - with self.assertRaisesRegex(RuntimeError, "no Markdown counterpart"): - publisher.assemble_publication(html, markdown, self.root / "combined") - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5871439f..d12915aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,9 +29,6 @@ jobs: install: true install_args: actionlint shellcheck - - name: Test documentation publisher - run: PYTHONPATH=.github/scripts python3 -m unittest discover -s .github/scripts/tests -v - - name: Check workflow syntax run: mise exec -- actionlint diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4ac2bb33..67825e86 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,7 +38,7 @@ jobs: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - - name: Checkout publisher from main + - name: Checkout source from main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main @@ -57,21 +57,15 @@ jobs: validate-wrappers: true dependency-graph: disabled - - name: Validate publisher - run: python3 -m unittest discover -s .github/scripts/tests -v - - name: Reconcile and save documentation archive id: docs + uses: connectbot/actions/docs@main env: GH_TOKEN: ${{ github.token }} - RETRY_FAILED: ${{ inputs.retry_failed || false }} - run: >- - python3 .github/scripts/publish-docs.py - --output "$RUNNER_TEMP/docs-site" - --logs "$RUNNER_TEMP/docs-logs" - --retry-failed "$RETRY_FAILED" - --require-ci - --publish + with: + retry-failed: ${{ inputs.retry_failed || false }} + publish: 'true' + require-ci: 'true' - name: Preserve build logs if: always()