diff --git a/.github/dockerfiles/Dockerfile_extension b/.github/dockerfiles/Dockerfile_extension index c0317e502..4c05975e6 100644 --- a/.github/dockerfiles/Dockerfile_extension +++ b/.github/dockerfiles/Dockerfile_extension @@ -31,9 +31,14 @@ RUN set -eux && \ postgresql-${PG_MAJOR}-pgvector \ postgresql-${PG_MAJOR}-postgis-3 -# Install the DocumentDB extension from a pre-built .deb +# Install the DocumentDB extension via apt (not dpkg -i) so its +# dependencies (e.g. rum) resolve, then verify the installed version. COPY ${DEB_PACKAGE_REL_PATH} /tmp/documentdb.deb -RUN dpkg -i /tmp/documentdb.deb && \ +RUN set -eux && \ + DEB_VERSION=$(dpkg-deb -f /tmp/documentdb.deb Version) && \ + apt-get install -y --no-install-recommends /tmp/documentdb.deb && \ + INSTALLED_VERSION=$(dpkg-query -W -f='${Version}' "postgresql-${PG_MAJOR}-documentdb") && \ + test "$INSTALLED_VERSION" = "$DEB_VERSION" && \ rm -f /tmp/documentdb.deb # Gather system library dependencies not present in the CNPG base image diff --git a/.github/workflows/build_documentdb_images.yml b/.github/workflows/build_documentdb_images.yml index 9c780735e..55ccca090 100644 --- a/.github/workflows/build_documentdb_images.yml +++ b/.github/workflows/build_documentdb_images.yml @@ -1,7 +1,8 @@ name: RELEASE - Build DocumentDB Candidate Images -# Builds documentdb extension and gateway images from public DocumentDB release artifacts. -# - documentdb image: public deb13 PostgreSQL 18 extension package +# Builds documentdb extension and gateway images from published DocumentDB artifacts. +# - documentdb image: Debian 13 (trixie) PostgreSQL 18 extension package from the +# PGDG APT repository (apt.postgresql.org) # - gateway image: public documentdb-local image payload # These images follow the DATABASE version track (documentDbVersion in values.yaml). # For operator/sidecar images, see build_operator_images.yml. @@ -13,10 +14,14 @@ on: description: 'Released DocumentDB version to package (for example 0.113.0)' required: false default: '0.113.0' - documentdb_extension_github_repo: - description: 'GitHub owner/repo for DocumentDB extension releases' + documentdb_apt_base_url: + description: 'Base URL of the APT repository serving the DocumentDB extension package' required: false - default: 'documentdb/documentdb' + default: 'https://apt.postgresql.org/pub/repos/apt' + documentdb_apt_suite: + description: 'APT suite (distribution) to resolve the Debian 13 package from' + required: false + default: 'trixie-pgdg' documentdb_gateway_image_repo: description: 'Container image repo for gateway source (without tag)' required: false @@ -33,7 +38,10 @@ permissions: env: DEFAULT_DOCUMENTDB_VERSION: '0.113.0' - DOCUMENTDB_EXTENSION_GITHUB_REPO: ${{ github.event.inputs.documentdb_extension_github_repo || 'documentdb/documentdb' }} + DOCUMENTDB_PG_MAJOR: '18' + DOCUMENTDB_APT_BASE_URL: ${{ github.event.inputs.documentdb_apt_base_url || 'https://apt.postgresql.org/pub/repos/apt' }} + DOCUMENTDB_APT_SUITE: ${{ github.event.inputs.documentdb_apt_suite || 'trixie-pgdg' }} + DOCUMENTDB_APT_COMPONENT: 'main' DOCUMENTDB_GATEWAY_IMAGE_REPO: ${{ github.event.inputs.documentdb_gateway_image_repo || 'ghcr.io/documentdb/documentdb/documentdb-local' }} @@ -48,6 +56,8 @@ jobs: documentdb_version: ${{ steps.version.outputs.documentdb_version }} documentdb_version_dash: ${{ steps.version.outputs.documentdb_version_dash }} image_tag: ${{ steps.version.outputs.image_tag }} + extension_packages: ${{ steps.extension.outputs.extension_packages }} + extension_deb_version: ${{ steps.extension.outputs.extension_deb_version }} gateway_source_image: ${{ steps.version.outputs.gateway_source_image }} steps: - name: Resolve released DocumentDB version @@ -75,20 +85,88 @@ jobs: echo "gateway_source_image=$GATEWAY_SOURCE_IMAGE" } >> "$GITHUB_OUTPUT" echo "DocumentDB version: $VERSION" - echo "Release tag: v$VERSION_DASH" + echo "Extension package version: $VERSION_DASH" echo "Candidate image tag: $IMAGE_TAG" echo "Gateway source image: $GATEWAY_SOURCE_IMAGE" - - name: Verify public extension release assets + # Pin one package version (and SHA256) for both arches before any image build. + - name: Resolve extension package in APT repository + id: extension env: + PACKAGE: postgresql-${{ env.DOCUMENTDB_PG_MAJOR }}-documentdb VERSION_DASH: ${{ steps.version.outputs.documentdb_version_dash }} + shell: bash run: | set -euo pipefail - for ARCH in amd64 arm64; do - ASSET_URL="https://github.com/${{ env.DOCUMENTDB_EXTENSION_GITHUB_REPO }}/releases/download/v${VERSION_DASH}/deb13-postgresql-18-documentdb_${VERSION_DASH}_${ARCH}.deb" - echo "Checking $ASSET_URL" - curl -fsI -L "$ASSET_URL" >/dev/null - done + python3 - <<'PY' + import gzip + import json + import os + import subprocess + import urllib.request + + base = os.environ["DOCUMENTDB_APT_BASE_URL"].rstrip("/") + suite = os.environ["DOCUMENTDB_APT_SUITE"] + component = os.environ["DOCUMENTDB_APT_COMPONENT"] + package = os.environ["PACKAGE"] + upstream = os.environ["VERSION_DASH"] + + + def is_newer(candidate, current): + return subprocess.call( + ["dpkg", "--compare-versions", candidate, "gt", current] + ) == 0 + + + resolved = {} + for arch in ("amd64", "arm64"): + index_url = f"{base}/dists/{suite}/{component}/binary-{arch}/Packages.gz" + print(f"Reading {index_url}") + with urllib.request.urlopen(index_url, timeout=120) as response: + index = gzip.decompress(response.read()).decode("utf-8", "replace") + + best = None + for stanza in index.split("\n\n"): + fields = {} + for line in stanza.splitlines(): + if line.startswith((" ", "\t")) or ":" not in line: + continue + key, _, value = line.partition(":") + fields[key] = value.strip() + + if fields.get("Package") != package: + continue + + # PGDG appends a packaging revision (0.116-0 -> 0.116-0-1.pgdg13+1). + version = fields.get("Version", "") + if version != upstream and not version.startswith(f"{upstream}-"): + continue + + if best is None or is_newer(version, best["version"]): + best = { + "version": version, + "url": f"{base}/{fields['Filename']}", + "sha256": fields["SHA256"], + } + + if best is None: + raise SystemExit( + f"{package} {upstream} is not published for {arch} in " + f"{suite}/{component} at {base}" + ) + print(f" {arch}: {best['version']} -> {best['url']}") + resolved[arch] = best + + versions = {entry["version"] for entry in resolved.values()} + if len(versions) != 1: + raise SystemExit( + f"architectures resolved to different package versions: {sorted(versions)}" + ) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"extension_packages={json.dumps(resolved, separators=(',', ':'))}\n") + output.write(f"extension_deb_version={versions.pop()}\n") + PY - name: Verify public gateway source image env: @@ -125,14 +203,51 @@ jobs: with: persist-credentials: false - - name: Download public extension package + - name: Download and validate extension package if: matrix.image.name == 'documentdb' + env: + PACKAGE: postgresql-${{ env.DOCUMENTDB_PG_MAJOR }}-documentdb + DEB_URL: ${{ fromJSON(needs.resolve-public-artifacts.outputs.extension_packages)[matrix.arch].url }} + DEB_SHA256: ${{ fromJSON(needs.resolve-public-artifacts.outputs.extension_packages)[matrix.arch].sha256 }} + DEB_VERSION: ${{ needs.resolve-public-artifacts.outputs.extension_deb_version }} + EXPECTED_SCHEMA_VERSION: ${{ needs.resolve-public-artifacts.outputs.documentdb_version_dash }} + EXPECTED_ARCH: ${{ matrix.arch }} + shell: bash run: | set -euo pipefail mkdir -p packages - DEB_FILE="deb13-postgresql-18-documentdb_${{ needs.resolve-public-artifacts.outputs.documentdb_version_dash }}_${{ matrix.arch }}.deb" - ASSET_URL="https://github.com/${{ env.DOCUMENTDB_EXTENSION_GITHUB_REPO }}/releases/download/v${{ needs.resolve-public-artifacts.outputs.documentdb_version_dash }}/${DEB_FILE}" - curl -fsSL -o "packages/${DEB_FILE}" -L "$ASSET_URL" + DEB_FILE="packages/${PACKAGE}_${DEB_VERSION}_${EXPECTED_ARCH}.deb" + + echo "Downloading $DEB_URL" + curl -fsSL -o "$DEB_FILE" "$DEB_URL" + echo "${DEB_SHA256} ${DEB_FILE}" | sha256sum -c - + + PACKAGE_NAME=$(dpkg-deb -f "$DEB_FILE" Package) + PACKAGE_VERSION=$(dpkg-deb -f "$DEB_FILE" Version) + PACKAGE_ARCH=$(dpkg-deb -f "$DEB_FILE" Architecture) + [[ "$PACKAGE_NAME" == "$PACKAGE" ]] || { + echo "Unexpected package name: $PACKAGE_NAME" >&2 + exit 1 + } + [[ "$PACKAGE_VERSION" == "$DEB_VERSION" ]] || { + echo "Unexpected package version: $PACKAGE_VERSION" >&2 + exit 1 + } + [[ "$PACKAGE_ARCH" == "$EXPECTED_ARCH" ]] || { + echo "Unexpected package architecture: $PACKAGE_ARCH" >&2 + exit 1 + } + + # Assert the extension schema version (Debian version may carry a packaging revision). + CONTROL_PATH="./usr/share/postgresql/${DOCUMENTDB_PG_MAJOR}/extension/documentdb.control" + SCHEMA_VERSION=$(dpkg-deb --fsys-tarfile "$DEB_FILE" | tar -xO "$CONTROL_PATH" | + sed -nE "s/^default_version[[:space:]]*=[[:space:]]*'([^']+)'.*/\1/p") + [[ "$SCHEMA_VERSION" == "$EXPECTED_SCHEMA_VERSION" ]] || { + echo "Package declares extension version '$SCHEMA_VERSION', expected '$EXPECTED_SCHEMA_VERSION'" >&2 + exit 1 + } + + echo "DEB_PACKAGE_REL_PATH=$DEB_FILE" >> "$GITHUB_ENV" ls -lh packages/ - name: Login to GHCR @@ -148,9 +263,8 @@ jobs: case "${{ matrix.image.name }}" in documentdb) - DEB_FILE="deb13-postgresql-18-documentdb_${{ needs.resolve-public-artifacts.outputs.documentdb_version_dash }}_${{ matrix.arch }}.deb" - echo "Using deb: $DEB_FILE" - BUILD_ARGS="--build-arg PG_MAJOR=18 --build-arg DEB_PACKAGE_REL_PATH=packages/$DEB_FILE" + echo "Using deb: $DEB_PACKAGE_REL_PATH" + BUILD_ARGS="--build-arg PG_MAJOR=${{ env.DOCUMENTDB_PG_MAJOR }} --build-arg DEB_PACKAGE_REL_PATH=$DEB_PACKAGE_REL_PATH" ;; gateway) echo "Using public gateway source image: ${{ needs.resolve-public-artifacts.outputs.gateway_source_image }}" @@ -221,7 +335,8 @@ jobs: echo "" echo "- **DocumentDB Version**: \`${{ needs.resolve-public-artifacts.outputs.documentdb_version }}\`" echo "- **Candidate Image Tag**: \`${{ needs.resolve-public-artifacts.outputs.image_tag }}\`" - echo "- **Extension Package Source**: \`https://github.com/${{ env.DOCUMENTDB_EXTENSION_GITHUB_REPO }}/releases/download/v${{ needs.resolve-public-artifacts.outputs.documentdb_version_dash }}/deb13-postgresql-18-documentdb_${{ needs.resolve-public-artifacts.outputs.documentdb_version_dash }}_{amd64,arm64}.deb\`" + echo "- **Extension Package**: \`postgresql-${{ env.DOCUMENTDB_PG_MAJOR }}-documentdb ${{ needs.resolve-public-artifacts.outputs.extension_deb_version }}\` (amd64, arm64)" + echo "- **Extension Package Source**: \`${{ env.DOCUMENTDB_APT_BASE_URL }} ${{ env.DOCUMENTDB_APT_SUITE }}/${{ env.DOCUMENTDB_APT_COMPONENT }}\`" echo "- **Gateway Source Image**: \`${{ needs.resolve-public-artifacts.outputs.gateway_source_image }}\`" echo "- **Images**: documentdb, gateway" echo "" diff --git a/AGENTS.md b/AGENTS.md index 4ba89e9f5..c71710f58 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ The project uses **two independent version tracks** for container images: | **Database** | documentdb (extension), gateway | `values.yaml` → `documentDbVersion` | `0.113.0` | - Operator images are built from this repo's Go source -- Database images are built from public `documentdb/documentdb` release artifacts (extension `.deb` + gateway payload from `documentdb-local`) +- Database images are built from the PGDG `postgresql-18-documentdb` package (Debian 13) and the public `documentdb-local` gateway image - Each track has its own build and release workflows - Database image defaults are also hardcoded in `constants.go` and `config.go` as fallbacks diff --git a/CHANGELOG.md b/CHANGELOG.md index 424044906..695e6c1a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Major Features - **Fail-fast ImageVolume capability check**: The operator now depends on the Kubernetes [ImageVolume](https://kubernetes.io/docs/concepts/storage/volumes/#image) feature to mount the DocumentDB extension into PostgreSQL pods. Instead of gating on a Kubernetes version number, the validating webhook performs a capability probe (a server-side dry-run) when a `DocumentDB` is created and **rejects the resource with an actionable error if ImageVolume is unavailable**, so you find out immediately instead of waiting for pods that never become ready. ImageVolume is GA (on by default) in Kubernetes **1.35+**; on **1.33/1.34** it is beta and must be enabled via the `ImageVolume` feature gate on a containerd/CRI-O runtime. The Helm chart's `kubeVersion` floor is relaxed to `>= 1.33.0-0` accordingly. See [Before you start](docs/operator-public-documentation/preview/getting-started/before-you-start.md). +- **DocumentDB extension packages from PGDG**: The database image build now resolves `postgresql-18-documentdb` from the [PGDG APT repository](https://apt.postgresql.org/) (`trixie-pgdg`), pinning one version and SHA256 across both architectures and validating `default_version` before building. Replaces the upstream Debian 13 release assets that stopped being published after `0.115`. ## [0.3.0] - 2026-07-15 diff --git a/docs/designs/image-management.md b/docs/designs/image-management.md index 492303ca0..c4ad63a5f 100644 --- a/docs/designs/image-management.md +++ b/docs/designs/image-management.md @@ -24,7 +24,7 @@ This document describes how the DocumentDB Kubernetes Operator manages, builds, The project manages **5 container images** across two independent version tracks: - **Operator track**: images built from Go source code in this repository -- **Database track**: images built from `.deb` packages produced by the upstream [`documentdb/documentdb`](https://github.com/documentdb/documentdb) repository +- **Database track**: extension `.deb` from [PGDG](https://apt.postgresql.org/) (`trixie-pgdg`), gateway payload from the upstream [`documentdb/documentdb`](https://github.com/documentdb/documentdb) `documentdb-local` image All images are published to **GitHub Container Registry (GHCR)** under `ghcr.io/documentdb/documentdb-kubernetes-operator/`. A sixth image (PostgreSQL) comes from the CloudNative-PG project and is consumed as-is. @@ -44,7 +44,7 @@ All images are published to **GitHub Container Registry (GHCR)** under `ghcr.io/ | Image | GHCR Path | Source | Dockerfile | Purpose | |-------|-----------|--------|------------|---------| -| **documentdb** | `.../documentdb` | Public `deb13` PostgreSQL 18 package from `documentdb/documentdb` releases | `.github/dockerfiles/Dockerfile_extension` | DocumentDB PostgreSQL extension files for CNPG ImageVolume mode | +| **documentdb** | `.../documentdb` | `postgresql-18-documentdb` from the PGDG APT repository (`trixie-pgdg`) | `.github/dockerfiles/Dockerfile_extension` | DocumentDB PostgreSQL extension files for CNPG ImageVolume mode | | **gateway** | `.../gateway` | Public gateway payload copied from `ghcr.io/documentdb/documentdb/documentdb-local:pg17-` | `.github/dockerfiles/Dockerfile_gateway_public_image` | MongoDB wire-protocol gateway binary (Rust) | ### External Image (Not Built Here) @@ -199,7 +199,7 @@ Builds operator and sidecar images from this repo's Go source. ### Database Image Build (`build_documentdb_images.yml`) -Builds documentdb extension and gateway images from public DocumentDB release artifacts. +Builds documentdb extension and gateway images from published DocumentDB artifacts. | Aspect | Details | |--------|---------| @@ -207,17 +207,23 @@ Builds documentdb extension and gateway images from public DocumentDB release ar | **Images** | documentdb, gateway | | **Dockerfiles** | `.github/dockerfiles/Dockerfile_extension`, `.github/dockerfiles/Dockerfile_gateway_public_image` | | **Tag pattern** | `{documentdb_version}-build-{run_id}-{attempt}-{sha}` (candidate) | -| **Build time** | ~5 minutes (public artifact download + image build) | +| **Build time** | ~5 minutes (package download + image build) | | **Multi-arch** | amd64 + arm64 → multi-arch manifest | | **Signing** | cosign keyless (OIDC) | | **Version detection** | Workflow input / repository dispatch payload (defaults to released `0.113.0`) | The build process: -1. Resolves the released DocumentDB version to package -2. Downloads the public `deb13` PostgreSQL 18 extension package from `documentdb/documentdb` release assets -3. Verifies the public multi-arch `documentdb-local:pg17-` image exists -4. Builds `Dockerfile_extension` using the public extension `.deb` (installs pg_cron, pgvector, postgis alongside) -5. Builds `Dockerfile_gateway_public_image` by copying the gateway binary and runtime files from the public upstream image +1. Resolves the released DocumentDB version +2. Pins `postgresql-18-documentdb` from the PGDG `trixie-pgdg` APT index (one version + SHA256 for both arches) +3. Verifies the public `documentdb-local:pg17-` image exists +4. Downloads and validates each `.deb` (checksum, name, version, arch, `default_version`) +5. Builds `Dockerfile_extension` (installs pg_cron, pgvector, postgis alongside) +6. Builds `Dockerfile_gateway_public_image` from the upstream gateway payload + +> **Why PGDG?** Upstream stopped publishing Debian 13 `.deb` assets after `0.115`. PGDG publishes +> the same package for `trixie` on both architectures. PGDG versions carry a packaging revision +> (`0.116-0` → `0.116-0-1.pgdg13+1`), so the workflow resolves the index rather than constructing +> a URL, and asserts `default_version` rather than relying on the Debian version string. ### Dockerfile Details diff --git a/docs/developer-guides/testing-with-fork-images.md b/docs/developer-guides/testing-with-fork-images.md index f88071599..62445e5fa 100644 --- a/docs/developer-guides/testing-with-fork-images.md +++ b/docs/developer-guides/testing-with-fork-images.md @@ -7,7 +7,7 @@ It covers two independent image tracks — pick whichever you actually changed: | Track | What it ships | Repo to fork & build from | Workflow to run | |---|---|---|---| | **Operator track** | `operator`, `sidecar` | This repo (`documentdb/documentdb-kubernetes-operator`) | [`RELEASE - Build Operator Candidate Images`](../../.github/workflows/build_operator_images.yml) | -| **Database track** | `documentdb` (extension), `gateway` | Upstream [`documentdb/documentdb`](https://github.com/documentdb/documentdb) **then** this repo | DocumentDB release pipeline → [`RELEASE - Build DocumentDB Candidate Images`](../../.github/workflows/build_documentdb_images.yml) | +| **Database track** | `documentdb` (extension), `gateway` | Published packages/images (PGDG APT + upstream [`documentdb/documentdb`](https://github.com/documentdb/documentdb)) | [`RELEASE - Build DocumentDB Candidate Images`](../../.github/workflows/build_documentdb_images.yml) | If your change is purely Go controller code, skip Step 1 entirely and use the upstream `0.110.0` (or any released) database images. @@ -24,24 +24,21 @@ If your change is purely Go controller code, skip Step 1 entirely and use the up --- -## Step 1 — (Database track only) Build extension + gateway from a documentdb fork +## Step 1 — (Database track only) Build the extension + gateway images Skip this step if you don't need to change the DocumentDB extension or gateway. -1. **Fork** [`documentdb/documentdb`](https://github.com/documentdb/documentdb) and push your changes. -2. **Run the DocumentDB release pipeline** on your fork (typically `Release` workflow). This must publish: - - A GitHub release named `v.-` (note the dash before patch — for example `v0.110-0`). - - Per-arch `.deb` assets attached to that release: `deb13-postgresql-18-documentdb_.-_amd64.deb` and `_arm64.deb`. - - A `documentdb-local` GHCR image: `ghcr.io//documentdb/documentdb-local:pg17-..`. +The workflow resolves `postgresql-18-documentdb` from [PGDG](https://apt.postgresql.org/) (`trixie-pgdg`) — no extension build needed. The gateway comes from an upstream `documentdb-local` image. - The operator-side workflow probes for these exact paths in [its verify steps](../../.github/workflows/build_documentdb_images.yml) (`Verify public extension release assets` and `Verify public gateway source image`). +1. **In your operator fork**, run **Actions → `RELEASE - Build DocumentDB Candidate Images` → Run workflow**: + - `version`: the released DocumentDB version (e.g. `0.116.0`) + - `documentdb_gateway_image_repo`: override if using a forked gateway image -3. **In your operator fork**, run **Actions → `RELEASE - Build DocumentDB Candidate Images` → Run workflow**. Provide these inputs: - - `version`: `0.110.0` (or whatever you released in step 2) - - `documentdb_extension_github_repo`: `/documentdb` - - `documentdb_gateway_image_repo`: `ghcr.io//documentdb/documentdb-local` + The resolve step fails fast if the version is not published for both architectures. -4. After the run finishes, your fork has the candidate tag (and per-arch variants): +2. **For unreleased extension changes**, override `documentdb_apt_base_url` / `documentdb_apt_suite` to point at your own APT repository. The package must be named `postgresql-18-documentdb` with a matching `default_version`. For a custom gateway, publish a `documentdb-local` GHCR image from your fork and override `documentdb_gateway_image_repo`. + +3. After the run finishes, your fork has the candidate tag (and per-arch variants): ```text ghcr.io//documentdb-kubernetes-operator/documentdb:-build---