From 2b6fe64098cecfe558e8515528329856e1446942 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 11 Sep 2026 17:31:20 -0400 Subject: [PATCH 1/2] Migrate local and CI object storage to pinned PGSTY Silo --- .github/workflows/ci.yml | 27 ++-- README.md | 10 +- docker-compose.yaml | 12 +- docs/silo-local-storage.md | 131 ++++++++++++++++++ justfile | 13 +- k8s/README.md | 10 +- k8s/local-config-store.compose.yaml | 4 +- scripts/seed_ducklake.sh | 22 +-- scripts/test_silo.sh | 73 ++++++++++ tests/integration/README.md | 8 +- tests/integration/composefile/compose_test.go | 44 +++++- .../credential_rotation_pin_test.go | 8 +- tests/integration/docker-compose.yml | 20 +-- tests/integration/trino/ducklake.properties | 2 +- tests/trino-ducklake-smoke/README.md | 9 +- tests/trino-ducklake-smoke/smoke_test.go | 4 +- 16 files changed, 329 insertions(+), 68 deletions(-) create mode 100644 docs/silo-local-storage.md create mode 100644 scripts/test_silo.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b86770de3..0a50b4a51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,27 +173,16 @@ jobs: - name: Build run: go build -o duckgres . - - name: Start MinIO + - name: Start PGSTY Silo run: | - docker run -d --name minio \ - -p 39000:9000 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - quay.io/minio/minio:latest server /data - # Wait for MinIO to be ready - for i in {1..30}; do - curl -sf http://localhost:39000/minio/health/live && break - sleep 1 - done - - - name: Create MinIO bucket + docker compose -f tests/integration/docker-compose.yml up -d --wait minio + + - name: Create Silo bucket run: | - # Install mc client - curl -sL https://dl.min.io/client/mc/release/linux-arm64/mc -o /tmp/mc - chmod +x /tmp/mc - # Configure and create bucket - /tmp/mc alias set minio http://localhost:39000 minioadmin minioadmin - /tmp/mc mb minio/ducklake --ignore-existing + docker compose -f tests/integration/docker-compose.yml run --rm --no-deps minio-init + + - name: Verify Silo storage and credential revocation + run: just test-silo - name: Run integration tests run: just test-integration diff --git a/README.md b/README.md index a9ad2b2b7..8e82c83dd 100644 --- a/README.md +++ b/README.md @@ -725,7 +725,7 @@ larger steady-state metadata connection footprint. The easiest way to get started with DuckLake is using the included Docker Compose setup: ```bash -# Start PostgreSQL (metadata) and MinIO (object storage) +# Start PostgreSQL (metadata) and PGSTY Silo (object storage) docker compose up -d # Wait for services to be ready @@ -746,7 +746,7 @@ The `docker-compose.yaml` creates: - Database: `ducklake` - User/Password: `ducklake` / `ducklake` -**MinIO** (S3-compatible object storage): +**PGSTY Silo** (S3-compatible object storage): - S3 API: `localhost:9000` - Web Console: `http://localhost:9001` - Access Key: `minioadmin` @@ -755,6 +755,12 @@ The `docker-compose.yaml` creates: The included `duckgres.yaml` is pre-configured to use both services. +Local and CI object storage use a pinned PGSTY Silo release with its bundled +`mcli` admin client. The Compose service names `minio` and `minio-init`, +`MINIO_*` settings, and existing endpoints are retained for compatibility. +See the [Silo local storage runbook](docs/silo-local-storage.md) for exact +image pins, migration, validation, and rollback instructions. + ### Object Storage Configuration DuckLake can store data files in S3-compatible object storage (AWS S3, MinIO, etc.). Two credential providers are supported: diff --git a/docker-compose.yaml b/docker-compose.yaml index f7a3e4408..df8d1d476 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,7 +17,7 @@ services: retries: 5 minio: - image: minio/minio:latest + image: docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 container_name: ducklake-storage command: server /data --console-address ":9001" environment: @@ -29,23 +29,23 @@ services: volumes: - minio-data:/data healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "mcli", "ready", "local"] interval: 5s timeout: 5s retries: 5 # Creates the ducklake bucket on startup minio-init: - image: minio/mc:latest + image: docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 container_name: ducklake-storage-init depends_on: minio: condition: service_healthy entrypoint: > /bin/sh -c " - mc alias set minio http://minio:9000 minioadmin minioadmin; - mc mb minio/ducklake --ignore-existing; - mc anonymous set download minio/ducklake; + mcli alias set minio http://minio:9000 minioadmin minioadmin; + mcli mb minio/ducklake --ignore-existing; + mcli anonymous set download minio/ducklake; echo 'Bucket ducklake created successfully'; " diff --git a/docs/silo-local-storage.md b/docs/silo-local-storage.md new file mode 100644 index 000000000..b0baad832 --- /dev/null +++ b/docs/silo-local-storage.md @@ -0,0 +1,131 @@ +# Local and CI object storage with PGSTY Silo + +Duckgres uses [PGSTY Silo](https://github.com/pgsty/silo), a maintained MinIO +fork, for local development and test object storage. Duckgres continues to use +the generic S3 API. This migration does not change production storage or the +real-S3 performance workflows. + +## Release pins + +Server and initialization containers use the same immutable multi-platform image: + +```text +docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 +``` + +The [server release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z) +bundles `mcli` version `RELEASE.2026-09-03T07-13-05Z`. Using the bundled client +also pins the admin executable by the image digest; no runtime download of an +unversioned client is needed. + +| Platform | Image manifest digest | +| --- | --- | +| `linux/amd64` | `sha256:885275e0f42acfdf80304c577d2e46c2c3978a276619d60b9eedd7486f104b30` | +| `linux/arm64` | `sha256:c35123a06f2147372523ffc5ce42a0efc4a239405481cbc2a3534bd883257b06` | + +Release provenance was checked with `gh attestation verify` against `pgsty/silo` +and `pgsty/mc` for their respective release checksum manifests. Downloaded Linux +archives matched those manifests, and the extracted server and client binary +hashes matched `/usr/bin/silo` and `/usr/bin/mcli` in the container on both +architectures. The [client release](https://github.com/pgsty/mc/releases/tag/RELEASE.2026-09-03T07-13-05Z) +and server release archives have these SHA256 hashes: + +| Archive | SHA256 | +| --- | --- | +| Silo Linux amd64 | `cbe5c01eac0a97ccb22fa252eafa432e8608bfde7e3ea27a324cb5ed625a1e96` | +| Silo Linux arm64 | `311846ca9387de36f8e34daa8bf1a130684cc7b8c61aa39581264249eb8df0cf` | +| mcli Linux amd64 | `cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8` | +| mcli Linux arm64 | `7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8` | + +The image shell, entrypoint, and executable versions were verified. The isolated +S3/admin contract passed on both architectures, with amd64 run under emulation. + +The server executable is `silo`; initialization and administrative operations +use `mcli` instead of `mc`. Silo preserves `MINIO_*` environment variables, +`/minio/*` routes, and the on-disk format according to upstream. The Compose +service names `minio` and `minio-init`, container names, volume names, and S3 +endpoints are deliberately retained so existing local configurations and +container-to-container connections continue to work. + +## Defaults and local development + +The root Compose stack still exposes S3 at `localhost:9000` and its console at +`http://localhost:9001`, with local credentials `minioadmin` / `minioadmin`. +It initializes the `ducklake` bucket with the existing anonymous-download +policy. Integration and local Kubernetes fixtures retain their existing port +mapping, buckets, credentials, reader policy, and metadata configuration; refer +to their Compose files for the separate fixture defaults. + +For a new root Compose environment: + +```bash +docker compose up -d +docker compose ps -a +docker compose logs minio minio-init +just build +./duckgres --config duckgres.yaml +# In a separate terminal once Duckgres is ready: +just seed-ducklake +``` + +Before running the integration storage contract, explicitly reconcile the +server with the pinned Compose image, wait for health, and initialize it: + +```bash +docker compose -f tests/integration/docker-compose.yml up -d --wait minio +docker compose -f tests/integration/docker-compose.yml run --rm --no-deps minio-init +just test-silo +just test-integration +``` + +`just test-silo` checks the initialized fixture's S3 and admin behavior. CI uses +the same Compose server and initializer before running this recipe. The +integration harness checks TCP reachability when reusing an existing fixture; +`just test-integration` alone does not upgrade an already-running MinIO server. +The contract script defaults `SILO_TEST_CONTAINER` to `duckgres-test-minio`. +For an isolated fixture, override that container name. Its credential defaults +are `SILO_TEST_ROOT_USER=minioadmin`, `SILO_TEST_ROOT_PASSWORD=minioadmin`, +`SILO_TEST_READER_USER=trino-reader`, and `SILO_TEST_READER_PASSWORD=trino-reader`. +Use `just trino-ducklake-smoke`, `just perf-trino-ducklake`, and +`just perf-trino-ducklake-realistic` for the opt-in Trino paths. +Kind and OrbStack setup remain `just run-multitenant-kind` and +`just run-multitenant-local`, respectively. + +## Migrate an existing local environment + +1. Record the current checkout and the actual image digests of existing server + and client containers before replacing them. Older configuration used mutable + tags, so a tag alone is not a reproducible rollback target. +2. Stop Duckgres, Trino, and other writers, then stop the affected Compose stack. + Back up its existing object-storage data and metadata together while stopped. + Preserve the entire object-store volume, including hidden server metadata, + and the PostgreSQL data or a verified database backup. Do not use `down -v`. + Local Kubernetes dependencies use tmpfs for object and metadata storage; + export any fixture data you need before stopping those containers. +3. Switch to the migration checkout and pull the pinned image. Start only the + intended local stack with its existing Compose project name and volumes. + For root Compose, run `docker compose up -d`; for test and Kubernetes + fixtures, use the explicit integration setup commands or Kubernetes recipes + above. +4. Check that the object store becomes healthy and initialization exits with + status zero. Verify bucket access and run the affected test recipes before + resuming local work. Credential rotation tests must still reject disabled + credentials and accept the replacement credentials; S3 compatibility alone + does not establish admin API compatibility. + +## Failure recovery and rollback + +Inspect `docker compose ps -a` and `docker compose logs minio minio-init` for +the root stack; add `-f tests/integration/docker-compose.yml` for the integration +stack. A failing healthcheck or nonzero initialization exit should be fixed +before starting clients. Confirm the pinned image was pulled for the intended +architecture, the configured credentials agree, and the configured ports are +available. Keep `MINIO_*` settings and `/minio/*` routes unchanged. + +To roll back, stop writers and the affected stack, preserve the failed state +for diagnosis, restore the pre-migration object-store and metadata backups as +a consistent pair, and restore the previous checkout and recorded image +digests. Then start the previous stack and verify access before resuming work. +Do not assume that starting an older MinIO image against data already written +by Silo is a safe rollback. For disposable test fixtures, explicitly discard +and reseed only that fixture's data instead of restoring a backup. diff --git a/justfile b/justfile index de1baf840..25cbb32d6 100644 --- a/justfile +++ b/justfile @@ -152,15 +152,15 @@ check-multitenant-kind-ports: [group('dev')] multitenant-config-store-up: check-multitenant-local-ports docker compose -f k8s/local-config-store.compose.yaml -f k8s/orbstack/dependency-ports.overlay.yaml up -d --wait - docker exec duckgres-local-minio mc alias set local http://127.0.0.1:9000 minioadmin minioadmin - docker exec duckgres-local-minio mc mb local/duckgres-local --ignore-existing + docker exec duckgres-local-minio mcli alias set local http://127.0.0.1:9000 minioadmin minioadmin + docker exec duckgres-local-minio mcli mb local/duckgres-local --ignore-existing # Start the local PostgreSQL config store used by the kind-backed multi-tenant K8s flow [group('dev')] multitenant-config-store-up-kind: check-multitenant-kind-ports docker compose -f k8s/local-config-store.compose.yaml -f k8s/kind/config-store.overlay.yaml up -d --wait - docker exec duckgres-local-minio mc alias set local http://127.0.0.1:9000 minioadmin minioadmin - docker exec duckgres-local-minio mc mb local/duckgres-local --ignore-existing + docker exec duckgres-local-minio mcli alias set local http://127.0.0.1:9000 minioadmin minioadmin + docker exec duckgres-local-minio mcli mb local/duckgres-local --ignore-existing # Stop the local PostgreSQL config store used by the multi-tenant K8s flow [group('dev')] @@ -317,6 +317,11 @@ test-cache-proxy: test-integration: go test -v ./tests/integration/... +# Verify Silo S3 permissions and admin credential revocation after starting the integration stack. +[group('test')] +test-silo: + bash scripts/test_silo.sh + # Run shared/process control plane tests [group('test')] test-controlplane: diff --git a/k8s/README.md b/k8s/README.md index c51d2f691..6086265a4 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -53,7 +53,7 @@ The control plane handles TLS, authentication, PostgreSQL wire protocol, and SQL | `kind/config-store.overlay.yaml` | Compose overlay that attaches local dependency containers to the external Docker `kind` network | | `kind/config-store.seed.sql` | Kind-oriented managed-warehouse seed for the shared-worker flow | | `kind/control-plane.yaml` | Kind-first shared-worker control-plane manifest used by local dev and CI | -| `orbstack/dependency-ports.overlay.yaml` | Optional OrbStack overlay that publishes local DuckLake and MinIO dependency ports on the host | +| `orbstack/dependency-ports.overlay.yaml` | Optional OrbStack overlay that publishes local DuckLake and PGSTY Silo dependency ports on the host | ## Configuration @@ -104,7 +104,13 @@ PGPASSWORD=postgres psql "host=127.0.0.1 port=5432 user=postgres dbname=duckgres `just multitenant-port-forward-pg` forwards pgwire on `5432`. -`just run-multitenant-kind` recreates a local kind cluster, starts the config store plus the local warehouse DB, DuckLake metadata DB, and MinIO backing the seeded managed-warehouse contract, attaches those dependency containers to the Docker `kind` network, loads the locally built image into kind, and deploys the shared-worker control plane. +`just run-multitenant-kind` recreates a local kind cluster, starts the config store plus the local warehouse DB, DuckLake metadata DB, and PGSTY Silo backing the seeded managed-warehouse contract, attaches those dependency containers to the Docker `kind` network, loads the locally built image into kind, and deploys the shared-worker control plane. + +The local object store retains its `minio` service name, `MINIO_*` settings, +and existing dependency endpoints. Both kind and OrbStack use the pinned Silo +image and its bundled `mcli` admin client. See the +[Silo local storage runbook](../docs/silo-local-storage.md) before migrating +existing data or recovering a failed local setup. Default login: `postgres / postgres` diff --git a/k8s/local-config-store.compose.yaml b/k8s/local-config-store.compose.yaml index 70f6c7381..9d1ecd173 100644 --- a/k8s/local-config-store.compose.yaml +++ b/k8s/local-config-store.compose.yaml @@ -47,14 +47,14 @@ services: - /var/lib/postgresql/data minio: - image: minio/minio:latest + image: docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 container_name: duckgres-local-minio command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "mcli", "ready", "local"] interval: 5s timeout: 5s retries: 10 diff --git a/scripts/seed_ducklake.sh b/scripts/seed_ducklake.sh index 2090cbf7b..c541b8155 100755 --- a/scripts/seed_ducklake.sh +++ b/scripts/seed_ducklake.sh @@ -1,12 +1,12 @@ #!/bin/bash -# Seed script for DuckLake catalog with sample data stored in MinIO +# Seed script for DuckLake catalog with sample data stored in PGSTY Silo # # This script connects to a running Duckgres server with DuckLake configured -# (PostgreSQL metadata + MinIO object storage) and creates tables with sample -# data. DuckLake automatically stores the data as Parquet files in MinIO. +# (PostgreSQL metadata + Silo object storage) and creates tables with sample +# data. DuckLake automatically stores the data as Parquet files in Silo. # # Prerequisites: -# - docker compose up -d (PostgreSQL + MinIO running) +# - docker compose up -d (PostgreSQL + Silo running) # - ./duckgres --config duckgres.yaml (Duckgres server running) # # Usage: @@ -54,7 +54,7 @@ while [[ $# -gt 0 ]]; do -h|--help) echo "Usage: $0 [options]" echo "" - echo "Seeds the DuckLake catalog with sample data stored as Parquet in MinIO." + echo "Seeds the DuckLake catalog with sample data stored as Parquet in Silo." echo "" echo "Options:" echo " --host HOST Server host (default: 127.0.0.1)" @@ -79,7 +79,7 @@ done CONN="host=$HOST port=$PORT user=$USER sslmode=require" echo "=== DuckLake Catalog Seed Script ===" -echo "Seeding DuckLake with Parquet data in MinIO" +echo "Seeding DuckLake with Parquet data in Silo" echo "" echo "Connecting to $HOST:$PORT as $USER..." @@ -113,7 +113,7 @@ if ! run_sql "SHOW ALL TABLES" 2>&1 | grep -q "ducklake"; then echo "" echo "Make sure duckgres.yaml has DuckLake configured with:" echo " - metadata_store pointing to PostgreSQL" - echo " - object_store pointing to MinIO (s3://ducklake/data/)" + echo " - object_store pointing to Silo (s3://ducklake/data/)" echo " - S3 credentials configured" exit 1 fi @@ -135,7 +135,7 @@ if [ "$CLEAN" = true ]; then fi echo "=== Creating Tables in DuckLake Catalog ===" -echo "(Data will be stored as Parquet files in MinIO)" +echo "(Data will be stored as Parquet files in Silo)" echo "" # Create categories table @@ -240,7 +240,7 @@ EOF echo "" echo "=== Inserting Data into DuckLake ===" -echo "(This creates Parquet files in MinIO)" +echo "(This creates Parquet files in Silo)" echo "" # Seed categories @@ -410,9 +410,9 @@ echo "=== Seed Complete! ===" echo "" echo "Data is now stored in DuckLake with:" echo " - Metadata in PostgreSQL (localhost:5433)" -echo " - Parquet files in MinIO (localhost:9000, bucket: ducklake)" +echo " - Parquet files in Silo (localhost:9000, bucket: ducklake)" echo "" -echo "View MinIO console at: http://localhost:9001" +echo "View Silo console at: http://localhost:9001" echo " Username: minioadmin" echo " Password: minioadmin" echo "" diff --git a/scripts/test_silo.sh b/scripts/test_silo.sh new file mode 100644 index 000000000..a9ea00602 --- /dev/null +++ b/scripts/test_silo.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Exercise the shipped client and the integration stack's S3/admin contract. +# Setup: docker compose -f tests/integration/docker-compose.yml up -d --wait minio +# Then: docker compose -f tests/integration/docker-compose.yml run --rm --no-deps minio-init +# Run against initialized integration infrastructure; SILO_TEST_CONTAINER selects +# an isolated stack instead of the default duckgres-test-minio container. +set -euo pipefail + +container=${SILO_TEST_CONTAINER:-duckgres-test-minio} +root_user=${SILO_TEST_ROOT_USER:-minioadmin} +root_password=${SILO_TEST_ROOT_PASSWORD:-minioadmin} +reader_user=${SILO_TEST_READER_USER:-trino-reader} +reader_password=${SILO_TEST_READER_PASSWORD:-trino-reader} +fixture="silo-contract-$(date +%s)-$$" +config="/tmp/$fixture" +object="ducklake/data/$fixture.txt" +outside_object="ducklake/$fixture.txt" +password="$fixture-secret" +payload="Silo storage contract $fixture" + +# This intentionally fails on an image without the maintained bundled client. +docker exec "$container" mcli --version +docker exec "$container" mkdir -p "$config" +mcli() { + docker exec -i "$container" mcli --config-dir "$config" "$@" +} +cleanup() { + mcli rm "root/$object" "root/$outside_object" >/dev/null 2>&1 || true + mcli admin user remove root "$fixture" >/dev/null 2>&1 || true + docker exec "$container" rm -rf "$config" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +expect_denied() { + local output + if output=$("$@" 2>&1); then + printf 'FAIL: unauthorized operation succeeded: %s\n' "$*" >&2 + exit 1 + fi + # A transport failure or missing executable is not evidence of enforcement. + if ! printf '%s\n' "$output" | grep -Eiq 'Access Denied|AccessDenied|Insufficient permissions|InvalidAccessKeyId|access key.*(does not exist|disabled)|account.*disabled'; then + printf 'FAIL: expected an authorization error, got: %s\n' "$output" >&2 + exit 1 + fi +} + +mcli alias set root http://127.0.0.1:9000 "$root_user" "$root_password" +mcli ready root +# Require the initialized bucket; never make a missing initialization pass. +mcli stat root/ducklake +mcli anonymous get root/ducklake | grep -Eq '`private`|private' +mcli admin user add root "$fixture" "$password" +mcli admin policy attach root readwrite --user "$fixture" +mcli alias set writer http://127.0.0.1:9000 "$fixture" "$password" +printf '%s' "$payload" | mcli pipe "writer/$object" +test "$(mcli cat "writer/$object")" = "$payload" + +mcli admin user disable root "$fixture" +expect_denied mcli cat "writer/$object" +mcli admin user enable root "$fixture" +test "$(mcli cat "writer/$object")" = "$payload" +mcli admin user remove root "$fixture" +expect_denied mcli cat "writer/$object" + +# Exercise the exact reader configured for Trino, including the data/ boundary. +mcli alias set reader http://127.0.0.1:9000 "$reader_user" "$reader_password" +test "$(mcli cat "reader/$object")" = "$payload" +mcli ls "reader/ducklake/data/" >/dev/null +expect_denied mcli cp "root/$object" "reader/$object" +printf '%s' "$payload" | mcli pipe "root/$outside_object" +expect_denied mcli cat "reader/$outside_object" +test "$(mcli cat "root/$object")" = "$payload" +printf 'PASS: Silo bucket privacy, read/write, credential revocation, and Trino reader policy\n' diff --git a/tests/integration/README.md b/tests/integration/README.md index 773c11ea9..9034fdcaf 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -230,13 +230,19 @@ The test suite includes benchmarks that measure DuckLake transaction conflict ra ### Prerequisites -The DuckLake benchmarks require the metadata PostgreSQL and MinIO infrastructure: +The DuckLake benchmarks require the metadata PostgreSQL and PGSTY Silo infrastructure: ```bash # Start DuckLake infrastructure docker compose -f tests/integration/docker-compose.yml up -d ducklake-metadata minio minio-init ``` +The service names `minio` and `minio-init` remain stable; their pinned Silo +image supplies both the `silo` server and `mcli` admin client. Existing buckets, +reader policies, test credentials, and credential revocation assertions remain +part of the fixture. See the [Silo local storage runbook](../../docs/silo-local-storage.md) +for release pins and migration/recovery instructions. + ### Running concurrency benchmarks ```bash diff --git a/tests/integration/composefile/compose_test.go b/tests/integration/composefile/compose_test.go index 2885bff6b..2ca0fe627 100644 --- a/tests/integration/composefile/compose_test.go +++ b/tests/integration/composefile/compose_test.go @@ -3,6 +3,7 @@ package composefile import ( "os" "path/filepath" + "strings" "testing" "gopkg.in/yaml.v3" @@ -10,7 +11,7 @@ import ( // TestDockerComposeCoreStack guards the shape of the integration compose // stack: the comparison Postgres, the DuckLake metadata Postgres, and the -// MinIO object store the DuckLake tests run against. +// Silo object store the DuckLake tests run against. func TestDockerComposeCoreStack(t *testing.T) { compose := readCompose(t) @@ -44,6 +45,37 @@ func TestDockerComposeCoreStack(t *testing.T) { } } +// All local stacks use the same server and bundled admin client, so health and +// initialization exercise the exact release used by credential-rotation tests. +func TestDockerComposeSilo(t *testing.T) { + const image = "docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3" + for _, path := range []string{ + filepath.Join("..", "docker-compose.yml"), + filepath.Join("..", "..", "..", "docker-compose.yaml"), + filepath.Join("..", "..", "..", "k8s", "local-config-store.compose.yaml"), + } { + t.Run(path, func(t *testing.T) { + compose := readComposePath(t, path) + storage := serviceNamed(t, compose, "minio") + if storage.Image != image { + t.Errorf("storage image = %q, want pinned Silo %q", storage.Image, image) + } + if got := strings.Join(storage.Healthcheck.Test, " "); got != "CMD mcli ready local" { + t.Errorf("healthcheck = %q, want bundled mcli readiness check", got) + } + if init, ok := compose.Services["minio-init"]; ok { + if init.Image != image { + t.Errorf("init image = %q, want same bundled client release as server", init.Image) + } + entrypoint, _ := init.Entrypoint.(string) + if !strings.Contains(entrypoint, "/bin/sh") || !strings.Contains(entrypoint, "mcli mb minio/ducklake --ignore-existing") { + t.Errorf("init must override server entrypoint and create existing DuckLake bucket with mcli: %q", init.Entrypoint) + } + } + }) + } +} + type composeFile struct { Services map[string]service `yaml:"services"` } @@ -53,12 +85,20 @@ type service struct { Ports []string `yaml:"ports"` Environment map[string]any `yaml:"environment"` DependsOn map[string]any `yaml:"depends_on"` + Entrypoint any `yaml:"entrypoint"` + Healthcheck struct { + Test []string `yaml:"test"` + } `yaml:"healthcheck"` } func readCompose(t *testing.T) composeFile { t.Helper() + return readComposePath(t, filepath.Join("..", "docker-compose.yml")) +} - raw, err := os.ReadFile(filepath.Join("..", "docker-compose.yml")) +func readComposePath(t *testing.T, path string) composeFile { + t.Helper() + raw, err := os.ReadFile(path) if err != nil { t.Fatalf("read docker-compose.yml: %v", err) } diff --git a/tests/integration/credential_rotation_pin_test.go b/tests/integration/credential_rotation_pin_test.go index 5e634cf68..9b4a1fec8 100644 --- a/tests/integration/credential_rotation_pin_test.go +++ b/tests/integration/credential_rotation_pin_test.go @@ -63,16 +63,16 @@ func runRotationScenario(t *testing.T, sc rotationScenario) rotationResult { } mc := func(args ...string) (string, error) { - out, err := exec.Command("docker", append([]string{"exec", rotationMinioContainer, "mc"}, args...)...).CombinedOutput() + out, err := exec.Command("docker", append([]string{"exec", rotationMinioContainer, "mcli"}, args...)...).CombinedOutput() return string(out), err } if out, err := mc("ready", "local"); err != nil { - t.Skipf("mc not available in MinIO container (%v): %s", err, out) + t.Fatalf("mcli not ready in Silo container (%v): %s", err, out) } // The image's built-in `local` alias is anonymous (enough for the compose // healthcheck, not for admin ops) — register an authenticated alias. if out, err := mc("alias", "set", "localadmin", "http://localhost:9000", "minioadmin", "minioadmin"); err != nil { - t.Skipf("could not configure authenticated mc alias (%v): %s", err, out) + t.Fatalf("could not configure authenticated mcli alias (%v): %s", err, out) } // --- MinIO fixtures: private bucket + two users with readwrite policy --- @@ -253,7 +253,7 @@ func loadStockHTTPFS(t *testing.T, db *sql.DB) { // recovery (at which point the fork patch and the freshness-floor caveats in // controlplane/sts_broker.go can both be revisited). // -// Requires the integration docker compose stack (MinIO with admin `mc` in the +// Requires the integration docker compose stack (Silo with admin `mcli` in the // container); skips otherwise. func TestInFlightScanDiesOnCredentialRotation(t *testing.T) { r := runRotationScenario(t, rotationScenario{loadHTTPFS: loadStockHTTPFS}) diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml index f69c9fd06..a41136f49 100644 --- a/tests/integration/docker-compose.yml +++ b/tests/integration/docker-compose.yml @@ -40,9 +40,9 @@ services: tmpfs: - /var/lib/postgresql/data - # MinIO for DuckLake object storage + # PGSTY Silo for DuckLake object storage (legacy service name retained) minio: - image: minio/minio:latest + image: docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 container_name: duckgres-test-minio command: server /data --console-address ":9001" environment: @@ -52,7 +52,7 @@ services: - "39000:9000" # S3 API - "39001:9001" # Web console healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "mcli", "ready", "local"] interval: 2s timeout: 5s retries: 10 @@ -61,18 +61,18 @@ services: # Creates the ducklake bucket on startup minio-init: - image: minio/mc:latest + image: docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 container_name: duckgres-test-minio-init depends_on: minio: condition: service_healthy entrypoint: > - /bin/sh -c " - mc alias set minio http://minio:9000 minioadmin minioadmin; - mc mb minio/ducklake --ignore-existing; - mc admin policy create minio trino-ducklake-read /policies/trino-ducklake-read.json || true; - mc admin user add minio trino-reader trino-reader || true; - mc admin policy attach minio trino-ducklake-read --user trino-reader; + /bin/sh -ec " + mcli alias set minio http://minio:9000 minioadmin minioadmin; + mcli mb minio/ducklake --ignore-existing; + mcli admin policy create minio trino-ducklake-read /policies/trino-ducklake-read.json || true; + mcli admin user add minio trino-reader trino-reader || true; + mcli admin policy attach minio trino-ducklake-read --user trino-reader; echo 'Bucket ducklake created successfully'; " volumes: diff --git a/tests/integration/trino/ducklake.properties b/tests/integration/trino/ducklake.properties index b4055d07a..f7d1e8194 100644 --- a/tests/integration/trino/ducklake.properties +++ b/tests/integration/trino/ducklake.properties @@ -8,7 +8,7 @@ ducklake.catalog.database-user=trino_reader ducklake.catalog.database-password=${ENV:TRINO_DUCKLAKE_DB_PASSWORD} ducklake.data-path=s3://ducklake/data/ -# Brikk’s connector uses Trino's native S3 filesystem. The MinIO policy grants +# Brikk’s connector uses Trino's native S3 filesystem. The Silo policy grants # only ListBucket for data/ and GetObject for data/*; no writes are permitted. fs.native-s3.enabled=true s3.endpoint=http://minio:9000 diff --git a/tests/trino-ducklake-smoke/README.md b/tests/trino-ducklake-smoke/README.md index 71ba0571d..317a72340 100644 --- a/tests/trino-ducklake-smoke/README.md +++ b/tests/trino-ducklake-smoke/README.md @@ -3,12 +3,17 @@ `just trino-ducklake-smoke` creates the existing local DuckLake fixture through Duckgres, then starts one pinned Trino coordinator with the pinned Brikk DuckLake connector. It verifies that read-only Trino credentials can discover -and query the same PostgreSQL metadata catalog and MinIO data path. +and query the same PostgreSQL metadata catalog and PGSTY Silo data path. The test is intentionally separate from the normal integration suite because it downloads and starts Trino. It uses only local Docker services and test credentials; it does not contact managed-warehouse infrastructure. +The shared object-storage fixture retains the `minio` endpoint and service +name while using pinned Silo and its bundled `mcli` client. Trino's existing +read-only S3 policy is unchanged. Release pins and local migration/rollback +instructions are in the [Silo local storage runbook](../../docs/silo-local-storage.md). + The `just` recipe writes the version artifact to `artifacts/trino-ducklake-smoke/`. Set `TRINO_DUCKLAKE_SMOKE_ARTIFACT_DIR` to override it, for example: @@ -35,7 +40,7 @@ to change the dataset size. Artifacts are written to set. This is a local end-to-end comparison, including PGWire or HTTP client overhead -and local Postgres/MinIO access. It is not a production capacity benchmark. +and local Postgres/Silo access. It is not a production capacity benchmark. For wide, PostHog-shaped synthetic events, use the explicit realistic profile: diff --git a/tests/trino-ducklake-smoke/smoke_test.go b/tests/trino-ducklake-smoke/smoke_test.go index 570e72f51..51809157e 100644 --- a/tests/trino-ducklake-smoke/smoke_test.go +++ b/tests/trino-ducklake-smoke/smoke_test.go @@ -139,8 +139,8 @@ func assertS3ReaderCannotWrite(t *testing.T, composeFile string) { t.Helper() cmdArgs := []string{ "compose", "-f", composeFile, "run", "--rm", "--no-deps", "--entrypoint", "/bin/sh", "minio-init", "-ec", - "mc alias set reader http://minio:9000 trino-reader trino-reader && " + - "printf blocked | mc pipe reader/ducklake/data/trino-smoke-write-probe", + "mcli alias set reader http://minio:9000 trino-reader trino-reader && " + + "printf blocked | mcli pipe reader/ducklake/data/trino-smoke-write-probe", } cmd := exec.Command("docker", cmdArgs...) output, err := cmd.CombinedOutput() From 1ba9817849b3c25d992c24c576e4be26a330783f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 11 Sep 2026 17:42:45 -0400 Subject: [PATCH 2/2] Trim Silo migration test scaffolding and runbook --- docs/silo-local-storage.md | 129 +++++------------- scripts/test_silo.sh | 8 +- tests/integration/composefile/compose_test.go | 21 +-- 3 files changed, 45 insertions(+), 113 deletions(-) diff --git a/docs/silo-local-storage.md b/docs/silo-local-storage.md index b0baad832..6d0a740a3 100644 --- a/docs/silo-local-storage.md +++ b/docs/silo-local-storage.md @@ -13,39 +13,10 @@ Server and initialization containers use the same immutable multi-platform image docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3 ``` -The [server release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z) -bundles `mcli` version `RELEASE.2026-09-03T07-13-05Z`. Using the bundled client -also pins the admin executable by the image digest; no runtime download of an -unversioned client is needed. - -| Platform | Image manifest digest | -| --- | --- | -| `linux/amd64` | `sha256:885275e0f42acfdf80304c577d2e46c2c3978a276619d60b9eedd7486f104b30` | -| `linux/arm64` | `sha256:c35123a06f2147372523ffc5ce42a0efc4a239405481cbc2a3534bd883257b06` | - -Release provenance was checked with `gh attestation verify` against `pgsty/silo` -and `pgsty/mc` for their respective release checksum manifests. Downloaded Linux -archives matched those manifests, and the extracted server and client binary -hashes matched `/usr/bin/silo` and `/usr/bin/mcli` in the container on both -architectures. The [client release](https://github.com/pgsty/mc/releases/tag/RELEASE.2026-09-03T07-13-05Z) -and server release archives have these SHA256 hashes: - -| Archive | SHA256 | -| --- | --- | -| Silo Linux amd64 | `cbe5c01eac0a97ccb22fa252eafa432e8608bfde7e3ea27a324cb5ed625a1e96` | -| Silo Linux arm64 | `311846ca9387de36f8e34daa8bf1a130684cc7b8c61aa39581264249eb8df0cf` | -| mcli Linux amd64 | `cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8` | -| mcli Linux arm64 | `7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8` | - -The image shell, entrypoint, and executable versions were verified. The isolated -S3/admin contract passed on both architectures, with amd64 run under emulation. - -The server executable is `silo`; initialization and administrative operations -use `mcli` instead of `mc`. Silo preserves `MINIO_*` environment variables, -`/minio/*` routes, and the on-disk format according to upstream. The Compose -service names `minio` and `minio-init`, container names, volume names, and S3 -endpoints are deliberately retained so existing local configurations and -container-to-container connections continue to work. +The image supports `linux/amd64` and `linux/arm64` and bundles `mcli` version +`RELEASE.2026-09-03T07-13-05Z`, pinning both server and admin client together. +The executables are `silo` and `mcli`. Existing `MINIO_*` variables, `/minio/*` +routes, Compose service/container/volume names, and S3 endpoints are retained. ## Defaults and local development @@ -56,20 +27,17 @@ policy. Integration and local Kubernetes fixtures retain their existing port mapping, buckets, credentials, reader policy, and metadata configuration; refer to their Compose files for the separate fixture defaults. -For a new root Compose environment: +Start and initialize root Compose storage: ```bash -docker compose up -d +docker compose up -d --wait minio +docker compose run --rm --no-deps minio-init docker compose ps -a docker compose logs minio minio-init -just build -./duckgres --config duckgres.yaml -# In a separate terminal once Duckgres is ready: -just seed-ducklake ``` -Before running the integration storage contract, explicitly reconcile the -server with the pinned Compose image, wait for health, and initialize it: +Duckgres application configuration and TLS setup are separate from this storage +setup. For integration storage, reconcile the pinned server and initialize it: ```bash docker compose -f tests/integration/docker-compose.yml up -d --wait minio @@ -78,54 +46,31 @@ just test-silo just test-integration ``` -`just test-silo` checks the initialized fixture's S3 and admin behavior. CI uses -the same Compose server and initializer before running this recipe. The -integration harness checks TCP reachability when reusing an existing fixture; -`just test-integration` alone does not upgrade an already-running MinIO server. -The contract script defaults `SILO_TEST_CONTAINER` to `duckgres-test-minio`. -For an isolated fixture, override that container name. Its credential defaults -are `SILO_TEST_ROOT_USER=minioadmin`, `SILO_TEST_ROOT_PASSWORD=minioadmin`, -`SILO_TEST_READER_USER=trino-reader`, and `SILO_TEST_READER_PASSWORD=trino-reader`. -Use `just trino-ducklake-smoke`, `just perf-trino-ducklake`, and -`just perf-trino-ducklake-realistic` for the opt-in Trino paths. -Kind and OrbStack setup remain `just run-multitenant-kind` and -`just run-multitenant-local`, respectively. - -## Migrate an existing local environment - -1. Record the current checkout and the actual image digests of existing server - and client containers before replacing them. Older configuration used mutable - tags, so a tag alone is not a reproducible rollback target. -2. Stop Duckgres, Trino, and other writers, then stop the affected Compose stack. - Back up its existing object-storage data and metadata together while stopped. - Preserve the entire object-store volume, including hidden server metadata, - and the PostgreSQL data or a verified database backup. Do not use `down -v`. - Local Kubernetes dependencies use tmpfs for object and metadata storage; - export any fixture data you need before stopping those containers. -3. Switch to the migration checkout and pull the pinned image. Start only the - intended local stack with its existing Compose project name and volumes. - For root Compose, run `docker compose up -d`; for test and Kubernetes - fixtures, use the explicit integration setup commands or Kubernetes recipes - above. -4. Check that the object store becomes healthy and initialization exits with - status zero. Verify bucket access and run the affected test recipes before - resuming local work. Credential rotation tests must still reject disabled - credentials and accept the replacement credentials; S3 compatibility alone - does not establish admin API compatibility. - -## Failure recovery and rollback - -Inspect `docker compose ps -a` and `docker compose logs minio minio-init` for -the root stack; add `-f tests/integration/docker-compose.yml` for the integration -stack. A failing healthcheck or nonzero initialization exit should be fixed -before starting clients. Confirm the pinned image was pulled for the intended -architecture, the configured credentials agree, and the configured ports are -available. Keep `MINIO_*` settings and `/minio/*` routes unchanged. - -To roll back, stop writers and the affected stack, preserve the failed state -for diagnosis, restore the pre-migration object-store and metadata backups as -a consistent pair, and restore the previous checkout and recorded image -digests. Then start the previous stack and verify access before resuming work. -Do not assume that starting an older MinIO image against data already written -by Silo is a safe rollback. For disposable test fixtures, explicitly discard -and reseed only that fixture's data instead of restoring a backup. +`just test-silo` checks S3 access, reader policy, and credential revocation. +`SILO_TEST_CONTAINER` defaults to `duckgres-test-minio`; override it for an +isolated fixture using the same credentials. `just test-integration` alone +does not upgrade an already-running MinIO server. Trino checks use +`just trino-ducklake-smoke`, `just perf-trino-ducklake`, and +`just perf-trino-ducklake-realistic`. Kind and OrbStack dependencies use +`just run-multitenant-kind` and `just run-multitenant-local`. + +## Migration and recovery + +1. Record the previous checkout and actual server/client image digests; mutable + tags alone cannot reproduce the old stack. Stop writers, export any needed + data from tmpfs fixtures, then stop the affected stack. +2. Back up the full object volume (including hidden metadata) and PostgreSQL + data together while stopped. Do not use `down -v`. Integration and local + Kubernetes object-storage fixtures use tmpfs rather than persistent volumes. +3. Switch checkout, pull the pinned image, and run the setup commands above for + the intended stack, preserving its Compose project name and volumes. +4. Confirm healthy storage and successful initialization, then verify bucket + access and affected tests before resuming writers. On failure inspect + `docker compose ps -a` and `docker compose logs minio minio-init` (add the + integration Compose `-f` option for that stack); check ports and credentials. + +To roll back, stop writers and the stack, preserve the failed state, and restore +the paired object/metadata backups, previous checkout, and recorded image +digests. Verify access before resuming work; do not assume an older MinIO image +can safely use data written by Silo. Disposable fixtures can instead be explicitly +discarded and reseeded, limited to that fixture's data. diff --git a/scripts/test_silo.sh b/scripts/test_silo.sh index a9ea00602..b71ed3027 100644 --- a/scripts/test_silo.sh +++ b/scripts/test_silo.sh @@ -7,10 +7,6 @@ set -euo pipefail container=${SILO_TEST_CONTAINER:-duckgres-test-minio} -root_user=${SILO_TEST_ROOT_USER:-minioadmin} -root_password=${SILO_TEST_ROOT_PASSWORD:-minioadmin} -reader_user=${SILO_TEST_READER_USER:-trino-reader} -reader_password=${SILO_TEST_READER_PASSWORD:-trino-reader} fixture="silo-contract-$(date +%s)-$$" config="/tmp/$fixture" object="ducklake/data/$fixture.txt" @@ -44,7 +40,7 @@ expect_denied() { fi } -mcli alias set root http://127.0.0.1:9000 "$root_user" "$root_password" +mcli alias set root http://127.0.0.1:9000 minioadmin minioadmin mcli ready root # Require the initialized bucket; never make a missing initialization pass. mcli stat root/ducklake @@ -63,7 +59,7 @@ mcli admin user remove root "$fixture" expect_denied mcli cat "writer/$object" # Exercise the exact reader configured for Trino, including the data/ boundary. -mcli alias set reader http://127.0.0.1:9000 "$reader_user" "$reader_password" +mcli alias set reader http://127.0.0.1:9000 trino-reader trino-reader test "$(mcli cat "reader/$object")" = "$payload" mcli ls "reader/ducklake/data/" >/dev/null expect_denied mcli cp "root/$object" "reader/$object" diff --git a/tests/integration/composefile/compose_test.go b/tests/integration/composefile/compose_test.go index 2ca0fe627..2975168cd 100644 --- a/tests/integration/composefile/compose_test.go +++ b/tests/integration/composefile/compose_test.go @@ -3,7 +3,7 @@ package composefile import ( "os" "path/filepath" - "strings" + "regexp" "testing" "gopkg.in/yaml.v3" @@ -45,10 +45,12 @@ func TestDockerComposeCoreStack(t *testing.T) { } } -// All local stacks use the same server and bundled admin client, so health and -// initialization exercise the exact release used by credential-rotation tests. +// Keep every local server and bundled admin client on the same immutable release. func TestDockerComposeSilo(t *testing.T) { - const image = "docker.io/pgsty/silo:RELEASE.2026-09-03T13-18-01Z@sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3" + image := serviceNamed(t, readCompose(t), "minio").Image + if !regexp.MustCompile(`^docker\.io/pgsty/silo:RELEASE\.[^@]+@sha256:[0-9a-f]{64}$`).MatchString(image) { + t.Fatalf("storage image must pin a Silo release and digest: %q", image) + } for _, path := range []string{ filepath.Join("..", "docker-compose.yml"), filepath.Join("..", "..", "..", "docker-compose.yaml"), @@ -60,17 +62,10 @@ func TestDockerComposeSilo(t *testing.T) { if storage.Image != image { t.Errorf("storage image = %q, want pinned Silo %q", storage.Image, image) } - if got := strings.Join(storage.Healthcheck.Test, " "); got != "CMD mcli ready local" { - t.Errorf("healthcheck = %q, want bundled mcli readiness check", got) - } if init, ok := compose.Services["minio-init"]; ok { if init.Image != image { t.Errorf("init image = %q, want same bundled client release as server", init.Image) } - entrypoint, _ := init.Entrypoint.(string) - if !strings.Contains(entrypoint, "/bin/sh") || !strings.Contains(entrypoint, "mcli mb minio/ducklake --ignore-existing") { - t.Errorf("init must override server entrypoint and create existing DuckLake bucket with mcli: %q", init.Entrypoint) - } } }) } @@ -85,10 +80,6 @@ type service struct { Ports []string `yaml:"ports"` Environment map[string]any `yaml:"environment"` DependsOn map[string]any `yaml:"depends_on"` - Entrypoint any `yaml:"entrypoint"` - Healthcheck struct { - Test []string `yaml:"test"` - } `yaml:"healthcheck"` } func readCompose(t *testing.T) composeFile {