diff --git a/.github/workflows/action-test.yml b/.github/workflows/action-test.yml new file mode 100644 index 0000000..1524880 --- /dev/null +++ b/.github/workflows/action-test.yml @@ -0,0 +1,53 @@ +name: Action test +# Exercises integrations/github-action against the latest release. Runs only when +# the action changes (or by hand) to keep CI minutes down. +on: + pull_request: + paths: + - "integrations/github-action/**" + - ".github/workflows/action-test.yml" + push: + branches: [main] + paths: + - "integrations/github-action/**" + - ".github/workflows/action-test.yml" + workflow_dispatch: + inputs: + version: + description: HomeCloud release to test (e.g. v0.3.0) + default: latest + +permissions: + contents: read + +jobs: + action: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Input validation + run: bash integrations/github-action/test/inputs.sh + - id: homecloud + uses: ./integrations/github-action + with: + version: ${{ inputs.version || 'latest' }} + services-wait: s3 + - name: AWS CLI against HomeCloud + run: | + set -eux + test "$AWS_ENDPOINT_URL" = "${{ steps.homecloud.outputs.endpoint }}" + aws sts get-caller-identity + aws s3 mb s3://action-test + echo hello | aws s3 cp - s3://action-test/hello.txt + test "$(aws s3 cp s3://action-test/hello.txt -)" = hello + url=$(aws sqs create-queue --queue-name action-test --query QueueUrl --output text) + aws sqs send-message --queue-url "$url" --message-body hi + test "$(aws sqs receive-message --queue-url "$url" --query 'Messages[0].Body' --output text)" = hi + aws dynamodb create-table --table-name action-test \ + --attribute-definitions AttributeName=id,AttributeType=S \ + --key-schema AttributeName=id,KeyType=HASH --billing-mode PAY_PER_REQUEST + aws dynamodb put-item --table-name action-test --item '{"id":{"S":"1"}}' + aws dynamodb get-item --table-name action-test --key '{"id":{"S":"1"}}' + aws lambda list-functions + homecloud version diff --git a/CHANGELOG.md b/CHANGELOG.md index 14fe4da..2e0249d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- HomeCloud as a drop-in AWS for tests ([docs/integrations.md](docs/integrations.md)): a GitHub Action (`integrations/github-action`) that installs a checksum-verified release, starts the server and exports `AWS_ENDPOINT_URL` and credentials, and testcontainers modules for Go (`integrations/testcontainers-go`) and Python (`integrations/testcontainers-python`) that run the HomeCloud image and clean up every container it started. - KMS supports imported key material (`create-key --origin EXTERNAL`, `get-parameters-for-import`, `import-key-material` with `RSAES_OAEP_SHA_1/256` and `RSA_AES_KEY_WRAP_SHA_1/256`, `ValidTo` expiry, `delete-imported-key-material`; only the same material can be imported again) and multi-Region primary keys (`--multi-region`, `mrk-` ids). `ReplicateKey` and `UpdatePrimaryRegion` return `UnsupportedOperationException` because HomeCloud runs a single region. - `homecloud configure` accepts `--ca-file` and `--region`, and keeps the region and `ca_file` already in the credentials file when it rewrites it (#79). - With `--addr 0.0.0.0:8080` (or `[::]`) the credentials file records `127.0.0.1` (or the public URL) as the endpoint instead of `0.0.0.0`; an existing file is repaired at the next start (#79). diff --git a/README.md b/README.md index b53e537..e12f9ee 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Live demo console · Quick start · AWS compatibility · + Testing & CI · vs. LocalStack, moto, MinIO… · Discord

@@ -70,6 +71,8 @@ eval "$(homecloud aws-env)" S3 starts in the background on first boot; if your tests use S3 right away, wait for `s3: MinIO ready` in the log (see [`.github/workflows/ci.yml`](.github/workflows/ci.yml)). +Or use the ready-made pieces in **[docs/integrations.md](docs/integrations.md)**: a GitHub Action that does the above in one step, and testcontainers modules for Go and Python. + --- ## Works with diff --git a/docs/integrations.md b/docs/integrations.md new file mode 100644 index 0000000..eabb3f8 --- /dev/null +++ b/docs/integrations.md @@ -0,0 +1,145 @@ +# Using HomeCloud in tests and CI + +HomeCloud speaks the AWS protocols (SigV4) on one port, so any project can use it as a +drop-in AWS for integration tests: point `AWS_ENDPOINT_URL` at it and use the root +credentials it creates on first start. This page covers the ready-made integrations in +[`integrations/`](../integrations): + +| | | +| --- | --- | +| [GitHub Action](#github-actions) | `integrations/github-action`: install, start, export `AWS_*` | +| [testcontainers-go](#testcontainers-go) | `integrations/testcontainers-go`: start HomeCloud from a Go test | +| [testcontainers-python](#testcontainers-python) | `integrations/testcontainers-python`: start HomeCloud from pytest | +| [Plain binary or Docker](#plain-binary-or-docker) | anything else | + +Everything HomeCloud runs (MinIO for S3, Lambda runtimes, databases) is a container on +the Docker host, so all of these need Docker. Only **one HomeCloud runs per Docker host** +at a time: its helper containers have fixed names (`homecloud-s3`, ...) and host ports +(MinIO 9500/9501, ECR 5500, DNS 8053), and it refuses to start next to another +installation's containers. Do not run test suites that each start HomeCloud in parallel +on the same Docker host. + +## GitHub Actions + +```yaml +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: solinode/homecloud/integrations/github-action@main + with: + version: latest # or v0.3.0 + services-wait: s3 # wait for MinIO too + - run: | + aws s3 mb s3://artifacts + aws sqs create-queue --queue-name jobs + pytest # boto3 reads AWS_ENDPOINT_URL +``` + +The action downloads the release, checks it against `checksums.txt`, runs `homecloud +serve` in the background (data and log under `$RUNNER_TEMP`), waits for +`/api/v1/health`, and exports `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, +`AWS_SECRET_ACCESS_KEY` (masked), `AWS_REGION` and `AWS_DEFAULT_REGION`. After the job, +a post step prints the server log (collapsed), stops HomeCloud and removes its Docker +resources. Inputs, outputs and boto3/Terraform examples: +[integrations/github-action/README.md](../integrations/github-action/README.md). + +## testcontainers-go + +```go +import homecloud "github.com/solinode/homecloud/integrations/testcontainers-go" + +hc, err := homecloud.Run(ctx, homecloud.DefaultImage) +testcontainers.CleanupContainer(t, hc) +if err != nil { + t.Fatal(err) +} +s3c := s3.NewFromConfig(hc.AWSConfig(), func(o *s3.Options) { o.UsePathStyle = true }) +``` + +`hc.EndpointURL()`, `hc.Credentials()` and `hc.Env()` give the connection details; +terminating the container also removes every container, network and volume HomeCloud +created. See [integrations/testcontainers-go/README.md](../integrations/testcontainers-go/README.md). + +## testcontainers-python + +```python +from testcontainers_homecloud import HomeCloudContainer + +@pytest.fixture(scope="session") +def homecloud(): + with HomeCloudContainer() as hc: + yield hc + +def test_upload(homecloud): + s3 = homecloud.get_client("s3") + s3.create_bucket(Bucket="test") +``` + +See [integrations/testcontainers-python/README.md](../integrations/testcontainers-python/README.md). + +### How the testcontainers modules run HomeCloud + +Both modules start the image (default `ghcr.io/solinode/homecloud:latest`, configurable) +with the Docker socket mounted and **host networking**, and the API on +`127.0.0.1:18080` of the Docker host (configurable; not 8080, which is often taken). +HomeCloud then reaches the containers it starts on the host's loopback ports, and they +call it back through `host.docker.internal`, exactly as when it runs on the host. This +works on Linux (GitHub Actions included), OrbStack, and Docker Desktop with host +networking enabled (Settings > Resources > Network). They wait for the API and S3, read +the credentials with `homecloud aws-env` inside the container, and on stop remove +everything labelled with the account HomeCloud created. + +Until the official image is published, build one from a checkout: + +```sh +docker build -f integrations/testdata/Dockerfile -t homecloud:test cli +HOMECLOUD_IMAGE=homecloud:test go test ./... # in integrations/testcontainers-go +HOMECLOUD_IMAGE=homecloud:test pytest # in integrations/testcontainers-python +``` + +## Plain binary or Docker + +With the binary (any CI with Docker, or a laptop): + +```sh +curl -fsSL https://homecloud.pages.dev/scripts/install.sh | sh +export HOMECLOUD_DATA_DIR=$(mktemp -d) +homecloud serve --data-dir "$HOMECLOUD_DATA_DIR" > homecloud.log 2>&1 & +until curl -fs http://127.0.0.1:8080/api/v1/health; do sleep 1; done +eval "$(homecloud aws-env)" # AWS_ENDPOINT_URL, keys, region +aws s3 ls +``` + +With Docker only, run the image the same way the testcontainers modules do. Until +`ghcr.io/solinode/homecloud` is published, use the image built above (`homecloud:test`); +afterwards, replace it with `ghcr.io/solinode/homecloud`: + +```sh +docker build -f integrations/testdata/Dockerfile -t homecloud:test cli +docker run -d --name homecloud --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + homecloud:test serve --data-dir /data --addr 127.0.0.1:18080 +until curl -fs http://127.0.0.1:18080/api/v1/health; do sleep 1; done +docker exec homecloud homecloud aws-env # credentials; the endpoint is http://127.0.0.1:18080 +``` + +To clean up afterwards, remove the HomeCloud container with its data volume (`-v`), then +everything labelled with its account (`docker logs homecloud | grep "created account"` +shows it): + +```sh +docker rm -fv homecloud +docker rm -fv $(docker ps -aq --filter label=homecloud.account=ACCOUNT) +docker network rm $(docker network ls -q --filter label=homecloud.account=ACCOUNT) +docker volume rm $(docker volume ls -q --filter label=homecloud.account=ACCOUNT) +``` + +## SDK notes + +- AWS CLI v2, boto3 1.28+, aws-sdk-go-v2, the JavaScript SDK v3 and the Terraform AWS + provider 5.x read `AWS_ENDPOINT_URL`. +- Use path-style S3 addressing (`UsePathStyle`, `addressing_style: path`, + `s3_use_path_style = true`) when the endpoint is an IP address or `localhost`. +- The region is `us-east-1` unless the server was started with another one. diff --git a/integrations/github-action/README.md b/integrations/github-action/README.md new file mode 100644 index 0000000..179ed63 --- /dev/null +++ b/integrations/github-action/README.md @@ -0,0 +1,126 @@ +# Set up HomeCloud (GitHub Action) + +Runs [HomeCloud](https://github.com/solinode/homecloud), a self-hosted AWS, inside your +workflow so tests can call S3, SQS, DynamoDB, Lambda and the rest of the AWS API without +an AWS account. + +The action: + +1. downloads the HomeCloud release you ask for and verifies it against the release's `checksums.txt`, +2. starts `homecloud serve` in the background (data in `$RUNNER_TEMP/homecloud-data`, log in `$RUNNER_TEMP/homecloud.log`), +3. waits for `/api/v1/health` and for any services listed in `services-wait`, +4. exports `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` (masked), `AWS_REGION`, `AWS_DEFAULT_REGION` and `HOMECLOUD_DATA_DIR` for the following steps, and puts `homecloud` on `PATH`, +5. in a post step after every job, prints the server log (collapsed), stops HomeCloud and removes the containers, networks and volumes it created (so self-hosted runners stay clean). + +HomeCloud runs every service on Docker, so use a runner with Docker: `ubuntu-latest` works +as is. macOS and Windows hosted runners have no Docker. + +## Usage + +```yaml +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: solinode/homecloud/integrations/github-action@main + with: + services-wait: s3 + - run: aws s3 mb s3://my-bucket && aws s3 ls +``` + +### Inputs + +| Input | Default | Description | +| --- | --- | --- | +| `version` | `latest` | Release to install, e.g. `v0.3.0` (the `v` is optional). | +| `port` | `8080` | Port the API listens on, on `127.0.0.1`. | +| `services-wait` | (none) | Comma or newline separated services to wait for. `s3`, `ecr` and `route53` start containers in the background; in-process services (`sqs`, `dynamodb`, `lambda`, ...) are ready with the API; an unknown name fails the step. | +| `wait-timeout` | `180` | Seconds to wait for the API and the services. | + +### Outputs + +| Output | Description | +| --- | --- | +| `endpoint` | e.g. `http://127.0.0.1:8080` (same as `AWS_ENDPOINT_URL`) | +| `access-key-id` | root access key ID | +| `region` | region HomeCloud serves (`us-east-1`) | +| `log-file` | path of the server log | + +AWS CLI v2 and current AWS SDKs read `AWS_ENDPOINT_URL`, so most tools need no configuration. + +## Examples + +### AWS CLI + +```yaml + - uses: solinode/homecloud/integrations/github-action@main + with: + services-wait: s3 + - run: | + aws s3 mb s3://artifacts + aws sqs create-queue --queue-name jobs + aws dynamodb list-tables + aws lambda list-functions +``` + +### Python (boto3) + +boto3 1.28 and later pick up `AWS_ENDPOINT_URL`: + +```yaml + - uses: solinode/homecloud/integrations/github-action@main + with: + services-wait: s3 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install boto3 pytest && pytest +``` + +```python +import boto3 + +def test_upload(): + s3 = boto3.client("s3") # endpoint, keys and region come from the environment + s3.create_bucket(Bucket="test") + s3.put_object(Bucket="test", Key="a.txt", Body=b"hi") + assert s3.get_object(Bucket="test", Key="a.txt")["Body"].read() == b"hi" +``` + +### Terraform + +The AWS provider (5.x and later) honours `AWS_ENDPOINT_URL`; tell it not to look for a real account: + +```yaml + - uses: solinode/homecloud/integrations/github-action@main + with: + services-wait: s3 + - uses: hashicorp/setup-terraform@v3 + - run: terraform init && terraform apply -auto-approve +``` + +```hcl +provider "aws" { + skip_credentials_validation = true + skip_requesting_account_id = true + s3_use_path_style = true +} +``` + +### Using the step outputs + +```yaml + - id: homecloud + uses: solinode/homecloud/integrations/github-action@main + - run: ./run-tests.sh --endpoint "${{ steps.homecloud.outputs.endpoint }}" +``` + +## Notes + +- One HomeCloud per Docker host: it names its helper containers (`homecloud-s3`, ...) and + publishes MinIO on 9500/9501, ECR on 5500 and DNS on 8053 on `127.0.0.1`. Those ports + must be free on the runner. +- The root console password from the first-start log is masked as well. +- This is a JavaScript action (`node24`) only because composite actions cannot declare a + post step; all the work is in `setup.sh` and `post.sh`, and it has no dependencies to install. diff --git a/integrations/github-action/action.yml b/integrations/github-action/action.yml new file mode 100644 index 0000000..6000f3e --- /dev/null +++ b/integrations/github-action/action.yml @@ -0,0 +1,48 @@ +name: Set up HomeCloud +description: Run HomeCloud, a self-hosted AWS, in the job and point the AWS CLI, SDKs and Terraform at it. +author: solinode +branding: + icon: cloud + color: blue + +inputs: + version: + description: HomeCloud release to install, e.g. v0.3.0, or "latest". + required: false + default: latest + port: + description: Port the HomeCloud API listens on (127.0.0.1). + required: false + default: "8080" + services-wait: + description: >- + Services to wait for besides the API, comma or newline separated (s3, ecr, route53). + Services that run inside the HomeCloud process (sqs, dynamodb, lambda, ...) are + ready as soon as the API is; unknown names fail the step. + required: false + default: "" + wait-timeout: + description: Seconds to wait for the API and every service in services-wait. + required: false + default: "180" + +outputs: + endpoint: + description: The HomeCloud endpoint URL (also exported as AWS_ENDPOINT_URL). + access-key-id: + description: The root access key ID (also exported as AWS_ACCESS_KEY_ID). + region: + description: The region HomeCloud serves (also exported as AWS_REGION and AWS_DEFAULT_REGION). + log-file: + description: Path of the server log. + +# A JavaScript action rather than a composite one: only JavaScript (and Docker) +# actions can declare a post step. It runs after every job: it prints the server +# log (collapsed), stops HomeCloud and removes its Docker resources, so +# self-hosted runners are left clean. The work is in setup.sh and post.sh; +# main.js and post.js only run them. +runs: + using: node24 + main: main.js + post: post.js + post-if: always() diff --git a/integrations/github-action/main.js b/integrations/github-action/main.js new file mode 100644 index 0000000..9fa7713 --- /dev/null +++ b/integrations/github-action/main.js @@ -0,0 +1,19 @@ +// Runs setup.sh with the action's inputs. No dependencies, so nothing to build. +const { spawnSync } = require("child_process"); +const path = require("path"); + +const input = (name, def) => (process.env[`INPUT_${name.toUpperCase()}`] || def).trim(); + +const env = { + ...process.env, + HC_VERSION: input("version", "latest"), + HC_PORT: input("port", "8080"), + HC_SERVICES_WAIT: input("services-wait", ""), + HC_WAIT_TIMEOUT: input("wait-timeout", "180"), +}; +const r = spawnSync("bash", [path.join(__dirname, "setup.sh")], { stdio: "inherit", env }); +if (r.error) { + console.log(`::error::${r.error.message}`); + process.exit(1); +} +process.exit(r.status === null ? 1 : r.status); diff --git a/integrations/github-action/post.js b/integrations/github-action/post.js new file mode 100644 index 0000000..9ad0b74 --- /dev/null +++ b/integrations/github-action/post.js @@ -0,0 +1,14 @@ +// Post step (every job): runs post.sh with the state setup.sh saved. +const { spawnSync } = require("child_process"); +const path = require("path"); + +const env = { + ...process.env, + HC_LOG: process.env.STATE_log || "", + HC_PID: process.env.STATE_pid || "", + HC_DATA: process.env.STATE_data || "", +}; +const r = spawnSync("bash", [path.join(__dirname, "post.sh")], { stdio: "inherit", env }); +if (r.error) console.log(`::warning::HomeCloud clean-up: ${r.error.message}`); +// Clean-up problems never fail the job. +process.exit(0); diff --git a/integrations/github-action/post.sh b/integrations/github-action/post.sh new file mode 100755 index 0000000..70ff9f0 --- /dev/null +++ b/integrations/github-action/post.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Post step, run after every job: prints the server log (collapsed), stops this +# HomeCloud and removes the containers, networks and volumes of its account, so +# a self-hosted runner is left as it was. Run by post.js; state arrives as HC_*. +set -uo pipefail + +log="${HC_LOG:-}" +pid="${HC_PID:-}" +data="${HC_DATA:-}" +[ -n "$log" ] || { echo "HomeCloud was not started"; exit 0; } + +if [ -f "$log" ]; then + # Masks registered by setup.sh hold for the whole job; masking again covers a + # setup step that failed before it could. + pw=$(sed -n 's/.*password: \([^ ]*\).*/\1/p' "$log" | head -n1) + [ -n "$pw" ] && echo "::add-mask::$pw" + echo "::group::HomeCloud server log" + cat "$log" + echo "::endgroup::" + account=$(sed -n 's/.*first start: created account \([0-9]*\).*/\1/p' "$log" | head -n1) +fi + +if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + echo "Stopping HomeCloud (pid $pid)" + kill "$pid" 2>/dev/null + for _ in $(seq 1 20); do + kill -0 "$pid" 2>/dev/null || break + sleep 0.5 + done + kill -9 "$pid" 2>/dev/null || true +fi + +if [ -n "${account:-}" ] && command -v docker >/dev/null 2>&1; then + echo "Removing the Docker resources of HomeCloud account $account" + f="label=homecloud.account=$account" + ids=$(docker ps -aq --filter "$f") + [ -n "$ids" ] && docker rm -fv $ids >/dev/null + ids=$(docker network ls -q --filter "$f") + [ -n "$ids" ] && docker network rm $ids >/dev/null + ids=$(docker volume ls -q --filter "$f") + [ -n "$ids" ] && docker volume rm -f $ids >/dev/null +fi +[ -n "$data" ] && rm -rf "$data" +exit 0 diff --git a/integrations/github-action/setup.sh b/integrations/github-action/setup.sh new file mode 100755 index 0000000..c303c1e --- /dev/null +++ b/integrations/github-action/setup.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Installs HomeCloud, starts `homecloud serve` in the background and exports the +# AWS_* variables for the rest of the job. Run by main.js; inputs arrive as HC_*. +# Runs on Linux and macOS runners that have Docker (ubuntu-* runners do). +set -euo pipefail + +REPO="solinode/homecloud" +version="${HC_VERSION:-latest}" +port="${HC_PORT:-8080}" +timeout="${HC_WAIT_TIMEOUT:-180}" +tmp="${RUNNER_TEMP:-$(mktemp -d)}" +bin_dir="$tmp/homecloud-bin" +data_dir="$tmp/homecloud-data" +log="$tmp/homecloud.log" +out="${GITHUB_OUTPUT:-/dev/null}" +envf="${GITHUB_ENV:-/dev/null}" + +fail() { echo "::error::$*"; exit 1; } + +# Inputs are checked before anything starts, so a bad value never leaves a +# half-started server (and its first-start log) behind. +case "$port" in '' | *[!0-9]*) fail "port must be a number, got '$port'" ;; esac +port=$((10#$port)) +[ "$port" -ge 1 ] && [ "$port" -le 65535 ] || fail "port must be between 1 and 65535, got '$HC_PORT'" +case "$timeout" in '' | *[!0-9]*) fail "wait-timeout must be a whole number of seconds, got '$timeout'" ;; esac +timeout=$((10#$timeout)) +[ "$timeout" -ge 1 ] || fail "wait-timeout must be at least 1 second" + +# Services that start containers in the background log a line when they are ready; +# the others run inside the HomeCloud process and are ready with the API. +in_process=" acm apigateway autoscaling cloudformation cloudtrail cloudwatch cognito-idp dynamodb ec2 ecs efs elb elbv2 elasticloadbalancing events eventbridge iam kms lambda logs rds secretsmanager sns sqs ssm states stepfunctions sts " +waits=() +for svc in $(echo "${HC_SERVICES_WAIT:-}" | tr ',\n' ' ' | tr '[:upper:]' '[:lower:]'); do + case "$svc" in + s3 | ecr | route53 | dns) waits+=("$svc") ;; + *) case "$in_process" in *" $svc "*) ;; *) fail "services-wait: unknown service '$svc' (background: s3, ecr, route53; in-process:$in_process)" ;; esac ;; + esac +done +[ "${HC_VALIDATE_ONLY:-}" = 1 ] && { echo "inputs OK: port=$port wait-timeout=$timeout waits=${waits[*]:-}"; exit 0; } + +command -v docker >/dev/null 2>&1 || fail "HomeCloud needs Docker on the runner (ubuntu-* runners have it)" +docker info >/dev/null 2>&1 || fail "Docker is installed but not running" + +os=$(uname -s | tr '[:upper:]' '[:lower:]') +case "$(uname -m)" in + x86_64 | amd64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) fail "unsupported architecture $(uname -m)" ;; +esac +name="homecloud-$os-$arch" +if [ "$version" = latest ]; then + base="https://github.com/$REPO/releases/latest/download" +else + case "$version" in v*) ;; *) version="v$version" ;; esac + base="https://github.com/$REPO/releases/download/$version" +fi + +echo "::group::Install HomeCloud ($version, $os/$arch)" +dl="$tmp/homecloud-dl" +rm -rf "$dl" && mkdir -p "$dl" "$bin_dir" +curl -fsSL --retry 3 "$base/$name.tar.gz" -o "$dl/$name.tar.gz" || fail "download $base/$name.tar.gz failed" +curl -fsSL --retry 3 "$base/checksums.txt" -o "$dl/checksums.txt" || fail "download $base/checksums.txt failed" +expected=$(awk -v f="$name.tar.gz" '$2 == f || $2 == "*" f {print $1}' "$dl/checksums.txt") +[ -n "$expected" ] || fail "checksums.txt has no entry for $name.tar.gz" +if command -v sha256sum >/dev/null 2>&1; then + actual=$(sha256sum "$dl/$name.tar.gz" | cut -d' ' -f1) +else + actual=$(shasum -a 256 "$dl/$name.tar.gz" | cut -d' ' -f1) +fi +[ "$expected" = "$actual" ] || fail "checksum mismatch for $name.tar.gz (expected $expected, got $actual)" +echo "checksum OK ($actual)" +tar -xzf "$dl/$name.tar.gz" -C "$dl" +install -m 0755 "$dl/$name/homecloud" "$bin_dir/homecloud" +rm -rf "$dl" +[ -n "${GITHUB_PATH:-}" ] && echo "$bin_dir" >> "$GITHUB_PATH" +"$bin_dir/homecloud" version +echo "::endgroup::" + +echo "Starting homecloud serve on 127.0.0.1:$port (data: $data_dir, log: $log)" +mkdir -p "$data_dir" +export HOMECLOUD_DATA_DIR="$data_dir" +nohup "$bin_dir/homecloud" serve --data-dir "$data_dir" --addr "127.0.0.1:$port" > "$log" 2>&1 & +pid=$! +echo "$pid" > "$tmp/homecloud.pid" +if [ -n "${GITHUB_STATE:-}" ]; then + { echo "log=$log"; echo "pid=$pid"; echo "data=$data_dir"; } >> "$GITHUB_STATE" +fi + +# The first-start log carries the root console password: mask it as soon as it +# appears (post.sh masks it again before it prints the log). +masked= +mask_password() { + [ -n "$masked" ] && return 0 + local pw + pw=$(sed -n 's/.*password: \([^ ]*\).*/\1/p' "$log" | head -n1) + if [ -n "$pw" ]; then echo "::add-mask::$pw"; masked=1; fi + return 0 +} + +endpoint="http://127.0.0.1:$port" +deadline=$((SECONDS + timeout)) +until curl -fsS --connect-timeout 2 --max-time 5 "$endpoint/api/v1/health" >/dev/null 2>&1; do + mask_password + kill -0 "$pid" 2>/dev/null || fail "homecloud serve exited before it became healthy (the post step prints its log)" + [ "$SECONDS" -lt "$deadline" ] || fail "HomeCloud was not healthy after ${timeout}s" + sleep 1 +done +mask_password +echo "HomeCloud is healthy at $endpoint" + +for svc in ${waits[@]+"${waits[@]}"}; do + case "$svc" in + s3) ready="s3: MinIO ready" ;; + ecr) ready="ecr: registry ready" ;; + route53 | dns) ready="route53: DNS ready" ;; + esac + echo "Waiting for $svc" + until grep -q "$ready" "$log"; do + kill -0 "$pid" 2>/dev/null || fail "homecloud serve exited while waiting for $svc" + [ "$SECONDS" -lt "$deadline" ] || fail "$svc was not ready after ${timeout}s" + sleep 1 + done + echo "$svc is ready" +done + +# Credentials and region, as `homecloud aws-env` prints them. +creds=$("$bin_dir/homecloud" aws-env | sed -n 's/^export \([A-Z_]*\)="\(.*\)"$/\1=\2/p') +get() { echo "$creds" | sed -n "s/^$1=//p"; } +key=$(get AWS_ACCESS_KEY_ID) +secret=$(get AWS_SECRET_ACCESS_KEY) +region=$(get AWS_REGION) +[ -n "$key" ] && [ -n "$secret" ] || fail "could not read HomeCloud credentials from $data_dir" +echo "::add-mask::$secret" +{ + echo "AWS_ENDPOINT_URL=$endpoint" + echo "AWS_ACCESS_KEY_ID=$key" + echo "AWS_SECRET_ACCESS_KEY=$secret" + echo "AWS_REGION=$region" + echo "AWS_DEFAULT_REGION=$region" + echo "HOMECLOUD_DATA_DIR=$data_dir" +} >> "$envf" +{ + echo "endpoint=$endpoint" + echo "access-key-id=$key" + echo "region=$region" + echo "log-file=$log" +} >> "$out" +echo "Exported AWS_ENDPOINT_URL=$endpoint, AWS_ACCESS_KEY_ID=$key, AWS_REGION=$region" diff --git a/integrations/github-action/test/inputs.sh b/integrations/github-action/test/inputs.sh new file mode 100755 index 0000000..e2386bc --- /dev/null +++ b/integrations/github-action/test/inputs.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Checks setup.sh's input validation without installing anything: +# bash integrations/github-action/test/inputs.sh +set -u +setup="$(cd "$(dirname "$0")/.." && pwd)/setup.sh" +fails=0 + +# expect ok|fail PORT TIMEOUT SERVICES +expect() { + local want=$1 out got=ok + out=$(HC_VALIDATE_ONLY=1 HC_PORT="$2" HC_WAIT_TIMEOUT="$3" HC_SERVICES_WAIT="$4" \ + GITHUB_OUTPUT=/dev/null GITHUB_ENV=/dev/null GITHUB_STATE= RUNNER_TEMP="${TMPDIR:-/tmp}" bash "$setup" 2>&1) || got=fail + if [ "$got" != "$want" ]; then + echo "FAIL: port='$2' timeout='$3' services='$4': want $want, got $got: $out" + fails=$((fails + 1)) + else + echo "ok: port='$2' timeout='$3' services='$4' -> $want" + fi +} + +expect ok 8080 180 "" +expect ok 8080 08 "s3, sqs" # leading zero is decimal, not octal +expect ok 08080 180 "S3" +expect ok 8080 180 $'s3\necr\ndynamodb,lambda' +expect fail 8080 abc "" +expect fail 8080 0 "" +expect ok 8080 "" "" # empty means the default (180) +expect fail 8080 -5 "" +expect fail http 180 "" +expect fail 70000 180 "" +expect fail 8080 180 "s33" +expect fail 8080 180 "s3,nosuchservice" + +[ "$fails" -eq 0 ] && echo "all input checks passed" || { echo "$fails input checks failed"; exit 1; } diff --git a/integrations/testcontainers-go/README.md b/integrations/testcontainers-go/README.md new file mode 100644 index 0000000..81e5830 --- /dev/null +++ b/integrations/testcontainers-go/README.md @@ -0,0 +1,73 @@ +# HomeCloud module for testcontainers-go + +Start [HomeCloud](https://github.com/solinode/homecloud), a self-hosted AWS, from a Go +test and talk to it with aws-sdk-go-v2. + +```sh +go get github.com/solinode/homecloud/integrations/testcontainers-go +``` + +```go +import ( + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/testcontainers/testcontainers-go" + homecloud "github.com/solinode/homecloud/integrations/testcontainers-go" +) + +func TestUpload(t *testing.T) { + ctx := context.Background() + hc, err := homecloud.Run(ctx, homecloud.DefaultImage) + testcontainers.CleanupContainer(t, hc) // also removes MinIO, Lambda runtimes, ... it started + if err != nil { + t.Fatal(err) + } + cfg := hc.AWSConfig() // endpoint, root keys and region + s3c := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true }) + // ... +} +``` + +## API + +| | | +| --- | --- | +| `Run(ctx, image, opts...)` | starts HomeCloud and waits for the API (and S3, by default) | +| `hc.EndpointURL()` | `http://127.0.0.1:18080` | +| `hc.Endpoint(ctx, "http")` | same, through the `testcontainers.Container` interface | +| `hc.Credentials()` | root `AccessKeyID`, `SecretAccessKey`, `Region` | +| `hc.AWSConfig()` | `aws.Config` for any `NewFromConfig` | +| `hc.Env()` | `AWS_ENDPOINT_URL`, keys and region for subprocesses (AWS CLI, Terraform) | +| `hc.AccountID()` | the account HomeCloud created | +| `hc.Terminate(ctx)` | stops HomeCloud and removes every container, network and volume it created | +| `RemoveResources(ctx, account)` | the clean-up alone, for runs that were killed | + +Options: `WithPort(n)` (default 18080), `WithWaitForServices("s3", "ecr", "route53")` +(default `s3`; pass none to wait only for the API), `WithDockerSocket(path)`, plus any +generic `testcontainers` option. + +## How it runs + +HomeCloud starts each service's backing containers (MinIO for S3, Lambda runtimes, +databases) on the Docker host, so the container gets the Docker socket +(`/var/run/docker.sock`) and **host networking**: it reaches those containers on the +host's loopback ports and they call it back through `host.docker.internal`. The API +listens on `127.0.0.1:` of the Docker host. This works on Linux (including GitHub +Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > +Resources > Network). + +Only one HomeCloud can run per Docker host: its helper containers have fixed names +(`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that +use it in parallel, nor next to a HomeCloud you run yourself on the same Docker host +(HomeCloud refuses to start then). + +## Image + +`DefaultImage` is `ghcr.io/solinode/homecloud:latest`. Any image whose entrypoint is +the `homecloud` binary works; to build one from a checkout: + +```sh +docker build -f integrations/testdata/Dockerfile -t homecloud:test cli +HOMECLOUD_IMAGE=homecloud:test go test ./... +``` + +The module's own test reads `HOMECLOUD_IMAGE` (default `DefaultImage`). diff --git a/integrations/testcontainers-go/go.mod b/integrations/testcontainers-go/go.mod new file mode 100644 index 0000000..489ec01 --- /dev/null +++ b/integrations/testcontainers-go/go.mod @@ -0,0 +1,77 @@ +module github.com/solinode/homecloud/integrations/testcontainers-go + +go 1.25.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.47.1 + github.com/aws/aws-sdk-go-v2/credentials v1.20.7 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1 + github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 + github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 + github.com/moby/moby/api v1.55.0 + github.com/moby/moby/client v0.5.0 + github.com/testcontainers/testcontainers-go v0.44.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect + github.com/aws/smithy-go v1.28.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.7 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.3.3 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.1 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sys v0.47.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/integrations/testcontainers-go/go.sum b/integrations/testcontainers-go/go.sum new file mode 100644 index 0000000..2889f67 --- /dev/null +++ b/integrations/testcontainers-go/go.sum @@ -0,0 +1,173 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/credentials v1.20.7 h1:mit8rRVnmt+wvT4U6putiWSlwe3eKD2n6ElgyfcP/Mw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.7/go.mod h1:/2pPi2qhPug21z6l3M0+AKUqAT1zn2MhRCtnQv8R5Zg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1 h1:kEs6rZI/z6aD9JBIjXzaXA2UyGYqhpQrxDnmcpU9fGA= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1/go.mod h1:Gm+i2GlUsFNlzoBq8VXF44XHbKANn3tV8nYBBp3rN8Q= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 h1:6HvmOQ1rBRrZ4qPJSWxd5szPKUsngXCwSw+V3UaJHmw= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4/go.mod h1:zv2N29aiQUhG2XZNM9zgwCnAyVBdTBbcIpfNAlNmA20= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0 h1:/P13I8ouMzMIa2yY32aUh+GIh0wYbJ/auQb1KAEhB8k= +github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0/go.mod h1:jUmFXtUKRVCKTaKap+NgL32pmSkVehamqqMENlGMApk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2 h1:6ZXe/lG2mbGhw10y/aiKbe/poytLoYJsdbKV1pT9+ws= +github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2/go.mod h1:+TDqZ1h8CLkW9ewfQkSPWHYRjm7/wDThKeDlR46qyvE= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 h1:AWSPqynwEJV8J2pl/p7aizCf7a23/NGHR+rK/KLOxi4= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.2/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.3.3 h1:OxxR9paxsluYi+zDUEXTTaIxtkK3viymW+Ka7vRhhME= +github.com/moby/go-archive v0.3.3/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00= +github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= +github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= +github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/integrations/testcontainers-go/homecloud.go b/integrations/testcontainers-go/homecloud.go new file mode 100644 index 0000000..89aaeab --- /dev/null +++ b/integrations/testcontainers-go/homecloud.go @@ -0,0 +1,331 @@ +// Package homecloud is a testcontainers-go module for HomeCloud, a self-hosted +// AWS: one container that speaks the AWS APIs (S3, SQS, DynamoDB, Lambda, ...) +// on a single port with SigV4. +// +// hc, err := homecloud.Run(ctx, homecloud.DefaultImage) +// defer testcontainers.TerminateContainer(hc) +// cfg := hc.AWSConfig() +// s3c := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true }) +// +// HomeCloud runs each service's backing containers (MinIO, Lambda runtimes, +// databases) on the Docker host itself, so the container mounts the Docker +// socket and uses host networking: it reaches those containers on the host's +// loopback ports, and they reach it back through host.docker.internal. That +// works on Linux, OrbStack and Docker Desktop with host networking enabled. +// +// One HomeCloud runs per Docker host at a time (its helper containers have fixed +// names and host ports), so tests that use it must not run in parallel with +// each other or next to a HomeCloud you run yourself on the same Docker host. +package homecloud + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + // DefaultImage is the official HomeCloud image. + DefaultImage = "ghcr.io/solinode/homecloud:latest" + // DefaultPort is the port the API listens on, on the Docker host. It is not + // HomeCloud's usual 8080, which is often taken on developer machines. + DefaultPort = 18080 + // DefaultDockerSocket is the Docker socket on the Docker host. + DefaultDockerSocket = "/var/run/docker.sock" + + dataDir = "/data" + accountLabel = "homecloud.account" +) + +// readyLines are the log lines of services that start containers in the background. +var readyLines = map[string]string{ + "s3": "s3: MinIO ready", + "ecr": "ecr: registry ready", + "route53": "route53: DNS ready", +} + +// Credentials are the root access key HomeCloud creates on first start. +type Credentials struct { + AccessKeyID string + SecretAccessKey string + Region string +} + +// Container is a running HomeCloud. +type Container struct { + testcontainers.Container + endpoint string + creds Credentials + account string +} + +type options struct { + port int + socket string + services []string +} + +// Option configures Run. It also satisfies testcontainers.ContainerCustomizer, +// so it can be passed next to the generic testcontainers options. +type Option func(*options) + +// Customize is a no-op: Options are read by Run itself. +func (Option) Customize(*testcontainers.GenericContainerRequest) error { return nil } + +// WithPort sets the port the API listens on, on the Docker host (default DefaultPort). +func WithPort(port int) Option { return func(o *options) { o.port = port } } + +// WithDockerSocket sets the path of the Docker socket on the Docker host +// (default /var/run/docker.sock, which is right for Linux, OrbStack and Docker Desktop). +func WithDockerSocket(path string) Option { return func(o *options) { o.socket = path } } + +// WithWaitForServices sets which background services Run waits for besides the +// API: "s3", "ecr", "route53". The default is "s3". Every other service runs +// inside the HomeCloud process and is ready with the API. +func WithWaitForServices(services ...string) Option { + return func(o *options) { o.services = services } +} + +// Run starts HomeCloud from img (DefaultImage, or a locally built image) and +// waits until the API and the requested services are ready. +func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustomizer) (*Container, error) { + o := options{port: DefaultPort, socket: DefaultDockerSocket, services: []string{"s3"}} + for _, opt := range opts { + if f, ok := opt.(Option); ok { + f(&o) + } + } + strategies := []wait.Strategy{wait.ForLog("listening on")} + for _, s := range o.services { + line, ok := readyLines[strings.ToLower(s)] + if !ok { + continue + } + strategies = append(strategies, wait.ForLog(line)) + } + base := []testcontainers.ContainerCustomizer{ + testcontainers.WithCmd("serve", "--data-dir", dataDir, "--addr", "127.0.0.1:"+strconv.Itoa(o.port)), + testcontainers.WithEnv(map[string]string{"HOMECLOUD_DATA_DIR": dataDir}), + testcontainers.WithHostConfigModifier(func(hc *container.HostConfig) { + hc.NetworkMode = "host" + hc.Binds = append(hc.Binds, o.socket+":/var/run/docker.sock") + }), + testcontainers.WithWaitStrategyAndDeadline(3*time.Minute, wait.ForAll(strategies...)), + } + ctr, err := testcontainers.Run(ctx, img, append(base, opts...)...) + if ctr == nil { + return nil, fmt.Errorf("run homecloud: %w", err) + } + c := &Container{Container: ctr} + // The account comes first, even when the start failed: HomeCloud may already + // have created helper containers, and Terminate finds them by account. + account, accErr := c.readAccount() + c.account = account + if err != nil { + return c, fmt.Errorf("run homecloud: %w", err) + } + if accErr != nil { + return c, accErr + } + host, err := ctr.Host(ctx) + if err != nil { + return c, err + } + if host == "localhost" { + host = "127.0.0.1" // keep SDKs from trying IPv6 first or bucket.localhost names + } + c.endpoint = "http://" + host + ":" + strconv.Itoa(o.port) + if err := c.waitHealthy(ctx, 30*time.Second); err != nil { + return c, err + } + if c.creds, err = c.readCredentials(ctx); err != nil { + return c, err + } + return c, nil +} + +// EndpointURL is the URL of the HomeCloud API, e.g. http://127.0.0.1:18080: +// what AWS_ENDPOINT_URL or an SDK's base endpoint should be set to. +func (c *Container) EndpointURL() string { return c.endpoint } + +// Endpoint returns the API address as testcontainers.Container.Endpoint does +// ("127.0.0.1:18080", or "http://127.0.0.1:18080" for proto "http"). It +// replaces the generic one, which looks for a mapped port and so does not work +// with host networking. +func (c *Container) Endpoint(_ context.Context, proto string) (string, error) { + addr := strings.TrimPrefix(c.endpoint, "http://") + if proto == "" { + return addr, nil + } + return proto + "://" + addr, nil +} + +// Credentials returns the root access key and the region. +func (c *Container) Credentials() Credentials { return c.creds } + +// AccountID is the AWS account ID HomeCloud created on first start. +func (c *Container) AccountID() string { return c.account } + +// AWSConfig returns an aws-sdk-go-v2 config pointed at this HomeCloud. Pass +// it to any service client's NewFromConfig. For S3 also set UsePathStyle. +func (c *Container) AWSConfig() aws.Config { + return aws.Config{ + Region: c.creds.Region, + Credentials: credentials.NewStaticCredentialsProvider(c.creds.AccessKeyID, c.creds.SecretAccessKey, ""), + BaseEndpoint: aws.String(c.endpoint), + } +} + +// Env returns the AWS_* environment variables for tools that read them (AWS +// CLI, Terraform, SDKs in a subprocess). +func (c *Container) Env() map[string]string { + return map[string]string{ + "AWS_ENDPOINT_URL": c.endpoint, + "AWS_ACCESS_KEY_ID": c.creds.AccessKeyID, + "AWS_SECRET_ACCESS_KEY": c.creds.SecretAccessKey, + "AWS_REGION": c.creds.Region, + "AWS_DEFAULT_REGION": c.creds.Region, + } +} + +// Terminate stops HomeCloud and removes every container, network and volume it +// created on the Docker host (MinIO, Lambda runtimes, databases, ...). +func (c *Container) Terminate(ctx context.Context, opts ...testcontainers.TerminateOption) error { + if c == nil || c.Container == nil { + return nil + } + err := c.Container.Terminate(ctx, opts...) + return errors.Join(err, RemoveResources(ctx, c.account)) +} + +// RemoveResources removes the containers, networks and volumes HomeCloud +// created for account on the Docker host. Terminate calls it; call it yourself +// to clean up after a test process that was killed. +func RemoveResources(ctx context.Context, account string) error { + if account == "" { + return nil + } + cli, err := testcontainers.NewDockerClientWithOpts(ctx) + if err != nil { + return err + } + defer cli.Close() + f := client.Filters{}.Add("label", accountLabel+"="+account) + var errs []error + cs, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: f}) + errs = append(errs, err) + for _, ct := range cs.Items { + _, err := cli.ContainerRemove(ctx, ct.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}) + errs = append(errs, err) + } + ns, err := cli.NetworkList(ctx, client.NetworkListOptions{Filters: f}) + errs = append(errs, err) + for _, n := range ns.Items { + _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}) + errs = append(errs, err) + } + vs, err := cli.VolumeList(ctx, client.VolumeListOptions{Filters: f}) + errs = append(errs, err) + for _, v := range vs.Items { + _, err := cli.VolumeRemove(ctx, v.Name, client.VolumeRemoveOptions{Force: true}) + errs = append(errs, err) + } + return errors.Join(errs...) +} + +var accountRE = regexp.MustCompile(`first start: created account (\d+)`) + +// readAccount finds the account in the first-start log. It uses its own +// context, so it still works when the caller's expired during a failed start. +func (c *Container) readAccount() (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + rc, err := c.Logs(ctx) + if err != nil { + return "", fmt.Errorf("homecloud: read log: %w", err) + } + defer rc.Close() + b, err := io.ReadAll(rc) + if err != nil { + return "", fmt.Errorf("homecloud: read log: %w", err) + } + return accountFromLog(b) +} + +func accountFromLog(b []byte) (string, error) { + m := accountRE.FindSubmatch(b) + if m == nil { + return "", errors.New("homecloud: no account in the first-start log") + } + return string(m[1]), nil +} + +func (c *Container) waitHealthy(ctx context.Context, d time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, d) + defer cancel() + var last error + for { + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint+"/api/v1/health", nil) + resp, err := http.DefaultClient.Do(req) + if err == nil { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + err = fmt.Errorf("status %s", resp.Status) + } + last = err + select { + case <-ctx.Done(): + return fmt.Errorf("homecloud: %s/api/v1/health not reachable (does this Docker setup support host networking?): %w", c.endpoint, last) + case <-time.After(250 * time.Millisecond): + } + } +} + +// readCredentials runs `homecloud aws-env` in the container. +func (c *Container) readCredentials(ctx context.Context) (Credentials, error) { + code, r, err := c.Exec(ctx, []string{"homecloud", "aws-env"}, tcexec.Multiplexed()) + if err != nil { + return Credentials{}, fmt.Errorf("homecloud aws-env: %w", err) + } + out, _ := io.ReadAll(r) + if code != 0 { + return Credentials{}, fmt.Errorf("homecloud aws-env exited %d: %s", code, out) + } + vars := map[string]string{} + sc := bufio.NewScanner(strings.NewReader(string(out))) + for sc.Scan() { + k, v, ok := strings.Cut(strings.TrimPrefix(sc.Text(), "export "), "=") + if !ok { + continue + } + if u, err := strconv.Unquote(v); err == nil { + v = u + } + vars[k] = v + } + cr := Credentials{AccessKeyID: vars["AWS_ACCESS_KEY_ID"], SecretAccessKey: vars["AWS_SECRET_ACCESS_KEY"], Region: vars["AWS_REGION"]} + if cr.AccessKeyID == "" || cr.SecretAccessKey == "" { + return cr, fmt.Errorf("homecloud aws-env printed no credentials: %s", out) + } + if cr.Region == "" { + cr.Region = "us-east-1" + } + return cr, nil +} diff --git a/integrations/testcontainers-go/homecloud_test.go b/integrations/testcontainers-go/homecloud_test.go new file mode 100644 index 0000000..127cde4 --- /dev/null +++ b/integrations/testcontainers-go/homecloud_test.go @@ -0,0 +1,180 @@ +package homecloud_test + +import ( + "context" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/aws-sdk-go-v2/service/lambda" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/moby/moby/client" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + homecloud "github.com/solinode/homecloud/integrations/testcontainers-go" +) + +// image is HOMECLOUD_IMAGE (e.g. a local build of integrations/testdata/Dockerfile) +// or the official image. +func image() string { + if v := os.Getenv("HOMECLOUD_IMAGE"); v != "" { + return v + } + return homecloud.DefaultImage +} + +func TestHomeCloud(t *testing.T) { + testcontainers.SkipIfProviderIsNotHealthy(t) + ctx := context.Background() + + hc, err := homecloud.Run(ctx, image()) + if hc != nil { + t.Cleanup(func() { + if err := testcontainers.TerminateContainer(hc); err != nil { + t.Errorf("terminate: %v", err) + } + assertNoResources(t, hc.AccountID()) + }) + } + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(hc.EndpointURL(), "http://") || hc.Credentials().AccessKeyID == "" || hc.AccountID() == "" { + t.Fatalf("endpoint %q, credentials %+v, account %q", hc.EndpointURL(), hc.Credentials().AccessKeyID, hc.AccountID()) + } + cfg := hc.AWSConfig() + + id, err := sts.NewFromConfig(cfg).GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + t.Fatalf("sts: %v", err) + } + if aws.ToString(id.Account) != hc.AccountID() { + t.Errorf("caller account %s, want %s", aws.ToString(id.Account), hc.AccountID()) + } + + s3c := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true }) + if _, err := s3c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String("tc-go")}); err != nil { + t.Fatalf("s3 create bucket: %v", err) + } + if _, err := s3c.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String("tc-go"), Key: aws.String("a.txt"), Body: strings.NewReader("hello")}); err != nil { + t.Fatalf("s3 put: %v", err) + } + obj, err := s3c.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String("tc-go"), Key: aws.String("a.txt")}) + if err != nil { + t.Fatalf("s3 get: %v", err) + } + b, _ := io.ReadAll(obj.Body) + obj.Body.Close() + if string(b) != "hello" { + t.Errorf("s3 body %q", b) + } + + sq := sqs.NewFromConfig(cfg) + q, err := sq.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("tc-go")}) + if err != nil { + t.Fatalf("sqs create: %v", err) + } + if _, err := sq.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: q.QueueUrl, MessageBody: aws.String("hi")}); err != nil { + t.Fatalf("sqs send: %v", err) + } + msgs, err := sq.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: q.QueueUrl, WaitTimeSeconds: 1}) + if err != nil || len(msgs.Messages) != 1 || aws.ToString(msgs.Messages[0].Body) != "hi" { + t.Fatalf("sqs receive: %v %+v", err, msgs) + } + + ddb := dynamodb.NewFromConfig(cfg) + if _, err := ddb.CreateTable(ctx, &dynamodb.CreateTableInput{ + TableName: aws.String("tc-go"), + AttributeDefinitions: []ddbtypes.AttributeDefinition{{AttributeName: aws.String("id"), AttributeType: ddbtypes.ScalarAttributeTypeS}}, + KeySchema: []ddbtypes.KeySchemaElement{{AttributeName: aws.String("id"), KeyType: ddbtypes.KeyTypeHash}}, + BillingMode: ddbtypes.BillingModePayPerRequest, + }); err != nil { + t.Fatalf("dynamodb create: %v", err) + } + item := map[string]ddbtypes.AttributeValue{"id": &ddbtypes.AttributeValueMemberS{Value: "1"}} + if _, err := ddb.PutItem(ctx, &dynamodb.PutItemInput{TableName: aws.String("tc-go"), Item: item}); err != nil { + t.Fatalf("dynamodb put: %v", err) + } + got, err := ddb.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String("tc-go"), Key: item}) + if err != nil || got.Item["id"] == nil { + t.Fatalf("dynamodb get: %v %+v", err, got) + } + + if _, err := lambda.NewFromConfig(cfg).ListFunctions(ctx, &lambda.ListFunctionsInput{}); err != nil { + t.Fatalf("lambda list: %v", err) + } +} + +// TestRunFailureStillCleansUp makes Run fail after HomeCloud has started (and +// created its helper containers): the returned container must still know its +// account, so that Terminate removes them. +func TestRunFailureStillCleansUp(t *testing.T) { + testcontainers.SkipIfProviderIsNotHealthy(t) + ctx := context.Background() + + hc, err := homecloud.Run(ctx, image(), testcontainers.WithAdditionalWaitStrategyAndDeadline( + 20*time.Second, wait.ForLog("s3: MinIO ready"), wait.ForLog("a line HomeCloud never logs"))) + if err == nil { + t.Error("Run succeeded; want a wait-strategy failure") + } + if hc == nil { + t.Fatal("Run returned no container") + } + account := hc.AccountID() + if account == "" { + t.Error("no account after a failed start") + } + if n := countResources(t, account); n == 0 { + t.Error("HomeCloud created no helper resources before the failure; the test proves nothing") + } + if err := testcontainers.TerminateContainer(hc); err != nil { + t.Errorf("terminate: %v", err) + } + assertNoResources(t, account) +} + +// assertNoResources checks that Terminate left nothing of the account behind. +func assertNoResources(t *testing.T, account string) { + t.Helper() + if n := countResources(t, account); n > 0 { + t.Errorf("left behind: %d containers, networks and volumes of account %s", n, account) + } +} + +// countResources counts the containers, networks and volumes of account. +func countResources(t *testing.T, account string) int { + t.Helper() + if account == "" { + t.Error("no account to look up") + return -1 + } + ctx := context.Background() + cli, err := testcontainers.NewDockerClientWithOpts(ctx) + if err != nil { + t.Error(err) + return -1 + } + defer cli.Close() + f := client.Filters{}.Add("label", "homecloud.account="+account) + cs, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: f}) + if err != nil { + t.Errorf("list containers: %v", err) + } + ns, err := cli.NetworkList(ctx, client.NetworkListOptions{Filters: f}) + if err != nil { + t.Errorf("list networks: %v", err) + } + vs, err := cli.VolumeList(ctx, client.VolumeListOptions{Filters: f}) + if err != nil { + t.Errorf("list volumes: %v", err) + } + return len(cs.Items) + len(ns.Items) + len(vs.Items) +} diff --git a/integrations/testcontainers-python/.gitignore b/integrations/testcontainers-python/.gitignore new file mode 100644 index 0000000..47584ad --- /dev/null +++ b/integrations/testcontainers-python/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.egg-info/ +build/ +dist/ +.venv/ diff --git a/integrations/testcontainers-python/README.md b/integrations/testcontainers-python/README.md new file mode 100644 index 0000000..6b31a39 --- /dev/null +++ b/integrations/testcontainers-python/README.md @@ -0,0 +1,60 @@ +# testcontainers-homecloud + +A [testcontainers-python](https://testcontainers-python.readthedocs.io/) module for +[HomeCloud](https://github.com/solinode/homecloud), a self-hosted AWS: start it from a +test and talk to it with boto3. + +```sh +pip install "testcontainers-homecloud @ git+https://github.com/solinode/homecloud#subdirectory=integrations/testcontainers-python" +``` + +```python +import pytest +from testcontainers_homecloud import HomeCloudContainer + +@pytest.fixture(scope="session") +def homecloud(): + with HomeCloudContainer() as hc: # stop() also removes MinIO, Lambda runtimes, ... it started + yield hc + +def test_upload(homecloud): + s3 = homecloud.get_client("s3") # boto3 client: endpoint, root keys, region, path-style S3 + s3.create_bucket(Bucket="test") + s3.put_object(Bucket="test", Key="a.txt", Body=b"hi") +``` + +## API + +| | | +| --- | --- | +| `HomeCloudContainer(image=DEFAULT_IMAGE, port=18080, docker_socket="/var/run/docker.sock", wait_for_services=("s3",), startup_timeout=180)` | `wait_for_services` takes `s3`, `ecr`, `route53`; everything else is ready with the API | +| `get_endpoint()` | `http://127.0.0.1:18080` | +| `get_credentials()` | `Credentials(access_key_id, secret_access_key, region)` | +| `get_client(service, **kw)` / `get_resource(service, **kw)` | boto3 client / resource for HomeCloud | +| `aws_env()` | `AWS_ENDPOINT_URL`, keys and region for subprocesses (AWS CLI, Terraform) | +| `account_id` | the account HomeCloud created | +| `stop()` | stops HomeCloud and removes every container, network and volume it created | +| `remove_resources(docker_client, account_id)` | the clean-up alone, for runs that were killed | + +## How it runs + +HomeCloud starts each service's backing containers on the Docker host, so the container +gets the Docker socket and **host networking**; the API listens on `127.0.0.1:` of +the Docker host. This works on Linux (including GitHub Actions), OrbStack, and Docker +Desktop with host networking turned on. + +Only one HomeCloud can run per Docker host: its helper containers have fixed names and +host ports (9500, 9501, 5500, 8053). Use one session-scoped fixture, do not run it under +`pytest-xdist`, and stop any HomeCloud you run yourself on the same Docker host first. + +## Image + +`DEFAULT_IMAGE` is `ghcr.io/solinode/homecloud:latest`. To test against a build from a +checkout: + +```sh +docker build -f integrations/testdata/Dockerfile -t homecloud:test cli +cd integrations/testcontainers-python +pip install -e '.[test]' +HOMECLOUD_IMAGE=homecloud:test pytest +``` diff --git a/integrations/testcontainers-python/pyproject.toml b/integrations/testcontainers-python/pyproject.toml new file mode 100644 index 0000000..34d03dd --- /dev/null +++ b/integrations/testcontainers-python/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "testcontainers-homecloud" +version = "0.1.0" +description = "testcontainers module for HomeCloud, a self-hosted AWS" +readme = "README.md" +license = "AGPL-3.0-only" +requires-python = ">=3.9" +dependencies = ["testcontainers>=4.0", "boto3>=1.28"] + +[project.optional-dependencies] +test = ["pytest>=7"] + +[project.urls] +Homepage = "https://github.com/solinode/homecloud" +Source = "https://github.com/solinode/homecloud/tree/main/integrations/testcontainers-python" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/integrations/testcontainers-python/src/testcontainers_homecloud/__init__.py b/integrations/testcontainers-python/src/testcontainers_homecloud/__init__.py new file mode 100644 index 0000000..f0550db --- /dev/null +++ b/integrations/testcontainers-python/src/testcontainers_homecloud/__init__.py @@ -0,0 +1,222 @@ +"""testcontainers module for HomeCloud, a self-hosted AWS. + + from testcontainers_homecloud import HomeCloudContainer + + with HomeCloudContainer() as hc: + s3 = hc.get_client("s3") + s3.create_bucket(Bucket="test") + +HomeCloud starts each service's backing containers (MinIO for S3, Lambda runtimes, +databases) on the Docker host, so the container mounts the Docker socket and uses +host networking: it reaches those containers on the host's loopback ports and they +call it back through host.docker.internal. That works on Linux, OrbStack and Docker +Desktop with host networking enabled. + +Only one HomeCloud runs per Docker host at a time (its helper containers have fixed +names and host ports): don't run tests that use it in parallel. +""" + +from __future__ import annotations + +import re +import shlex +import time +import urllib.request +from typing import Any, Iterable, NamedTuple, Optional + +from testcontainers.core.container import DockerContainer + +__all__ = ["HomeCloudContainer", "Credentials", "DEFAULT_IMAGE", "DEFAULT_PORT", "remove_resources"] + +DEFAULT_IMAGE = "ghcr.io/solinode/homecloud:latest" +# Not HomeCloud's usual 8080, which is often taken on developer machines. +DEFAULT_PORT = 18080 +DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock" + +_DATA_DIR = "/data" +_ACCOUNT_LABEL = "homecloud.account" +# Log lines of the services that start containers in the background. +_READY_LINES = { + "s3": "s3: MinIO ready", + "ecr": "ecr: registry ready", + "route53": "route53: DNS ready", +} +_ACCOUNT_RE = re.compile(r"first start: created account (\d+)") + + +class Credentials(NamedTuple): + access_key_id: str + secret_access_key: str + region: str + + +class HomeCloudContainer(DockerContainer): + """A HomeCloud server for tests. + + :param image: HomeCloud image; any image whose entrypoint is the ``homecloud`` binary. + :param port: port the API listens on, on the Docker host's 127.0.0.1. + :param docker_socket: path of the Docker socket on the Docker host. + :param wait_for_services: background services to wait for besides the API + ("s3", "ecr", "route53"); every other service is ready with the API. + :param startup_timeout: seconds to wait for all of it. + """ + + def __init__( + self, + image: str = DEFAULT_IMAGE, + port: int = DEFAULT_PORT, + docker_socket: str = DEFAULT_DOCKER_SOCKET, + wait_for_services: Iterable[str] = ("s3",), + startup_timeout: float = 180, + **kwargs: Any, + ) -> None: + # Host networking is required (see the module docstring); every other + # Docker option the caller passes (platform, mem_limit, ...) is kept. + kwargs["network_mode"] = "host" + super().__init__(image, **kwargs) + self.port = port + self.wait_for_services = [s.lower() for s in wait_for_services] + self.startup_timeout = startup_timeout + self.account_id: Optional[str] = None + self._credentials: Optional[Credentials] = None + self.with_command(["serve", "--data-dir", _DATA_DIR, "--addr", f"127.0.0.1:{port}"]) + self.with_env("HOMECLOUD_DATA_DIR", _DATA_DIR) + self.with_volume_mapping(docker_socket, "/var/run/docker.sock", "rw") + + # Lifecycle + + def start(self) -> "HomeCloudContainer": + super().start() + deadline = time.monotonic() + self.startup_timeout + wanted = ["listening on"] + [_READY_LINES[s] for s in self.wait_for_services if s in _READY_LINES] + while True: + logs = self._logs() + m = _ACCOUNT_RE.search(logs) + if m: # known as early as possible, so stop() cleans up after a failed start too + self.account_id = m.group(1) + if all(w in logs for w in wanted): + break + self.reload() + if self.status not in ("created", "running"): + raise RuntimeError(f"HomeCloud exited during startup:\n{logs}") + if time.monotonic() > deadline: + raise TimeoutError(f"HomeCloud not ready after {self.startup_timeout}s (waiting for {wanted}):\n{logs}") + time.sleep(0.5) + self._wait_healthy(deadline) + self._credentials = self._read_credentials() + return self + + def stop(self, force: bool = True, delete_volume: bool = True) -> None: + """Stop HomeCloud and remove every container, network and volume it created.""" + # Stop HomeCloud first so it does not recreate what is being removed; + # super().stop() then only closes the Docker client. + if self._container: + self._container.remove(force=force, v=delete_volume) + self._container = None + try: + if self.account_id: + remove_resources(self.get_docker_client().client, self.account_id) + finally: + super().stop(force=force, delete_volume=delete_volume) + + # Connection details + + def get_endpoint(self) -> str: + """The API URL, e.g. http://127.0.0.1:18080 (what AWS_ENDPOINT_URL should be).""" + host = self.get_container_host_ip() + if host == "localhost": + host = "127.0.0.1" + return f"http://{host}:{self.port}" + + def get_credentials(self) -> Credentials: + """The root access key HomeCloud created, and its region.""" + if self._credentials is None: + raise RuntimeError("HomeCloud is not started") + return self._credentials + + def aws_env(self) -> dict[str, str]: + """AWS_* environment variables for the AWS CLI, Terraform or a subprocess.""" + c = self.get_credentials() + return { + "AWS_ENDPOINT_URL": self.get_endpoint(), + "AWS_ACCESS_KEY_ID": c.access_key_id, + "AWS_SECRET_ACCESS_KEY": c.secret_access_key, + "AWS_REGION": c.region, + "AWS_DEFAULT_REGION": c.region, + } + + def _session_kwargs(self) -> dict[str, Any]: + c = self.get_credentials() + return { + "endpoint_url": self.get_endpoint(), + "aws_access_key_id": c.access_key_id, + "aws_secret_access_key": c.secret_access_key, + "region_name": c.region, + } + + def get_client(self, service: str, **kwargs: Any) -> Any: + """A boto3 client for service, pointed at HomeCloud (S3 uses path-style addressing).""" + import boto3 + from botocore.config import Config + + if service == "s3" and "config" not in kwargs: + kwargs["config"] = Config(s3={"addressing_style": "path"}) + return boto3.client(service, **{**self._session_kwargs(), **kwargs}) + + def get_resource(self, service: str, **kwargs: Any) -> Any: + """A boto3 resource (s3, dynamodb, sqs, ...) pointed at HomeCloud.""" + import boto3 + from botocore.config import Config + + if service == "s3" and "config" not in kwargs: + kwargs["config"] = Config(s3={"addressing_style": "path"}) + return boto3.resource(service, **{**self._session_kwargs(), **kwargs}) + + # Internals + + def _logs(self) -> str: + out, err = self.get_logs() + return out.decode(errors="replace") + err.decode(errors="replace") + + def _wait_healthy(self, deadline: float) -> None: + url = self.get_endpoint() + "/api/v1/health" + last: Exception = RuntimeError("not tried") + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as r: + if r.status == 200: + return + except Exception as e: # noqa: BLE001 + last = e + time.sleep(0.25) + raise TimeoutError(f"{url} not reachable (does this Docker setup support host networking?): {last}") + + def _read_credentials(self) -> Credentials: + code, out = self.exec(["homecloud", "aws-env"]) + text = out.decode(errors="replace") + if code != 0: + raise RuntimeError(f"homecloud aws-env exited {code}: {text}") + env: dict[str, str] = {} + for line in text.splitlines(): + line = line.removeprefix("export ") + key, sep, value = line.partition("=") + if sep: + env[key] = (shlex.split(value) or [""])[0] + if not env.get("AWS_ACCESS_KEY_ID") or not env.get("AWS_SECRET_ACCESS_KEY"): + raise RuntimeError(f"homecloud aws-env printed no credentials: {text}") + return Credentials(env["AWS_ACCESS_KEY_ID"], env["AWS_SECRET_ACCESS_KEY"], env.get("AWS_REGION") or "us-east-1") + + +def remove_resources(client: Any, account_id: str) -> None: + """Remove the containers, networks and volumes HomeCloud created for account_id. + + ``client`` is a ``docker.DockerClient``. HomeCloudContainer.stop() calls this; + call it yourself to clean up after a test run that was killed. + """ + flt = {"label": f"{_ACCOUNT_LABEL}={account_id}"} + for c in client.containers.list(all=True, filters=flt): + c.remove(force=True, v=True) + for n in client.networks.list(filters=flt): + n.remove() + for v in client.volumes.list(filters=flt): + v.remove(force=True) diff --git a/integrations/testcontainers-python/tests/test_homecloud.py b/integrations/testcontainers-python/tests/test_homecloud.py new file mode 100644 index 0000000..c792ce3 --- /dev/null +++ b/integrations/testcontainers-python/tests/test_homecloud.py @@ -0,0 +1,73 @@ +import os + +import docker +import pytest + +from testcontainers_homecloud import DEFAULT_IMAGE, HomeCloudContainer + +# A local build of integrations/testdata/Dockerfile, or the official image. +IMAGE = os.environ.get("HOMECLOUD_IMAGE", DEFAULT_IMAGE) + + +def leftovers(account_id): + flt = {"label": f"homecloud.account={account_id}"} + dc = docker.from_env() + try: + return ( + len(dc.containers.list(all=True, filters=flt)), + len(dc.networks.list(filters=flt)), + len(dc.volumes.list(filters=flt)), + ) + finally: + dc.close() + + +@pytest.fixture(scope="module") +def homecloud(): + hc = HomeCloudContainer(IMAGE) + with hc: + yield hc + assert hc.account_id + assert leftovers(hc.account_id) == (0, 0, 0), "stop() left HomeCloud resources behind" + + +def test_connection_details(homecloud): + assert homecloud.get_endpoint().startswith("http://") + creds = homecloud.get_credentials() + assert creds.access_key_id and creds.secret_access_key and creds.region + assert homecloud.aws_env()["AWS_ENDPOINT_URL"] == homecloud.get_endpoint() + ident = homecloud.get_client("sts").get_caller_identity() + assert ident["Account"] == homecloud.account_id + + +def test_s3(homecloud): + s3 = homecloud.get_client("s3") + s3.create_bucket(Bucket="tc-python") + s3.put_object(Bucket="tc-python", Key="a.txt", Body=b"hello") + assert s3.get_object(Bucket="tc-python", Key="a.txt")["Body"].read() == b"hello" + bucket = homecloud.get_resource("s3").Bucket("tc-python") + assert [o.key for o in bucket.objects.all()] == ["a.txt"] + + +def test_sqs(homecloud): + sqs = homecloud.get_client("sqs") + url = sqs.create_queue(QueueName="tc-python")["QueueUrl"] + sqs.send_message(QueueUrl=url, MessageBody="hi") + msgs = sqs.receive_message(QueueUrl=url, WaitTimeSeconds=1)["Messages"] + assert [m["Body"] for m in msgs] == ["hi"] + + +def test_dynamodb(homecloud): + ddb = homecloud.get_resource("dynamodb") + table = ddb.create_table( + TableName="tc-python", + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + table.put_item(Item={"id": "1", "n": 2}) + assert table.get_item(Key={"id": "1"})["Item"]["n"] == 2 + + +def test_lambda_list(homecloud): + assert homecloud.get_client("lambda").list_functions()["Functions"] == [] diff --git a/integrations/testcontainers-python/tests/test_options.py b/integrations/testcontainers-python/tests/test_options.py new file mode 100644 index 0000000..4185beb --- /dev/null +++ b/integrations/testcontainers-python/tests/test_options.py @@ -0,0 +1,28 @@ +"""Unit tests that need no running container.""" + +from unittest import mock + +from testcontainers_homecloud import HomeCloudContainer + + +def make(**kwargs): + # Creating a DockerContainer connects to Docker; these tests only inspect options. + with mock.patch("testcontainers.core.container.DockerClient"): + return HomeCloudContainer("homecloud:test", **kwargs) + + +def test_caller_docker_options_are_kept(): + hc = make(mem_limit="1g", platform="linux/amd64") + assert hc._kwargs["mem_limit"] == "1g" + assert hc._kwargs["platform"] == "linux/amd64" + assert hc._kwargs["network_mode"] == "host" + + +def test_host_network_cannot_be_overridden(): + assert make(network_mode="bridge")._kwargs["network_mode"] == "host" + + +def test_command_and_socket(): + hc = make(port=19090, docker_socket="/run/user/1000/docker.sock") + assert hc._command == ["serve", "--data-dir", "/data", "--addr", "127.0.0.1:19090"] + assert hc.volumes["/run/user/1000/docker.sock"]["bind"] == "/var/run/docker.sock" diff --git a/integrations/testdata/Dockerfile b/integrations/testdata/Dockerfile new file mode 100644 index 0000000..4c478ed --- /dev/null +++ b/integrations/testdata/Dockerfile @@ -0,0 +1,23 @@ +# Test-only HomeCloud image for the testcontainers modules, used until the +# official ghcr.io/solinode/homecloud image is published. Build context: cli/ +# +# docker build -f integrations/testdata/Dockerfile -t homecloud:test cli +# +# The image's entrypoint is the homecloud binary, so `docker run IMAGE serve ...` +# and `docker exec CONTAINER homecloud aws-env` both work. HomeCloud drives the +# host's Docker through the mounted socket and must run with host networking +# (see docs/integrations.md). +FROM golang:1.25-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X github.com/homecloudhq/homecloud/cli/cmd.Version=test" -o /out/homecloud . + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates +COPY --from=build /out/homecloud /usr/local/bin/homecloud +ENV HOMECLOUD_DATA_DIR=/data +VOLUME /data +ENTRYPOINT ["homecloud"] +CMD ["serve", "--data-dir", "/data", "--addr", "0.0.0.0:8080"]