diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 277fe8c..b88929f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: strategy: fail-fast: false matrix: - module: ["dbkit/postgres", "gonsole/auth/postgres"] + module: ["dbkit/postgres", "gonsole/auth/postgres", "postgresblob"] defaults: run: working-directory: ${{ matrix.module }} @@ -99,7 +99,7 @@ jobs: strategy: fail-fast: false matrix: - module: ["dbkit", "dbkit/postgres", "dbkit/sqlite", "mailkit", "goncierge", "gonsole", "gonsole/auth", "gonsole/auth/postgres", "gonsole/auth/sqlite", "pluginkit", "pluginkit/graphwire"] + module: ["dbkit", "dbkit/postgres", "dbkit/sqlite", "mailkit", "goncierge", "gonsole", "gonsole/auth", "gonsole/auth/postgres", "gonsole/auth/sqlite", "pluginkit", "pluginkit/graphwire", "postgresblob"] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -121,7 +121,7 @@ jobs: strategy: fail-fast: false matrix: - module: ["dbkit", "dbkit/postgres", "dbkit/sqlite", "mailkit", "goncierge", "gonsole", "gonsole/auth", "gonsole/auth/postgres", "gonsole/auth/sqlite", "pluginkit", "pluginkit/graphwire"] + module: ["dbkit", "dbkit/postgres", "dbkit/sqlite", "mailkit", "goncierge", "gonsole", "gonsole/auth", "gonsole/auth/postgres", "gonsole/auth/sqlite", "pluginkit", "pluginkit/graphwire", "postgresblob"] defaults: run: working-directory: ${{ matrix.module }} diff --git a/.github/workflows/gocloud-master.yml b/.github/workflows/gocloud-master.yml new file mode 100644 index 0000000..93eb497 --- /dev/null +++ b/.github/workflows/gocloud-master.yml @@ -0,0 +1,45 @@ +name: gocloud-master + +on: + schedule: + - cron: "17 4 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + postgresblob: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + DBKIT_TEST_POSTGRES_URL: "postgres://postgres:postgres@localhost:5434/postgres?sslmode=disable" + defaults: + run: + working-directory: postgresblob + services: + postgres: + image: postgres:18 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5434:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 2s + --health-timeout 2s + --health-retries 15 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: postgresblob/go.mod + cache-dependency-path: | + postgresblob/go.mod + postgresblob/go.sum + - run: go get gocloud.dev@master + - run: go mod tidy + - run: go test -race ./... + - run: go vet ./... diff --git a/README.md b/README.md index d6c4ad9..8238a14 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ need and ignore the rest. wiring. - [`pluginkit/graphwire`](pluginkit/graphwire/) generates the GraphQL resolver root of an application from the schemas of its plugins. +- [`postgresblob`](postgresblob/) is a gocloud.dev/blob driver that + keeps a program's files in two tables of its PostgreSQL database. ## Design diff --git a/dbkit/postgres/compose.yaml b/dbkit/postgres/compose.yaml index 70e8d9e..3e371ee 100644 --- a/dbkit/postgres/compose.yaml +++ b/dbkit/postgres/compose.yaml @@ -6,7 +6,7 @@ services: environment: POSTGRES_PASSWORD: postgres ports: - - "5434:5432" + - "127.0.0.1:5434:5432" volumes: - postgres-data:/var/lib/postgresql healthcheck: diff --git a/postgresblob/.golangci.yml b/postgresblob/.golangci.yml new file mode 100644 index 0000000..c340011 --- /dev/null +++ b/postgresblob/.golangci.yml @@ -0,0 +1,52 @@ +version: "2" + +linters: + default: standard + enable: + - cyclop + - depguard + - gocognit + - lll + - misspell + - revive + - unconvert + - unparam + settings: + cyclop: + max-complexity: 10 + gocognit: + min-complexity: 15 + lll: + line-length: 120 + revive: + rules: + - name: blank-imports + disabled: true + - name: exported + depguard: + rules: + dependency-purity: + list-mode: strict + allow: + - $gostd + - gocloud.dev + - github.com/google/uuid + - github.com/gopherium/framework/dbkit/postgres + - github.com/gopherium/framework/postgresblob + - github.com/jackc/pgx/v5 + - github.com/pressly/goose/v3 + exclusions: + rules: + - path: _test\.go + linters: + - cyclop + - gocognit + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/gopherium/framework diff --git a/postgresblob/CHANGELOG.md b/postgresblob/CHANGELOG.md new file mode 100644 index 0000000..f02db52 --- /dev/null +++ b/postgresblob/CHANGELOG.md @@ -0,0 +1,38 @@ +# Changelog + +All notable changes to the `postgresblob` module are documented in this +file. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the +module follows [Semantic Versioning](https://semver.org/). While at +v0.x, minor releases may contain breaking changes. + +Releases of this module are tagged `postgresblob/vX.Y.Z`. + +## [Unreleased] + +### Added + +- A driver of `driver.Bucket` from gocloud.dev/blob v0.47.0 that keeps each object in two PostgreSQL tables of one schema. +- `OpenBucket`, a bucket on a `*pgxpool.Pool` it never closes, refused when the database encoding is not UTF8. +- `Options` with `Schema`, `ChunkSize`, `MaxSize` and `CleanupTimeout`, each zero field taking its default. +- `MaxChunkSize`, 256 MiB, the largest `ChunkSize` a bucket takes. +- `Migrate`, which creates the schema and its `objects` and `chunks` tables through `dbkit/postgres` under goose's lock. +- Writes stream in chunks of 1 MiB by default and stay hidden until `Close`, which replaces the old object in one transaction. +- Writes, copies and deletes take a transaction lock of each key they touch, so `IfNotExist` lets exactly one of many racing writers win. +- Keys are bytes, so any UTF-8 key up to `MaxKeyBytes` (1,024 bytes) works, NUL included, and listings follow byte order. +- Each `Read` runs one statement for at most one chunk, or 1 MiB when chunks are smaller. +- A read whose object was replaced or deleted fails with `FailedPrecondition`. +- `MaxSize` caps each object, and a write or copy past it fails with `ResourceExhausted` and keeps the old object. +- A writer whose context ends or whose write fails deletes its pending row within `CleanupTimeout`. +- `Sweep` and `SweepOptions`, which delete at most `Limit` unfinished uploads untouched for longer than `OlderThan`. +- An upload `Sweep` deleted fails with `FailedPrecondition` at its next chunk or at `Close`. +- `Copy` in one transaction, onto the same key too. +- `URLOpener` on the `postgres` scheme, which takes `schema` out of the query and closes the pool it opened. +- A URL pgx cannot parse fails with one message that holds no part of the URL. +- `ErrorCode` names 42P01, 23505, 42501, 22021, the 53 class, 54000 and 57014, and `ErrorAs` exposes `*pgconn.PgError`. +- `SignedURL` answers `Unimplemented`. +- The tests run go-cloud's `drivertest` unmodified three ways: plain, through `driver.NewPrefixedBucket` and with 7 byte chunks. + +### Security + +- Requires `golang.org/x/net` v0.60.0, which fixes GO-2026-6603, GO-2026-6610, GO-2026-6611, GO-2026-6612 and GO-2026-6617. diff --git a/postgresblob/README.md b/postgresblob/README.md new file mode 100644 index 0000000..af5dd9c --- /dev/null +++ b/postgresblob/README.md @@ -0,0 +1,167 @@ +# postgresblob + +A [gocloud.dev/blob](https://gocloud.dev/howto/blob/) driver that keeps +files in PostgreSQL. + +gocloud.dev/blob is one Go API for buckets of files, with drivers for +S3, Google Cloud Storage, Azure, a local folder and memory. postgresblob +adds PostgreSQL, so a program that already runs PostgreSQL can keep its +files there without a second service. + +If you know Laravel, gocloud.dev/blob plays the part of Flysystem, and +postgresblob is one more adapter behind it. Flysystem's own adapters +offer no disk made of database tables, and that is what this one is. + +The guide is at . + +## How it stores files + +Each object is one row in the table `objects`, and its bytes are rows +in the table `chunks`, 1 MiB each by default. Writes send one chunk at +a time. Each read fetches at most one chunk, or 1 MiB when chunks are +smaller, so a large file never sits whole in memory unless you read it +all at once. + +A new object stays hidden until its writer closes. Then it replaces +the old object under that key in one transaction. Writes, copies and +deletes of the same key wait for each other, so `IfNotExist` lets +exactly one of many writers win. + +Keys are stored as bytes. Any UTF-8 key of up to 1,024 bytes works, a +NUL byte included, and a listing follows byte order. + +## Creating the tables + +```go +err := postgresblob.Migrate(ctx, db, "files") +``` + +`Migrate` takes a `*sql.DB` and creates the schema and both tables +through `dbkit/postgres`, under goose's migration lock. The schema +keeps its own version table, `files.goose_db_version`. + +The schema must be a lowercase plain name of up to 63 bytes, made of +letters, digits and underscores, that does not start with a digit. It +must not be a word PostgreSQL reserves, such as `user` or `order`, and +must not start with `pg_`. An empty one means `blob`. + +## Opening a bucket on your pool + +```go +bucket, err := postgresblob.OpenBucket(ctx, pool, &postgresblob.Options{ + Schema: "files", + MaxSize: 25 << 20, +}) +if err != nil { + return err +} +defer bucket.Close() +``` + +`OpenBucket` takes the program's `*pgxpool.Pool` and never closes it. +It refuses a database whose encoding is not UTF8. Each option left at +zero takes its default: + +| Option | Default | Meaning | +| --- | --- | --- | +| `Schema` | `blob` | the schema that holds the tables | +| `ChunkSize` | 1 MiB | the most bytes one chunk row holds, up to `MaxChunkSize` (256 MiB) | +| `MaxSize` | no cap | the most bytes one object may hold | +| `CleanupTimeout` | 10 seconds | how long a failed upload may take to delete its rows | + +## Opening a bucket from a URL + +```go +import _ "github.com/gopherium/framework/postgresblob" + +bucket, err := blob.OpenBucket(ctx, "postgres://app@db.example.com/site?schema=files") +``` + +The `postgres` scheme opens a pool of its own, and `bucket.Close` +closes it. The opener takes `schema` out of the query and hands the +rest to `pgxpool.ParseConfig`, so pgx settings such as `sslmode` and +`pool_max_conns` work. pgx sends every other query parameter to the +server as a setting, and the server turns down a name it does not +know. go-cloud's own `prefix` parameter works too. + +When pgx cannot parse the URL, the error holds no part of it. A URL +that Go's `net/url` cannot parse never reaches the driver, and +go-cloud's error for it repeats the whole URL, password included. So +keep those errors out of shared logs. + +## The size cap + +With `MaxSize` set, a write or a copy that would take an object past +the cap fails with `gcerrors.ResourceExhausted`. The old object under +that key stays as it was. + +## Unfinished uploads + +A writer whose context ends, or whose write fails, deletes its own +rows. A program that stops in the middle of an upload leaves a pending +row behind, and `Sweep` deletes those: + +```go +deleted, err := postgresblob.Sweep(ctx, pool, postgresblob.SweepOptions{ + Schema: "files", + OlderThan: time.Hour, + Limit: 100, +}) +``` + +`Sweep` deletes at most `Limit` uploads that got no new chunk for +longer than `OlderThan`, the oldest first. Run it from a timer. When it +returns `Limit`, more may be waiting. An upload that `Sweep` deleted +fails with `gcerrors.FailedPrecondition` when it writes its next chunk +or closes. + +## Errors + +| Code | When | +| --- | --- | +| `NotFound` | no object has the key, or the schema has no tables (42P01) | +| `FailedPrecondition` | `IfNotExist` found an object, the object changed during a read, `Sweep` deleted the upload, or 23505 | +| `InvalidArgument` | an empty key, a key over 1,024 bytes, or 22021 | +| `ResourceExhausted` | a write or copy past `MaxSize`, the class 53, or 54000 | +| `PermissionDenied` | 42501 | +| `Canceled` | a canceled context, or 57014 | +| `Unimplemented` | `SignedURL` | +| `Unknown` | every other error | + +Since a schema with no tables answers `NotFound`, call +`bucket.IsAccessible` when your program starts. + +## As + +`As` answers false on the bucket, its readers, attributes and list +objects. `ErrorAs` sets a `*pgconn.PgError` when the error came from +the server. + +## What is not supported + +- `SignedURL` answers `gcerrors.Unimplemented`. +- A read takes no snapshot. When another write replaces or deletes the + object during a read, the read fails with + `gcerrors.FailedPrecondition`, and never returns short data. +- Files grow the database and its backups, and the chunks are stored + without compression. +- A large `Copy` runs as one long transaction, and writes to its + source or its destination wait until it ends. + +## Testing + +The tests need a PostgreSQL server where they may create databases. +`compose.yaml` starts one on port 5434: + +```sh +docker compose up -d +DBKIT_TEST_POSTGRES_URL='postgres://postgres:postgres@localhost:5434/postgres?sslmode=disable' go test ./... +``` + +They run go-cloud's `drivertest` three ways: plain, through +`driver.NewPrefixedBucket`, and with 7 byte chunks so the small test +files cross chunk edges. + +## License + +Apache-2.0. diff --git a/postgresblob/bucket.go b/postgresblob/bucket.go new file mode 100644 index 0000000..fe81373 --- /dev/null +++ b/postgresblob/bucket.go @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "hash/fnv" + "slices" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "gocloud.dev/blob" + "gocloud.dev/blob/driver" + "gocloud.dev/gcerrors" +) + +const ( + // DefaultSchema is the schema of a bucket whose options name none. + DefaultSchema = "blob" + // DefaultChunkSize is the chunk size in bytes of a bucket whose options set none. + DefaultChunkSize = 1 << 20 + // MaxChunkSize is the largest chunk size in bytes that Options.ChunkSize may set. + MaxChunkSize = 256 << 20 + // DefaultCleanupTimeout is the cleanup timeout of a bucket whose options set none. + DefaultCleanupTimeout = 10 * time.Second + // utf8Encoding is the server encoding the driver needs. + utf8Encoding = "UTF8" +) + +// Options configures one bucket, each zero field taking its default. +type Options struct { + // Schema names the schema that holds the tables, DefaultSchema when empty. + Schema string + // ChunkSize is the most bytes one chunk row holds, DefaultChunkSize when zero. + ChunkSize int + // MaxSize is the most bytes one object may hold, with no cap when zero. + MaxSize int64 + // CleanupTimeout bounds the removal of an upload whose context ended, DefaultCleanupTimeout when zero. + CleanupTimeout time.Duration +} + +// errNoPool refuses a nil pool. +var errNoPool = errors.New("postgresblob: the driver needs a pool, got nil") + +// resolved returns o with each zero field set to its default, or the error for the first field out of range. +func (o Options) resolved() (Options, error) { + switch { + case o.ChunkSize < 0 || o.ChunkSize > MaxChunkSize: + return o, fmt.Errorf("postgresblob: the option ChunkSize must be from 0 to %d, got %d", MaxChunkSize, o.ChunkSize) + case o.MaxSize < 0: + return o, fmt.Errorf("postgresblob: the option MaxSize must be 0 or more, got %d", o.MaxSize) + case o.CleanupTimeout < 0: + return o, fmt.Errorf("postgresblob: the option CleanupTimeout must be 0 or more, got %v", o.CleanupTimeout) + } + o.Schema = schemaOr(o.Schema) + if o.ChunkSize == 0 { + o.ChunkSize = DefaultChunkSize + } + if o.CleanupTimeout == 0 { + o.CleanupTimeout = DefaultCleanupTimeout + } + return o, nil +} + +// bucket is the driver of the tables of one schema. +type bucket struct { + // pool reaches the database. + pool *pgxpool.Pool + // opts are the options with their defaults. + opts Options + // sql are the statements of the tables of the schema. + sql statements + // ownsPool reports whether the bucket opened the pool and closes it. + ownsPool bool +} + +// OpenBucket returns a bucket on the tables of opts.Schema reached through pool, which it never closes. +func OpenBucket(ctx context.Context, pool *pgxpool.Pool, opts *Options) (*blob.Bucket, error) { + b, err := openBucket(ctx, pool, opts) + if err != nil { + return nil, err + } + return blob.NewBucket(b), nil +} + +// openBucket returns the driver behind OpenBucket, once the server encoding is UTF8. +func openBucket(ctx context.Context, pool *pgxpool.Pool, opts *Options) (*bucket, error) { + if pool == nil { + return nil, errNoPool + } + var o Options + if opts != nil { + o = *opts + } + o, err := o.resolved() + if err != nil { + return nil, err + } + var encoding string + if err := pool.QueryRow(ctx, serverEncoding).Scan(&encoding); err != nil { + return nil, fmt.Errorf("postgresblob: read the server encoding: %w", err) + } + if encoding != utf8Encoding { + return nil, fmt.Errorf("postgresblob: the database encoding is %s, the driver needs %s", encoding, utf8Encoding) + } + return &bucket{pool: pool, opts: o, sql: statementsOf(o.Schema)}, nil +} + +// noAs is the As function of every hook, which exposes no type. +func noAs(any) bool { return false } + +// runHook calls hook with noAs when hook is set, and returns its error. +func runHook(hook func(asFunc func(any) bool) error) error { + if hook == nil { + return nil + } + return hook(noAs) +} + +// finished is a pending row ready to become the live object of its key. +type finished struct { + // key is the key the row becomes the live object of. + key string + // id is the id of the row. + id uuid.UUID + // size is the size of the object in bytes. + size int64 + // md5 is the MD5 hash of the object. + md5 []byte +} + +// lockOf returns the advisory lock key of key in schema. +func lockOf(schema, key string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte(schema)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(key)) + return int64(h.Sum64()) +} + +// lockIn takes the transaction lock of each of keys in tx, in ascending order of lock key. +func (b *bucket) lockIn(ctx context.Context, tx pgx.Tx, keys ...string) error { + locks := make([]int64, 0, len(keys)) + for _, key := range keys { + locks = append(locks, lockOf(b.opts.Schema, key)) + } + slices.Sort(locks) + for _, lock := range slices.Compact(locks) { + if _, err := tx.Exec(ctx, lockKey, lock); err != nil { + return err + } + } + return nil +} + +// checkSize returns the error for an object of size bytes past Options.MaxSize, or nil. +func (b *bucket) checkSize(size int64) error { + if limit := b.opts.MaxSize; limit > 0 && size > limit { + return fmt.Errorf("%w of %d bytes", errTooLarge, limit) + } + return nil +} + +// replaceIn makes the pending row of row the live object of its key in tx, in place of any live one. +func (b *bucket) replaceIn(ctx context.Context, tx pgx.Tx, row finished) error { + var created *time.Time + err := tx.QueryRow(ctx, b.sql.deleteLive, []byte(row.key)).Scan(&created) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + _, err = tx.Exec(ctx, b.sql.markLive, row.id, row.size, row.md5, created) + return err +} + +// etagOf returns the ETag of the object row id. +func etagOf(id uuid.UUID) string { + return `"` + hex.EncodeToString(id[:]) + `"` +} + +// ErrorCode returns the code of err. +func (*bucket) ErrorCode(err error) gcerrors.ErrorCode { return errorCode(err) } + +// As answers false for every type. +func (*bucket) As(any) bool { return false } + +// ErrorAs sets target, a **pgconn.PgError, to the server error err holds, and reports whether it held one. +func (*bucket) ErrorAs(err error, target any) bool { + serverErr, ok := target.(**pgconn.PgError) + return ok && errors.As(err, serverErr) +} + +// Attributes returns the attributes of the live object of key. +func (b *bucket) Attributes(ctx context.Context, key string) (*driver.Attributes, error) { + var id uuid.UUID + attrs := &driver.Attributes{} + err := b.pool.QueryRow(ctx, b.sql.readAttributes, []byte(key)).Scan(&id, &attrs.Size, &attrs.ContentType, + &attrs.CacheControl, &attrs.ContentDisposition, &attrs.ContentEncoding, &attrs.ContentLanguage, + &attrs.Metadata, &attrs.MD5, &attrs.CreateTime, &attrs.ModTime) + if errors.Is(err, pgx.ErrNoRows) { + return nil, errNotFound + } + if err != nil { + return nil, err + } + attrs.ETag = etagOf(id) + return attrs, nil +} + +// Copy copies the live object of srcKey to dstKey in one transaction, in place of any live object of dstKey. +func (b *bucket) Copy(ctx context.Context, dstKey, srcKey string, opts *driver.CopyOptions) error { + if err := checkKey(dstKey); err != nil { + return err + } + if err := runHook(opts.BeforeCopy); err != nil { + return err + } + return pgx.BeginTxFunc(ctx, b.pool, pgx.TxOptions{}, func(tx pgx.Tx) error { + return b.copyIn(ctx, tx, dstKey, srcKey) + }) +} + +// copyIn copies the live object of srcKey to dstKey in tx, under the locks of both keys and within Options.MaxSize. +func (b *bucket) copyIn(ctx context.Context, tx pgx.Tx, dstKey, srcKey string) error { + if err := b.lockIn(ctx, tx, dstKey, srcKey); err != nil { + return err + } + var srcID uuid.UUID + var size int64 + var md5 []byte + err := tx.QueryRow(ctx, b.sql.copySource, []byte(srcKey)).Scan(&srcID, &size, &md5) + if errors.Is(err, pgx.ErrNoRows) { + return errNotFound + } + if err != nil { + return err + } + if err := b.checkSize(size); err != nil { + return err + } + row := finished{key: dstKey, id: uuid.Must(uuid.NewV7()), size: size, md5: md5} + if _, err := tx.Exec(ctx, b.sql.copyObject, row.id, []byte(dstKey), srcID); err != nil { + return err + } + return b.replaceIn(ctx, tx, row) +} + +// Delete deletes the live object of key in one transaction. +func (b *bucket) Delete(ctx context.Context, key string, opts *driver.DeleteOptions) error { + if err := runHook(opts.BeforeDelete); err != nil { + return err + } + return pgx.BeginTxFunc(ctx, b.pool, pgx.TxOptions{}, func(tx pgx.Tx) error { + return b.deleteIn(ctx, tx, key) + }) +} + +// deleteIn deletes the live object of key in tx, under the lock of key. +func (b *bucket) deleteIn(ctx context.Context, tx pgx.Tx, key string) error { + if err := b.lockIn(ctx, tx, key); err != nil { + return err + } + tag, err := tx.Exec(ctx, b.sql.deleteLive, []byte(key)) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return errNotFound + } + return nil +} + +// SignedURL answers Unimplemented. +func (*bucket) SignedURL(context.Context, string, *driver.SignedURLOptions) (string, error) { + return "", errUnimplemented +} + +// Close closes the pool when the bucket opened it, and leaves a pool it was given open. +func (b *bucket) Close() error { + if b.ownsPool { + b.pool.Close() + } + return nil +} diff --git a/postgresblob/compose.yaml b/postgresblob/compose.yaml new file mode 100644 index 0000000..aca3fea --- /dev/null +++ b/postgresblob/compose.yaml @@ -0,0 +1,19 @@ +name: postgresblob + +services: + postgres: + image: postgres:18 + environment: + POSTGRES_PASSWORD: postgres + ports: + - "127.0.0.1:5434:5432" + volumes: + - postgres-data:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 2s + retries: 15 + +volumes: + postgres-data: diff --git a/postgresblob/concurrency_test.go b/postgresblob/concurrency_test.go new file mode 100644 index 0000000..86dbba6 --- /dev/null +++ b/postgresblob/concurrency_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "gocloud.dev/blob" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +const ( + // concurrentWriters is how many writers race for one key. + concurrentWriters = 8 + // lockWaitBound is how long a test waits for sessions to queue on a lock. + lockWaitBound = 10 * time.Second + // lockWaits counts the sessions of the current database that wait on a lock. + lockWaits = "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock'" + // liveRowLock locks for update the live rows of the keys its one argument lists. + liveRowLock = "SELECT FROM files.objects WHERE key = ANY($1) AND state = 'live' FOR UPDATE" + // firstVersion is the text a key holds before a replace. + firstVersion = "the first version" + // secondVersion is the text a replace writes. + secondVersion = "the second version" +) + +// awaitLockWaits fails the test unless at least n sessions of db wait on a lock within lockWaitBound. +func (db testDB) awaitLockWaits(n int) { + db.t.Helper() + deadline := time.Now().Add(lockWaitBound) + for db.count(lockWaits) < n { + if time.Now().After(deadline) { + db.t.Fatalf("fewer than %d sessions wait on a lock after %v", n, lockWaitBound) + } + time.Sleep(10 * time.Millisecond) + } +} + +// holdRows begins a transaction on a new connection that locks the live rows of keys for update. +func (db testDB) holdRows(keys ...string) pgx.Tx { + db.t.Helper() + tx, err := db.holder().Begin(db.t.Context()) + if err != nil { + db.t.Fatalf("begin the holding transaction: %v", err) + } + raw := make([][]byte, len(keys)) + for i, key := range keys { + raw[i] = []byte(key) + } + tag, err := tx.Exec(db.t.Context(), liveRowLock, raw) + if err != nil || tag.RowsAffected() != int64(len(keys)) { + db.t.Fatalf("lock the live rows of %q: %v, %v", keys, tag, err) + } + return tx +} + +// release rolls back the holding transaction tx, failing the test when it cannot. +func (db testDB) release(tx pgx.Tx) { + db.t.Helper() + if err := tx.Rollback(db.t.Context()); err != nil { + db.t.Fatalf("roll back the holding transaction: %v", err) + } +} + +// writeTexts writes each text of texts under its key on b, failing the test at the first write that fails. +func writeTexts(t *testing.T, b *blob.Bucket, texts map[string]string) { + t.Helper() + for key, text := range texts { + if err := b.WriteAll(t.Context(), key, []byte(text), textOptions); err != nil { + t.Fatalf("WriteAll(%q) = %v, want nil", key, err) + } + } +} + +// duringAReplace returns the error of call, run while a write of secondVersion waits to replace firstVersion at key. +func (db testDB) duringAReplace(b *blob.Bucket, key string, call func() error) error { + db.t.Helper() + writeTexts(db.t, b, map[string]string{key: firstVersion}) + holder := db.holdRows(key) + writeErr := make(chan error, 1) + go func() { writeErr <- b.WriteAll(db.t.Context(), key, []byte(secondVersion), textOptions) }() + db.awaitLockWaits(1) + callErr := make(chan error, 1) + go func() { callErr <- call() }() + db.awaitLockWaits(2) + db.release(holder) + if err := <-writeErr; err != nil { + db.t.Fatalf("WriteAll() of the second version = %v, want nil", err) + } + return <-callErr +} + +// raceWrites runs concurrentWriters writes of key on b at once with opts, and returns their errors. +func raceWrites(b *blob.Bucket, key string, opts *blob.WriterOptions) []error { + errs := make([]error, concurrentWriters) + var wg sync.WaitGroup + for i := range concurrentWriters { + wg.Go(func() { + errs[i] = b.WriteAll(context.Background(), key, fmt.Appendf(nil, "version %d", i), opts) + }) + } + wg.Wait() + return errs +} + +func TestConcurrentWritesToOneKeyAllSucceedAndLeaveOneObject(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{}) + const key = "notes/shared.txt" + + errs := raceWrites(b, key, textOptions) + + for i, err := range errs { + if err != nil { + t.Errorf("concurrent WriteAll() %d = %v, want nil", i, err) + } + } + if n := db.count(liveRows, []byte(key)); n != 1 { + t.Errorf("the key holds %d live rows after the race, want 1", n) + } +} + +func TestConcurrentWritesUnderIfNotExistLetExactlyOneWin(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{}) + opts := &blob.WriterOptions{ContentType: "text/plain", IfNotExist: true} + + errs := raceWrites(b, "notes/claimed.txt", opts) + + wins := 0 + for i, err := range errs { + switch code := gcerrors.Code(err); code { + case gcerrors.OK: + wins++ + case gcerrors.FailedPrecondition: + default: + t.Errorf("concurrent WriteAll() %d under IfNotExist = %v, the code %v, want nil or FailedPrecondition", + i, err, code) + } + } + if wins != 1 { + t.Errorf("%d concurrent writes under IfNotExist succeeded, want exactly 1", wins) + } +} + +func TestCopiesThatSwapTwoKeysBothSucceed(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{}) + const first, second = "swap/first.txt", "swap/second.txt" + writeTexts(t, b, map[string]string{first: "the first text", second: "the second text"}) + holder := db.holdRows(first, second) + errs := make(chan error, 2) + go func() { errs <- b.Copy(t.Context(), second, first, nil) }() + go func() { errs <- b.Copy(t.Context(), first, second, nil) }() + db.awaitLockWaits(2) + + db.release(holder) + + for range 2 { + if err := <-errs; err != nil { + t.Errorf("Copy() in a swap of two keys = %v, the code %v, want nil", err, gcerrors.Code(err)) + } + } + firstText, err := b.ReadAll(t.Context(), first) + if err != nil { + t.Fatalf("ReadAll(%q) after the swap = %v, want the text", first, err) + } + expectContent(t, b, second, firstText) +} + +func TestACopyDuringAReplaceOfItsSourceCopiesTheNewObject(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{}) + const source, destination = "race/source.txt", "race/destination.txt" + + err := db.duringAReplace(b, source, func() error { return b.Copy(t.Context(), destination, source, nil) }) + + if err != nil { + t.Errorf("Copy() of a source that held an object throughout = %v, the code %v, want nil", err, gcerrors.Code(err)) + } + expectContent(t, b, destination, []byte(secondVersion)) +} + +func TestADeleteDuringAReplaceDeletesTheNewObject(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{}) + const key = "race/deleted.txt" + + err := db.duringAReplace(b, key, func() error { return b.Delete(t.Context(), key) }) + + if err != nil { + t.Errorf("Delete() of a key that held an object throughout = %v, the code %v, want nil", err, gcerrors.Code(err)) + } + if n := db.count(keyRows, []byte(key)); n != 0 { + t.Errorf("the key holds %d rows after the delete, want none", n) + } +} diff --git a/postgresblob/copy_test.go b/postgresblob/copy_test.go new file mode 100644 index 0000000..b12ea02 --- /dev/null +++ b/postgresblob/copy_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "testing" + + "gocloud.dev/blob" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +func TestCopyOntoItselfKeepsTheObject(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const key = "notes/report.txt" + content := []byte("a report longer than one chunk") + opts := &blob.WriterOptions{ContentType: "text/plain", Metadata: map[string]string{"author": "Ada Lovelace"}} + if err := b.WriteAll(t.Context(), key, content, opts); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + + if err := b.Copy(t.Context(), key, key, nil); err != nil { + t.Fatalf("Copy() of a key onto itself = %v, want nil", err) + } + + expectContent(t, b, key, content) + attrs, err := b.Attributes(t.Context(), key) + if err != nil { + t.Fatalf("Attributes() after the copy = %v, want the attributes", err) + } + if attrs.Metadata["author"] != "Ada Lovelace" { + t.Errorf("the metadata after the copy = %v, want the author kept", attrs.Metadata) + } + if attrs.ContentType != "text/plain" { + t.Errorf("the content type after the copy = %q, want text/plain", attrs.ContentType) + } + if n := db.count(liveRows, []byte(key)); n != 1 { + t.Errorf("the key holds %d live rows after the copy, want 1", n) + } +} + +func TestACopyTheRoleMayNotWriteFailsWithPermissionDeniedAndKeepsTheDestination(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{}) + const source, destination = "notes/source.txt", "notes/destination.txt" + for key, text := range map[string]string{source: "the source", destination: "the destination"} { + if err := b.WriteAll(t.Context(), key, []byte(text), textOptions); err != nil { + t.Fatalf("WriteAll(%q) = %v, want nil", key, err) + } + } + db.exec("REVOKE INSERT ON files.chunks FROM CURRENT_USER") + + err := b.Copy(t.Context(), destination, source, nil) + + expectCode(t, "Copy() by a role that may not insert chunks", err, gcerrors.PermissionDenied) + expectContent(t, b, destination, []byte("the destination")) +} + +func TestACopyPastTheCapFailsAndKeepsTheDestination(t *testing.T) { + t.Parallel() + db := freshDB(t) + const large, small, destination = "notes/large.txt", "notes/small.txt", "notes/destination.txt" + writeTexts(t, db.open(postgresblob.Options{}), + map[string]string{large: "a text past the cap", small: "four", destination: "old"}) + capped := db.open(postgresblob.Options{MaxSize: 4}) + + pastErr := capped.Copy(t.Context(), destination, large, nil) + + expectCode(t, "Copy() of 19 bytes through a cap of 4", pastErr, gcerrors.ResourceExhausted) + expectContent(t, capped, destination, []byte("old")) + if n := db.count(pendingRows); n != 0 { + t.Errorf("%d pending rows stay after the copy past the cap, want none", n) + } + if err := capped.Copy(t.Context(), destination, small, nil); err != nil { + t.Errorf("Copy() of exactly the cap = %v, want nil", err) + } +} diff --git a/postgresblob/doc.go b/postgresblob/doc.go new file mode 100644 index 0000000..3dd0dee --- /dev/null +++ b/postgresblob/doc.go @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package postgresblob is a gocloud.dev/blob driver on two PostgreSQL tables, opened by OpenBucket or a postgres URL. +package postgresblob diff --git a/postgresblob/drivertest_test.go b/postgresblob/drivertest_test.go new file mode 100644 index 0000000..cbefdbb --- /dev/null +++ b/postgresblob/drivertest_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "context" + "net/http" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "gocloud.dev/blob/driver" + "gocloud.dev/blob/drivertest" + + "github.com/gopherium/framework/postgresblob" +) + +// harness is the drivertest harness of one fresh test database. +type harness struct { + // pool reaches the fresh test database. + pool *pgxpool.Pool + // opts are the options of every driver the harness makes. + opts postgresblob.Options + // prefix starts every key of every driver the harness makes, and an empty prefix adds nothing. + prefix string +} + +// harnessMaker returns a maker of harnesses whose drivers take opts in testSchema and prefix every key with prefix. +func harnessMaker(opts postgresblob.Options, prefix string) drivertest.HarnessMaker { + opts.Schema = testSchema + return func(_ context.Context, t *testing.T) (drivertest.Harness, error) { + return &harness{pool: freshDB(t).pool, opts: opts, prefix: prefix}, nil + } +} + +// MakeDriver returns a driver on the tables of the fresh test database. +func (h *harness) MakeDriver(ctx context.Context) (driver.Bucket, error) { + opts := h.opts + drv, err := postgresblob.OpenDriver(ctx, h.pool, &opts) + if err != nil || h.prefix == "" { + return drv, err + } + return driver.NewPrefixedBucket(drv, h.prefix), nil +} + +// MakeDriverForNonexistentBucket returns a driver on a schema the fresh test database lacks. +func (h *harness) MakeDriverForNonexistentBucket(ctx context.Context) (driver.Bucket, error) { + opts := h.opts + opts.Schema = unmigratedSchema + return postgresblob.OpenDriver(ctx, h.pool, &opts) +} + +// HTTPClient returns nil. +func (*harness) HTTPClient() *http.Client { return nil } + +// SupportsCopy reports that the driver copies. +func (*harness) SupportsCopy() bool { return true } + +// SupportsNonSlashDelimiters reports that the driver lists with any delimiter. +func (*harness) SupportsNonSlashDelimiters() bool { return true } + +// Close closes the pool of the harness. +func (h *harness) Close() { h.pool.Close() } + +func TestConformance(t *testing.T) { + t.Parallel() + + drivertest.RunConformanceTests(t, harnessMaker(postgresblob.Options{}, ""), nil) +} + +func TestConformanceThroughAPrefixedBucket(t *testing.T) { + t.Parallel() + + drivertest.RunConformanceTests(t, harnessMaker(postgresblob.Options{}, "tenant/"), nil) +} + +func TestConformanceWithSevenByteChunks(t *testing.T) { + t.Parallel() + + drivertest.RunConformanceTests(t, harnessMaker(postgresblob.Options{ChunkSize: 7}, ""), nil) +} diff --git a/postgresblob/errors.go b/postgresblob/errors.go new file mode 100644 index 0000000..7966637 --- /dev/null +++ b/postgresblob/errors.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgconn" + "gocloud.dev/gcerrors" +) + +// MaxKeyBytes is the most bytes a key may hold. +const MaxKeyBytes = 1024 + +// insufficientResources is the SQLSTATE class of a server short of disk, memory or connections. +const insufficientResources = "53" + +var ( + // errNotFound answers a key that holds no live object. + errNotFound = errors.New("postgresblob: no object has the key") + // errExists answers a write under IfNotExist to a key that holds a live object. + errExists = errors.New("postgresblob: an object already has the key") + // errUnimplemented answers SignedURL. + errUnimplemented = errors.New("postgresblob: signed URLs are not supported") + // errEmptyKey refuses an empty key. + errEmptyKey = errors.New("postgresblob: the key is empty") + // errLongKey refuses a key longer than MaxKeyBytes. + errLongKey = fmt.Errorf("postgresblob: the key is longer than %d bytes", MaxKeyBytes) + // errTooLarge refuses a write or a copy that would take an object past Options.MaxSize. + errTooLarge = errors.New("postgresblob: the object would be larger than the size cap") + // errReplaced ends a read whose object was replaced or deleted since the reader opened. + errReplaced = errors.New("postgresblob: the object was replaced or deleted during the read") + // errSwept ends an upload whose pending row Sweep deleted. + errSwept = errors.New("postgresblob: Sweep deleted the upload before it finished") +) + +// sentinelCodes pairs each error of this package with its code. +var sentinelCodes = []struct { + // err is the error of this package. + err error + // code is its code. + code gcerrors.ErrorCode +}{ + {err: errNotFound, code: gcerrors.NotFound}, + {err: errExists, code: gcerrors.FailedPrecondition}, + {err: errUnimplemented, code: gcerrors.Unimplemented}, + {err: errEmptyKey, code: gcerrors.InvalidArgument}, + {err: errLongKey, code: gcerrors.InvalidArgument}, + {err: errTooLarge, code: gcerrors.ResourceExhausted}, + {err: errReplaced, code: gcerrors.FailedPrecondition}, + {err: errSwept, code: gcerrors.FailedPrecondition}, +} + +// serverCodes are the codes of the SQLSTATE values the driver names one by one. +var serverCodes = map[string]gcerrors.ErrorCode{ + "42P01": gcerrors.NotFound, + "23505": gcerrors.FailedPrecondition, + "42501": gcerrors.PermissionDenied, + "22021": gcerrors.InvalidArgument, + "54000": gcerrors.ResourceExhausted, + "57014": gcerrors.Canceled, +} + +// errorCode returns the code of err, Unknown for an error the driver does not name. +func errorCode(err error) gcerrors.ErrorCode { + for _, sentinel := range sentinelCodes { + if errors.Is(err, sentinel.err) { + return sentinel.code + } + } + var serverErr *pgconn.PgError + if !errors.As(err, &serverErr) { + return gcerrors.Unknown + } + if code, named := serverCodes[serverErr.Code]; named { + return code + } + if strings.HasPrefix(serverErr.Code, insufficientResources) { + return gcerrors.ResourceExhausted + } + return gcerrors.Unknown +} + +// checkKey returns the error for a key the driver refuses. +func checkKey(key string) error { + switch { + case key == "": + return errEmptyKey + case len(key) > MaxKeyBytes: + return errLongKey + } + return nil +} diff --git a/postgresblob/errors_test.go b/postgresblob/errors_test.go new file mode 100644 index 0000000..0a7cd2f --- /dev/null +++ b/postgresblob/errors_test.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "errors" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +// serverError returns a server error with the SQLSTATE code. +func serverError(code string) error { + return &pgconn.PgError{Code: code, Message: "a server error"} +} + +func TestErrorCodeNamesTheCodeOfEachServerError(t *testing.T) { + t.Parallel() + drv, err := postgresblob.OpenDriver(t.Context(), freshDB(t).pool, &postgresblob.Options{Schema: testSchema}) + if err != nil { + t.Fatalf("OpenDriver() = %v, want a driver", err) + } + tests := map[string]struct { + err error + want gcerrors.ErrorCode + }{ + "an undefined table": {serverError("42P01"), gcerrors.NotFound}, + "a unique violation": {serverError("23505"), gcerrors.FailedPrecondition}, + "an insufficient privilege": {serverError("42501"), gcerrors.PermissionDenied}, + "a character not in repertoire": {serverError("22021"), gcerrors.InvalidArgument}, + "a full disk": {serverError("53100"), gcerrors.ResourceExhausted}, + "an out of memory error": {serverError("53200"), gcerrors.ResourceExhausted}, + "too many connections": {serverError("53300"), gcerrors.ResourceExhausted}, + "an exceeded program limit": {serverError("54000"), gcerrors.ResourceExhausted}, + "a canceled query": {serverError("57014"), gcerrors.Canceled}, + "a syntax error": {serverError("42601"), gcerrors.Unknown}, + "a wrapped server error": {fmt.Errorf("a step: %w", serverError("42501")), gcerrors.PermissionDenied}, + "an error of another kind": {errors.New("an error of another kind"), gcerrors.Unknown}, + } + + for name, tt := range tests { + if got := drv.ErrorCode(tt.err); got != tt.want { + t.Errorf("ErrorCode() of %s = %v, want %v", name, got, tt.want) + } + } +} + +func TestErrorAsExposesTheServerError(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{Schema: unmigratedSchema}) + + _, err := b.Attributes(t.Context(), "notes/report.txt") + + var serverErr *pgconn.PgError + if !b.ErrorAs(err, &serverErr) || serverErr.Code != "42P01" { + t.Errorf("ErrorAs() of the error %v of a schema with no tables found %+v, want the server error 42P01", + err, serverErr) + } + if code := gcerrors.Code(err); code != gcerrors.NotFound { + t.Errorf("Attributes() on a schema with no tables failed with the code %v, want NotFound", code) + } +} diff --git a/postgresblob/expect_test.go b/postgresblob/expect_test.go new file mode 100644 index 0000000..883b8e3 --- /dev/null +++ b/postgresblob/expect_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "bytes" + "context" + "slices" + "strings" + "testing" + + "gocloud.dev/blob" + "gocloud.dev/gcerrors" +) + +// mentions reports whether err is set and its message holds every one of words. +func mentions(err error, words ...string) bool { + return err != nil && !slices.ContainsFunc(words, lacks(err)) +} + +// mentionsAny reports whether err is set and its message holds one of words at least. +func mentionsAny(err error, words ...string) bool { + return err != nil && slices.ContainsFunc(words, func(word string) bool { return !lacks(err)(word) }) +} + +// lacks returns a test of whether the message of err lacks a word. +func lacks(err error) func(word string) bool { + return func(word string) bool { return !strings.Contains(err.Error(), word) } +} + +// expectCode fails the test unless err, returned by call, carries the code want. +func expectCode(t *testing.T, call string, err error, want gcerrors.ErrorCode) { + t.Helper() + if code := gcerrors.Code(err); code != want { + t.Errorf("%s = %v, the code %v, want %v", call, err, code, want) + } +} + +// expectContent fails the test unless the live object of key on b holds want. +func expectContent(t *testing.T, b *blob.Bucket, key string, want []byte) { + t.Helper() + got, err := b.ReadAll(context.Background(), key) + if err != nil || !bytes.Equal(got, want) { + t.Errorf("ReadAll(%q) = %q, %v, want %q", key, got, err, want) + } +} diff --git a/postgresblob/export_test.go b/postgresblob/export_test.go new file mode 100644 index 0000000..e2ebbe9 --- /dev/null +++ b/postgresblob/export_test.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" + "gocloud.dev/blob/driver" +) + +// OpenDriver returns the driver OpenBucket wraps. +func OpenDriver(ctx context.Context, pool *pgxpool.Pool, opts *Options) (driver.Bucket, error) { + b, err := openBucket(ctx, pool, opts) + if err != nil { + return nil, err + } + return b, nil +} + +// MigrationStatements is migrationStatements under an exported name. +var MigrationStatements = migrationStatements + +// LockOf is lockOf under an exported name. +var LockOf = lockOf diff --git a/postgresblob/failures_test.go b/postgresblob/failures_test.go new file mode 100644 index 0000000..1654cf7 --- /dev/null +++ b/postgresblob/failures_test.go @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "context" + "errors" + "net/url" + "slices" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "gocloud.dev/blob" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +// lockNotAvailable is the SQLSTATE of a lock wait that lock_timeout ended. +const lockNotAvailable = "55P03" + +// exec runs each statement on db, failing the test at the first that fails. +func (db testDB) exec(statements ...string) { + db.t.Helper() + for _, statement := range statements { + if _, err := db.pool.Exec(context.Background(), statement); err != nil { + db.t.Fatalf("%s: %v", statement, err) + } + } +} + +// withLockTimeout returns db reached through a pool whose lock waits end after a tenth of a second. +func (db testDB) withLockTimeout() testDB { + db.t.Helper() + address := addressWith(db.t, db.address, url.Values{"lock_timeout": {"100ms"}}) + return testDB{t: db.t, address: address, pool: poolOn(db.t, address)} +} + +// holder returns a connection to db, closed when the test ends. +func (db testDB) holder() *pgx.Conn { + db.t.Helper() + conn, err := pgx.Connect(context.Background(), db.address) + if err != nil { + db.t.Fatalf("connect to the test database: %v", err) + } + db.t.Cleanup(func() { _ = conn.Close(context.Background()) }) + return conn +} + +// serverCode returns the SQLSTATE of the server error err holds, or nothing. +func serverCode(err error) string { + var serverErr *pgconn.PgError + if errors.As(err, &serverErr) { + return serverErr.Code + } + return "" +} + +func TestAFailingHookStopsItsCall(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{}) + const key = "notes/report.txt" + if err := b.WriteAll(t.Context(), key, []byte("text"), textOptions); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + hookErr := errors.New("the hook says no") + hook := func(func(any) bool) error { return hookErr } + ctx := t.Context() + calls := map[string]func() error{ + "BeforeWrite": func() error { + return b.WriteAll(ctx, "notes/other.txt", []byte("text"), + &blob.WriterOptions{ContentType: "text/plain", BeforeWrite: hook}) + }, + "BeforeRead": func() error { + _, err := b.NewReader(ctx, key, &blob.ReaderOptions{BeforeRead: hook}) + return err + }, + "BeforeList": func() error { + _, err := b.List(&blob.ListOptions{BeforeList: hook}).Next(ctx) + return err + }, + "BeforeCopy": func() error { + return b.Copy(ctx, "notes/copy.txt", key, &blob.CopyOptions{BeforeCopy: hook}) + }, + } + + for name, call := range calls { + if err := call(); !errors.Is(err, hookErr) { + t.Errorf("the call with a failing %s = %v, want the error of the hook", name, err) + } + } + if keys := keysOf(t, b, "notes/"); !slices.Equal(keys, []string{key}) { + t.Errorf("the bucket holds %q after the stopped calls, want only %q", keys, key) + } +} + +func TestTheBucketExposesNoTypeThroughAs(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{}) + + var pool *pgxpool.Pool + if b.As(&pool) { + t.Error("As() of a pool answered true, want false") + } +} + +func TestEveryCallOnASchemaWithNoTablesFailsWithNotFound(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{Schema: unmigratedSchema, ChunkSize: 4}) + ctx := t.Context() + const key = "notes/report.txt" + claim := &blob.WriterOptions{ContentType: "text/plain", IfNotExist: true} + calls := map[string]func() error{ + "NewReader": func() error { + _, err := b.NewReader(ctx, key, nil) + return err + }, + "Copy": func() error { return b.Copy(ctx, "notes/copy.txt", key, nil) }, + "Delete": func() error { return b.Delete(ctx, key) }, + "WriteAll of one chunk": func() error { return b.WriteAll(ctx, key, []byte("one"), textOptions) }, + "WriteAll of many chunks": func() error { return b.WriteAll(ctx, key, []byte("many chunks"), textOptions) }, + "WriteAll under IfNotExist": func() error { return b.WriteAll(ctx, key, []byte("one"), claim) }, + } + + for name, call := range calls { + if code := gcerrors.Code(call()); code != gcerrors.NotFound { + t.Errorf("%s on a schema with no tables failed with the code %v, want NotFound", name, code) + } + } + _, err := postgresblob.Sweep(ctx, db.pool, + postgresblob.SweepOptions{Schema: unmigratedSchema, OlderThan: time.Hour, Limit: 1}) + if serverCode(err) != "42P01" || !mentions(err, unmigratedSchema) { + t.Errorf("Sweep() of a schema with no tables = %v, want the server error 42P01 and the schema named", err) + } +} + +func TestACanceledContextEndsARead(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{}) + const key = "notes/report.txt" + if err := b.WriteAll(t.Context(), key, []byte("text"), textOptions); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + ctx, cancel := context.WithCancel(t.Context()) + r, err := b.NewReader(ctx, key, nil) + if err != nil { + t.Fatalf("NewReader() = %v, want a reader", err) + } + defer func() { _ = r.Close() }() + + cancel() + _, err = r.Read(make([]byte, 4)) + + if !errors.Is(err, context.Canceled) { + t.Errorf("Read() after the cancel = %v, want context.Canceled", err) + } +} + +func TestACanceledContextEndsAnUploadBetweenChunks(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const key = "notes/draft.txt" + ctx, cancel := context.WithCancel(t.Context()) + w, err := b.NewWriter(ctx, key, textOptions) + if err != nil { + t.Fatalf("NewWriter() = %v, want a writer", err) + } + if _, err := w.Write([]byte("first!")); err != nil { + t.Fatalf("Write() of the first chunk = %v, want nil", err) + } + + cancel() + _, writeErr := w.Write([]byte("second")) + closeErr := w.Close() + + if !errors.Is(writeErr, context.Canceled) || !errors.Is(closeErr, context.Canceled) { + t.Errorf("Write() and Close() after the cancel = %v and %v, want context.Canceled", writeErr, closeErr) + } + if n := db.count(keyRows, []byte(key)); n != 0 { + t.Errorf("the key holds %d rows after the canceled upload, want none", n) + } +} + +func TestOpenBucketWithACanceledContextFails(t *testing.T) { + t.Parallel() + pool := freshDB(t).pool + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + b, err := postgresblob.OpenBucket(ctx, pool, nil) + + if !errors.Is(err, context.Canceled) || !mentions(err, "server encoding") { + t.Errorf("OpenBucket() with a canceled context = %v, %v, want context.Canceled while reading the encoding", b, err) + } +} + +func TestMigrateFailsAndCreatesNothingWhenATableOfTheSchemaExists(t *testing.T) { + t.Parallel() + db := emptyDB(t) + db.exec("CREATE SCHEMA archive", "CREATE TABLE archive.objects (note text)") + + err := postgresblob.Migrate(t.Context(), db.sqlDB(), "archive") + + if serverCode(err) != "42P07" || !mentions(err, "archive") { + t.Errorf("Migrate() over an existing archive.objects = %v, want the server error 42P07 and the schema named", err) + } + if db.hasTable("archive.chunks") { + t.Error("the database holds archive.chunks after the failed migration, want none") + } +} + +func TestAWriteCopyOrDeleteWaitsForTheLockOfItsKeyUntilLockTimeout(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.withLockTimeout().open(postgresblob.Options{}) + const key, source = "notes/locked.txt", "notes/source.txt" + if err := b.WriteAll(t.Context(), source, []byte("text"), textOptions); err != nil { + t.Fatalf("WriteAll() of the source = %v, want nil", err) + } + lock := postgresblob.LockOf(testSchema, key) + if _, err := db.holder().Exec(t.Context(), "SELECT pg_advisory_lock($1)", lock); err != nil { + t.Fatalf("hold the lock of the key: %v", err) + } + + writeErr := b.WriteAll(t.Context(), key, []byte("text"), textOptions) + copyErr := b.Copy(t.Context(), key, source, nil) + deleteErr := b.Delete(t.Context(), key) + + for call, err := range map[string]error{"WriteAll()": writeErr, "Copy()": copyErr, "Delete()": deleteErr} { + if serverCode(err) != lockNotAvailable { + t.Errorf("%s of a locked key = %v, want the server error %s", call, err, lockNotAvailable) + } + } + if n := db.count(keyRows, []byte(key)); n != 0 { + t.Errorf("the locked key holds %d rows, want none", n) + } +} + +func TestAWriteThatCannotDeleteTheOldObjectKeepsIt(t *testing.T) { + t.Parallel() + db := freshDB(t) + const key = "notes/report.txt" + if err := db.open(postgresblob.Options{}).WriteAll(t.Context(), key, []byte("old text"), textOptions); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + holder, err := db.holder().Begin(t.Context()) + if err != nil { + t.Fatalf("begin the holding transaction: %v", err) + } + if _, err := holder.Exec(t.Context(), "SELECT FROM files.objects WHERE key = $1 FOR UPDATE", []byte(key)); err != nil { + t.Fatalf("lock the row of the old object: %v", err) + } + + err = db.withLockTimeout().open(postgresblob.Options{}).WriteAll(t.Context(), key, []byte("new text"), textOptions) + + if serverCode(err) != lockNotAvailable { + t.Errorf("WriteAll() over a row another transaction locks = %v, want the server error %s", err, lockNotAvailable) + } + if got, err := db.open(postgresblob.Options{}).ReadAll(t.Context(), key); err != nil || string(got) != "old text" { + t.Errorf("ReadAll() after the failed write = %q, %v, want the old text", got, err) + } +} diff --git a/postgresblob/go.mod b/postgresblob/go.mod new file mode 100644 index 0000000..9f34efc --- /dev/null +++ b/postgresblob/go.mod @@ -0,0 +1,42 @@ +module github.com/gopherium/framework/postgresblob + +go 1.27.3 + +require ( + github.com/google/uuid v1.6.0 + github.com/gopherium/framework/dbkit/postgres v0.3.0 + github.com/jackc/pgx/v5 v5.10.0 + github.com/pressly/goose/v3 v3.27.2 + gocloud.dev v0.47.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/googleapis/gax-go/v2 v2.24.1 // indirect + github.com/gopherium/framework/dbkit v0.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/peterldowns/pgtestdb v0.1.1 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.46.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.297.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/postgresblob/go.sum b/postgresblob/go.sum new file mode 100644 index 0000000..adbeece --- /dev/null +++ b/postgresblob/go.sum @@ -0,0 +1,203 @@ +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo= +cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.13.0 h1:ufT3FPT5rFFXu6UtLkNoxaOaV5EuA1dsSkmemCSTo6U= +cloud.google.com/go/iam v1.13.0/go.mod h1:gHXdDEiPDvqd1q1KwBDGQlgZY/BwY760zU2LhOZS5w0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8= +cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.37.0 h1:edOuLWehuDlQ68roBF900Y09jvBoJP5W5d/wusz9nWY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.37.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y= +github.com/aws/aws-sdk-go-v2 v1.46.0 h1:1kt7m/EKcEHt5mlyyxx9cSlMddRPIKbjb6DIQsu4HPk= +github.com/aws/aws-sdk-go-v2 v1.46.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.33.3 h1:h090b3O5S17bF87/0ysHZuIT/7DCb4EBRFQX2PMVPCw= +github.com/aws/aws-sdk-go-v2/config v1.33.3/go.mod h1:YYDB1kTejxbfAbEVUqgCtkVp26xvNCHev9cLKABMGAk= +github.com/aws/aws-sdk-go-v2/credentials v1.20.3 h1:tToOYM/LXev4NpfWlIYGDvBvjHmJ3HXpRU9ppl+pM6k= +github.com/aws/aws-sdk-go-v2/credentials v1.20.3/go.mod h1:wfGneWyncO7p67wqXV2IQhPk14JqIc25woKlaArT3WI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2 h1:Ldv7RPHs7qwwTscRjAl3YBud32f3BvdAGRmSvAx5L38= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2/go.mod h1:XyK6UV8xbo66ysVqLd2783C09pBYHOm8aKTRV5DVJ30= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2 h1:q/PSLGuRWCChWg+dLnb9dWOnrCxJtnboXbBtFoqqRrI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2/go.mod h1:TD1jvU2LvXkJexct5vBqcd8QlNXh5EmRUeL/Z32p0n4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2 h1:6fl86IPqKEXoySqiOWdfgbEp9OVbn44zTfEICNEBDhY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2/go.mod h1:63HDfhFkdzBpI8WGXTSKUHPKS6mqldj4u3LJW7RZtSU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2 h1:XMgIRS+uW9F3yFKnXGRrI9pkHi99CXTmoz2kz2/TGBA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2/go.mod h1:vorxDzK+n3jiv9a5ST/LG0Eu9cSv1CRdKTpG6pDMs+M= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.2 h1:ZtHYnumr6QyxhzEzNZwzQTFJEXOswrZqTTkRxthwvr4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.2/go.mod h1:a1NXrYpBd311gBzn1UI5UzJyyvXktM4xNh/ydPiPpqY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= +github.com/aws/aws-sdk-go-v2/service/signin v1.9.0 h1:c3k+k/CS4L+sAIH6fxikL+g5g2LpeNczaoyjjw1iMKI= +github.com/aws/aws-sdk-go-v2/service/signin v1.9.0/go.mod h1:AGIoQg99fBrOIQnF78TLx4lj18mc4gZ0hJx1UaLIFM4= +github.com/aws/aws-sdk-go-v2/service/sso v1.37.0 h1:+rqBaOq7jzInjY8M12hr+zEe85JpRll9BjMx38r33Ok= +github.com/aws/aws-sdk-go-v2/service/sso v1.37.0/go.mod h1:XFlVwUsw3sYh8Hw37umYVJnrcWrwXWRxbNw/JY0bblw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.42.0 h1:hzM3GslEAOBcLn3DHH6ENToFi+vXP+n02W+x6zejAIM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.42.0/go.mod h1:588e7skMkYIYkSUseT8E3WKFfbAfrA1bj3Zf+qxJtJY= +github.com/aws/aws-sdk-go-v2/service/sts v1.49.0 h1:N7Ey8obY3uSui+cxl0OUzFlFmkxSucoJnrniFhw+cLc= +github.com/aws/aws-sdk-go-v2/service/sts v1.49.0/go.mod h1:zMBwjSf4Pt8a1OHYiZ5rPD0PJRK1kQrUaxhS/Dbld8E= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.39.0 h1:1uwRDYPYG8BIBU9Mj1sUAebNmlM6beu/ZKKweSLDxk8= +github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA= +github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= +github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU= +github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.24.1 h1:AtqTN21IXMMWo99LiEVAiBfNNQmO40d8xUfZI640mc0= +github.com/googleapis/gax-go/v2 v2.24.1/go.mod h1:bWeBei0NVwaNZKb2y1HUBS7gLXIF3/Tu3pq7j8D2Tb0= +github.com/gopherium/framework/dbkit v0.1.0 h1:jeFld4uZu555PkgbZQRKkDK1WOOB8am+Wu2Ar4xX9Jw= +github.com/gopherium/framework/dbkit v0.1.0/go.mod h1:M837GTBK4MLEt0oqvEKhCQydKFNhVLWvsDlAcqT21co= +github.com/gopherium/framework/dbkit/postgres v0.3.0 h1:W9SoXXu9Y3YRum5vIG10iBHlMMIhmbuUmOPLmZEnW3U= +github.com/gopherium/framework/dbkit/postgres v0.3.0/go.mod h1:ZUsvp2lOkhX+08sstBErt+jN6FM5HkhxV7hXgToRd+A= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= +github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= +github.com/peterldowns/testy v0.0.1 h1:9a6LzvnKcL52Crzud1z7jbsAojTntCh89ho6mgsr4KU= +github.com/peterldowns/testy v0.0.1/go.mod h1:J4sm75UEzbfBIcq0zbrshWWjsJQiJ5RrhTPYKVY2Ww8= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.27.2 h1:FjKNzcmMdGrQlSIu5alMSmakQtJFBgtw+A0bb1p/LC8= +github.com/pressly/goose/v3 v3.27.2/go.mod h1:qWW+/8dkVtJYjJrbIpwD5xxnEJTUKvxkQ9JKQp9LaIM= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo= +go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU= +go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA= +gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.297.0 h1:WktxTsnnx0yZNnsR6j0q6hR21RnnK81FHTOPy/ux4OE= +google.golang.org/api v0.297.0/go.mod h1:S4m8x0M6OkQpkOzGk1y9JG2sm4fFQrMh6dxzjCTszhE= +google.golang.org/genproto v0.0.0-20260904194346-d0f1323225a4 h1:6QRNo34FxJcE1LG6XfnmaQ17m6w21oXaKlKoBaF+UqE= +google.golang.org/genproto v0.0.0-20260904194346-d0f1323225a4/go.mod h1:eifJ91nefXnfP/bMkZsACVGm14ZtEL4eUXRLTryfbY4= +google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4 h1:NCe/UiklGd/9xjT+ROBVhJ1kf6TRQaFedsR+z7u1gvo= +google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4/go.mod h1:fJ2lYaWjqNknJyQBOCd0fA3HnEElJqGplH71a2txi+g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 h1:5t+ZydAFj5kGVLrgCvLmpmCf9ylGRd64hpEronfRaws= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= diff --git a/postgresblob/list.go b/postgresblob/list.go new file mode 100644 index 0000000..778ac15 --- /dev/null +++ b/postgresblob/list.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "gocloud.dev/blob/driver" +) + +// DefaultPageSize is the page size of a listing that sets none. +const DefaultPageSize = 1000 + +// lastByte is the highest byte value, which no UTF-8 key holds. +const lastByte = 0xFF + +// listing is one ListPaged call in progress. +type listing struct { + // b is the bucket being listed. + b *bucket + // prefix starts every key the listing answers. + prefix []byte + // upper is the bound every key under prefix stays below. + upper []byte + // delimiter ends a directory, and an empty delimiter lists every key flat. + delimiter string + // pageSize is the most objects the page holds. + pageSize int + // from is the bound the next key starts at or after. + from []byte + // page is the page so far. + page driver.ListPage +} + +// ListPaged returns one page of the live objects under the prefix of opts, in byte order, after BeforeList. +func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driver.ListPage, error) { + if err := runHook(opts.BeforeList); err != nil { + return nil, err + } + l := &listing{ + b: b, + prefix: append([]byte{}, opts.Prefix...), + upper: append([]byte(opts.Prefix), lastByte), + delimiter: opts.Delimiter, + pageSize: opts.PageSize, + from: opts.PageToken, + } + if l.pageSize == 0 { + l.pageSize = DefaultPageSize + } + return l.run(ctx) +} + +// run queries until the page is full or the prefix holds no more keys, and returns the page. +func (l *listing) run(ctx context.Context) (*driver.ListPage, error) { + again := true + for again { + var err error + if again, err = l.query(ctx); err != nil { + return nil, err + } + } + return &l.page, nil +} + +// listRow is one row a listing reads. +type listRow struct { + // Key is the key of the object. + Key []byte + // Size is the size of the object in bytes. + Size int64 + // MD5 is the MD5 hash of the object. + MD5 []byte + // ModTime is the time the object was last written. + ModTime time.Time +} + +// query reads the keys from l.from, and reports whether it stopped at a directory. +func (l *listing) query(ctx context.Context) (bool, error) { + rows, _ := l.b.pool.Query(ctx, l.b.sql.listLive, l.prefix, l.from, l.upper, l.pageSize-len(l.page.Objects)+1) + found, err := pgx.CollectRows(rows, pgx.RowToStructByPos[listRow]) + if err != nil { + return false, err + } + for _, row := range found { + if len(l.page.Objects) == l.pageSize { + l.page.NextPageToken = l.from + return false, nil + } + obj := &driver.ListObject{Key: string(row.Key), ModTime: row.ModTime, Size: row.Size, MD5: row.MD5} + next := append(row.Key, 0) + if dir, isDir := l.dirOf(obj.Key); isDir { + obj = &driver.ListObject{Key: dir, IsDir: true} + next = append([]byte(dir), lastByte) + } + l.page.Objects = append(l.page.Objects, obj) + l.from = next + if obj.IsDir { + return true, nil + } + } + return false, nil +} + +// dirOf returns the directory key holds under the prefix, and reports whether it is in one. +func (l *listing) dirOf(key string) (string, bool) { + if l.delimiter == "" { + return "", false + } + i := strings.Index(key[len(l.prefix):], l.delimiter) + if i < 0 { + return "", false + } + return key[:len(l.prefix)+i+len(l.delimiter)], true +} diff --git a/postgresblob/list_test.go b/postgresblob/list_test.go new file mode 100644 index 0000000..c605699 --- /dev/null +++ b/postgresblob/list_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "slices" + "testing" + + "github.com/gopherium/framework/postgresblob" +) + +func TestListFollowsByteOrderAcrossATwoByteRune(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{}) + want := []string{"names/e", "names/z", "names/é", "names/\U0010FFFF"} + + for _, key := range []string{"names/é", "names/\U0010FFFF", "names/z", "names/e"} { + if err := b.WriteAll(t.Context(), key, []byte("text"), nil); err != nil { + t.Fatalf("WriteAll(%q) = %v, want nil", key, err) + } + } + got := keysOf(t, b, "names/") + + if !slices.Equal(got, want) { + t.Errorf("List() keys = %q, want the byte order %q", got, want) + } +} diff --git a/postgresblob/main_test.go b/postgresblob/main_test.go new file mode 100644 index 0000000..ff031fb --- /dev/null +++ b/postgresblob/main_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "io" + "net/url" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "gocloud.dev/blob" + + "github.com/gopherium/framework/dbkit/postgres/pgtest" + "github.com/gopherium/framework/postgresblob" +) + +const ( + // serverVariable names the environment variable that holds the address of the test server. + serverVariable = "DBKIT_TEST_POSTGRES_URL" + // testSchema is the schema that holds the tables of every fresh test database. + testSchema = "files" + // unmigratedSchema is a schema no test database holds. + unmigratedSchema = "never_migrated" + // poolSize is the most connections the pool of one test database holds. + poolSize = 4 + // liveRows counts the live rows of the key its one argument names. + liveRows = "SELECT count(*) FROM files.objects WHERE key = $1 AND state = 'live'" + // pendingRows counts the pending rows of every key. + pendingRows = "SELECT count(*) FROM files.objects WHERE state = 'pending'" + // keyRows counts the rows of any state of the key its one argument names. + keyRows = "SELECT count(*) FROM files.objects WHERE key = $1" + // tableLookup answers whether the table its one argument names exists. + tableLookup = "SELECT to_regclass($1) IS NOT NULL" +) + +// server is the address of the test server. +var server string + +// TestMain runs the tests once serverVariable holds an address pgtest takes. +func TestMain(m *testing.M) { + server = os.Getenv(serverVariable) + if err := pgtest.CheckAddress(server); err != nil { + _, _ = fmt.Fprintf(os.Stderr, + "postgresblob: the tests need %s, the address of a PostgreSQL server pgtest takes: %v\n", serverVariable, err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +// migrator builds the template of every fresh test database, with the tables in testSchema. +var migrator = pgtest.Migrator(templateHash(), func(ctx context.Context, db *sql.DB) error { + return postgresblob.Migrate(ctx, db, testSchema) +}) + +// emptyMigrator builds a template that holds no table. +var emptyMigrator = pgtest.Migrator("postgresblob-empty", func(context.Context, *sql.DB) error { return nil }) + +// templateHash returns the hash of the statements that build the template of migrator. +func templateHash() string { + sum := sha256.Sum256([]byte(strings.Join(postgresblob.MigrationStatements(testSchema), "\n"))) + return "postgresblob-" + hex.EncodeToString(sum[:]) +} + +// testDB is one fresh test database of one test. +type testDB struct { + // t is the test the database belongs to. + t *testing.T + // address is the address of the database. + address string + // pool reaches the database. + pool *pgxpool.Pool +} + +// freshDB returns a fresh database that holds the tables in testSchema. +func freshDB(t *testing.T) testDB { + t.Helper() + address := pgtest.New(t, server, migrator) + return testDB{t: t, address: address, pool: poolOn(t, address)} +} + +// emptyDB returns a fresh database that holds no table. +func emptyDB(t *testing.T) testDB { + t.Helper() + address := pgtest.New(t, server, emptyMigrator) + return testDB{t: t, address: address, pool: poolOn(t, address)} +} + +// poolOn returns a pool of poolSize connections on address, closed when the test ends. +func poolOn(t testing.TB, address string) *pgxpool.Pool { + t.Helper() + config, err := pgxpool.ParseConfig(address) + if err != nil { + t.Fatalf("parse the test database address: %v", err) + } + config.MaxConns = poolSize + pool, err := pgxpool.NewWithConfig(context.Background(), config) + if err != nil { + t.Fatalf("open a pool on the test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +// open returns a bucket on db with opts, in testSchema when opts names no schema, closed when the test ends. +func (db testDB) open(opts postgresblob.Options) *blob.Bucket { + db.t.Helper() + if opts.Schema == "" { + opts.Schema = testSchema + } + b, err := postgresblob.OpenBucket(db.t.Context(), db.pool, &opts) + if err != nil { + db.t.Fatalf("OpenBucket() = %v, want a bucket", err) + } + db.t.Cleanup(func() { _ = b.Close() }) + return b +} + +// count returns the count lookup answers for args on db. +func (db testDB) count(lookup string, args ...any) int { + db.t.Helper() + var n int + if err := db.pool.QueryRow(context.Background(), lookup, args...).Scan(&n); err != nil { + db.t.Fatalf("%s: %v", lookup, err) + } + return n +} + +// hasTable reports whether db holds the table named table. +func (db testDB) hasTable(table string) bool { + db.t.Helper() + var held bool + if err := db.pool.QueryRow(context.Background(), tableLookup, table).Scan(&held); err != nil { + db.t.Fatalf("%s: %v", tableLookup, err) + } + return held +} + +// addressWith returns address with each query parameter of params set. +func addressWith(t testing.TB, address string, params url.Values) string { + t.Helper() + parsed, err := url.Parse(address) + if err != nil { + t.Fatalf("parse the test database address: %v", err) + } + query := parsed.Query() + for key, values := range params { + query[key] = values + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +// keysOf returns the keys a listing of b under prefix answers, in its order. +func keysOf(t testing.TB, b *blob.Bucket, prefix string) []string { + t.Helper() + var keys []string + iter := b.List(&blob.ListOptions{Prefix: prefix}) + for { + obj, err := iter.Next(context.Background()) + if err == io.EOF { + return keys + } + if err != nil { + t.Fatalf("List().Next() = %v, want an object or io.EOF", err) + } + keys = append(keys, obj.Key) + } +} diff --git a/postgresblob/migrate.go b/postgresblob/migrate.go new file mode 100644 index 0000000..b9c3551 --- /dev/null +++ b/postgresblob/migrate.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "database/sql" + "fmt" + + "github.com/pressly/goose/v3" + + "github.com/gopherium/framework/dbkit/postgres" +) + +// versionTable is the goose version table each schema keeps. +const versionTable = "goose_db_version" + +// Migrate creates schema, DefaultSchema when empty, and its tables on db under goose's migration lock. +func Migrate(ctx context.Context, db *sql.DB, schema string) error { + schema = schemaOr(schema) + err := postgres.Migrate(ctx, db, postgres.Migrations{ + Table: schema + "." + versionTable, + Go: []*goose.Migration{goose.NewGoMigration(1, &goose.GoFunc{RunTx: runAll(migrationStatements(schema))}, nil)}, + }) + if err != nil { + return fmt.Errorf("postgresblob: migrate the schema %s: %w", schema, err) + } + return nil +} + +// runAll returns a migration step that runs each of statements in its transaction. +func runAll(statements []string) func(ctx context.Context, tx *sql.Tx) error { + return func(ctx context.Context, tx *sql.Tx) error { + for _, statement := range statements { + if _, err := tx.ExecContext(ctx, statement); err != nil { + return err + } + } + return nil + } +} + +// schemaOr returns schema, or DefaultSchema when schema is empty. +func schemaOr(schema string) string { + if schema == "" { + return DefaultSchema + } + return schema +} diff --git a/postgresblob/migrate_test.go b/postgresblob/migrate_test.go new file mode 100644 index 0000000..ea34ae2 --- /dev/null +++ b/postgresblob/migrate_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "database/sql" + "strings" + "testing" + + "github.com/jackc/pgx/v5/stdlib" + + "github.com/gopherium/framework/postgresblob" +) + +// sqlDB returns a database/sql view of the pool of db, closed when the test ends. +func (db testDB) sqlDB() *sql.DB { + view := stdlib.OpenDBFromPool(db.pool) + db.t.Cleanup(func() { _ = view.Close() }) + return view +} + +func TestMigrateCreatesTheSchemaAndBothTablesOnce(t *testing.T) { + t.Parallel() + db := emptyDB(t) + + for run := range 2 { + if err := postgresblob.Migrate(t.Context(), db.sqlDB(), "archive"); err != nil { + t.Fatalf("Migrate() run %d = %v, want nil", run+1, err) + } + } + + for _, table := range []string{"archive.objects", "archive.chunks"} { + if !db.hasTable(table) { + t.Errorf("the database lacks %s after Migrate(), want it", table) + } + } +} + +func TestMigrateAndOpenBucketShareTheDefaultSchema(t *testing.T) { + t.Parallel() + db := emptyDB(t) + + if err := postgresblob.Migrate(t.Context(), db.sqlDB(), ""); err != nil { + t.Fatalf("Migrate() with no schema = %v, want nil", err) + } + b, err := postgresblob.OpenBucket(t.Context(), db.pool, nil) + if err != nil { + t.Fatalf("OpenBucket() with no options = %v, want a bucket", err) + } + defer func() { _ = b.Close() }() + + if err := b.WriteAll(t.Context(), "notes/report.txt", []byte("text"), nil); err != nil { + t.Errorf("WriteAll() on the default schema = %v, want nil", err) + } + if !db.hasTable(postgresblob.DefaultSchema + ".objects") { + t.Errorf("the database lacks %s.objects, want the default schema to hold the tables", postgresblob.DefaultSchema) + } +} + +func TestMigrateRefusesASchemaThatIsNotALowercasePlainName(t *testing.T) { + t.Parallel() + db := emptyDB(t) + + err := postgresblob.Migrate(t.Context(), db.sqlDB(), "Files") + + if err == nil || !strings.Contains(err.Error(), "Files") { + t.Errorf("Migrate() with the schema Files = %v, want an error that names it", err) + } + if db.hasTable(`"Files".objects`) { + t.Error("the database holds \"Files\".objects, want no table") + } +} diff --git a/postgresblob/must_internal_test.go b/postgresblob/must_internal_test.go new file mode 100644 index 0000000..f67e5d9 --- /dev/null +++ b/postgresblob/must_internal_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestMustPanicsOnTheErrorOfAPoolWithNoRoom(t *testing.T) { + t.Parallel() + + config, err := pgxpool.ParseConfig("postgres://reader@127.0.0.1:1/files") + if err != nil { + t.Fatalf("ParseConfig() = %v, want a configuration", err) + } + config.MaxConns = 0 + defer func() { + if recover() == nil { + t.Error("must() returned, want a panic with the error of a pool capped at zero") + } + }() + + must(pgxpool.NewWithConfig(context.Background(), config)) +} diff --git a/postgresblob/open_test.go b/postgresblob/open_test.go new file mode 100644 index 0000000..1558655 --- /dev/null +++ b/postgresblob/open_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "context" + "crypto/rand" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/gopherium/framework/postgresblob" +) + +// serverExec runs statement on the database postgres of the test server. +func serverExec(t testing.TB, statement string) { + t.Helper() + conn, err := pgx.Connect(context.Background(), server) + if err != nil { + t.Fatalf("connect to the test server: %v", err) + } + defer func() { _ = conn.Close(context.Background()) }() + if _, err := conn.Exec(context.Background(), statement); err != nil { + t.Fatalf("%s: %v", statement, err) + } +} + +// asciiDatabase returns the address of a new database whose encoding is SQL_ASCII, dropped when the test ends. +func asciiDatabase(t *testing.T) string { + t.Helper() + name := "postgresblob_ascii_" + strings.ToLower(rand.Text()) + ident := pgx.Identifier{name}.Sanitize() + serverExec(t, "CREATE DATABASE "+ident+" ENCODING 'SQL_ASCII' LOCALE 'C' TEMPLATE template0") + t.Cleanup(func() { serverExec(t, "DROP DATABASE "+ident+" WITH (FORCE)") }) + address, err := url.Parse(server) + if err != nil { + t.Fatalf("parse the test server address: %v", err) + } + address.Path = "/" + name + return address.String() +} + +func TestOpenBucketRefusesOptionsOutOfRange(t *testing.T) { + t.Parallel() + pool := freshDB(t).pool + largest := strconv.Itoa(postgresblob.MaxChunkSize) + tests := map[string]struct { + opts postgresblob.Options + want []string + }{ + "a negative chunk size": {opts: postgresblob.Options{ChunkSize: -1}, want: []string{"ChunkSize", largest}}, + "a chunk size past MaxChunkSize": { + opts: postgresblob.Options{ChunkSize: postgresblob.MaxChunkSize + 1}, + want: []string{"ChunkSize", largest}, + }, + "a negative size cap": {opts: postgresblob.Options{MaxSize: -1}, want: []string{"MaxSize"}}, + "a negative cleanup timeout": { + opts: postgresblob.Options{CleanupTimeout: -time.Second}, + want: []string{"CleanupTimeout"}, + }, + } + + for name, tt := range tests { + b, err := postgresblob.OpenBucket(t.Context(), pool, &tt.opts) + if !mentions(err, tt.want...) { + t.Errorf("OpenBucket() with %s = %v, %v, want an error that names %q", name, b, err, tt.want) + } + } +} + +func TestOpenBucketTakesTheLargestChunkSize(t *testing.T) { + t.Parallel() + pool := freshDB(t).pool + + b, err := postgresblob.OpenBucket(t.Context(), pool, + &postgresblob.Options{Schema: testSchema, ChunkSize: postgresblob.MaxChunkSize}) + + if err != nil { + t.Fatalf("OpenBucket() with a chunk size of MaxChunkSize = %v, want a bucket", err) + } + _ = b.Close() +} + +func TestOpenBucketNeverClosesItsPool(t *testing.T) { + t.Parallel() + pool := freshDB(t).pool + b, err := postgresblob.OpenBucket(t.Context(), pool, &postgresblob.Options{Schema: testSchema}) + if err != nil { + t.Fatalf("OpenBucket() = %v, want a bucket", err) + } + + if err := b.Close(); err != nil { + t.Fatalf("Close() = %v, want nil", err) + } + + if err := pool.Ping(t.Context()); err != nil { + t.Errorf("Ping() on the pool after the bucket closed = %v, want nil", err) + } +} + +func TestOpenBucketAndSweepRefuseANilPool(t *testing.T) { + t.Parallel() + + b, openErr := postgresblob.OpenBucket(t.Context(), nil, nil) + n, sweepErr := postgresblob.Sweep(t.Context(), nil, postgresblob.SweepOptions{OlderThan: time.Hour, Limit: 1}) + + if !mentions(openErr, "pool") { + t.Errorf("OpenBucket() with a nil pool = %v, %v, want an error that names the pool", b, openErr) + } + if !mentions(sweepErr, "pool") { + t.Errorf("Sweep() with a nil pool = %d, %v, want an error that names the pool", n, sweepErr) + } +} + +func TestOpenBucketRefusesADatabaseWhoseEncodingIsNotUTF8(t *testing.T) { + t.Parallel() + pool := poolOn(t, asciiDatabase(t)) + + b, err := postgresblob.OpenBucket(t.Context(), pool, &postgresblob.Options{Schema: testSchema}) + + if !mentions(err, "SQL_ASCII", "UTF8") { + t.Errorf("OpenBucket() on a SQL_ASCII database = %v, %v, want an error that names SQL_ASCII and UTF8", b, err) + } +} diff --git a/postgresblob/reader.go b/postgresblob/reader.go new file mode 100644 index 0000000..2005a29 --- /dev/null +++ b/postgresblob/reader.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "errors" + "io" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "gocloud.dev/blob/driver" +) + +// reader reads a range of one object, one statement per Read. +type reader struct { + // ctx is the context of every read. + ctx context.Context + // b is the bucket the object lives in. + b *bucket + // id is the id of the row of the object. + id uuid.UUID + // chunkSize is the chunk size the object was written with. + chunkSize int64 + // perRead is the most bytes one Read fetches, the chunk size or DefaultChunkSize when that is larger. + perRead int64 + // pos is the offset of the next byte to read. + pos int64 + // end is the offset after the last byte to read. + end int64 + // attrs are the attributes of the object. + attrs driver.ReaderAttributes +} + +// NewRangeReader returns a reader of at most length bytes of the live object of key from offset, after BeforeRead. +func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length int64, + opts *driver.ReaderOptions) (driver.Reader, error) { + r := &reader{ctx: ctx, b: b} + err := b.pool.QueryRow(ctx, b.sql.readObject, []byte(key)).Scan(&r.id, &r.attrs.Size, &r.chunkSize, + &r.attrs.ContentType, &r.attrs.ModTime) + if errors.Is(err, pgx.ErrNoRows) { + return nil, errNotFound + } + if err != nil { + return nil, err + } + if err := runHook(opts.BeforeRead); err != nil { + return nil, err + } + r.perRead = max(r.chunkSize, DefaultChunkSize) + r.pos = min(offset, r.attrs.Size) + r.end = r.attrs.Size + if length >= 0 && length < r.end-r.pos { + r.end = r.pos + length + } + return r, nil +} + +// Read fills p with at most perRead of the next bytes of the range, from the chunks that hold them. +func (r *reader) Read(p []byte) (int, error) { + if r.pos >= r.end { + return 0, io.EOF + } + stop := r.pos + min(int64(len(p)), r.end-r.pos, r.perRead) + var data []byte + if err := r.b.pool.QueryRow(r.ctx, r.b.sql.readRange, r.id, r.pos, stop, r.chunkSize).Scan(&data); err != nil { + return 0, err + } + if int64(len(data)) < stop-r.pos { + return 0, errReplaced + } + copied := copy(p, data) + r.pos += int64(copied) + return copied, nil +} + +// Close closes the reader, which holds no connection. +func (*reader) Close() error { return nil } + +// Attributes returns the attributes of the object. +func (r *reader) Attributes() *driver.ReaderAttributes { return &r.attrs } + +// As answers false for every type. +func (*reader) As(any) bool { return false } diff --git a/postgresblob/reader_test.go b/postgresblob/reader_test.go new file mode 100644 index 0000000..a367ef3 --- /dev/null +++ b/postgresblob/reader_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "bytes" + "strconv" + "testing" + + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +// digits returns size bytes of the ten digits in a loop. +func digits(size int) []byte { + return bytes.Repeat([]byte("0123456789"), size/10+1)[:size] +} + +func TestOneReadFetchesAtMostOneChunkOrTheDefaultChunkSize(t *testing.T) { + t.Parallel() + db := freshDB(t) + tests := map[string]struct { + chunkSize int + want int + }{ + "chunks smaller than the default": {chunkSize: 4096, want: postgresblob.DefaultChunkSize}, + "chunks larger than the default": { + chunkSize: 2 * postgresblob.DefaultChunkSize, + want: 2 * postgresblob.DefaultChunkSize, + }, + } + + for name, tt := range tests { + b := db.open(postgresblob.Options{ChunkSize: tt.chunkSize}) + key := "large/" + strconv.Itoa(tt.chunkSize) + ".txt" + content := digits(tt.want + 10) + if err := b.WriteAll(t.Context(), key, content, textOptions); err != nil { + t.Fatalf("WriteAll() of %d bytes = %v, want nil", len(content), err) + } + r, err := b.NewReader(t.Context(), key, nil) + if err != nil { + t.Fatalf("NewReader() = %v, want a reader", err) + } + buf := make([]byte, 2*len(content)) + + n, err := r.Read(buf) + + _ = r.Close() + if n != tt.want || err != nil { + t.Errorf("Read() into %d bytes with %s = %d bytes, %v, want %d bytes", len(buf), name, n, err, tt.want) + } + if !bytes.Equal(buf[:n], content[:n]) { + t.Errorf("Read() with %s filled %d bytes that differ from the start of the object", name, n) + } + } +} + +func TestAReadOfAReplacedObjectFails(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{ChunkSize: 4}) + const key = "notes/report.txt" + if err := b.WriteAll(t.Context(), key, []byte("the first version"), textOptions); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + r, err := b.NewReader(t.Context(), key, nil) + if err != nil { + t.Fatalf("NewReader() = %v, want a reader", err) + } + defer func() { _ = r.Close() }() + start := make([]byte, 4) + if n, err := r.Read(start); n != 4 || err != nil { + t.Fatalf("Read() of the first chunk = %d, %v, want 4 bytes", n, err) + } + if err := b.WriteAll(t.Context(), key, []byte("the second version"), textOptions); err != nil { + t.Fatalf("WriteAll() of the replacement = %v, want nil", err) + } + + n, err := r.Read(make([]byte, 64)) + + if n != 0 || gcerrors.Code(err) != gcerrors.FailedPrecondition { + t.Errorf("Read() after the replacement = %d, %v, want no bytes and FailedPrecondition", n, err) + } +} diff --git a/postgresblob/statements.go b/postgresblob/statements.go new file mode 100644 index 0000000..f9daf35 --- /dev/null +++ b/postgresblob/statements.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "strings" + + "github.com/jackc/pgx/v5" +) + +const ( + // createObjects creates the table of objects. + createObjects = `CREATE TABLE {objects} ( + id uuid PRIMARY KEY, + key bytea NOT NULL, + state text NOT NULL CHECK (state IN ('pending', 'live')), + size bigint NOT NULL DEFAULT 0, + chunk_size integer NOT NULL CHECK (chunk_size > 0), + content_type text NOT NULL, + cache_control text NOT NULL DEFAULT '', + content_disposition text NOT NULL DEFAULT '', + content_encoding text NOT NULL DEFAULT '', + content_language text NOT NULL DEFAULT '', + metadata json NOT NULL DEFAULT '{}', + md5 bytea, + created_at timestamptz NOT NULL DEFAULT now(), + modified_at timestamptz NOT NULL DEFAULT now() +)` + // createLiveKeyIndex lets each key hold one live object at most. + createLiveKeyIndex = `CREATE UNIQUE INDEX objects_live_key ON {objects} (key) WHERE state = 'live'` + // createPendingAgeIndex orders the unfinished uploads by age. + createPendingAgeIndex = `CREATE INDEX objects_pending_age ON {objects} (modified_at) WHERE state = 'pending'` + // createChunks creates the table of chunks. + createChunks = `CREATE TABLE {chunks} ( + object_id uuid NOT NULL REFERENCES {objects} (id) ON DELETE CASCADE, + seq integer NOT NULL, + data bytea NOT NULL, + PRIMARY KEY (object_id, seq) +)` + // storeChunksExternal keeps the chunk bytes uncompressed and out of line. + storeChunksExternal = `ALTER TABLE {chunks} ALTER COLUMN data SET STORAGE EXTERNAL` + + // serverEncoding reads the encoding of the database. + serverEncoding = `SELECT current_setting('server_encoding')` + // lockKey takes the transaction lock of one key. + lockKey = `SELECT pg_advisory_xact_lock($1)` + // liveExists answers whether a key holds a live object. + liveExists = `SELECT EXISTS (SELECT FROM {objects} WHERE key = $1 AND state = 'live')` + // insertPending inserts the pending row of an upload. + insertPending = `INSERT INTO {objects} (id, key, state, chunk_size, content_type, cache_control, + content_disposition, content_encoding, content_language, metadata) +VALUES ($1, $2, 'pending', $3, $4, $5, $6, $7, $8, $9)` + // insertChunk inserts one chunk of a pending upload and marks the upload as touched. + insertChunk = `WITH touched AS ( + UPDATE {objects} SET modified_at = now() WHERE id = $1 AND state = 'pending' RETURNING id +) +INSERT INTO {chunks} (object_id, seq, data) SELECT id, $2, $3 FROM touched` + // deleteLive deletes the live object of a key and returns when it was created. + deleteLive = `DELETE FROM {objects} WHERE key = $1 AND state = 'live' RETURNING created_at` + // markLive makes a pending row live with its size and hash, keeping the creation time it is given. + markLive = `UPDATE {objects} SET state = 'live', size = $2, md5 = $3, modified_at = now(), + created_at = COALESCE($4, created_at) +WHERE id = $1` + // deletePending deletes the pending row of an upload. + deletePending = `DELETE FROM {objects} WHERE id = $1 AND state = 'pending'` + // readAttributes reads the attributes of the live object of a key. + readAttributes = `SELECT id, size, content_type, cache_control, content_disposition, content_encoding, + content_language, metadata, md5, created_at, modified_at +FROM {objects} WHERE key = $1 AND state = 'live'` + // readObject reads what a reader needs of the live object of a key. + readObject = `SELECT id, size, chunk_size, content_type, modified_at FROM {objects} WHERE key = $1 AND state = 'live'` + // readRange reads the bytes from $2 up to $3 of an object of chunk size $4, from every chunk the range crosses. + readRange = `SELECT string_agg(substring(c.data + FROM (GREATEST(r.start_at, c.seq * r.chunk_size) - c.seq * r.chunk_size + 1)::integer + FOR (LEAST(r.stop_at, (c.seq + 1) * r.chunk_size) - GREATEST(r.start_at, c.seq * r.chunk_size))::integer), + ''::bytea ORDER BY c.seq) +FROM {chunks} c, (SELECT $2::bigint AS start_at, $3::bigint AS stop_at, $4::bigint AS chunk_size) r +WHERE c.object_id = $1 AND c.seq BETWEEN r.start_at / r.chunk_size AND (r.stop_at - 1) / r.chunk_size` + // listLive lists the live objects from a bound up to the end of a prefix, in byte order. + listLive = `SELECT key, size, md5, modified_at FROM {objects} +WHERE state = 'live' AND key >= GREATEST($1::bytea, $2::bytea) AND key < $3 +ORDER BY key LIMIT $4` + // copySource reads the id, size and hash of the live object of a key. + copySource = `SELECT id, size, md5 FROM {objects} WHERE key = $1 AND state = 'live'` + // copyObject inserts a pending copy of an object and of its chunks under a new id and key. + copyObject = `WITH copied AS ( + INSERT INTO {objects} (id, key, state, size, chunk_size, content_type, cache_control, + content_disposition, content_encoding, content_language, metadata, md5) + SELECT $1, $2, 'pending', size, chunk_size, content_type, cache_control, + content_disposition, content_encoding, content_language, metadata, md5 + FROM {objects} WHERE id = $3 + RETURNING id +) +INSERT INTO {chunks} (object_id, seq, data) +SELECT copied.id, c.seq, c.data FROM copied, {chunks} c WHERE c.object_id = $3` +) + +// statements are the SQL statements of one schema. +type statements struct { + // liveExists answers whether a key holds a live object. + liveExists string + // insertPending inserts the pending row of an upload. + insertPending string + // insertChunk inserts one chunk of a pending upload. + insertChunk string + // deleteLive deletes the live object of a key. + deleteLive string + // markLive makes a pending row live. + markLive string + // deletePending deletes the pending row of an upload. + deletePending string + // readAttributes reads the attributes of a live object. + readAttributes string + // readObject reads what a reader needs of a live object. + readObject string + // readRange reads a range of an object. + readRange string + // listLive lists live objects in byte order. + listLive string + // copySource reads the source of a copy. + copySource string + // copyObject inserts a pending copy of an object and its chunks. + copyObject string +} + +// tablesOf returns the replacer that writes the quoted names of the tables of schema. +func tablesOf(schema string) *strings.Replacer { + return strings.NewReplacer( + "{objects}", pgx.Identifier{schema, "objects"}.Sanitize(), + "{chunks}", pgx.Identifier{schema, "chunks"}.Sanitize(), + ) +} + +// statementsOf returns the statements of the tables of schema. +func statementsOf(schema string) statements { + tables := tablesOf(schema) + return statements{ + liveExists: tables.Replace(liveExists), + insertPending: tables.Replace(insertPending), + insertChunk: tables.Replace(insertChunk), + deleteLive: tables.Replace(deleteLive), + markLive: tables.Replace(markLive), + deletePending: tables.Replace(deletePending), + readAttributes: tables.Replace(readAttributes), + readObject: tables.Replace(readObject), + readRange: tables.Replace(readRange), + listLive: tables.Replace(listLive), + copySource: tables.Replace(copySource), + copyObject: tables.Replace(copyObject), + } +} + +// migrationStatements returns the statements of the first migration of schema. +func migrationStatements(schema string) []string { + tables := tablesOf(schema) + return []string{ + tables.Replace(createObjects), + tables.Replace(createLiveKeyIndex), + tables.Replace(createPendingAgeIndex), + tables.Replace(createChunks), + tables.Replace(storeChunksExternal), + } +} diff --git a/postgresblob/sweep.go b/postgresblob/sweep.go new file mode 100644 index 0000000..d7beff8 --- /dev/null +++ b/postgresblob/sweep.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// sweepPending deletes the oldest pending rows untouched for an interval that no transaction holds, up to a limit. +const sweepPending = `DELETE FROM {objects} WHERE id IN ( + SELECT id FROM {objects} WHERE state = 'pending' AND modified_at < now() - $1::interval + ORDER BY modified_at LIMIT $2 FOR UPDATE SKIP LOCKED +)` + +// SweepOptions name the unfinished uploads one Sweep call deletes. +type SweepOptions struct { + // Schema names the schema that holds the tables, DefaultSchema when empty. + Schema string + // OlderThan is how long an upload must go without a new chunk before Sweep deletes it, above zero. + OlderThan time.Duration + // Limit is the most uploads one call deletes, 1 or more. + Limit int +} + +// Sweep deletes the oldest unfinished uploads opts names, at most opts.Limit, and returns how many it deleted. +func Sweep(ctx context.Context, pool *pgxpool.Pool, opts SweepOptions) (int, error) { + switch { + case pool == nil: + return 0, errNoPool + case opts.OlderThan <= 0: + return 0, fmt.Errorf("postgresblob: the option OlderThan must be above zero, got %v", opts.OlderThan) + case opts.Limit < 1: + return 0, fmt.Errorf("postgresblob: the option Limit must be 1 or more, got %d", opts.Limit) + } + schema := schemaOr(opts.Schema) + tag, err := pool.Exec(ctx, tablesOf(schema).Replace(sweepPending), opts.OlderThan, opts.Limit) + if err != nil { + return 0, fmt.Errorf("postgresblob: sweep the schema %s: %w", schema, err) + } + return int(tag.RowsAffected()), nil +} diff --git a/postgresblob/sweep_test.go b/postgresblob/sweep_test.go new file mode 100644 index 0000000..2c22921 --- /dev/null +++ b/postgresblob/sweep_test.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "testing" + "time" + + "gocloud.dev/blob" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +// staleAge is an age every pending row of a finished statement is older than. +const staleAge = time.Microsecond + +// uploadText is the text every pending upload writes. +const uploadText = "an upload longer than one chunk" + +// pendingUpload starts an upload of key on b that leaves its pending row in the database, closed when the test ends. +func pendingUpload(t *testing.T, b *blob.Bucket, key string) *blob.Writer { + t.Helper() + w, err := b.NewWriter(t.Context(), key, textOptions) + if err != nil { + t.Fatalf("NewWriter() = %v, want a writer", err) + } + t.Cleanup(func() { _ = w.Close() }) + if _, err := w.Write([]byte(uploadText)); err != nil { + t.Fatalf("Write() = %v, want nil", err) + } + return w +} + +// sweepOf returns the options of a sweep of testSchema. +func sweepOf(olderThan time.Duration, limit int) postgresblob.SweepOptions { + return postgresblob.SweepOptions{Schema: testSchema, OlderThan: olderThan, Limit: limit} +} + +// pendingAge sets how long ago the pending row of the key its first argument names was last touched. +const pendingAge = "UPDATE files.objects SET modified_at = now() - $2::interval WHERE key = $1 AND state = 'pending'" + +// ageUpload makes the pending row of key on db look last touched age ago. +func (db testDB) ageUpload(key string, age time.Duration) { + db.t.Helper() + tag, err := db.pool.Exec(db.t.Context(), pendingAge, []byte(key), age) + if err != nil || tag.RowsAffected() != 1 { + db.t.Fatalf("age the upload of %s: %v, %v", key, tag, err) + } +} + +func TestSweepDeletesTheOldestUploadsFirst(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const youngest = "uploads/one-hour.txt" + ages := map[string]time.Duration{ + "uploads/four-hours.txt": 4 * time.Hour, + "uploads/three-hours.txt": 3 * time.Hour, + youngest: time.Hour, + } + for key, age := range ages { + pendingUpload(t, b, key) + db.ageUpload(key, age) + } + + n, err := postgresblob.Sweep(t.Context(), db.pool, sweepOf(30*time.Minute, 2)) + + if n != 2 || err != nil { + t.Errorf("Sweep() of three stale uploads with the limit 2 = %d, %v, want 2", n, err) + } + if left := db.count(keyRows, []byte(youngest)); left != 1 { + t.Errorf("the youngest stale upload holds %d rows after the sweep, want its pending row", left) + } +} + +func TestSweepCountsTheAgeFromTheLastChunk(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const key = "uploads/slow.txt" + w := pendingUpload(t, b, key) + db.ageUpload(key, time.Hour) + if _, err := w.Write([]byte("one more chunk of text")); err != nil { + t.Fatalf("Write() of one more chunk = %v, want nil", err) + } + + n, err := postgresblob.Sweep(t.Context(), db.pool, sweepOf(30*time.Minute, 10)) + + if n != 0 || err != nil { + t.Errorf("Sweep() of an upload that wrote a chunk just now = %d, %v, want 0", n, err) + } +} + +func TestSweepDeletesStaleUploadsLimitAtATime(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + for _, key := range []string{"uploads/a.txt", "uploads/b.txt", "uploads/c.txt"} { + pendingUpload(t, b, key) + } + + n, err := postgresblob.Sweep(t.Context(), db.pool, sweepOf(staleAge, 2)) + + if n != 2 || err != nil { + t.Errorf("Sweep() of three stale uploads with the limit 2 = %d, %v, want 2", n, err) + } + if left := db.count(pendingRows); left != 1 { + t.Errorf("%d pending rows stay after the sweep, want 1", left) + } +} + +func TestSweepSparesFreshUploads(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const key = "uploads/fresh.txt" + w := pendingUpload(t, b, key) + + n, err := postgresblob.Sweep(t.Context(), db.pool, sweepOf(time.Hour, 10)) + + if n != 0 || err != nil { + t.Errorf("Sweep() of a fresh upload = %d, %v, want 0", n, err) + } + if err := w.Close(); err != nil { + t.Errorf("Close() of the spared upload = %v, want nil", err) + } + expectContent(t, b, key, []byte(uploadText)) +} + +func TestTheWriterOfASweptUploadFailsAndLeavesNoObject(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const key = "uploads/stale.txt" + w := pendingUpload(t, b, key) + if n, err := postgresblob.Sweep(t.Context(), db.pool, sweepOf(staleAge, 10)); n != 1 || err != nil { + t.Fatalf("Sweep() of a stale upload = %d, %v, want 1", n, err) + } + + err := w.Close() + + expectCode(t, "Close() of a swept upload", err, gcerrors.FailedPrecondition) + if n := db.count(keyRows, []byte(key)); n != 0 { + t.Errorf("the key holds %d rows after the swept upload closed, want none", n) + } +} + +func TestSweepRefusesAnAgeAtOrBelowZeroAndALimitBelowOne(t *testing.T) { + t.Parallel() + db := freshDB(t) + tests := map[string]struct { + opts postgresblob.SweepOptions + want string + }{ + "a zero age": {opts: sweepOf(0, 1), want: "OlderThan"}, + "a negative age": {opts: sweepOf(-time.Second, 1), want: "OlderThan"}, + "a zero limit": {opts: sweepOf(time.Hour, 0), want: "Limit"}, + } + + for name, tt := range tests { + n, err := postgresblob.Sweep(t.Context(), db.pool, tt.opts) + if n != 0 || !mentions(err, tt.want) { + t.Errorf("Sweep() with %s = %d, %v, want an error that names %s", name, n, err, tt.want) + } + } +} diff --git a/postgresblob/url.go b/postgresblob/url.go new file mode 100644 index 0000000..75bb432 --- /dev/null +++ b/postgresblob/url.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "errors" + "net/url" + + "github.com/jackc/pgx/v5/pgxpool" + "gocloud.dev/blob" +) + +const ( + // Scheme is the URL scheme postgresblob registers its URLOpener under on blob.DefaultURLMux. + Scheme = "postgres" + // schemaParameter is the query parameter of a URL that names the schema. + schemaParameter = "schema" +) + +// errUnparsable refuses a URL pgx cannot parse, and holds none of pgx's text. +var errUnparsable = errors.New("postgresblob: pgx cannot parse the URL or one of its settings") + +// init registers a URLOpener with no options for Scheme on blob.DefaultURLMux. +func init() { + blob.DefaultURLMux().RegisterBucket(Scheme, &URLOpener{}) +} + +// URLOpener opens buckets from postgres URLs, each on a pool of its own that the bucket closes. +type URLOpener struct { + // Options are the options of every bucket the opener opens. + Options Options +} + +// OpenBucketURL opens a bucket on a new pool for u, in the schema its query parameter schema names. +func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) { + opts := o.Options + address := *u + query := address.Query() + if query.Has(schemaParameter) { + opts.Schema = query.Get(schemaParameter) + query.Del(schemaParameter) + } + address.RawQuery = query.Encode() + config, err := pgxpool.ParseConfig(address.String()) + if err != nil { + return nil, errUnparsable + } + pool := must(pgxpool.NewWithConfig(ctx, config)) + b, err := openBucket(ctx, pool, &opts) + if err != nil { + pool.Close() + return nil, err + } + b.ownsPool = true + return blob.NewBucket(b), nil +} + +// must returns value, and panics with err when err is set. +func must[T any](value T, err error) T { + if err != nil { + panic(err) + } + return value +} diff --git a/postgresblob/url_test.go b/postgresblob/url_test.go new file mode 100644 index 0000000..8e5a880 --- /dev/null +++ b/postgresblob/url_test.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "crypto/rand" + "net/url" + "strings" + "testing" + "time" + + "gocloud.dev/blob" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +// sessionBound is how long a test waits for the sessions of a closed pool to end. +const sessionBound = 5 * time.Second + +// namedSessions counts the sessions whose application name is its one argument. +const namedSessions = "SELECT count(*) FROM pg_stat_activity WHERE application_name = $1" + +// inTestSchema is the query that names testSchema. +var inTestSchema = url.Values{"schema": {testSchema}} + +// awaitNoSession fails the test unless no session on the server is named name within sessionBound. +func (db testDB) awaitNoSession(name string) { + db.t.Helper() + deadline := time.Now().Add(sessionBound) + for db.count(namedSessions, name) != 0 { + if time.Now().After(deadline) { + db.t.Fatalf("a session named %s stayed %v after Close(), want the pool closed", name, sessionBound) + } + time.Sleep(10 * time.Millisecond) + } +} + +// sessionName returns a new application name no other session has. +func sessionName() string { + return "postgresblob_" + strings.ToLower(rand.Text()) +} + +func TestTheURLOpenerOpensTheSchemaTheURLNames(t *testing.T) { + t.Parallel() + db := freshDB(t) + const key = "notes/report.txt" + + b, err := blob.OpenBucket(t.Context(), addressWith(t, db.address, inTestSchema)) + if err != nil { + t.Fatalf("blob.OpenBucket() of a postgres URL = %v, want a bucket", err) + } + defer func() { _ = b.Close() }() + + if err := b.WriteAll(t.Context(), key, []byte("text"), nil); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + if n := db.count(liveRows, []byte(key)); n != 1 { + t.Errorf("%s holds %d live rows of the key, want 1", testSchema, n) + } +} + +func TestTheURLOpenerRefusesAURLPgxCannotParseWithAFixedMessage(t *testing.T) { + t.Parallel() + + b, err := blob.OpenBucket(t.Context(), "postgres://reader:hunter2@db.example.com/files?sslmode=sometimes") + + if err == nil || mentionsAny(err, "db.example.com", "hunter2") { + t.Errorf("blob.OpenBucket() of a URL with a bad sslmode = %v, %v, want an error that holds no part of the URL", + b, err) + } +} + +func TestTheURLOpenerClosesItsPoolWhenTheDatabaseIsRefused(t *testing.T) { + t.Parallel() + db := freshDB(t) + name := sessionName() + + b, err := blob.OpenBucket(t.Context(), addressWith(t, asciiDatabase(t), url.Values{"application_name": {name}})) + + if !mentions(err, "UTF8") { + t.Errorf("blob.OpenBucket() of a SQL_ASCII database = %v, %v, want an error that names UTF8", b, err) + } + db.awaitNoSession(name) +} + +func TestTheURLOpenerGivesEachBucketItsOptions(t *testing.T) { + t.Parallel() + db := freshDB(t) + mux := new(blob.URLMux) + mux.RegisterBucket(postgresblob.Scheme, &postgresblob.URLOpener{Options: postgresblob.Options{MaxSize: 4}}) + b, err := mux.OpenBucket(t.Context(), addressWith(t, db.address, inTestSchema)) + if err != nil { + t.Fatalf("OpenBucket() of a postgres URL = %v, want a bucket", err) + } + defer func() { _ = b.Close() }() + + err = b.WriteAll(t.Context(), "notes/report.txt", []byte("five!"), textOptions) + + expectCode(t, "WriteAll() of 5 bytes past the cap of 4", err, gcerrors.ResourceExhausted) +} + +func TestTheURLOpenerClosesThePoolItOpened(t *testing.T) { + t.Parallel() + db := freshDB(t) + name := sessionName() + address := addressWith(t, db.address, url.Values{"schema": {testSchema}, "application_name": {name}}) + b, err := blob.OpenBucket(t.Context(), address) + if err != nil { + t.Fatalf("blob.OpenBucket() of a postgres URL = %v, want a bucket", err) + } + if err := b.WriteAll(t.Context(), "notes/report.txt", []byte("text"), nil); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + if db.count(namedSessions, name) == 0 { + t.Fatalf("no session is named %s after a write, want the pool of the opener on the server", name) + } + + if err := b.Close(); err != nil { + t.Fatalf("Close() = %v, want nil", err) + } + + db.awaitNoSession(name) +} diff --git a/postgresblob/writer.go b/postgresblob/writer.go new file mode 100644 index 0000000..6459efd --- /dev/null +++ b/postgresblob/writer.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "crypto/md5" + "hash" + "maps" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "gocloud.dev/blob/driver" +) + +// execer runs one statement, on the pool or in a transaction. +type execer interface { + // Exec runs sql with args and returns its command tag. + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +// writer streams one object into chunk rows and makes it live on Close. +type writer struct { + // ctx is the context of the upload. + ctx context.Context + // b is the bucket the object goes to. + b *bucket + // key is the key of the object. + key string + // id is the id of the row of the object. + id uuid.UUID + // contentType is the MIME type of the object. + contentType string + // opts are the write options of the object. + opts *driver.WriterOptions + // metadata is a copy of the metadata of the object. + metadata map[string]string + // buf holds the bytes of the chunk not yet written. + buf []byte + // seq is the number of the next chunk. + seq int32 + // size counts the bytes written so far. + size int64 + // md5 hashes the bytes written so far. + md5 hash.Hash + // pending reports whether the pending row of the object was written. + pending bool + // err is the first error of a write, which ends the upload. + err error +} + +// NewTypedWriter returns a writer of the object at key, after BeforeWrite. +func (b *bucket) NewTypedWriter(ctx context.Context, key, contentType string, + opts *driver.WriterOptions) (driver.Writer, error) { + if err := checkKey(key); err != nil { + return nil, err + } + if err := runHook(opts.BeforeWrite); err != nil { + return nil, err + } + metadata := make(map[string]string, len(opts.Metadata)) + maps.Copy(metadata, opts.Metadata) + return &writer{ + ctx: ctx, + b: b, + key: key, + id: uuid.Must(uuid.NewV7()), + contentType: contentType, + opts: opts, + metadata: metadata, + buf: []byte{}, + md5: md5.New(), + }, nil +} + +// Write buffers p, writing each full chunk as one row once more bytes follow it, and refuses p past the size cap. +func (w *writer) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + if w.err = w.b.checkSize(w.size + int64(len(p))); w.err != nil { + return 0, w.err + } + written := 0 + for len(p) > 0 { + if len(w.buf) == w.b.opts.ChunkSize { + if w.err = w.flush(w.b.pool); w.err != nil { + return written, w.err + } + } + n := min(len(p), w.b.opts.ChunkSize-len(w.buf)) + w.buf = append(w.buf, p[:n]...) + _, _ = w.md5.Write(p[:n]) + w.size += int64(n) + written += n + p = p[n:] + } + return written, nil +} + +// ensurePending writes the pending row of the object through db unless it was written. +func (w *writer) ensurePending(db execer) error { + if w.pending { + return nil + } + _, err := db.Exec(w.ctx, w.b.sql.insertPending, w.id, []byte(w.key), int32(w.b.opts.ChunkSize), w.contentType, + w.opts.CacheControl, w.opts.ContentDisposition, w.opts.ContentEncoding, w.opts.ContentLanguage, w.metadata) + w.pending = err == nil + return err +} + +// flush writes the buffered chunk through db, after the pending row of the object. +func (w *writer) flush(db execer) error { + if err := w.ensurePending(db); err != nil { + return err + } + tag, err := db.Exec(w.ctx, w.b.sql.insertChunk, w.id, w.seq, w.buf) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return errSwept + } + w.seq++ + w.buf = w.buf[:0] + return nil +} + +// Close makes the object live, or deletes its pending row when the context ended or a write failed. +func (w *writer) Close() error { + err := w.ctx.Err() + if err == nil { + err = w.err + } + if err == nil { + err = pgx.BeginTxFunc(w.ctx, w.b.pool, pgx.TxOptions{}, w.commitIn) + } + if err != nil && w.pending { + w.dropPending() + } + return err +} + +// dropPending deletes the pending row of the object within the cleanup timeout, even once its context ended. +func (w *writer) dropPending() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(w.ctx), w.b.opts.CleanupTimeout) + defer cancel() + _, _ = w.b.pool.Exec(ctx, w.b.sql.deletePending, w.id) +} + +// commitIn writes the last chunk in tx under the lock of the key, and makes the object live in place of any other. +func (w *writer) commitIn(tx pgx.Tx) error { + if err := w.b.lockIn(w.ctx, tx, w.key); err != nil { + return err + } + if w.opts.IfNotExist { + var exists bool + if err := tx.QueryRow(w.ctx, w.b.sql.liveExists, []byte(w.key)).Scan(&exists); err != nil { + return err + } + if exists { + return errExists + } + } + if err := w.flush(tx); err != nil { + return err + } + return w.b.replaceIn(w.ctx, tx, finished{key: w.key, id: w.id, size: w.size, md5: w.md5.Sum(nil)}) +} diff --git a/postgresblob/writer_internal_test.go b/postgresblob/writer_internal_test.go new file mode 100644 index 0000000..27a8178 --- /dev/null +++ b/postgresblob/writer_internal_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob + +import ( + "context" + "testing" + + "gocloud.dev/blob/driver" +) + +func TestASmallWriteHoldsNoMoreThanItsBytesNeed(t *testing.T) { + t.Parallel() + + b := &bucket{opts: Options{ChunkSize: DefaultChunkSize}} + w, err := b.NewTypedWriter(context.Background(), "notes/small.txt", "text/plain", &driver.WriterOptions{}) + if err != nil { + t.Fatalf("NewTypedWriter() = %v, want a writer", err) + } + if _, err := w.Write([]byte("hello")); err != nil { + t.Fatalf("Write() = %v, want nil", err) + } + + if held := cap(w.(*writer).buf); held >= DefaultChunkSize { + t.Errorf("a 5 byte write holds a buffer of %d bytes, want less than the chunk size %d", held, DefaultChunkSize) + } +} diff --git a/postgresblob/writer_test.go b/postgresblob/writer_test.go new file mode 100644 index 0000000..9fd38c4 --- /dev/null +++ b/postgresblob/writer_test.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgresblob_test + +import ( + "context" + "errors" + "strings" + "testing" + + "gocloud.dev/blob" + "gocloud.dev/gcerrors" + + "github.com/gopherium/framework/postgresblob" +) + +// textOptions are the options of a plain text write. +var textOptions = &blob.WriterOptions{ContentType: "text/plain"} + +func TestACanceledWriteLeavesNoPendingRow(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4}) + const key = "notes/draft.txt" + ctx, cancel := context.WithCancel(t.Context()) + w, err := b.NewWriter(ctx, key, textOptions) + if err != nil { + t.Fatalf("NewWriter() = %v, want a writer", err) + } + if _, err := w.Write([]byte("a draft longer than one chunk")); err != nil { + t.Fatalf("Write() = %v, want nil", err) + } + if n := db.count(keyRows, []byte(key)); n != 1 { + t.Fatalf("the key holds %d rows during the upload, want its pending row", n) + } + + cancel() + err = w.Close() + + if !errors.Is(err, context.Canceled) { + t.Errorf("Close() after the cancel = %v, want context.Canceled", err) + } + if n := db.count(keyRows, []byte(key)); n != 0 { + t.Errorf("the key holds %d rows after the canceled upload, want none", n) + } +} + +func TestAKeyOverTheByteLimitIsRefused(t *testing.T) { + t.Parallel() + b := freshDB(t).open(postgresblob.Options{}) + longest := strings.Repeat("k", postgresblob.MaxKeyBytes) + if err := b.WriteAll(t.Context(), longest, []byte("text"), textOptions); err != nil { + t.Fatalf("WriteAll() with a key of %d bytes = %v, want nil", len(longest), err) + } + + writeErr := b.WriteAll(t.Context(), longest+"k", []byte("text"), textOptions) + copyErr := b.Copy(t.Context(), longest+"k", longest, nil) + + expectCode(t, "WriteAll() with a key one byte past the limit", writeErr, gcerrors.InvalidArgument) + expectCode(t, "Copy() to a key one byte past the limit", copyErr, gcerrors.InvalidArgument) +} + +func TestAWritePastTheCapFailsAndKeepsTheOldObject(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{ChunkSize: 4, MaxSize: 8}) + const key = "notes/short.txt" + old := []byte("old text") + if err := b.WriteAll(t.Context(), key, old, textOptions); err != nil { + t.Fatalf("WriteAll() of exactly the cap = %v, want nil", err) + } + w, err := b.NewWriter(t.Context(), key, textOptions) + if err != nil { + t.Fatalf("NewWriter() = %v, want a writer", err) + } + + _, writeErr := w.Write([]byte("new te")) + if writeErr == nil { + _, writeErr = w.Write([]byte("xt past")) + } + _, againErr := w.Write([]byte("x")) + closeErr := w.Close() + + expectCode(t, "Write() past the cap", writeErr, gcerrors.ResourceExhausted) + expectCode(t, "Write() after the refused write", againErr, gcerrors.ResourceExhausted) + expectCode(t, "Close() after a write past the cap", closeErr, gcerrors.ResourceExhausted) + expectContent(t, b, key, old) + if n := db.count(pendingRows); n != 0 { + t.Errorf("%d pending rows stay after the refused write, want none", n) + } +} + +// earlierCreateTime moves the create time of the live object of the key its one argument names back one day. +const earlierCreateTime = "UPDATE files.objects SET created_at = created_at - interval '24 hours' " + + "WHERE key = $1 AND state = 'live'" + +func TestAnOverwriteKeepsTheCreateTimeOfTheFirstWrite(t *testing.T) { + t.Parallel() + db := freshDB(t) + b := db.open(postgresblob.Options{}) + const key = "notes/report.txt" + if err := b.WriteAll(t.Context(), key, []byte("old text"), textOptions); err != nil { + t.Fatalf("WriteAll() = %v, want nil", err) + } + if _, err := db.pool.Exec(t.Context(), earlierCreateTime, []byte(key)); err != nil { + t.Fatalf("%s: %v", earlierCreateTime, err) + } + before, err := b.Attributes(t.Context(), key) + if err != nil { + t.Fatalf("Attributes() before the overwrite = %v, want the attributes", err) + } + + if err := b.WriteAll(t.Context(), key, []byte("new text"), textOptions); err != nil { + t.Fatalf("WriteAll() of the overwrite = %v, want nil", err) + } + + after, err := b.Attributes(t.Context(), key) + if err != nil { + t.Fatalf("Attributes() after the overwrite = %v, want the attributes", err) + } + if !after.CreateTime.Equal(before.CreateTime) { + t.Errorf("CreateTime after the overwrite = %v, want the first write's %v", after.CreateTime, before.CreateTime) + } +}