diff --git a/.env.example b/.env.example index a7a90ab8..0ebb2b9a 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,13 @@ GOPHENBERG_DATABASE_URL=postgres://postgres:gophenberg@localhost:5435/postgres?s # server run programs: warn logs a warning and serves, refuse stops it. A SQLite file ignores it. # GOPHENBERG_SUPERUSER_POLICY=warn +# How long serve waits for a PostgreSQL server that is not up yet, such as one still starting at +# boot, and the pause between two tries. Only a refused connection, a socket that is not there yet +# or a server still starting up is waited for. A wrong password, a missing role or a missing +# database stops serve at once. 0 does not wait. Only serve waits, every other command stops at once. +# GOPHENBERG_DATABASE_WAIT=60s +# GOPHENBERG_DATABASE_WAIT_INTERVAL=1s + # Only a sqlite: address reads these seven, and a PostgreSQL address ignores them. How long a # connection waits for a lock another one holds, in whole milliseconds. # GOPHENBERG_SQLITE_BUSY_TIMEOUT=5s diff --git a/cmd/releasetool/main.go b/cmd/releasetool/main.go index caf4231f..0b765ea6 100644 --- a/cmd/releasetool/main.go +++ b/cmd/releasetool/main.go @@ -21,7 +21,7 @@ const usage = `usage: releasetool sign -key [-passphrase-file ] ... releasetool verify -public ... releasetool sums ... - releasetool record -version -download-url ... + releasetool record -version -download-url
... releasetool keys -release -standby -timestamp [-serial ] [-valid ] releasetool sign-keys -key [-passphrase-file ] releasetool index -site -release -standby -timestamp [-root ]... diff --git a/cmd/releasetool/record.go b/cmd/releasetool/record.go index 12d7f4b5..6fd4dfe5 100644 --- a/cmd/releasetool/record.go +++ b/cmd/releasetool/record.go @@ -7,9 +7,10 @@ import ( "errors" "fmt" "io" - "net/url" "slices" "strings" + + "github.com/gopherium/gophenberg/internal/update" ) // productName is the name every release archive starts with. @@ -61,7 +62,8 @@ func record(args []string, stdout, stderr io.Writer) error { var opts recordOptions flags := newFlagSet("record", stderr) flags.StringVar(&opts.version, "version", "", "the version the archives carry") - flags.StringVar(&opts.base, "download-url", "", "the https address the release serves its files under") + flags.StringVar(&opts.base, "download-url", "", + "the address the release serves its files under, https, or http on a loopback host") if err := parseFlags(flags, args); err != nil { return err } @@ -80,7 +82,7 @@ func record(args []string, stdout, stderr io.Writer) error { return encoder.Encode(releaseRecord{Version: opts.version, Artifacts: recordArtifacts{Package: entries}}) } -// validate refuses a run with no version, no archive, or a download address that is not https. +// validate refuses a run with no version, no archive, or a download address a site refuses to download from. func (o recordOptions) validate(archives int) error { if o.version == "" { return usageError{"record needs -version, the version the archives carry"} @@ -88,22 +90,12 @@ func (o recordOptions) validate(archives int) error { if archives == 0 { return usageError{"record needs at least one archive to list"} } - if !httpsAddress(o.base) { - return usageError{fmt.Sprintf("record -download-url %q is not the https address the release serves its files under", - o.base)} + if _, err := update.ParseAddress(o.base); err != nil { + return usageError{fmt.Sprintf("record -download-url: %v", err)} } return nil } -// httpsAddress reports whether text is an https address naming a host. -func httpsAddress(text string) bool { - address, err := url.Parse(text) - if err != nil { - return false - } - return address.Scheme == "https" && address.Host != "" -} - // artifacts returns the record entry of every archive, in the order given. func (o recordOptions) artifacts(files []releaseFile) ([]artifact, error) { entries := make([]artifact, len(files)) diff --git a/cmd/releasetool/record_test.go b/cmd/releasetool/record_test.go index 6436c8c2..cdab4491 100644 --- a/cmd/releasetool/record_test.go +++ b/cmd/releasetool/record_test.go @@ -48,6 +48,29 @@ func TestRecordReadsAnyNameOnThePattern(t *testing.T) { } } +func TestRecordTakesPlainHTTPOnALoopbackHost(t *testing.T) { + t.Parallel() + + archives := signedArchives(t, "gophenberg_0.20.0_linux_amd64.tar.gz") + for name, base := range map[string]string{ + "an IPv4 loopback": "http://127.0.0.1:8090/releases", + "localhost": "http://localhost:8090", + "an IPv6 loopback": "http://[::1]:8090/v0.20.0", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := runTool("record", "-version", "0.20.0", "-download-url", base, archives[0]) + + listed := readRecord(t, got).Artifacts.Package + want := base + "/gophenberg_0.20.0_linux_amd64.tar.gz" + if len(listed) != 1 || listed[0].URL != want { + t.Errorf("record under %s lists %+v, want the archive at %s", base, listed, want) + } + }) + } +} + func TestRecordRefusesAnArchiveWithoutASignature(t *testing.T) { t.Parallel() @@ -115,6 +138,12 @@ func TestRecordExitsTwoWhenMisused(t *testing.T) { {"no archive", []string{"-version", "0.20.0", "-download-url", downloadBase}, "archive"}, {"an address that is not https", []string{"-version", "0.20.0", "-download-url", "http://github.com/gopherium/gophenberg/releases/download/v0.20.0", archives[0]}, "https"}, + {"plain http off loopback", []string{"-version", "0.20.0", "-download-url", "http://example.com/releases", + archives[0]}, "http on a loopback host"}, + {"an address with a query", []string{"-version", "0.20.0", "-download-url", + "https://example.com/releases?tag=v0.20.0", archives[0]}, "query"}, + {"an address that names a user", []string{"-version", "0.20.0", "-download-url", + "https://maria@example.com/releases", archives[0]}, "user"}, {"an address with no host", []string{"-version", "0.20.0", "-download-url", "https:///releases", archives[0]}, "https"}, {"an address that does not parse", []string{"-version", "0.20.0", "-download-url", "https://example.com/%zz", diff --git a/docs/src/content/docs/self-hosting/commands.md b/docs/src/content/docs/self-hosting/commands.md index 6836d43e..108396d1 100644 --- a/docs/src/content/docs/self-hosting/commands.md +++ b/docs/src/content/docs/self-hosting/commands.md @@ -280,8 +280,8 @@ gophenberg about [flags] Prints what this binary is in one line, such as `Gophenberg %VERSION%, official build, commit 4c3ba60, theme kits 0.9.0 to %KIT_VERSION%`. With `-json` it answers one JSON document holding `schema`, `name`, -`version`, `commit`, `stamp`, `minimum_version`, `postgresql` and -`kits`. It needs no database. +`version`, `commit`, `stamp`, `minimum_version`, `postgresql`, `node` +and `kits`. It needs no database. - A binary from a release archive or the release image says `official build`, and names the commit it was built from. @@ -292,6 +292,10 @@ With `-json` it answers one JSON document holding `schema`, `name`, database this binary migrated, see [going back to an older version](/self-hosting/updates-and-backups/#going-back-to-an-older-version). - `postgresql` is the oldest PostgreSQL major this binary runs on. +- `node` is the oldest Node.js major this binary runs a theme on. The + release image ships this major. Outside the image, point + [`GOPHENBERG_NODE_BIN`](/self-hosting/configuration/) at a Node.js of + this major or newer. ### check @@ -387,6 +391,10 @@ with `serve`, since a run with no command only prints the listing. On a SQLite database `serve` exits with code 1 before it opens anything, see [on a SQLite database](#on-a-sqlite-database). +When the PostgreSQL server is not up yet, `serve` first waits for it, +up to `GOPHENBERG_DATABASE_WAIT`, see +[a database that starts late](/self-hosting/configuration/#a-database-that-starts-late). + Once an update an administrator started from the admin switched builds, `serve` stops as it does on `SIGTERM` and exits with code 75, so the service manager starts the new build, see @@ -556,7 +564,11 @@ gophenberg: restart Gophenberg to start 1.4.1 the minimum version, `update:apply` takes the backup anyway, since only the backup can take the site back then. - `-json` answers the state the update ends at, holding `step`, - `from`, `to`, `attempts`, `long_jump`, `backup` and `refused`. + `from`, `to`, `attempts`, `database_reached`, `long_jump`, `backup` + and `refused`. `database_reached` turns `true` once a start of the + new build reached the database, just before it migrates, or once any + other command of the new build migrates it, such as `migrate` run by + hand. Before it changes anything it refuses, with exit code 1 and what to do: a custom build, an install with no install root such as the @@ -584,10 +596,10 @@ the update again, adds the version it left to the refused versions, so `update:apply` never takes it again on its own, and ends asking for a restart with exit code 75. It refuses when no update switched builds since the last rollback, and when the update raised the minimum -version and its new build already started, since the older build -cannot open the database the newer one may have migrated. Then it -prints the steps that restore the backup taken before the update -instead. It never opens the database, so it works when the database is +version and its new build already reached the database, since the +older build cannot open the database the newer one may have migrated. +Then it prints the steps that restore the backup taken before the +update instead. It never opens the database, so it works when the database is what broke. `-after-failure` is the form the rollback unit of the service manager @@ -598,10 +610,12 @@ systemd says the starts hit the start limit, `start-limit-hit`, or the new build exited with code 76. It then exits with code 0, so the unit starts the service again. Otherwise it changes nothing and exits with code 1. It takes back each update once, so two failing builds never -take turns. An update that raised the minimum version and already -started stays down: it writes that the update failed, so -`update:status` shows the steps that restore the backup, prints them -and exits with code 1. +take turns. An update that raised the minimum version and whose new +build already reached the database stays down: it writes that the +update failed, so `update:status` shows the steps that restore the +backup, prints them and exits with code 1. One whose new build never +reached the database, such as one that started while PostgreSQL was +still down, goes back like any other update. `-json` answers the state the rollback ends at, as `update:apply` does. diff --git a/docs/src/content/docs/self-hosting/configuration.md b/docs/src/content/docs/self-hosting/configuration.md index 497d896c..68d1d402 100644 --- a/docs/src/content/docs/self-hosting/configuration.md +++ b/docs/src/content/docs/self-hosting/configuration.md @@ -20,6 +20,8 @@ so it takes the default. | `GOPHENBERG_DATABASE_URL` | Yes | | The PostgreSQL connection string, or `sqlite:` and the absolute path of a SQLite file. [The PostgreSQL address](#the-postgresql-address) lists what a connection string may not hold. [A SQLite database](#a-sqlite-database) says what runs on one and lists its settings | | `GOPHENBERG_DATABASE_MAX_CONNS` | No | The larger of 4 and the number of CPUs | How many connections to the database stay open at most, from 2 to 2147483647, see [the database connections](#the-database-connections) | | `GOPHENBERG_SUPERUSER_POLICY` | No | `warn` | What the server does when it connects to PostgreSQL as a superuser: `warn` logs a warning and serves, `refuse` stops it, see [the PostgreSQL server](#the-postgresql-server) | +| `GOPHENBERG_DATABASE_WAIT` | No | `60s` | How long `serve` waits for a PostgreSQL server that is not up yet before it stops. `0` does not wait, see [a database that starts late](#a-database-that-starts-late) | +| `GOPHENBERG_DATABASE_WAIT_INTERVAL` | No | `1s` | The pause between two tries while `serve` waits for the database | | `GOPHENBERG_ADDR` | No | `localhost:8081` | Address the server listens on. The container image sets `0.0.0.0:8081` | | `GOPHENBERG_WEB_DIR` | No | | A built admin folder to serve in place of the admin built into the binary. It must hold `index.html` | | `GOPHENBERG_SITE_TITLE` | No | `Gophenberg` | The site name shown by the built-in renderer | @@ -89,7 +91,7 @@ so it takes the default. | `GOPHENBERG_UPDATE_DISK_MARGIN_MB` | No | `64` | How many megabytes of free disk an update asks for beyond `GOPHENBERG_UPDATE_DOWNLOAD_CAP_MB` and, when it takes a backup, the size of the database, so the site still has room once the update is done. `0` asks for none | | `GOPHENBERG_UPDATE_BACKUPS_KEEP` | No | `3` | How many database backups the `backups` folder of the install root keeps | | `GOPHENBERG_PG_DUMP_BIN` | No | `pg_dump` | The `pg_dump` an update backs up the database with, a name found on the path or a full path. Its major version must be at least the server's | -| `GOPHENBERG_UPDATE_ATTEMPTS` | No | `3` | How many times the build an update switched to may try to start. The next start goes back to the build before it, or stays down when the update raised the minimum version | +| `GOPHENBERG_UPDATE_ATTEMPTS` | No | `3` | How many times the build an update switched to may try to start. The next start goes back to the build before it, or stays down when the update raised the minimum version and its new build reached the database | | `GOPHENBERG_UPDATE_VERIFY_TIMEOUT` | No | `1m` | How long the first start of a new build has to check itself: the server listens, then `/admin/` and `/api/locale` answer 200 and `/` answers below 500 over loopback | | `GOPHENBERG_UPDATE_VERSIONS_KEEP` | No | `2` | How many of the newest builds the `versions` folder keeps once a new build passes its check. The builds the `current` and `previous` links name always stay | | `GOPHENBERG_FEED_TITLE` | No | `Gophenberg` | The RSS channel title | @@ -131,6 +133,41 @@ it logs a warning and serves. Set `GOPHENBERG_SUPERUSER_POLICY` to up from it shows the warning. A SQLite file has no roles, so the setting does nothing there. +### A database that starts late + +A server often starts Gophenberg before PostgreSQL answers, at boot or +right after an update. `serve` then tries the database again every +`GOPHENBERG_DATABASE_WAIT_INTERVAL`, 1 second by default, for up to +`GOPHENBERG_DATABASE_WAIT`, 60 seconds by default, before it checks the +role it connects as and migrates. It logs +`waiting for the database to answer` once when it starts to wait, with +the reason, and `the database answered` once it does, with how long +that took. + +Only a database that is not up yet is worth the wait: a connection +refused, a socket that is not there yet, or a server that says it is +still starting up. A wrong password, a missing role or a missing +database stops `serve` at once with its own message, since waiting +cannot fix them. Each try gets only the time the wait has left, so a +server that takes the connection and never answers holds `serve` no +longer than the wait. A database that still does not answer once the +wait runs out stops `serve` with code 1, and the message ends with the +last reason: + +```text +gophenberg: the database did not answer within 1m0s, the time GOPHENBERG_DATABASE_WAIT allows: failed to connect to `user=gophenberg database=gophenberg`: ... +``` + +`GOPHENBERG_DATABASE_WAIT=0` stops at the first try instead, and that +one try has no time bound of its own. Only +`serve` waits. `migrate`, `seed -yes` and the `account:` commands stop +at once, so a person running them sees the reason straight away. + +Right after an update, a start that waits still counts as one attempt +of the new build, so a database down for longer than about three waits +takes the update back, see +[a database that starts late after an update](/self-hosting/updates-and-backups/#a-database-that-starts-late-after-an-update). + ## The database connections `GOPHENBERG_DATABASE_MAX_CONNS` caps how many connections to the @@ -544,8 +581,10 @@ The server refuses to start, and says why, when: Write them as durations, `1h`, `90s`, `5m`. Part of a second is refused, because the header counts in whole seconds. - Any of the theme timings, HTTP timeouts, shutdown graces, cron job - timings, update timings or the upload timeout is not a positive - duration. Write them the way Go does, `30s`, `500ms`, `1m`. + timings, update timings, the upload timeout or + `GOPHENBERG_DATABASE_WAIT_INTERVAL` is not a positive duration. + Write them the way Go does, `30s`, `500ms`, `1m`. +- `GOPHENBERG_DATABASE_WAIT` is not a duration of `0` or more. - `GOPHENBERG_UPDATE_URL` is not an `https` address, or an `http` one on a loopback host, or it names a user, a query or a fragment. - `GOPHENBERG_UPDATE_CHANNEL` holds anything but lowercase letters, diff --git a/docs/src/content/docs/self-hosting/updates-and-backups.md b/docs/src/content/docs/self-hosting/updates-and-backups.md index 559263a7..ee05438d 100644 --- a/docs/src/content/docs/self-hosting/updates-and-backups.md +++ b/docs/src/content/docs/self-hosting/updates-and-backups.md @@ -748,10 +748,10 @@ The update ends in one of five ways: again, as `-retry` does, so fix what failed first. - **The update to 1.4.1 failed its start check, and Gophenberg cannot go back to 1.4.0 on its own.** An update that raised the minimum - version failed its start check, and the screen says so once the - service manager starts the new version again. Run `update:status` to - see the backup taken before the update, then restore it or fix what - failed, see + version failed its start check after it reached the database, and + the screen says so once the service manager starts the new version + again. Run `update:status` to see the backup taken before the update, + then restore it or fix what failed, see [an update that raised the minimum version](#an-update-that-raised-the-minimum-version). - **The update failed before anything changed.** A step before the switch failed, and the reason follows. The site keeps running the @@ -778,7 +778,11 @@ the build has not passed its check yet, the start checks it: 1. It counts the attempt. A build that already tried to start `GOPHENBERG_UPDATE_ATTEMPTS` times, 3 unless set, fails at once. -2. It migrates the database, as every start does. +2. It waits for a database that is not up yet, see + [a database that starts late after an update](#a-database-that-starts-late-after-an-update). + Once the database answers, it writes `database_reached` into + `state/update.json`, then migrates the database, as every start + does. 3. Within `GOPHENBERG_UPDATE_VERIFY_TIMEOUT`, one minute unless set, it checks that the public site serves through the theme the site chose. Then it asks itself over loopback for `/admin/` and `/api/locale`, @@ -823,10 +827,11 @@ migrated. `update:check` says `going back to this version after it needs the backup` for such a release. For it, `update:apply` always takes the backup, even with `-skip-backup`. -If such a build fails its check, it does not go back on its own. The -site stays down, and nothing restores the backup on its own, since -everything written after the backup would be lost. The error, the log -and `update:status` name the backup and the steps: +If such a build reached the database and then fails its check, it +does not go back on its own. The site stays down, and nothing restores +the backup on its own, since everything written after the backup would +be lost. The error, the log and `update:status` name the backup and the +steps: ```text gophenberg: updater: restore the backup taken before the update, /opt/gophenberg/backups/1.4.0-20261020T120000Z.dump, into an empty database, point current at versions/1.4.0 and start Gophenberg again, as https://docs.gophenberg.org/self-hosting/updates-and-backups/#restoring-on-a-server-install shows, or fix what failed and start Gophenberg 1.4.1 again @@ -837,6 +842,24 @@ fails, ends the same way. Under systemd, the starts hit the start limit, and the rollback unit writes that the update failed instead of going back, so `update:status` shows the same steps. +A build that never reached the database cannot have migrated it, so it +goes back like any other update, see +[a version that fails its check](#a-version-that-fails-its-check). +That happens when every start stopped before the database answered, +such as while PostgreSQL was still down. `update:status -json` then +shows `database_reached` as `false`. + +A start is not the only way the new build reaches the database. Every +command of the new build that migrates, such as `migrate`, `seed -yes`, +`account:create-admin` and `account:grant-role -yes`, sets +`database_reached` too, once the database lets that build on and +before the first schema step changes anything. So +once you run the new build's `migrate` by hand, the update counts as +one that may have migrated, and `update:rollback` prints the restore +steps instead of going back. The command marks the update only when it +runs with the settings of the service, `GOPHENBERG_INSTALL_ROOT` among +them. + [Restoring on a server install](#restoring-on-a-server-install) shows each step. Or fix what failed, such as a setting or a full disk, and start Gophenberg again. Either way, the build that starts runs the @@ -845,6 +868,46 @@ When the check passes, `update:status` says the update was kept, or taken back after a restore. When it fails, the site keeps serving and the log says `the update stays failed` with the reason. +### A database that starts late after an update + +A server may start Gophenberg before PostgreSQL answers, such as when +both start at boot. `serve` then waits up to `GOPHENBERG_DATABASE_WAIT`, +60 seconds unless set, and stops with code 1 when the database still +does not answer, so the service manager starts it again, see +[a database that starts late](/self-hosting/configuration/#a-database-that-starts-late). +With no update waiting for its check, the boot guard counts nothing, +so a late database never takes anything back. + +Right after an update, the boot guard counts each start of the new +build before it waits, so one late start costs one attempt. With the +defaults, 3 attempts of up to 60 seconds each, the database can stay +down for about three minutes after the first start, plus the time the +service manager waits between two starts. When it answers within that +time, the start goes on, migrates and runs its check as usual. The +bound is about `GOPHENBERG_UPDATE_ATTEMPTS` times +`GOPHENBERG_DATABASE_WAIT`, so raising either one gives the database +longer. + +A database down for longer than that takes a good update back. The +start after the last attempt, the fourth with the defaults, fails its +check at once, the new build goes back as +[a version that fails its check](#a-version-that-fails-its-check) +shows, and its version joins the refused versions. Once the database +answers again, take the update again: + +```sh +/opt/gophenberg/current/gophenberg update:apply -to 1.4.1 -retry +``` + +This holds for an update that raised the minimum version too. Its +starts never reached the database, so `database_reached` stays `false` +and the update goes back instead of staying down. + +`update:rollback` takes such an update back too while its new build +still waits, since that build has not reached the database yet. The +waiting start then stops before it migrates, and the next start runs +the build before the update. + ## Going back to an older version Every release carries a minimum version, the oldest version allowed to diff --git a/internal/app/about.go b/internal/app/about.go index f0a4d89c..e5fa2995 100644 --- a/internal/app/about.go +++ b/internal/app/about.go @@ -26,6 +26,7 @@ type buildFacts struct { Stamp string `json:"stamp"` MinimumVersion string `json:"minimum_version"` PostgreSQL int `json:"postgresql"` + Node int `json:"node"` Kits []string `json:"kits"` } @@ -43,7 +44,8 @@ func aboutCommand() gonsole.Command { func printAbout(_ context.Context, call gonsole.Call) error { facts := buildFacts{ Schema: 1, Name: "gophenberg", Version: version.Version(), Commit: version.Commit(), Stamp: version.Stamp(), - MinimumVersion: version.Minimum(), PostgreSQL: postgres.OldestMajor, Kits: themehost.ServedKits(), + MinimumVersion: version.Minimum(), PostgreSQL: postgres.OldestMajor, Node: themehost.OldestNodeMajor, + Kits: themehost.ServedKits(), } if call.JSON { return call.Encode(facts) diff --git a/internal/app/about_test.go b/internal/app/about_test.go index 34682f4d..060cc694 100644 --- a/internal/app/about_test.go +++ b/internal/app/about_test.go @@ -32,11 +32,22 @@ func TestAboutPrintsTheBuildFactsAsOneDocument(t *testing.T) { } want := buildFacts{ Schema: 1, Name: "gophenberg", Version: version.Version(), MinimumVersion: version.Minimum(), - PostgreSQL: postgres.OldestMajor, Kits: themehost.ServedKits(), + PostgreSQL: postgres.OldestMajor, Node: themehost.OldestNodeMajor, Kits: themehost.ServedKits(), } - if len(fields) != 8 || !facts.equal(want) { - t.Errorf("about -json = %s, want exactly schema, name, version, commit, stamp, minimum_version, postgresql "+ - "and kits as %+v", got.Stdout, want) + if len(fields) != 9 || !facts.equal(want) { + t.Errorf("about -json = %s, want exactly schema, name, version, commit, stamp, minimum_version, postgresql, "+ + "node and kits as %+v", got.Stdout, want) + } +} + +func TestAboutNamesNodeJS24AsTheLowestMajorAThemeRunsOn(t *testing.T) { + t.Parallel() + + got := testkit.Run(t, Program(testkit.Getenv(map[string]string{}), noPlugins), "", "about", "-json") + + var facts buildFacts + if err := json.Unmarshal([]byte(got.Stdout), &facts); err != nil || facts.Node != 24 { + t.Errorf("about -json = %s (%v), want node 24", got.Stdout, err) } } diff --git a/internal/app/bootguard_test.go b/internal/app/bootguard_test.go index d7a2a16d..fc693669 100644 --- a/internal/app/bootguard_test.go +++ b/internal/app/bootguard_test.go @@ -4,6 +4,7 @@ package app import ( "context" + "io" "os" "path/filepath" "strings" @@ -11,7 +12,9 @@ import ( "github.com/gopherium/framework/gonsole" "github.com/gopherium/framework/gonsole/testkit" + "github.com/gopherium/framework/pluginkit" + "github.com/gopherium/gophenberg/internal/testdb" "github.com/gopherium/gophenberg/internal/updater" "github.com/gopherium/gophenberg/internal/version" ) @@ -119,6 +122,122 @@ func TestServeKeepsAnUpdateWhoseFirstStartAnswersItsCheck(t *testing.T) { } } +func TestServeMarksThatTheNewBuildReachedTheDatabaseBeforeItAsksAboutTheRole(t *testing.T) { + t.Parallel() + + databaseURL := emptyDatabaseURL(t) + root := switchedRoot(t, func(*updater.State) {}) + + result := runServe(t, map[string]string{ + "GOPHENBERG_DATABASE_URL": testdb.AsServer(t, databaseURL), "GOPHENBERG_SUPERUSER_POLICY": "refuse", + installRootSetting: root, + }) + + if result.Code != gonsole.ExitFailed || !strings.Contains(result.Stderr, "connects to its database as a superuser") { + t.Errorf("serve = %d with stderr %q, want 1 and the superuser refused", result.Code, result.Stderr) + } + state, err := updater.Layout{Root: root}.ReadState() + if err != nil { + t.Fatalf("reading the update state: %v", err) + } + if state.Attempts != 1 || !state.DatabaseReached { + t.Errorf("the update state is %+v, want the start counted and the database marked reached", state) + } + if got := storedMinimumVersionAt(t, databaseURL); got != "" { + t.Errorf("stored minimum version = %q once serve refused the role, want nothing migrated", got) + } +} + +func TestServeMarksNothingWhenTheDatabaseNeverAnswers(t *testing.T) { + t.Parallel() + + root := switchedRoot(t, func(*updater.State) {}) + + result := runServe(t, map[string]string{ + "GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, "GOPHENBERG_DATABASE_WAIT": "0", installRootSetting: root, + }) + + if result.Code != gonsole.ExitFailed || !strings.Contains(result.Stderr, "the database did not answer within 0s") { + t.Errorf("serve = %d with stderr %q, want 1 and the wait run out", result.Code, result.Stderr) + } + state, err := updater.Layout{Root: root}.ReadState() + if err != nil { + t.Fatalf("reading the update state: %v", err) + } + if state.Attempts != 1 || state.DatabaseReached { + t.Errorf("the update state is %+v, want the start counted and the database not marked", state) + } +} + +// guardedBoot returns a start on the database at databaseURL whose boot guard counted a start on the install root. +func guardedBoot(t *testing.T, databaseURL, root string) boot { + t.Helper() + handle, err := openPostgres(settingsEnv(testkit.Getenv(nil)), databaseURL) + if err != nil { + t.Fatalf("opening the pool: %v", err) + } + t.Cleanup(func() { _ = handle.Close() }) + settings := timedConfig() + settings.serving, settings.databaseURL = servingDefaults, databaseURL + settings.install, err = updater.SettingsFrom(settingsEnv(testkit.Getenv(map[string]string{ + installRootSetting: root, + }))) + if err != nil { + t.Fatalf("reading the install settings: %v", err) + } + guard, err := updater.Start(settings.install, version.Version(), testLogger(io.Discard)) + if err != nil { + t.Fatalf("Start() = %v, want the start counted", err) + } + return boot{settings: settings, built: site{pgHandle: handle}, host: pluginkit.NewHost(), + logger: testLogger(io.Discard), guard: guard} +} + +func TestReadyStopsBeforeItMigratesWhenItCannotMarkTheDatabaseReached(t *testing.T) { + t.Parallel() + + if os.Geteuid() == 0 { + t.Skip("running as root, which writes to a folder whatever its mode says") + } + databaseURL := emptyDatabaseURL(t) + start := guardedBoot(t, databaseURL, switchedRoot(t, func(*updater.State) {})) + folder := filepath.Dir(updater.Layout{Root: start.settings.install.Root}.StateFile()) + if err := os.Chmod(folder, 0o500); err != nil { + t.Fatalf("locking %q: %v", folder, err) + } + t.Cleanup(func() { _ = os.Chmod(folder, 0o755) }) + + err := start.ready(t.Context()) + + if err == nil || !strings.Contains(err.Error(), "updater: write the update state") { + t.Errorf("ready() = %v, want the mark it could not write", err) + } + if got := storedMinimumVersionAt(t, databaseURL); got != "" { + t.Errorf("stored minimum version = %q after the mark failed, want nothing migrated", got) + } +} + +func TestReadyStopsBeforeItMigratesALongJumpTakenBackWhileItWaitedForTheDatabase(t *testing.T) { + t.Parallel() + + databaseURL := emptyDatabaseURL(t) + start := guardedBoot(t, databaseURL, switchedRoot(t, func(state *updater.State) { state.LongJump = true })) + if _, err := updater.Rollback(start.settings.install); err != nil { + t.Fatalf("Rollback() = %v, want the long jump taken back before its build reached the database", err) + } + + err := start.ready(t.Context()) + + want := "updater: another command took back the update to Gophenberg " + version.Version() + + " while this start waited for the database, so it stops before it migrates" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("ready() = %v, want %q", err, want) + } + if got := storedMinimumVersionAt(t, databaseURL); got != "" { + t.Errorf("stored minimum version = %q after the update was taken back, want nothing migrated", got) + } +} + func TestExitCodesAnswerTheCodeAnErrorAskedForOnlyInPlaceOfAFailure(t *testing.T) { t.Parallel() diff --git a/internal/app/core_updates_test.go b/internal/app/core_updates_test.go index 981dfcfc..4942b919 100644 --- a/internal/app/core_updates_test.go +++ b/internal/app/core_updates_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/cucumber/godog" + "github.com/gopherium/framework/gonsole" "github.com/gopherium/framework/gonsole/testkit" "github.com/gopherium/gophenberg/internal/updater" @@ -54,6 +55,9 @@ func initializeCoreUpdates(sc *godog.ScenarioContext, s *operatorScenario) { sc.Given(`^(?:this version|the new build) already tried to start (\d+) times$`, s.triedToStart) sc.Given(`^the update raised the minimum version, with a backup taken before the switch$`, s.raiseMinimum) sc.Given(`^the new build passed its check$`, s.passCheck) + sc.Given(`^the new build reached the database$`, s.markDatabaseReached) + sc.Given(`^the site failed to start (\d+) times$`, s.failStarts) + sc.Given(`^the start check of this version runs out of time$`, s.outlastCheck) sc.When(`^the site starts$`, s.startSite) sc.When(`^the service manager runs "([^"]*)" with (MONITOR_[A-Z_]+) set to "([^"]*)"$`, s.runAfterFailure) sc.Then(`^the command asks for a restart to start "([^"]*)"$`, s.asksForRestart) @@ -106,6 +110,11 @@ func (s *operatorScenario) passCheck() error { return editState(s.env[installRootSetting], func(state *updater.State) { state.Step = updater.StepCommitted }) } +// markDatabaseReached records that the build the update switched to reached the database at an earlier start. +func (s *operatorScenario) markDatabaseReached() error { + return editState(s.env[installRootSetting], func(state *updater.State) { state.DatabaseReached = true }) +} + // asksForRestart fails unless the command asked for a restart to start the version given. func (s *operatorScenario) asksForRestart(release string) error { return s.errorSays("restart Gophenberg to start " + release) @@ -232,6 +241,23 @@ func (s *operatorScenario) startSite() { s.result = testkit.Result{Code: exits.of(code), Stdout: stdout.String(), Stderr: stderr.String()} } +// failStarts starts the site the times given, failing unless each start stops with exit code 1. +func (s *operatorScenario) failStarts(times int) error { + for range times { + s.startSite() + if s.result.Code != gonsole.ExitFailed { + return fmt.Errorf("serve exited with code %d and stderr %q, want %d", s.result.Code, s.result.Stderr, + gonsole.ExitFailed) + } + } + return nil +} + +// outlastCheck leaves the start check of the build that starts no time, so it runs out before any page answers. +func (s *operatorScenario) outlastCheck() { + s.env["GOPHENBERG_UPDATE_VERIFY_TIMEOUT"] = "1ns" +} + // keptWatch records what a run writes to its error stream and calls cancel once the boot guard kept the update. type keptWatch struct { mu sync.Mutex diff --git a/internal/app/database_wait.go b/internal/app/database_wait.go new file mode 100644 index 00000000..041100db --- /dev/null +++ b/internal/app/database_wait.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "context" + "errors" + "fmt" + "log/slog" + "syscall" + "time" + + "github.com/gopherium/framework/gonsole" + "github.com/jackc/pgx/v5/pgconn" +) + +// databaseWaitSetting names the setting that bounds how long serve waits for a database that is not up yet. +const databaseWaitSetting = "DATABASE_WAIT" + +// cannotConnectNow is the SQLSTATE PostgreSQL answers while it starts up, shuts down or recovers. +const cannotConnectNow = "57P03" + +// waitingLine is what serve logs once it starts to wait for a database that is not up yet. +const waitingLine = "waiting for the database to answer" + +// answeredLine is what serve logs once the database it waited for answers. +const answeredLine = "the database answered" + +// databaseWait is how long serve waits for a database that is not up yet, and the pause between two tries. +type databaseWait struct { + limit time.Duration + interval time.Duration +} + +// databaseWaitFrom reads how long serve waits for its database, 0 waiting not at all, and the pause between two tries. +func databaseWaitFrom(env gonsole.Env) (databaseWait, error) { + limit, err := env.Duration(databaseWaitSetting, time.Minute, gonsole.AllowZero()) + if err != nil { + return databaseWait{}, err + } + interval, err := env.Duration("DATABASE_WAIT_INTERVAL", time.Second) + if err != nil { + return databaseWait{}, err + } + return databaseWait{limit: limit, interval: interval}, nil +} + +// await pings until the database answers, ctx ends or the wait runs out, logging when it waits and when it answered. +func (w databaseWait) await(ctx context.Context, ping func(context.Context) error, logger *slog.Logger) error { + started := time.Now() + deadline := started.Add(w.limit) + err := w.try(ctx, ping, deadline) + if !notUpYet(err) { + return err + } + if w.limit > 0 { + logger.Warn(waitingLine, "error", err, "wait", w.limit) + } + if err := w.retry(ctx, ping, deadline, err); err != nil { + return err + } + logger.Info(answeredLine, "after", time.Since(started).Round(time.Millisecond)) + return nil +} + +// retry pings again after each pause while failed says the database is not up yet, until deadline passes or ctx ends. +func (w databaseWait) retry( + ctx context.Context, ping func(context.Context) error, deadline time.Time, failed error, +) error { + for notUpYet(failed) { + if err := w.pauseBefore(ctx, deadline, failed); err != nil { + return err + } + failed = w.try(ctx, ping, deadline) + } + return failed +} + +// try pings once, within deadline while the wait stands above 0, answering the end of ctx as it is once ctx ended. +func (w databaseWait) try(ctx context.Context, ping func(context.Context) error, deadline time.Time) error { + if w.limit == 0 { + return ping(ctx) + } + bounded, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + err := ping(bounded) + if ctx.Err() != nil { + return ctx.Err() + } + if bounded.Err() == nil { + return err + } + if err == nil { + err = bounded.Err() + } + return unanswered{failed: err} +} + +// pauseBefore pauses before the next try, answering that the wait ran out when deadline passed before or after it. +func (w databaseWait) pauseBefore(ctx context.Context, deadline time.Time, failed error) error { + left := time.Until(deadline) + if left <= 0 { + return w.ranOut(failed) + } + if err := pause(ctx, min(w.interval, left)); err != nil { + return err + } + if time.Now().Before(deadline) { + return nil + } + return w.ranOut(failed) +} + +// ranOut says the database did not answer within the wait, naming the setting and keeping failed, the last failure. +func (w databaseWait) ranOut(failed error) error { + return fmt.Errorf("the database did not answer within %v, the time %s allows: %w", + w.limit, settingsPrefix+databaseWaitSetting, failed) +} + +// unanswered is a try the wait's end cut short, holding what it failed with, so it counts as not up yet. +type unanswered struct { + failed error +} + +// Error says what the try the wait's end cut short failed with. +func (u unanswered) Error() string { + return u.failed.Error() +} + +// Unwrap answers what the try the wait's end cut short failed with. +func (u unanswered) Unwrap() error { + return u.failed +} + +// pause waits for d, answering the end of ctx when it comes first. +func pause(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// notUpYet reports whether err says the database is not up yet: refused, no socket yet, 57P03 or a try cut short. +func notUpYet(err error) bool { + if errors.As(err, &unanswered{}) { + return true + } + var answered *pgconn.PgError + if errors.As(err, &answered) { + return answered.Code == cannotConnectNow + } + return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) +} diff --git a/internal/app/database_wait_steps_test.go b/internal/app/database_wait_steps_test.go new file mode 100644 index 00000000..db27a784 --- /dev/null +++ b/internal/app/database_wait_steps_test.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "context" + "fmt" + "io" + "net" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/cucumber/godog" + "github.com/gopherium/framework/gonsole/testkit" +) + +// databaseWaitFeature is the feature serve answers to while its database starts late. +const databaseWaitFeature = "../../test/features/features/database-wait.feature" + +// waitedLines are the lines serve logs while it waits for its database, by the words a scenario names them with. +var waitedLines = map[string]string{ + "it waits for the database": waitingLine, + "the database answered": answeredLine, +} + +// waitScenario is what one database wait scenario keeps beside the operator scenario: the stand-in for its database. +type waitScenario struct { + *operatorScenario + mu sync.Mutex + timer *time.Timer + listener net.Listener + ended bool +} + +// initializeDatabaseWait binds the steps of the scenarios where serve waits for a database that starts late to s. +func initializeDatabaseWait(sc *godog.ScenarioContext, s *operatorScenario) { + w := &waitScenario{operatorScenario: s} + sc.After(w.closeStandIn) + sc.Given(`^serve tries the database again every "([^"]*)"$`, w.tryEvery) + sc.Given(`^serve waits "([^"]*)" for the database$`, w.waitFor) + sc.Given(`^the database refuses every connection$`, w.refuseAll) + sc.Given(`^the database refuses every connection for (\d+) seconds$`, w.refuseFor) + sc.Given(`^the settings name a wrong password for the database$`, w.nameWrongPassword) + sc.When(`^the site starts and listens$`, w.startAndListen) + sc.Then(`^the site logged once that (it waits for the database|the database answered)$`, w.loggedOnce) + sc.Then(`^the site never waited for the database$`, w.neverWaited) +} + +// tryEvery names the pause serve takes between two tries of the database. +func (w *waitScenario) tryEvery(interval string) { + w.env["GOPHENBERG_DATABASE_WAIT_INTERVAL"] = interval +} + +// waitFor names how long serve waits for a database that is not up yet. +func (w *waitScenario) waitFor(wait string) { + w.env["GOPHENBERG_DATABASE_WAIT"] = wait +} + +// refuseAll points the settings at a loopback address nothing listens on, in place of the scenario's database. +func (w *waitScenario) refuseAll() error { + _, _, err := w.moveDatabase() + return err +} + +// refuseFor points the settings at a stand-in that refuses every connection for the seconds given, then forwards each. +func (w *waitScenario) refuseFor(seconds int) error { + address, upstream, err := w.moveDatabase() + if err != nil { + return err + } + w.mu.Lock() + defer w.mu.Unlock() + w.timer = time.AfterFunc(time.Duration(seconds)*time.Second, func() { w.openStandIn(address, upstream) }) + return nil +} + +// moveDatabase points the settings at a loopback address nothing listens on, answering it and the database's address. +func (w *waitScenario) moveDatabase() (string, string, error) { + held, err := url.Parse(w.env["GOPHENBERG_DATABASE_URL"]) + if err != nil { + return "", "", err + } + upstream := held.Host + held.Host = testkit.FreeAddr(w.t) + w.env["GOPHENBERG_DATABASE_URL"] = held.String() + return held.Host, upstream, nil +} + +// openStandIn listens at address and forwards every connection it accepts to upstream, unless the scenario ended. +func (w *waitScenario) openStandIn(address, upstream string) { + w.mu.Lock() + defer w.mu.Unlock() + if w.ended { + return + } + listener, err := net.Listen("tcp", address) + if err != nil { + w.t.Errorf("opening the database stand-in at %s: %v", address, err) + return + } + w.listener = listener + go forwardAll(listener, upstream) +} + +// closeStandIn keeps the stand-in from opening and closes it, once the scenario ends. +func (w *waitScenario) closeStandIn(ctx context.Context, _ *godog.Scenario, _ error) (context.Context, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.ended = true + if w.timer != nil { + w.timer.Stop() + } + if w.listener == nil { + return ctx, nil + } + return ctx, w.listener.Close() +} + +// forwardAll forwards every connection listener accepts to upstream, until listener closes. +func forwardAll(listener net.Listener, upstream string) { + for { + accepted, err := listener.Accept() + if err != nil { + return + } + go forward(accepted, upstream) + } +} + +// forward copies bytes both ways between accepted and a new connection to upstream, until either side ends. +func forward(accepted net.Conn, upstream string) { + defer func() { _ = accepted.Close() }() + dialed, err := net.Dial("tcp", upstream) + if err != nil { + return + } + defer func() { _ = dialed.Close() }() + ended := make(chan struct{}, 2) + go func() { _, _ = io.Copy(dialed, accepted); ended <- struct{}{} }() + go func() { _, _ = io.Copy(accepted, dialed); ended <- struct{}{} }() + <-ended +} + +// nameWrongPassword names the scenario's database with a password its role does not have. +func (w *waitScenario) nameWrongPassword() error { + held, err := url.Parse(w.env["GOPHENBERG_DATABASE_URL"]) + if err != nil { + return err + } + held.User = url.UserPassword(held.User.Username(), "wrongpassword") + w.env["GOPHENBERG_DATABASE_URL"] = held.String() + return nil +} + +// startAndListen serves the site over the scenario's settings until it listens, failing when it never does. +func (w *waitScenario) startAndListen() error { + w.env["GOPHENBERG_ADDR"] = "127.0.0.1:0" + ctx, cancel := context.WithTimeout(w.t.Context(), siteStartLimit) + defer cancel() + exits := &exitCodes{} + logged := &strings.Builder{} + code := commandLine(testkit.Getenv(w.env), w.loaded, exits, w.stamp).Run( + ctx, []string{"serve"}, strings.NewReader(""), io.Discard, listenRecorder{cancel: cancel, logged: logged}) + w.result = testkit.Result{Code: exits.of(code), Stderr: logged.String()} + if !strings.Contains(w.result.Stderr, "msg=listening") { + return fmt.Errorf("the site never listened, its stderr says %q", w.result.Stderr) + } + return nil +} + +// loggedOnce fails unless the site logged the line a scenario names by what exactly once. +func (w *waitScenario) loggedOnce(what string) error { + line := waitedLines[what] + if held := strings.Count(w.result.Stderr, line); held != 1 { + return fmt.Errorf("the site logged %q %d times, want once, its stderr says %q", line, held, w.result.Stderr) + } + return nil +} + +// neverWaited fails when the site logged that it waits for the database. +func (w *waitScenario) neverWaited() error { + if strings.Contains(w.result.Stderr, waitingLine) { + return fmt.Errorf("the site logged %q, want it stopped without waiting, its stderr says %q", waitingLine, + w.result.Stderr) + } + return nil +} + +func TestDatabaseWait(t *testing.T) { + t.Parallel() + + runOperatorFeature(t, databaseWaitFeature, postgresOperators()) +} diff --git a/internal/app/database_wait_test.go b/internal/app/database_wait_test.go new file mode 100644 index 00000000..7dab8e60 --- /dev/null +++ b/internal/app/database_wait_test.go @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" + "time" + + dbkitpg "github.com/gopherium/framework/dbkit/postgres" + "github.com/gopherium/framework/gonsole/testkit" + "github.com/gopherium/framework/pluginkit" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/gopherium/gophenberg/sdk" +) + +// errRefused is a connection the database server refused, as a dial answers it. +var errRefused = &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)} + +// errWrongPassword is PostgreSQL's answer to a password its role does not have. +var errWrongPassword = &pgconn.PgError{Severity: "FATAL", Code: "28P01", Message: "password authentication failed"} + +// pingAnswering returns a ping that fails with err the first fails times, then answers, counting each call in calls. +func pingAnswering(fails int, err error, calls *int) func(context.Context) error { + return func(context.Context) error { + *calls++ + if *calls <= fails { + return err + } + return nil + } +} + +func TestLoadRunConfigWaitsAMinuteForTheDatabaseTryingEverySecond(t *testing.T) { + t.Parallel() + + settings, err := loadRunConfig(testkit.Getenv(map[string]string{"GOPHENBERG_DATABASE_URL": unreachableDatabaseURL})) + + want := databaseWait{limit: time.Minute, interval: time.Second} + if err != nil || settings.databaseWait != want { + t.Errorf("loadRunConfig() = %+v, %v, want %+v", settings.databaseWait, err, want) + } +} + +func TestLoadRunConfigReadsTheDatabaseWaitFromTheEnvironment(t *testing.T) { + t.Parallel() + + for value, want := range map[string]databaseWait{ + "90s": {limit: 90 * time.Second, interval: 250 * time.Millisecond}, + "0": {limit: 0, interval: 250 * time.Millisecond}, + } { + settings, err := loadRunConfig(testkit.Getenv(map[string]string{ + "GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, "GOPHENBERG_DATABASE_WAIT": value, + "GOPHENBERG_DATABASE_WAIT_INTERVAL": "250ms", + })) + if err != nil || settings.databaseWait != want { + t.Errorf("loadRunConfig() with a wait of %q = %+v, %v, want %+v", value, settings.databaseWait, err, want) + } + } +} + +func TestLoadRunConfigRefusesADatabaseWaitItCannotStand(t *testing.T) { + t.Parallel() + + for name, asked := range map[string]struct { + key string + value string + want string + }{ + "a wait that is not a duration": {"GOPHENBERG_DATABASE_WAIT", "soon", "must be a duration like 30s"}, + "a wait below zero": {"GOPHENBERG_DATABASE_WAIT", "-1s", "must not be negative"}, + "an interval that is not a duration": {"GOPHENBERG_DATABASE_WAIT_INTERVAL", "often", "must be a duration like 30s"}, + "an interval standing at zero": {"GOPHENBERG_DATABASE_WAIT_INTERVAL", "0", "must stand above zero"}, + "an interval below zero": {"GOPHENBERG_DATABASE_WAIT_INTERVAL", "-1s", "must stand above zero"}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, err := loadRunConfig(testkit.Getenv(map[string]string{ + "GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, asked.key: asked.value, + })) + + want := fmt.Sprintf("%s: %s, got %q", asked.key, asked.want, asked.value) + if err == nil || err.Error() != want { + t.Errorf("loadRunConfig() error = %v, want %q", err, want) + } + }) + } +} + +// wantNothingLogged fails t when the wait logged anything. +func wantNothingLogged(t *testing.T, logged *bytes.Buffer) { + t.Helper() + if logged.Len() != 0 { + t.Errorf("await() logged %q, want nothing logged", logged.String()) + } +} + +func TestDatabaseWaitSaysNothingForADatabaseThatAnswersAtOnce(t *testing.T) { + t.Parallel() + var logged bytes.Buffer + calls := 0 + + err := databaseWait{limit: time.Minute, interval: time.Millisecond}.await( + t.Context(), pingAnswering(0, errRefused, &calls), testLogger(&logged)) + + if err != nil || calls != 1 { + t.Errorf("await() = %v after %d pings, want nil after one", err, calls) + } + wantNothingLogged(t, &logged) +} + +func TestDatabaseWaitTriesAgainUntilTheDatabaseAnswers(t *testing.T) { + t.Parallel() + var logged bytes.Buffer + calls := 0 + + err := databaseWait{limit: time.Minute, interval: time.Millisecond}.await( + t.Context(), pingAnswering(3, errRefused, &calls), testLogger(&logged)) + + if err != nil || calls != 4 { + t.Fatalf("await() = %v after %d pings, want nil once the fourth one answers", err, calls) + } + for _, line := range []string{"level=WARN msg=\"" + waitingLine + "\"", "connection refused", "wait=1m0s", + "level=INFO msg=\"" + answeredLine + "\" after="} { + if held := strings.Count(logged.String(), line); held != 1 { + t.Errorf("logged %q, holding %q %d times, want once", logged.String(), line, held) + } + } +} + +func TestDatabaseWaitStopsOnceTheWaitRunsOut(t *testing.T) { + t.Parallel() + var logged bytes.Buffer + calls := 0 + + err := databaseWait{limit: 50 * time.Millisecond, interval: 10 * time.Millisecond}.await( + t.Context(), pingAnswering(1<<30, errRefused, &calls), testLogger(&logged)) + + want := "the database did not answer within 50ms, the time GOPHENBERG_DATABASE_WAIT allows: " + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("await() = %v, want %q", err, want) + } + if !errors.Is(err, syscall.ECONNREFUSED) { + t.Errorf("await() = %v, want the last failure kept", err) + } + if calls < 2 || calls > 6 { + t.Errorf("await() pinged %d times, want from 2 to 6, one ping and one after each pause of 10ms in 50ms", calls) + } + if held := strings.Count(logged.String(), waitingLine); held != 1 { + t.Errorf("logged %q, holding the wait %d times, want once", logged.String(), held) + } + if strings.Contains(logged.String(), answeredLine) { + t.Errorf("logged %q, want no answer logged", logged.String()) + } +} + +func TestDatabaseWaitOfZeroStopsAtTheFirstFailure(t *testing.T) { + t.Parallel() + var logged bytes.Buffer + calls := 0 + + err := databaseWait{limit: 0, interval: time.Millisecond}.await( + t.Context(), pingAnswering(1, errRefused, &calls), testLogger(&logged)) + + want := "the database did not answer within 0s, the time GOPHENBERG_DATABASE_WAIT allows: " + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("await() = %v, want %q", err, want) + } + if calls != 1 { + t.Errorf("await() pinged %d times, want once", calls) + } + wantNothingLogged(t, &logged) +} + +func TestDatabaseWaitStopsAtOnceOnAFailureWaitingCannotFix(t *testing.T) { + t.Parallel() + var logged bytes.Buffer + calls := 0 + + err := databaseWait{limit: time.Minute, interval: time.Millisecond}.await( + t.Context(), pingAnswering(1, errWrongPassword, &calls), testLogger(&logged)) + + if !errors.Is(err, errWrongPassword) || calls != 1 { + t.Errorf("await() = %v after %d pings, want the wrong password after one", err, calls) + } + wantNothingLogged(t, &logged) +} + +// wantRanOut fails t unless err says the wait of within ran out, the setting it names included. +func wantRanOut(t *testing.T, err error, within string) { + t.Helper() + want := "the database did not answer within " + within + ", the time GOPHENBERG_DATABASE_WAIT allows: " + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("await() = %v, want %q", err, want) + } +} + +// pingHanging is a ping that hears nothing until its context ends, then fails with that end. +func pingHanging(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() +} + +// silentAddress listens on loopback and never takes a connection, so a client connects and then hears nothing. +func silentAddress(t *testing.T) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening on loopback: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + return listener.Addr().String() +} + +func TestDatabaseWaitBoundsEachTryByTheTimeTheWaitHasLeft(t *testing.T) { + t.Parallel() + var ends []time.Time + ping := func(ctx context.Context) error { + end, _ := ctx.Deadline() + ends = append(ends, end) + if len(ends) < 3 { + return errRefused + } + return nil + } + began := time.Now() + + err := databaseWait{limit: time.Minute, interval: time.Millisecond}.await( + t.Context(), ping, testLogger(&bytes.Buffer{})) + + if err != nil || len(ends) != 3 { + t.Fatalf("await() = %v after %d pings, want nil once the third one answers", err, len(ends)) + } + if ends[0].Before(began.Add(time.Minute)) || ends[0].After(time.Now().Add(time.Minute)) { + t.Errorf("the first ping ran until %v, want the wait's end, a minute after %v", ends[0], began) + } + for i, end := range ends { + if !end.Equal(ends[0]) { + t.Errorf("ping %d ran until %v, want the wait's end %v as every other ping", i+1, end, ends[0]) + } + } +} + +func TestDatabaseWaitEndsATryThatHangsOnceTheWaitRunsOut(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + began := time.Now() + + err := databaseWait{limit: 50 * time.Millisecond, interval: 10 * time.Millisecond}.await( + ctx, pingHanging, testLogger(&bytes.Buffer{})) + + took := time.Since(began) + wantRanOut(t, err, "50ms") + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("await() = %v, want the hanging try's failure kept", err) + } + if took > 300*time.Millisecond { + t.Errorf("await() took %v, want it to end soon after its wait of 50ms", took) + } +} + +func TestDatabaseWaitNeverTakesAnAnswerThatComesAfterTheWait(t *testing.T) { + t.Parallel() + late := func(context.Context) error { + time.Sleep(100 * time.Millisecond) + return nil + } + + err := databaseWait{limit: 50 * time.Millisecond, interval: 10 * time.Millisecond}.await( + t.Context(), late, testLogger(&bytes.Buffer{})) + + wantRanOut(t, err, "50ms") + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("await() = %v, want the wait's end kept as the failure", err) + } +} + +func TestDatabaseWaitPingsNoMoreOnceItsLastPauseReachesTheWaitsEnd(t *testing.T) { + t.Parallel() + calls := 0 + + err := databaseWait{limit: 50 * time.Millisecond, interval: time.Hour}.await( + t.Context(), pingAnswering(1, errRefused, &calls), testLogger(&bytes.Buffer{})) + + wantRanOut(t, err, "50ms") + if calls != 1 { + t.Errorf("await() pinged %d times, want once, as its only pause ends at the wait's end", calls) + } + if !errors.Is(err, syscall.ECONNREFUSED) { + t.Errorf("await() = %v, want the last failure kept", err) + } +} + +func TestDatabaseWaitOfZeroTriesOnceWithNoTimeBound(t *testing.T) { + t.Parallel() + var bounded []bool + ping := func(ctx context.Context) error { + _, held := ctx.Deadline() + bounded = append(bounded, held) + return errRefused + } + + err := databaseWait{limit: 0, interval: time.Millisecond}.await(t.Context(), ping, testLogger(&bytes.Buffer{})) + + wantRanOut(t, err, "0s") + if !slices.Equal(bounded, []bool{false}) { + t.Errorf("await() pinged with time bounds %v, want one ping with no time bound", bounded) + } +} + +func TestDatabaseWaitAnswersTheEndOfItsContextDuringATry(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + ping := func(tried context.Context) error { + cancel() + <-tried.Done() + return fmt.Errorf("failed to connect: %w", tried.Err()) + } + + err := databaseWait{limit: time.Minute, interval: time.Millisecond}.await(ctx, ping, testLogger(&bytes.Buffer{})) + + if !errors.Is(err, context.Canceled) || err.Error() != context.Canceled.Error() { + t.Errorf("await() = %v, want the end of its context as it is", err) + } +} + +func TestDatabaseWaitStopsOnceTheWaitRunsOutOnAServerThatNeverAnswers(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + handle, err := dbkitpg.Open("postgres://maria@"+silentAddress(t)+"/blog?sslmode=disable", + dbkitpg.Options{MaxConns: 2}) + if err != nil { + t.Fatalf("opening a pool: %v", err) + } + defer func() { _ = handle.Close() }() + began := time.Now() + + err = databaseWait{limit: 100 * time.Millisecond, interval: 10 * time.Millisecond}.await( + ctx, handle.Pool.Ping, testLogger(&bytes.Buffer{})) + + took := time.Since(began) + wantRanOut(t, err, "100ms") + if took > 300*time.Millisecond { + t.Errorf("await() took %v, want it to end soon after its wait of 100ms", took) + } +} + +func TestDatabaseWaitEndsWithTheContext(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + calls := 0 + ping := func(context.Context) error { + calls++ + cancel() + return errRefused + } + + err := databaseWait{limit: time.Minute, interval: time.Minute}.await(ctx, ping, testLogger(&bytes.Buffer{})) + + if !errors.Is(err, context.Canceled) || calls != 1 { + t.Errorf("await() = %v after %d pings, want the context's end after one", err, calls) + } +} + +// waitRecorder records what serve logs and cancels its context once serve logs that it waits for the database. +type waitRecorder struct { + cancel context.CancelFunc + logged *strings.Builder +} + +// Write records p and cancels on the line that says serve waits for the database. +func (w waitRecorder) Write(p []byte) (int, error) { + w.logged.Write(p) + if strings.Contains(string(p), waitingLine) { + w.cancel() + } + return len(p), nil +} + +// stopReportingPlugin is a plugin whose Stop records that it ran and returns the failure it holds. +type stopReportingPlugin struct { + failure error + stopped bool +} + +// ID names the plugin. +func (*stopReportingPlugin) ID() string { return "stop-reporting" } + +// Start starts nothing. +func (*stopReportingPlugin) Start(context.Context) error { return nil } + +// Stop records the stop and returns the failure the plugin holds. +func (p *stopReportingPlugin) Stop(context.Context) error { + p.stopped = true + return p.failure +} + +// stoppedDuringTheWait serves the plugin until serve logs that it waits for the database, then stops it, returning +// what serve logged and returned. +func stoppedDuringTheWait(t *testing.T, plugin sdk.Plugin) (string, error) { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + logged := &strings.Builder{} + env := map[string]string{"GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, "GOPHENBERG_DATABASE_WAIT": "1m"} + + err := run(ctx, testkit.Getenv(env), waitRecorder{cancel: cancel, logged: logged}, + func(sdk.Deps) ([]sdk.Plugin, error) { return []sdk.Plugin{plugin}, nil }) + + return logged.String(), err +} + +func TestServeStopsCleanlyWhenItEndsWhileItWaitsForTheDatabase(t *testing.T) { + t.Parallel() + plugin := &stopReportingPlugin{} + + logged, err := stoppedDuringTheWait(t, plugin) + + if err != nil { + t.Fatalf("run() = %v, want a clean stop as a stop while it serves ends", err) + } + if !strings.Contains(logged, `msg="shutting down"`) { + t.Errorf("serve logged %q, want it to say it shuts down", logged) + } + if !plugin.stopped { + t.Error("serve ended without stopping the plugin, want it stopped") + } +} + +func TestServeReturnsWhatAPluginFailedToStopWhenItEndsWhileItWaitsForTheDatabase(t *testing.T) { + t.Parallel() + errStuck := errors.New("the queue would not close") + + logged, err := stoppedDuringTheWait(t, &stopReportingPlugin{failure: errStuck}) + + if !errors.Is(err, errStuck) || errors.Is(err, context.Canceled) { + t.Errorf("run() = %v, want only the plugin's stop failure, as the stop itself ends serve cleanly", err) + } + if !strings.Contains(logged, `msg="shutting down"`) { + t.Errorf("serve logged %q, want it to say it shuts down", logged) + } +} + +func TestNotUpYetHoldsOnlyFailuresWaitingCanFix(t *testing.T) { + t.Parallel() + + for name, asked := range map[string]struct { + err error + want bool + }{ + "a connection refused": {errRefused, true}, + "a server still starting up": { + fmt.Errorf("connect: %w", &pgconn.PgError{Code: "57P03", Message: "the database system is starting up"}), true, + }, + "a wrong password": {fmt.Errorf("connect: %w", errWrongPassword), false}, + "a missing database": {&pgconn.PgError{Code: "3D000", Message: "database \"blog\" does not exist"}, false}, + "another failure": {errors.New("tls: handshake failure"), false}, + "no failure": {nil, false}, + } { + if held := notUpYet(asked.err); held != asked.want { + t.Errorf("notUpYet(%s) = %v, want %v", name, held, asked.want) + } + } +} + +func TestNotUpYetHoldsWhatAPoolAnswersWithNoServerThere(t *testing.T) { + t.Parallel() + + for name, address := range map[string]string{ + "a port nothing listens on": "postgres://maria@" + testkit.FreeAddr(t) + "/blog?sslmode=disable", + "a socket not made yet": "postgres://maria@/blog?host=" + filepath.Join(t.TempDir(), "run"), + } { + handle, err := dbkitpg.Open(address, dbkitpg.Options{MaxConns: 2}) + if err != nil { + t.Fatalf("opening a pool on %s: %v", name, err) + } + pinged := handle.Pool.Ping(t.Context()) + _ = handle.Close() + if !notUpYet(pinged) { + t.Errorf("notUpYet(%v) = false for %s, want true", pinged, name) + } + } +} + +func TestReadyWaitsForTheDatabaseBeforeItAsksAboutTheRole(t *testing.T) { + t.Parallel() + + handle, err := openPostgres(settingsEnv(testkit.Getenv(nil)), unreachableDatabaseURL) + if err != nil { + t.Fatalf("opening the pool: %v", err) + } + defer func() { _ = handle.Close() }() + settings := timedConfig() + settings.serving = servingDefaults + settings.databaseWait = databaseWait{limit: 0, interval: time.Millisecond} + start := boot{settings: settings, built: site{pgHandle: handle}, host: pluginkit.NewHost(), + logger: testLogger(&bytes.Buffer{})} + + err = start.ready(t.Context()) + + want := "the database did not answer within 0s, the time GOPHENBERG_DATABASE_WAIT allows: " + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("ready() = %v, want %q before the role is asked about", err, want) + } +} diff --git a/internal/app/features_test.go b/internal/app/features_test.go index 36a58376..6d65e61d 100644 --- a/internal/app/features_test.go +++ b/internal/app/features_test.go @@ -127,11 +127,13 @@ func initializeOperatorCommands(t *testing.T, engine operatorEngine) func(*godog sc.Given(`^the administrator "([^"]*)" gave "([^"]*)" the role "([^"]*)"$`, s.gaveRole) sc.Then(`^the answer names the version, the build stamp, the minimum version and the theme kits of this binary$`, s.namesTheBuildFacts) + sc.Then(`^the answer names Node.js (\d+) as the lowest major a theme runs on$`, s.namesTheNodeMajor) sc.Given(`^a Gophenberg migrated the database and left the minimum version "([^"]*)"$`, s.leaveMinimumVersion) sc.Then(`^the database holds the minimum version of this binary$`, s.holdsOwnMinimumVersion) sc.Then(`^the database still holds the minimum version "([^"]*)"$`, s.holdsMinimumVersion) initializeUpdateChecks(sc, s) initializeCoreUpdates(sc, s) + initializeDatabaseWait(sc, s) } } @@ -166,7 +168,8 @@ func (s *operatorScenario) namesTheBuildFacts() error { } want := buildFacts{ Schema: 1, Name: "gophenberg", Version: version.Version(), Commit: version.Commit(), Stamp: version.Stamp(), - MinimumVersion: version.Minimum(), PostgreSQL: postgres.OldestMajor, Kits: themehost.ServedKits(), + MinimumVersion: version.Minimum(), PostgreSQL: postgres.OldestMajor, Node: themehost.OldestNodeMajor, + Kits: themehost.ServedKits(), } if !facts.equal(want) { return fmt.Errorf("the answer %q holds %+v, want %+v", s.result.Stdout, facts, want) @@ -174,6 +177,20 @@ func (s *operatorScenario) namesTheBuildFacts() error { return nil } +// namesTheNodeMajor fails unless the answer names major as the lowest Node.js major a theme runs on. +func (s *operatorScenario) namesTheNodeMajor(major int) error { + var named struct { + Node *int `json:"node"` + } + if err := json.Unmarshal([]byte(s.result.Stdout), &named); err != nil { + return fmt.Errorf("the answer %q is no JSON document: %w", s.result.Stdout, err) + } + if named.Node == nil || *named.Node != major { + return fmt.Errorf("the answer %q names no Node.js major %d", s.result.Stdout, major) + } + return nil +} + // leaveMinimumVersion migrates the scenario's database, then stores minimum as the minimum version left there. func (s *operatorScenario) leaveMinimumVersion(ctx context.Context, minimum string) error { s.runLine("migrate") diff --git a/internal/app/postgres.go b/internal/app/postgres.go index fae4d55a..e3b9499b 100644 --- a/internal/app/postgres.go +++ b/internal/app/postgres.go @@ -4,6 +4,7 @@ package app import ( "context" + "database/sql" "errors" "fmt" "maps" @@ -16,6 +17,8 @@ import ( "github.com/gopherium/framework/gonsole" "github.com/gopherium/gophenberg/internal/dbconns" + "github.com/gopherium/gophenberg/internal/updater" + "github.com/gopherium/gophenberg/internal/version" ) // poolSettingPrefix starts every pool setting pgx reads. @@ -48,20 +51,32 @@ func poolSettings(address string) []string { return slices.DeleteFunc(keys, func(key string) bool { return !strings.HasPrefix(key, poolSettingPrefix) }) } -// checkedFirst returns step, run at an address once openPostgres takes it and the database lets this binary on. +// checkedFirst returns step, run at an address once openPostgres takes it and admitted lets this binary on it. func checkedFirst(env gonsole.Env, step gonsole.Step) gonsole.Step { return gonsole.Step{Name: step.Name, Run: func(ctx context.Context, address string) error { + install, err := updater.SettingsFrom(env) + if err != nil { + return err + } handle, err := openPostgres(env, address) if err != nil { return err } - if err := guardPostgres(ctx, handle.DB); err != nil { + if err := admitted(ctx, handle.DB, install); err != nil { return errors.Join(err, handle.Close()) } return errors.Join(handle.Close(), step.Run(ctx, address)) }} } +// admitted refuses a db that does not let this binary on, then marks it reached for an update that switched to it. +func admitted(ctx context.Context, db *sql.DB, install updater.Settings) error { + if err := guardPostgres(ctx, db); err != nil { + return err + } + return updater.MarkDatabaseReached(install, version.Version()) +} + // postgresCore returns the core schema step that opens a capped handle at the address, migrates on it, then closes it. func postgresCore(env gonsole.Env) gonsole.Step { return gonsole.Step{Name: "core", Run: func(ctx context.Context, address string) error { diff --git a/internal/app/postgres_migrate_test.go b/internal/app/postgres_migrate_test.go index bfc1967c..64405030 100644 --- a/internal/app/postgres_migrate_test.go +++ b/internal/app/postgres_migrate_test.go @@ -6,6 +6,8 @@ import ( "database/sql" "fmt" "net/url" + "os" + "path/filepath" "testing" "github.com/gopherium/framework/dbkit" @@ -14,6 +16,8 @@ import ( authkitpg "github.com/gopherium/gouncer/authkit/postgres" "github.com/gopherium/gophenberg/internal/testdb" + "github.com/gopherium/gophenberg/internal/updater" + "github.com/gopherium/gophenberg/internal/version" ) // coreBuilt reports whether the core schema stands in db. @@ -96,6 +100,59 @@ func TestMigrateNamesADatabaseAddressDbkitRefusesOnPostgreSQL(t *testing.T) { ": the query sets a key twice, set each key once, with dbname and database as one key\n") } +func TestMigrateNamesAnInstallRootItRefusesOnPostgreSQL(t *testing.T) { + t.Parallel() + + stopsBeforeAnySchema(t, func(_ *testing.T, address string) map[string]string { + return map[string]string{"GOPHENBERG_DATABASE_URL": address, installRootSetting: "opt/gophenberg"} + }, "gophenberg: migrate accounts: GOPHENBERG_INSTALL_ROOT: must be an absolute path, got \"opt/gophenberg\"\n") +} + +func TestMigrateStopsBeforeAnySchemaOnAnUpdateStateItCannotRead(t *testing.T) { + t.Parallel() + + root := t.TempDir() + state := updater.Layout{Root: root}.StateFile() + if err := os.MkdirAll(filepath.Dir(state), 0o755); err != nil { + t.Fatalf("making the state folder: %v", err) + } + if err := os.WriteFile(state, []byte("step: switched"), 0o644); err != nil { + t.Fatalf("writing the state file: %v", err) + } + + stopsBeforeAnySchema(t, func(_ *testing.T, address string) map[string]string { + return map[string]string{"GOPHENBERG_DATABASE_URL": address, installRootSetting: root} + }, "gophenberg: migrate accounts: updater: read the update state "+state+ + ": invalid character 's' looking for beginning of value\n") +} + +func TestEachCommandThatMigratesMarksTheDatabaseReachedForTheUpdateThatSwitchedToThisBuild(t *testing.T) { + t.Parallel() + + for name, args := range migratingCommands { + t.Run(name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + if err := switchedInstall(root, "0.18.0", version.Version()); err != nil { + t.Fatalf("making the install root: %v", err) + } + settings := map[string]string{"GOPHENBERG_DATABASE_URL": emptyDatabaseURL(t), installRootSetting: root} + + got := testkit.Run(t, Program(testkit.Getenv(settings), noPlugins), typedPassword+"\n", args...) + + state, err := updater.Layout{Root: root}.ReadState() + if got.Code != 0 || err != nil { + t.Fatalf("%s = %d with stderr %q, and reading the update state gave %v, want 0 and a readable state", + name, got.Code, got.Stderr, err) + } + if !state.DatabaseReached { + t.Errorf("%s left the update state %+v, want the database marked reached", name, state) + } + }) + } +} + func TestTheCoreStepOpensUnderTheConnectionCapSetting(t *testing.T) { t.Parallel() diff --git a/internal/app/run.go b/internal/app/run.go index 87c0c5af..7eb639f4 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -49,12 +49,15 @@ func serveSite(ctx context.Context, getenv func(string) string, stderr io.Writer return err } defer built.close() - b := boot{settings: settings, built: built, host: hostOf(built.registered), logger: logger, stamp: compiled.stamp} + b := boot{ + settings: settings, guard: guard, built: built, host: hostOf(built.registered), logger: logger, + stamp: compiled.stamp, + } serving, restart := context.WithCancelCause(ctx) defer restart(nil) b.admin = b.updatesFromAdmin(serving, restart, settingsEnv(getenv)) if err := b.ready(ctx); err != nil { - return err + return errors.Join(b.stopped(ctx, err), b.stopPlugins(ctx)) } jobs := cronOf(settings.cron, built, logger, updateJobs(settings.updates, b.updates())...) @@ -95,9 +98,10 @@ func started(getenv func(string) string, logger *slog.Logger) (runConfig, *updat return settings, guard, err } -// boot is one start of the site: its settings, what compose built, its plugins, logger, stamp and admin updates. +// boot is one start of the site: its settings, guard, what compose built, plugins, logger, stamp and admin updates. type boot struct { settings runConfig + guard *updater.Guard built site host *pluginkit.Host logger *slog.Logger @@ -125,14 +129,31 @@ func (b boot) updatesFromAdmin( } } -// ready refuses plugins that failed to register, then checks the database role and migrates, stopping them on failure. +// ready refuses plugins that failed to register, then waits for the database and readies it. func (b boot) ready(ctx context.Context) error { if b.built.failed != nil { - return errors.Join(fmt.Errorf("register plugins: %w", b.built.failed), b.stopPlugins(ctx)) + return fmt.Errorf("register plugins: %w", b.built.failed) } - if err := prepareDatabase(ctx, b.settings, b.built, b.logger); err != nil { - return errors.Join(err, b.stopPlugins(ctx)) + return b.reachDatabase(ctx) +} + +// reachDatabase waits for the database, marks it reached for the boot guard, then checks the role and migrates it. +func (b boot) reachDatabase(ctx context.Context) error { + if err := b.settings.databaseWait.await(ctx, b.built.pgHandle.Pool.Ping, b.logger); err != nil { + return err + } + if err := b.guard.ReachedDatabase(); err != nil { + return err } + return prepareDatabase(ctx, b.settings, b.built, b.logger) +} + +// stopped returns nil and logs the stop when err is the end of ctx, and err otherwise. +func (b boot) stopped(ctx context.Context, err error) error { + if !errors.Is(err, ctx.Err()) { + return err + } + b.logger.Info("shutting down") return nil } @@ -229,6 +250,8 @@ type runConfig struct { superuser superuserPolicy + databaseWait databaseWait + updates updateSettings install updater.Settings @@ -477,7 +500,7 @@ func loadRunConfig(getenv func(string) string) (runConfig, error) { return settings, nil } -// sectionsFrom reads the list, admin folder, cron, superuser, update check and server install settings into held. +// sectionsFrom reads the list, admin folder, cron, superuser, database wait, update and install settings into held. func sectionsFrom(env gonsole.Env, held *runConfig) error { lists, err := listSettingsFrom(env) if err != nil { @@ -493,6 +516,9 @@ func sectionsFrom(env gonsole.Env, held *runConfig) error { if held.superuser, err = superuserPolicyFrom(env); err != nil { return err } + if held.databaseWait, err = databaseWaitFrom(env); err != nil { + return err + } if held.updates, err = updateSettingsFrom(env); err != nil { return err } diff --git a/internal/app/run_test.go b/internal/app/run_test.go index 37696a5d..b6df3d90 100644 --- a/internal/app/run_test.go +++ b/internal/app/run_test.go @@ -50,7 +50,7 @@ func TestRunValidatesItsEnvironment(t *testing.T) { "GOPHENBERG_DATABASE_URL": "not a url \x00", }, "unreachable database": { - "GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, + "GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, "GOPHENBERG_DATABASE_WAIT": "0", }, "malformed trusted proxies": { "GOPHENBERG_DATABASE_URL": unreachableDatabaseURL, diff --git a/internal/themehost/image_test.go b/internal/themehost/image_test.go new file mode 100644 index 00000000..f8148956 --- /dev/null +++ b/internal/themehost/image_test.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 + +package themehost_test + +import ( + "os" + "regexp" + "strconv" + "testing" + + "github.com/gopherium/gophenberg/internal/themehost" +) + +// dockerfilePath is the file the release image is built from. +const dockerfilePath = "../../Dockerfile" + +// nodeImage matches a distroless Node.js image the Dockerfile names, holding its major. +var nodeImage = regexp.MustCompile(`distroless/nodejs(\d+)`) + +func TestTheImageShipsTheOldestNodeMajor(t *testing.T) { + t.Parallel() + + dockerfile, err := os.ReadFile(dockerfilePath) + if err != nil { + t.Fatalf("reading the Dockerfile: %v", err) + } + + named := nodeImage.FindAllSubmatch(dockerfile, -1) + if len(named) == 0 { + t.Fatalf("the Dockerfile names no distroless Node.js image, want nodejs%d", themehost.OldestNodeMajor) + } + want := strconv.Itoa(themehost.OldestNodeMajor) + for _, image := range named { + if major := string(image[1]); major != want { + t.Errorf("the Dockerfile names distroless/nodejs%s, want nodejs%s, the major about names", major, want) + } + } +} diff --git a/internal/themehost/supervisor.go b/internal/themehost/supervisor.go index 18174d8e..c2ed2de0 100644 --- a/internal/themehost/supervisor.go +++ b/internal/themehost/supervisor.go @@ -14,6 +14,9 @@ import ( "time" ) +// OldestNodeMajor is the oldest Node.js major version this release runs a theme server on. +const OldestNodeMajor = 24 + // healthPath is where a theme server reports that it is ready to serve. const healthPath = "/_gophenberg/health" diff --git a/internal/updater/database_mark_test.go b/internal/updater/database_mark_test.go new file mode 100644 index 00000000..c7263bdb --- /dev/null +++ b/internal/updater/database_mark_test.go @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 + +package updater_test + +import ( + "cmp" + "log/slog" + "net/http" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/gopherium/gophenberg/internal/updater" +) + +func TestReachedDatabaseMarksAStartOfTheBuildAnUpdateSwitchedTo(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, switchedTo) + + if err := f.started().ReachedDatabase(); err != nil { + t.Fatalf("ReachedDatabase() = %v, want the start marked", err) + } + + want := switchedTo + want.Attempts, want.DatabaseReached = 1, true + f.wantState(want) +} + +func TestReachedDatabaseLeavesAStateThatNamesNoStartOfThisBuildAlone(t *testing.T) { + t.Parallel() + + for name, state := range map[string]updater.State{ + "a kept update": {Step: updater.StepCommitted, From: running, To: offered}, + "a switch to another build": {Step: updater.StepSwitched, From: running, To: "0.22.0"}, + "a job that stopped": {Step: updater.StepBackup, From: running, To: offered}, + "an install never updated": {}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, state) + + if err := f.started().ReachedDatabase(); err != nil { + t.Fatalf("ReachedDatabase() = %v, want nil", err) + } + + f.wantState(state) + }) + } +} + +func TestReachedDatabaseMarksNothingOnAnInstallWithNoRoot(t *testing.T) { + t.Parallel() + + guard, err := updater.Start(settingsOf(t, map[string]string{}), offered, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatalf("Start() with no install root = %v, want nil", err) + } + + if err := guard.ReachedDatabase(); err != nil { + t.Errorf("ReachedDatabase() with no install root = %v, want nil", err) + } +} + +// spoiledState is one way to spoil the state of an install root, with what a command that marks the database says. +type spoiledState struct { + spoil func(t *testing.T, layout updater.Layout) + says string +} + +// spoiledStates are the ways to spoil the state of an install root that a command marking the database cannot get past. +var spoiledStates = map[string]spoiledState{ + "a state folder it cannot lock": { + spoil: func(t *testing.T, layout updater.Layout) { + t.Helper() + denied(t, filepath.Dir(layout.StateFile()), 0o300) + }, + says: "updater: lock the update state", + }, + "a state file it cannot read": { + spoil: func(t *testing.T, layout updater.Layout) { + t.Helper() + if err := os.WriteFile(layout.StateFile(), []byte("step: switched"), 0o644); err != nil { + t.Fatalf("writing the state file: %v", err) + } + }, + says: "updater: read the update state", + }, + "a state folder it cannot write": { + spoil: func(t *testing.T, layout updater.Layout) { + t.Helper() + denied(t, filepath.Dir(layout.StateFile()), 0o500) + }, + says: "updater: write the update state", + }, +} + +func TestReachedDatabaseRefusesAStateItCannotLockReadOrWrite(t *testing.T) { + t.Parallel() + + for name, tc := range spoiledStates { + t.Run(name, func(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, switchedTo) + guard := f.started() + tc.spoil(t, f.layout) + + err := guard.ReachedDatabase() + + wantRefused(t, err, tc.says) + }) + } +} + +func TestMarkDatabaseReachedMarksTheUpdateThatSwitchedToTheBuildThatRuns(t *testing.T) { + t.Parallel() + + for name, state := range map[string]updater.State{ + "a switch that waits for its check": switchedTo, + "an update that failed": {Step: updater.StepFailed, From: running, To: offered, LongJump: true}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, state) + + if err := updater.MarkDatabaseReached(settingsOf(t, f.env), offered); err != nil { + t.Fatalf("MarkDatabaseReached() = %v, want the update marked", err) + } + + want := state + want.DatabaseReached = true + f.wantState(want) + }) + } +} + +func TestMarkDatabaseReachedLeavesAStateThatNamesNoSwitchToTheBuildThatRunsAlone(t *testing.T) { + t.Parallel() + + for name, tc := range map[string]struct { + state updater.State + running string + }{ + "an install never updated": {running: offered}, + "a kept update": {state: updater.State{Step: updater.StepCommitted, From: running, To: offered}}, + "an update taken back": {state: takenBack}, + "a job that stopped": {state: updater.State{Step: updater.StepBackup, From: running, To: offered}}, + "a switch to another build": {state: switchedTo, running: "0.22.0"}, + "the build before a switch": {state: switchedTo, running: running}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, tc.state) + + if err := updater.MarkDatabaseReached(settingsOf(t, f.env), cmp.Or(tc.running, offered)); err != nil { + t.Fatalf("MarkDatabaseReached() = %v, want nil", err) + } + + f.wantState(tc.state) + }) + } +} + +func TestMarkDatabaseReachedMarksNothingOnAnInstallWithNoRoot(t *testing.T) { + folder := t.TempDir() + writeState(t, updater.Layout{Root: folder}, switchedTo) + t.Chdir(folder) + + if err := updater.MarkDatabaseReached(settingsOf(t, map[string]string{}), offered); err != nil { + t.Errorf("MarkDatabaseReached() with no install root = %v, want nil", err) + } + + if state := readState(t, updater.Layout{Root: folder}); !reflect.DeepEqual(state, switchedTo) { + t.Errorf("the state file in the working folder holds %+v, want %+v left alone", state, switchedTo) + } +} + +func TestMarkDatabaseReachedRefusesAStateItCannotLockReadOrWrite(t *testing.T) { + t.Parallel() + + for name, tc := range spoiledStates { + t.Run(name, func(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, switchedTo) + settings := settingsOf(t, f.env) + tc.spoil(t, f.layout) + + err := updater.MarkDatabaseReached(settings, offered) + + wantRefused(t, err, tc.says) + }) + } +} + +func TestStartGoesBackWhenALongJumpFailsBeforeItsBuildReachedTheDatabase(t *testing.T) { + t.Parallel() + + tried := switchedTo + tried.Attempts, tried.LongJump, tried.Backup = 3, true, longJumpBackup + f := newGuardFixture(t, tried) + + _, err := f.start() + + wantCheckFailed(t, err, + "updater: Gophenberg 0.21.0 failed its start check after the update from 0.20.0: it tried to start 4 times", + "updater: current names Gophenberg 0.20.0 again, which the next start runs") + f.wantCurrent(running) + want := tried + want.Attempts, want.Refused = 4, []string{"0.19.5", offered} + f.wantState(want) +} + +func TestServeKeepsTheSiteDownWhenTheBuildOfALongJumpReachedTheDatabaseAndFailsItsCheck(t *testing.T) { + t.Parallel() + + long := switchedTo + long.LongJump, long.Backup = true, longJumpBackup + f := newGuardFixture(t, long) + guard := f.started() + if err := guard.ReachedDatabase(); err != nil { + t.Fatalf("ReachedDatabase() = %v, want the start marked", err) + } + + err := served(t, guard, newTestSite(map[string]int{"/admin/": http.StatusInternalServerError}), passes) + + wantCheckFailed(t, err, + "updater: Gophenberg 0.21.0 failed its start check after the update from 0.20.0: /admin/ answered 500, want 200", + "so 0.20.0 cannot open the database and the site stays down") + f.wantCurrent(offered) + want := long + want.Step, want.Attempts, want.Refused, want.DatabaseReached = updater.StepFailed, 1, []string{"0.19.5", offered}, true + f.wantState(want) +} diff --git a/internal/updater/guard.go b/internal/updater/guard.go index 250ef55b..e3176947 100644 --- a/internal/updater/guard.go +++ b/internal/updater/guard.go @@ -114,6 +114,42 @@ func (g *Guard) count() error { return g.layout.WriteState(g.state) } +// ReachedDatabase writes under the state lock that the build of a start the guard checks reached the database. +func (g *Guard) ReachedDatabase() error { + if g.settles == "" { + return nil + } + state, unlock, err := g.layout.lockedState() + if err != nil { + return err + } + defer unlock() + if !g.pending(state) { + return fmt.Errorf("updater: another command took back the update to Gophenberg %s while this start waited "+ + "for the database, so it stops before it migrates", g.state.To) + } + state.DatabaseReached = true + return g.layout.WriteState(state) +} + +// MarkDatabaseReached writes under the state lock that running, the build an update switched to, reached the database. +func MarkDatabaseReached(settings Settings, running string) error { + if settings.Root == "" { + return nil + } + layout := Layout{Root: settings.Root} + state, unlock, err := layout.lockedState() + if err != nil { + return err + } + defer unlock() + if running != state.To || !slices.Contains(failedSteps, state.Step) { + return nil + } + state.DatabaseReached = true + return layout.WriteState(state) +} + // Serve runs serve with the start check once srv listens, ending it when a switched build fails it, or alone with none. func (g *Guard) Serve( ctx context.Context, srv *http.Server, inside func(context.Context) error, serve func(context.Context) error, @@ -281,7 +317,7 @@ func (l Layout) builds() []string { func (g *Guard) fail(reason error) error { failed := fmt.Errorf("updater: Gophenberg %s failed its start check after the update from %s: %w", g.state.To, g.state.From, reason) - if g.state.LongJump { + if mayHaveMigrated(g.state) { return CheckError{err: errors.Join(failed, g.layout.stayDown(g.state))} } return CheckError{err: errors.Join(failed, g.goBack(g.state))} diff --git a/internal/updater/guard_test.go b/internal/updater/guard_test.go index a47727b3..9adfa72e 100644 --- a/internal/updater/guard_test.go +++ b/internal/updater/guard_test.go @@ -336,11 +336,11 @@ func TestStartTakesTheAttemptsTheSettingNames(t *testing.T) { f.wantCurrent(offered) } -func TestStartKeepsTheSiteDownWhenAnUpdateThatRaisedTheMinimumVersionFails(t *testing.T) { +func TestStartKeepsTheSiteDownWhenALongJumpFailsAfterItsBuildReachedTheDatabase(t *testing.T) { t.Parallel() tried := switchedTo - tried.Attempts, tried.LongJump, tried.Backup = 3, true, longJumpBackup + tried.Attempts, tried.LongJump, tried.Backup, tried.DatabaseReached = 3, true, longJumpBackup, true f := newGuardFixture(t, tried) _, err := f.start() @@ -422,6 +422,7 @@ func TestStartStillFailsTheCheckWhenItCannotGoBack(t *testing.T) { tried := switchedTo tried.Attempts, tried.LongJump, tried.Backup = 3, deny.longJump, longJumpBackup + tried.DatabaseReached = deny.longJump f := newGuardFixture(t, tried) denied(t, filepath.Join(f.layout.Root, deny.folder), 0o500) diff --git a/internal/updater/rollback.go b/internal/updater/rollback.go index 1cb9e8c4..9bbcbc12 100644 --- a/internal/updater/rollback.go +++ b/internal/updater/rollback.go @@ -57,9 +57,9 @@ func RollbackAfterFailure(settings Settings, failure Failure) (State, error) { }) } -// mayHaveMigrated reports whether the build an update switched to raised the minimum version and has started once. +// mayHaveMigrated reports whether the build an update switched to raised the minimum version and reached the database. func mayHaveMigrated(state State) bool { - return state.LongJump && state.Attempts > 0 + return state.LongJump && state.DatabaseReached } // lastUpdate runs undo on the install root and its last update under the state lock, refusing an empty install root. diff --git a/internal/updater/rollback_test.go b/internal/updater/rollback_test.go index 183dae66..66e3f108 100644 --- a/internal/updater/rollback_test.go +++ b/internal/updater/rollback_test.go @@ -23,10 +23,11 @@ var takenBack = updater.State{ Step: updater.StepRolledBack, From: running, To: offered, Attempts: 1, Refused: []string{"0.19.5", offered}, } -// longJumped returns the state of an update from 0.20.0 to 0.21.0 that raised the minimum version and started, at step. +// longJumped returns the state at step of a long jump from 0.20.0 to 0.21.0 whose build reached the database. func longJumped(step updater.Step) updater.State { state := switchedTo state.Step, state.LongJump, state.Backup, state.Attempts = step, true, longJumpBackup, 1 + state.DatabaseReached = true return state } @@ -144,7 +145,7 @@ func TestRollbackRefusesAnUpdateThatRaisedTheMinimumVersionAndNamesItsBackup(t * } } -func TestRollbackTakesBackAnUpdateThatRaisedTheMinimumVersionBeforeItsBuildStarted(t *testing.T) { +func TestRollbackTakesBackAnUpdateThatRaisedTheMinimumVersionBeforeItsBuildReachedTheDatabase(t *testing.T) { t.Parallel() for name, rollback := range map[string]func(updater.Settings) (updater.State, error){ @@ -153,23 +154,25 @@ func TestRollbackTakesBackAnUpdateThatRaisedTheMinimumVersionBeforeItsBuildStart return updater.RollbackAfterFailure(settings, startLimitHit) }, } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - unstarted := longJumped(updater.StepSwitched) - unstarted.Attempts = 0 - f := newGuardFixture(t, unstarted) - - state, err := rollback(settingsOf(t, f.env)) - - want := unstarted - want.Step, want.Refused = updater.StepRolledBack, []string{"0.19.5", offered} - if err != nil || !reflect.DeepEqual(state, want) { - t.Errorf("the rollback = %+v, %v, want %+v", state, err, want) - } - f.wantCurrent(running) - f.wantState(want) - }) + for attempts, started := range map[int]string{0: "never started", 3: "started three times"} { + t.Run(name+", a build that "+started, func(t *testing.T) { + t.Parallel() + + unreached := longJumped(updater.StepSwitched) + unreached.Attempts, unreached.DatabaseReached = attempts, false + f := newGuardFixture(t, unreached) + + state, err := rollback(settingsOf(t, f.env)) + + want := unreached + want.Step, want.Refused = updater.StepRolledBack, []string{"0.19.5", offered} + if err != nil || !reflect.DeepEqual(state, want) { + t.Errorf("the rollback = %+v, %v, want %+v", state, err, want) + } + f.wantCurrent(running) + f.wantState(want) + }) + } } } diff --git a/internal/updater/state.go b/internal/updater/state.go index 6fbcb81c..95462ff3 100644 --- a/internal/updater/state.go +++ b/internal/updater/state.go @@ -43,14 +43,15 @@ const ( // State is what the state file holds: the last update, the versions that failed their start check, the last failure. type State struct { - Step Step `json:"step"` - From string `json:"from"` - To string `json:"to"` - Attempts int `json:"attempts"` - LongJump bool `json:"long_jump"` - Backup string `json:"backup"` - Refused []string `json:"refused"` - Failure *JobFailure `json:"failure,omitempty"` + Step Step `json:"step"` + From string `json:"from"` + To string `json:"to"` + Attempts int `json:"attempts"` + DatabaseReached bool `json:"database_reached"` + LongJump bool `json:"long_jump"` + Backup string `json:"backup"` + Refused []string `json:"refused"` + Failure *JobFailure `json:"failure,omitempty"` } // JobFailure is how a job failed after it wrote a step: when, from and to which versions, at which step and why. diff --git a/internal/updater/state_lock_test.go b/internal/updater/state_lock_test.go index 73afbe5a..f05dfe35 100644 --- a/internal/updater/state_lock_test.go +++ b/internal/updater/state_lock_test.go @@ -211,6 +211,56 @@ func TestStartWaitsForTheStateLockAndChecksWhatTheStateThenNames(t *testing.T) { f.wantState(takenBack) } +// databaseMarks return, for each command that marks the database reached, the mark it writes on the install root of f. +var databaseMarks = map[string]func(f *guardFixture) func() error{ + "a start the guard checks": func(f *guardFixture) func() error { + return f.started().ReachedDatabase + }, + "a command that migrates": func(f *guardFixture) func() error { + settings := settingsOf(f.t, f.env) + return func() error { return updater.MarkDatabaseReached(settings, offered) } + }, +} + +func TestTheDatabaseMarkWaitsForTheStateLockAndMarksWhatTheStateThenNames(t *testing.T) { + t.Parallel() + + for name, mark := range databaseMarks { + t.Run(name, func(t *testing.T) { + t.Parallel() + + counted := switchedTo + counted.Attempts = 2 + f := newGuardFixture(t, switchedTo) + marks := mark(f) + var err error + + waitsForTheLock(t, f.layout, counted, func() { err = marks() }) + + if err != nil { + t.Fatalf("the mark = %v, want nil", err) + } + marked := counted + marked.DatabaseReached = true + f.wantState(marked) + }) + } +} + +func TestReachedDatabaseRefusesAnUpdateTakenBackWhileItsStartWaitedForTheDatabase(t *testing.T) { + t.Parallel() + + f := newGuardFixture(t, switchedTo) + guard := f.started() + var err error + + waitsForTheLock(t, f.layout, takenBack, func() { err = guard.ReachedDatabase() }) + + wantRefused(t, err, "updater: another command took back the update to Gophenberg 0.21.0 while this start "+ + "waited for the database, so it stops before it migrates") + f.wantState(takenBack) +} + func TestRollbackWaitsForTheStateLockAndTakesBackWhatTheStateThenNames(t *testing.T) { t.Parallel() diff --git a/internal/updater/state_test.go b/internal/updater/state_test.go index fa8ffa6d..949123d2 100644 --- a/internal/updater/state_test.go +++ b/internal/updater/state_test.go @@ -14,10 +14,10 @@ import ( "github.com/gopherium/gophenberg/internal/updater" ) -// switchedState is the state of an update that switched from 0.20.0 to 0.21.0 and started twice. +// switchedState is the state of an update that switched from 0.20.0 to 0.21.0, started twice and reached the database. var switchedState = updater.State{ Step: updater.StepSwitched, From: "0.20.0", To: "0.21.0", Attempts: 2, LongJump: true, - Backup: "backups/0.20.0-20261020T120000Z.dump", Refused: []string{"0.20.5"}, + Backup: "backups/0.20.0-20261020T120000Z.dump", Refused: []string{"0.20.5"}, DatabaseReached: true, } // writeState writes state into the install root of layout, failing t when it cannot. @@ -65,7 +65,7 @@ func TestWriteStateKeepsEveryFieldUnderItsFrozenName(t *testing.T) { } want := map[string]any{ "step": "switched", "from": "0.20.0", "to": "0.21.0", "attempts": 2.0, "long_jump": true, - "backup": "backups/0.20.0-20261020T120000Z.dump", "refused": []any{"0.20.5"}, + "backup": "backups/0.20.0-20261020T120000Z.dump", "refused": []any{"0.20.5"}, "database_reached": true, } if !reflect.DeepEqual(fields, want) { t.Errorf("the state file holds %v, want %v", fields, want) @@ -99,8 +99,8 @@ func TestWriteStateKeepsTheFailureOfAJobBesideTheFrozenFields(t *testing.T) { "at": "2026-10-22T09:12:00Z", "from": "0.21.0", "to": "0.22.0", "step": "stage", "error": "updater: refused the archive", } - if !reflect.DeepEqual(fields["failure"], want) || len(fields) != 8 { - t.Errorf("the state file holds %v, want the seven frozen fields and the failure %v", fields, want) + if !reflect.DeepEqual(fields["failure"], want) || len(fields) != 9 { + t.Errorf("the state file holds %v, want the eight frozen fields and the failure %v", fields, want) } if read := readState(t, layout); !reflect.DeepEqual(read, failed) { t.Errorf("ReadState() = %+v, want the state written %+v", read, failed) diff --git a/test/features/features/core-updates.feature b/test/features/features/core-updates.feature index 22c7c45e..e12aa838 100644 --- a/test/features/features/core-updates.feature +++ b/test/features/features/core-updates.feature @@ -8,10 +8,14 @@ Feature: Operators and administrators update the core and see where it stands the attempt and checks itself. A healthy start keeps the update. A build that fails goes back to the previous one, and that version is never picked again on its own unless a person retries it. An update that - raised the minimum version never goes back on its own, since the older - build refuses the database the newer one migrated, so the site stays - down and shows how to restore the backup. An install with no install - root, like the Docker image, and a custom build keep the manual steps. + raised the minimum version, a long jump, goes back the same way while + its new build never reached the database. Once the new build reached + the database it may have migrated it, and the older build refuses a + database a newer one migrated, so the site stays down and shows how to + restore the backup. Any command of the new build that migrates, such as + a migrate run by hand, counts as reaching the database, the same as a + start. An install with no install root, like the Docker + image, and a custom build keep the manual steps. A step that fails leaves which update failed, when, at which step and why. An administrator asks the site for the same: the version that runs, whether this install updates itself, the refused versions, the @@ -117,10 +121,21 @@ Feature: Operators and administrators update the core and see where it stands And the current build is "0.18.0" And the update status lists this version as refused - Scenario: A failed update that raised the minimum version stays down and shows the restore steps + Scenario: A long jump whose new build never reached the database goes back Given an install root where an update switched from "0.18.0" to this version And the update raised the minimum version, with a backup taken before the switch - And this version already tried to start 3 times + And the database refuses every connection + And serve waits "0" for the database + And the site failed to start 3 times + When the service manager runs "update:rollback -after-failure" with MONITOR_SERVICE_RESULT set to "start-limit-hit" + Then the command succeeds + And the current build is "0.18.0" + And the update status lists this version as refused + + Scenario: A long jump that reached the database stays down + Given an install root where an update switched from "0.18.0" to this version + And the update raised the minimum version, with a backup taken before the switch + And the start check of this version runs out of time When the site starts Then the command exits with code 76 And the current build is still this version @@ -128,6 +143,17 @@ Feature: Operators and administrators update the core and see where it stands And the error says "restore the backup taken before the update" And the update status names the backup taken before the switch + Scenario: A migrate run by hand on the new build keeps a long jump from going back + Given an install root where an update switched from "0.18.0" to this version + And the update raised the minimum version, with a backup taken before the switch + And the operator ran "migrate" + When the operator runs "update:rollback" + Then the command exits with code 1 + And the current build is still this version + And the error says "update:rollback cannot go back to it" + And the error names the backup taken before the switch + And the error says "restore the backup taken before the update" + Scenario Outline: The service manager takes the site back after Given an install root where an update switched from this version to "99.0.0" When the service manager runs "update:rollback -after-failure" with set to "" @@ -147,10 +173,11 @@ Feature: Operators and administrators update the core and see where it stands Then the command exits with code 1 And the current build is still "99.0.0" - Scenario Outline: The service manager does not take back an update that raised the minimum version after + Scenario Outline: The service manager does not take back a long jump that reached the database after Given an install root where an update switched from this version to "99.0.0" And the update raised the minimum version, with a backup taken before the switch And the new build already tried to start 3 times + And the new build reached the database When the service manager runs "update:rollback -after-failure" with set to "" Then the command exits with code 1 And the current build is still "99.0.0" diff --git a/test/features/features/database-wait.feature b/test/features/features/database-wait.feature new file mode 100644 index 00000000..fa495930 --- /dev/null +++ b/test/features/features/database-wait.feature @@ -0,0 +1,45 @@ +Feature: A site waits for a database that starts late + A server often starts Gophenberg before PostgreSQL answers, at boot or + right after an update. serve then tries the database again, for up to + GOPHENBERG_DATABASE_WAIT, 60 seconds by default, with a pause of + GOPHENBERG_DATABASE_WAIT_INTERVAL, 1 second by default, between two + tries. It logs once when it starts to wait and once when the database + answers. Only a database that is not up yet is worth the wait: a + connection refused, a socket that is not there yet, or a server that + says it is still starting up. A wrong password, a missing role or a + missing database stops serve at once, since waiting cannot fix them. A + wait of 0 does not wait at all. + + Background: + Given the settings point at an empty database + And serve tries the database again every "100ms" + + Scenario: A site waits for a database that starts late + Given the database refuses every connection for 2 seconds + When the site starts and listens + Then the command succeeds + And the site logged once that it waits for the database + And the site logged once that the database answered + + Scenario: A site stops once the database stays down past the wait + Given the database refuses every connection + And serve waits "1s" for the database + When the site starts + Then the command exits with code 1 + And the error says "the database did not answer within 1s, the time GOPHENBERG_DATABASE_WAIT allows" + And the site logged once that it waits for the database + + Scenario: A site stops at once on a wrong password + Given the settings name a wrong password for the database + When the site starts + Then the command exits with code 1 + And the error says "password authentication failed" + And the site never waited for the database + + Scenario: A wait of 0 does not wait + Given the database refuses every connection + And serve waits "0" for the database + When the site starts + Then the command exits with code 1 + And the error says "the database did not answer within 0s, the time GOPHENBERG_DATABASE_WAIT allows" + And the site never waited for the database diff --git a/test/features/features/operator-commands.feature b/test/features/features/operator-commands.feature index 2eff4308..7788784f 100644 --- a/test/features/features/operator-commands.feature +++ b/test/features/features/operator-commands.feature @@ -35,6 +35,12 @@ Feature: Operators run Gophenberg from one command line Then the command succeeds And the answer names the version, the build stamp, the minimum version and the theme kits of this binary + Scenario: About names the lowest Node.js major + Given the settings name no database + When the operator runs "about -json" + Then the command succeeds + And the answer names Node.js 24 as the lowest major a theme runs on + Scenario: Migrating stores the minimum version of this binary When the operator runs "migrate" Then the command succeeds