Wait for a database that starts late

.env.example

@@ -24,6 +24,13 @@ GOPHENBERG_DATABASE_URL=postgres://postgres:gophenberg@localhost:5435/postgres?s
 # server run programs: warn logs a warning and serves, refuse stops it. A SQLite file ignores it.
 # GOPHENBERG_SUPERUSER_POLICY=warn
 
+# How long serve waits for a PostgreSQL server that is not up yet, such as one still starting at
+# boot, and the pause between two tries. Only a refused connection, a socket that is not there yet
+# or a server still starting up is waited for. A wrong password, a missing role or a missing
+# database stops serve at once. 0 does not wait. Only serve waits, every other command stops at once.
+# GOPHENBERG_DATABASE_WAIT=60s
+# GOPHENBERG_DATABASE_WAIT_INTERVAL=1s
+
 # Only a sqlite: address reads these seven, and a PostgreSQL address ignores them. How long a
 # connection waits for a lock another one holds, in whole milliseconds.
 # GOPHENBERG_SQLITE_BUSY_TIMEOUT=5s

cmd/releasetool/main.go

@@ -21,7 +21,7 @@ const usage = `usage:
   releasetool sign   -key <file> [-passphrase-file <file>] <file>...
   releasetool verify -public <file|multikey> <file>...
   releasetool sums   <file>...
-  releasetool record -version <version> -download-url <https address> <archive>...
+  releasetool record -version <version> -download-url <address> <archive>...
   releasetool keys   -release <multikey> -standby <multikey> -timestamp <multikey> [-serial <n>] [-valid <duration>]
   releasetool sign-keys -key <root file> [-passphrase-file <file>] <keys.json>
   releasetool index  -site <clone> -release <multikey> -standby <multikey> -timestamp <multikey> [-root <multikey>]...

cmd/releasetool/record.go

@@ -7,9 +7,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/url"
 	"slices"
 	"strings"
+
+	"github.com/gopherium/gophenberg/internal/update"
 )
 
 // productName is the name every release archive starts with.
@@ -61,7 +62,8 @@ func record(args []string, stdout, stderr io.Writer) error {
 	var opts recordOptions
 	flags := newFlagSet("record", stderr)
 	flags.StringVar(&opts.version, "version", "", "the version the archives carry")
-	flags.StringVar(&opts.base, "download-url", "", "the https address the release serves its files under")
+	flags.StringVar(&opts.base, "download-url", "",
+		"the address the release serves its files under, https, or http on a loopback host")
 	if err := parseFlags(flags, args); err != nil {
 		return err
 	}
@@ -80,30 +82,20 @@ func record(args []string, stdout, stderr io.Writer) error {
 	return encoder.Encode(releaseRecord{Version: opts.version, Artifacts: recordArtifacts{Package: entries}})
 }
 
-// validate refuses a run with no version, no archive, or a download address that is not https.
+// validate refuses a run with no version, no archive, or a download address a site refuses to download from.
 func (o recordOptions) validate(archives int) error {
 	if o.version == "" {
 		return usageError{"record needs -version, the version the archives carry"}
 	}
 	if archives == 0 {
 		return usageError{"record needs at least one archive to list"}
 	}
-	if !httpsAddress(o.base) {
-		return usageError{fmt.Sprintf("record -download-url %q is not the https address the release serves its files under",
-			o.base)}
+	if _, err := update.ParseAddress(o.base); err != nil {
+		return usageError{fmt.Sprintf("record -download-url: %v", err)}
 	}
 	return nil
 }
 
-// httpsAddress reports whether text is an https address naming a host.
-func httpsAddress(text string) bool {
-	address, err := url.Parse(text)
-	if err != nil {
-		return false
-	}
-	return address.Scheme == "https" && address.Host != ""
-}
-
 // artifacts returns the record entry of every archive, in the order given.
 func (o recordOptions) artifacts(files []releaseFile) ([]artifact, error) {
 	entries := make([]artifact, len(files))

cmd/releasetool/record_test.go

@@ -48,6 +48,29 @@ func TestRecordReadsAnyNameOnThePattern(t *testing.T) {
 	}
 }
 
+func TestRecordTakesPlainHTTPOnALoopbackHost(t *testing.T) {
+	t.Parallel()
+
+	archives := signedArchives(t, "gophenberg_0.20.0_linux_amd64.tar.gz")
+	for name, base := range map[string]string{
+		"an IPv4 loopback": "http://127.0.0.1:8090/releases",
+		"localhost":        "http://localhost:8090",
+		"an IPv6 loopback": "http://[::1]:8090/v0.20.0",
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := runTool("record", "-version", "0.20.0", "-download-url", base, archives[0])
+
+			listed := readRecord(t, got).Artifacts.Package
+			want := base + "/gophenberg_0.20.0_linux_amd64.tar.gz"
+			if len(listed) != 1 || listed[0].URL != want {
+				t.Errorf("record under %s lists %+v, want the archive at %s", base, listed, want)
+			}
+		})
+	}
+}
+
 func TestRecordRefusesAnArchiveWithoutASignature(t *testing.T) {
 	t.Parallel()
 
@@ -115,6 +138,12 @@ func TestRecordExitsTwoWhenMisused(t *testing.T) {
 		{"no archive", []string{"-version", "0.20.0", "-download-url", downloadBase}, "archive"},
 		{"an address that is not https", []string{"-version", "0.20.0", "-download-url",
 			"http://github.com/gopherium/gophenberg/releases/download/v0.20.0", archives[0]}, "https"},
+		{"plain http off loopback", []string{"-version", "0.20.0", "-download-url", "http://example.com/releases",
+			archives[0]}, "http on a loopback host"},
+		{"an address with a query", []string{"-version", "0.20.0", "-download-url",
+			"https://example.com/releases?tag=v0.20.0", archives[0]}, "query"},
+		{"an address that names a user", []string{"-version", "0.20.0", "-download-url",
+			"https://maria@example.com/releases", archives[0]}, "user"},
 		{"an address with no host", []string{"-version", "0.20.0", "-download-url", "https:///releases",
 			archives[0]}, "https"},
 		{"an address that does not parse", []string{"-version", "0.20.0", "-download-url", "https://example.com/%zz",

docs/src/content/docs/self-hosting/commands.md

@@ -280,8 +280,8 @@ gophenberg about [flags]
 Prints what this binary is in one line, such as
 `Gophenberg %VERSION%, official build, commit 4c3ba60, theme kits 0.9.0 to %KIT_VERSION%`.
 With `-json` it answers one JSON document holding `schema`, `name`,
-`version`, `commit`, `stamp`, `minimum_version`, `postgresql` and
-`kits`. It needs no database.
+`version`, `commit`, `stamp`, `minimum_version`, `postgresql`, `node`
+and `kits`. It needs no database.
 
 - A binary from a release archive or the release image says
   `official build`, and names the commit it was built from.
@@ -292,6 +292,10 @@ With `-json` it answers one JSON document holding `schema`, `name`,
   database this binary migrated, see
   [going back to an older version](/self-hosting/updates-and-backups/#going-back-to-an-older-version).
 - `postgresql` is the oldest PostgreSQL major this binary runs on.
+- `node` is the oldest Node.js major this binary runs a theme on. The
+  release image ships this major. Outside the image, point
+  [`GOPHENBERG_NODE_BIN`](/self-hosting/configuration/) at a Node.js of
+  this major or newer.
 
 ### check
 
@@ -387,6 +391,10 @@ with `serve`, since a run with no command only prints the listing.
 On a SQLite database `serve` exits with code 1 before it opens
 anything, see [on a SQLite database](#on-a-sqlite-database).
 
+When the PostgreSQL server is not up yet, `serve` first waits for it,
+up to `GOPHENBERG_DATABASE_WAIT`, see
+[a database that starts late](/self-hosting/configuration/#a-database-that-starts-late).
+
 Once an update an administrator started from the admin switched
 builds, `serve` stops as it does on `SIGTERM` and exits with code 75,
 so the service manager starts the new build, see
@@ -556,7 +564,11 @@ gophenberg: restart Gophenberg to start 1.4.1
   the minimum version, `update:apply` takes the backup anyway, since
   only the backup can take the site back then.
 - `-json` answers the state the update ends at, holding `step`,
-  `from`, `to`, `attempts`, `long_jump`, `backup` and `refused`.
+  `from`, `to`, `attempts`, `database_reached`, `long_jump`, `backup`
+  and `refused`. `database_reached` turns `true` once a start of the
+  new build reached the database, just before it migrates, or once any
+  other command of the new build migrates it, such as `migrate` run by
+  hand.
 
 Before it changes anything it refuses, with exit code 1 and what to
 do: a custom build, an install with no install root such as the
@@ -584,10 +596,10 @@ the update again, adds the version it left to the refused versions,
 so `update:apply` never takes it again on its own, and ends asking for
 a restart with exit code 75. It refuses when no update switched builds
 since the last rollback, and when the update raised the minimum
-version and its new build already started, since the older build
-cannot open the database the newer one may have migrated. Then it
-prints the steps that restore the backup taken before the update
-instead. It never opens the database, so it works when the database is
+version and its new build already reached the database, since the
+older build cannot open the database the newer one may have migrated.
+Then it prints the steps that restore the backup taken before the
+update instead. It never opens the database, so it works when the database is
 what broke.
 
 `-after-failure` is the form the rollback unit of the service manager
@@ -598,10 +610,12 @@ systemd says the starts hit the start limit, `start-limit-hit`, or the
 new build exited with code 76. It then exits with code 0, so the unit
 starts the service again. Otherwise it changes nothing and exits with
 code 1. It takes back each update once, so two failing builds never
-take turns. An update that raised the minimum version and already
-started stays down: it writes that the update failed, so
-`update:status` shows the steps that restore the backup, prints them
-and exits with code 1.
+take turns. An update that raised the minimum version and whose new
+build already reached the database stays down: it writes that the
+update failed, so `update:status` shows the steps that restore the
+backup, prints them and exits with code 1. One whose new build never
+reached the database, such as one that started while PostgreSQL was
+still down, goes back like any other update.
 
 `-json` answers the state the rollback ends at, as `update:apply`
 does.

docs/src/content/docs/self-hosting/configuration.md

@@ -20,6 +20,8 @@ so it takes the default.
 | `GOPHENBERG_DATABASE_URL` | Yes | | The PostgreSQL connection string, or `sqlite:` and the absolute path of a SQLite file. [The PostgreSQL address](#the-postgresql-address) lists what a connection string may not hold. [A SQLite database](#a-sqlite-database) says what runs on one and lists its settings |
 | `GOPHENBERG_DATABASE_MAX_CONNS` | No | The larger of 4 and the number of CPUs | How many connections to the database stay open at most, from 2 to 2147483647, see [the database connections](#the-database-connections) |
 | `GOPHENBERG_SUPERUSER_POLICY` | No | `warn` | What the server does when it connects to PostgreSQL as a superuser: `warn` logs a warning and serves, `refuse` stops it, see [the PostgreSQL server](#the-postgresql-server) |
+| `GOPHENBERG_DATABASE_WAIT` | No | `60s` | How long `serve` waits for a PostgreSQL server that is not up yet before it stops. `0` does not wait, see [a database that starts late](#a-database-that-starts-late) |
+| `GOPHENBERG_DATABASE_WAIT_INTERVAL` | No | `1s` | The pause between two tries while `serve` waits for the database |
 | `GOPHENBERG_ADDR` | No | `localhost:8081` | Address the server listens on. The container image sets `0.0.0.0:8081` |
 | `GOPHENBERG_WEB_DIR` | No | | A built admin folder to serve in place of the admin built into the binary. It must hold `index.html` |
 | `GOPHENBERG_SITE_TITLE` | No | `Gophenberg` | The site name shown by the built-in renderer |
@@ -89,7 +91,7 @@ so it takes the default.
 | `GOPHENBERG_UPDATE_DISK_MARGIN_MB` | No | `64` | How many megabytes of free disk an update asks for beyond `GOPHENBERG_UPDATE_DOWNLOAD_CAP_MB` and, when it takes a backup, the size of the database, so the site still has room once the update is done. `0` asks for none |
 | `GOPHENBERG_UPDATE_BACKUPS_KEEP` | No | `3` | How many database backups the `backups` folder of the install root keeps |
 | `GOPHENBERG_PG_DUMP_BIN` | No | `pg_dump` | The `pg_dump` an update backs up the database with, a name found on the path or a full path. Its major version must be at least the server's |
-| `GOPHENBERG_UPDATE_ATTEMPTS` | No | `3` | How many times the build an update switched to may try to start. The next start goes back to the build before it, or stays down when the update raised the minimum version |
+| `GOPHENBERG_UPDATE_ATTEMPTS` | No | `3` | How many times the build an update switched to may try to start. The next start goes back to the build before it, or stays down when the update raised the minimum version and its new build reached the database |
 | `GOPHENBERG_UPDATE_VERIFY_TIMEOUT` | No | `1m` | How long the first start of a new build has to check itself: the server listens, then `/admin/` and `/api/locale` answer 200 and `/` answers below 500 over loopback |
 | `GOPHENBERG_UPDATE_VERSIONS_KEEP` | No | `2` | How many of the newest builds the `versions` folder keeps once a new build passes its check. The builds the `current` and `previous` links name always stay |
 | `GOPHENBERG_FEED_TITLE` | No | `Gophenberg` | The RSS channel title |
@@ -131,6 +133,41 @@ it logs a warning and serves. Set `GOPHENBERG_SUPERUSER_POLICY` to
 up from it shows the warning. A SQLite file has no roles, so the
 setting does nothing there.
 
+### A database that starts late
+
+A server often starts Gophenberg before PostgreSQL answers, at boot or
+right after an update. `serve` then tries the database again every
+`GOPHENBERG_DATABASE_WAIT_INTERVAL`, 1 second by default, for up to
+`GOPHENBERG_DATABASE_WAIT`, 60 seconds by default, before it checks the
+role it connects as and migrates. It logs
+`waiting for the database to answer` once when it starts to wait, with
+the reason, and `the database answered` once it does, with how long
+that took.
+
+Only a database that is not up yet is worth the wait: a connection
+refused, a socket that is not there yet, or a server that says it is
+still starting up. A wrong password, a missing role or a missing
+database stops `serve` at once with its own message, since waiting
+cannot fix them. Each try gets only the time the wait has left, so a
+server that takes the connection and never answers holds `serve` no
+longer than the wait. A database that still does not answer once the
+wait runs out stops `serve` with code 1, and the message ends with the
+last reason:
+
+```text
+gophenberg: the database did not answer within 1m0s, the time GOPHENBERG_DATABASE_WAIT allows: failed to connect to `user=gophenberg database=gophenberg`: ...
+```
+
+`GOPHENBERG_DATABASE_WAIT=0` stops at the first try instead, and that
+one try has no time bound of its own. Only
+`serve` waits. `migrate`, `seed -yes` and the `account:` commands stop
+at once, so a person running them sees the reason straight away.
+
+Right after an update, a start that waits still counts as one attempt
+of the new build, so a database down for longer than about three waits
+takes the update back, see
+[a database that starts late after an update](/self-hosting/updates-and-backups/#a-database-that-starts-late-after-an-update).
+
 ## The database connections
 
 `GOPHENBERG_DATABASE_MAX_CONNS` caps how many connections to the
@@ -544,8 +581,10 @@ The server refuses to start, and says why, when:
   Write them as durations, `1h`, `90s`, `5m`. Part of a second is
   refused, because the header counts in whole seconds.
 - Any of the theme timings, HTTP timeouts, shutdown graces, cron job
-  timings, update timings or the upload timeout is not a positive
-  duration. Write them the way Go does, `30s`, `500ms`, `1m`.
+  timings, update timings, the upload timeout or
+  `GOPHENBERG_DATABASE_WAIT_INTERVAL` is not a positive duration.
+  Write them the way Go does, `30s`, `500ms`, `1m`.
+- `GOPHENBERG_DATABASE_WAIT` is not a duration of `0` or more.
 - `GOPHENBERG_UPDATE_URL` is not an `https` address, or an `http` one
   on a loopback host, or it names a user, a query or a fragment.
 - `GOPHENBERG_UPDATE_CHANNEL` holds anything but lowercase letters,