From 5cc19deca5291759cf6a612e381aa320aade995d Mon Sep 17 00:00:00 2001 From: Ocean Dekeyser Date: Thu, 22 Oct 2026 14:58:30 +0200 Subject: [PATCH] Restore test: compare with the database as it was during the dump After #99 the Deploy's restore test failed: it counted the live rows after restoring, and the site (now with the chat in use) had written meanwhile. The live counts are now taken just before and just after the dump; when they differ the backup is made again, at most 3 times. Co-Authored-By: Claude Opus 5.5 --- backup/backup.sh | 28 +++++++++++++++++++++++----- docs/CHANGELOG.md | 1 + docs/backups.md | 2 +- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/backup/backup.sh b/backup/backup.sh index c83065a..b5d56ab 100644 --- a/backup/backup.sh +++ b/backup/backup.sh @@ -8,8 +8,9 @@ # backup.sh once one backup now # backup.sh restore-test one backup now; the dump is restored into a scratch # database and compared table by table with the live -# database, the photo archive is compared file count -# with the photos folder +# database as it was during the dump (backed up again, +# up to 3 times, when it changed meanwhile); the photo +# archive is compared file count with the photos folder set -eu BACKUP_DIR=/backups @@ -73,17 +74,34 @@ row_counts() { } restore_test() { - backup + # The site keeps writing (Chat, logins), so the live row counts are taken just + # before and just after the dump: when they are equal, nothing changed during + # the dump, and the restored database must hold exactly those counts. + attempt=1 + while :; do + before="$(row_counts "$PGDATABASE")" + backup + after="$(row_counts "$PGDATABASE")" + [ "$before" = "$after" ] && break + if [ "$attempt" -ge 3 ]; then + log "restore test FAILED: the database kept changing during 3 backups" >&2 + exit 1 + fi + log "the database changed during the backup; backing up again" + attempt=$((attempt + 1)) + sleep 5 + done file="$(newest_backup_matching 'pmp-*.dump')" dropdb --if-exists "$SCRATCH_DB" createdb "$SCRATCH_DB" trap 'dropdb --if-exists "$SCRATCH_DB"' EXIT pg_restore --no-owner --exit-on-error --dbname="$SCRATCH_DB" "$file" - live="$(row_counts "$PGDATABASE")" restored="$(row_counts "$SCRATCH_DB")" echo "$restored" - if [ -z "$restored" ] || [ "$live" != "$restored" ]; then + if [ -z "$restored" ] || [ "$before" != "$restored" ]; then log "restore test FAILED: $file differs from the live database" >&2 + echo "live database during the backup:" >&2 + echo "$before" >&2 exit 1 fi log "restore test passed: $file" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 153ea96..c564dc6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -242,6 +242,7 @@ Grilling Q1–Q22 with the team, spec #91 and tickets #92–#98. Built one ticke | | What | Issue | PR | |---|---|---|---| | New | **Chat basics:** one chat per Project (Members, Product Owner; every Teacher and the Superuser can also read and write), text messages with clickable links, new messages within ~2 s (ADR 0008), Unread badge on the Chat / Chats item, Chats screen, Chat button on a Project's page; migration `0014` | #92 | #99 | +| Fixed | The restore test after a Deploy failed when the site wrote meanwhile (first seen after #99, with the new chat in use): it now compares with the row counts taken just before and after the dump, and backs up again (max 3 times) when they differ | – | #100 | ### 2026-10-22: team change requests diff --git a/docs/backups.md b/docs/backups.md index ea0374f..4dbf69b 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -23,7 +23,7 @@ docker exec pmp-backup-1 sh /backup.sh once ## Test a restore -Makes a backup, restores it into the scratch database `pmp_restore_test`, compares every table's row count with the live database and drops the scratch database again. It also checks that the new photo archive can be read and holds as many files as the photos folder. The Deploy workflow runs this after every deploy. +Makes a backup, restores it into the scratch database `pmp_restore_test`, compares every table's row count with the live database as it was during the backup, and drops the scratch database again. The site keeps writing (Chat, logins), so the live counts are taken just before and just after the dump; when they differ, the backup is made again (at most 3 times). It also checks that the new photo archive can be read and holds as many files as the photos folder. The Deploy workflow runs this after every deploy. ```sh docker exec pmp-backup-1 sh /backup.sh restore-test