ci(mobile): roll back or republish OTA updates from the release workflow, and gate publishes on a boot test

.github/workflows/mobile-release.yml

@@ -6,16 +6,24 @@ on:
   workflow_dispatch:
     inputs:
       mode:
-        description: auto publishes an update when the latest production build has the same runtime, and builds and submits otherwise.
+        description: auto publishes an update when the latest production build has the same runtime, and builds and submits otherwise. rollback and republish change what channel production serves.
         type: choice
-        options: [auto, update, build]
+        options: [auto, update, build, rollback, republish]
         default: auto
+      runtime:
+        description: rollback only. The runtime to roll back to its embedded update; empty takes the latest finished production build's.
+        type: string
+        default: ''
+      group:
+        description: republish only. The update group to publish again on channel production.
+        type: string
+        default: ''
 
 permissions:
   contents: read
 
 concurrency:
-  group: mobile-release
+  group: ${{ (inputs.mode == 'rollback' || inputs.mode == 'republish') && 'mobile-rollback' || 'mobile-release' }}
   cancel-in-progress: false
 
 jobs:
@@ -54,6 +62,10 @@ jobs:
       - name: Install before any secret is loaded
         run: pnpm install --frozen-lockfile --filter stim-mobile
 
+      - name: The app boots in the tests before anything is published
+        if: inputs.mode != 'rollback' && inputs.mode != 'republish'
+        run: pnpm test --ci
+
       - name: The tag matches the app version
         if: github.event_name == 'push'
         run: |
@@ -97,6 +109,53 @@ jobs:
           message="$(git log -1 --format=%s)"
           eas update --channel production --environment production --platform ios --message "$message" --non-interactive
 
+      - name: Roll back to the embedded update
+        if: steps.mode.outputs.mode == 'rollback'
+        env:
+          INPUT_RUNTIME: ${{ inputs.runtime }}
+        run: |
+          runtime="$INPUT_RUNTIME"
+          if [ -z "$runtime" ]; then
+            build="$(eas build:list --platform ios --build-profile production --status finished --limit 1 --json --non-interactive | jq -c '.[0] // {}')"
+            runtime="$(jq -r '.runtime.version // empty' <<<"$build")"
+            echo "Latest finished production build: $(jq -r '"\(.id // "none") build \(.appBuildVersion // "?") from \(.gitCommitHash // "?")"' <<<"$build")"
+          fi
+          if [ -z "$runtime" ]; then
+            echo "::error::No runtime given and no finished production build has one. Pass -f runtime=<runtime>."
+            exit 1
+          fi
+          eas update:roll-back-to-embedded --channel production --runtime-version "$runtime" --platform ios \
+            --message "Roll back to embedded (run $GITHUB_RUN_ID)" --non-interactive --json > rollback.json
+          group="$(jq -r '.[0].group' rollback.json)"
+          echo "::notice::Channel production on runtime $runtime now serves the embedded update (group $group)."
+
+      - name: Republish an update group
+        if: steps.mode.outputs.mode == 'republish'
+        env:
+          INPUT_GROUP: ${{ inputs.group }}
+        run: |
+          if ! [[ "$INPUT_GROUP" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]]; then
+            echo "::error::group must be an update group id, not '$INPUT_GROUP'. List them with eas update:list --branch production."
+            exit 1
+          fi
+          update="$(eas update:view "$INPUT_GROUP" --json | jq -c '[.[] | select(.platform == "ios")][0] // empty')"
+          if [ -z "$update" ]; then
+            echo "::error::Update group $INPUT_GROUP has no iOS update."
+            exit 1
+          fi
+          branch="$(jq -r .branch <<<"$update")"
+          runtime="$(jq -r .runtimeVersion <<<"$update")"
+          echo "Group $INPUT_GROUP: branch $branch, runtime $runtime, rollback to embedded $(jq -r .isRollBackToEmbedded <<<"$update"), message: $(jq -r .message <<<"$update")"
+          latest="$(eas build:list --platform ios --build-profile production --status finished --limit 1 --json --non-interactive | jq -r '.[0].runtime.version // empty')"
+          if [ "$runtime" != "$latest" ]; then
+            echo "::warning::Runtime $runtime is not the latest production build's runtime (${latest:-none}); only builds on $runtime get this update."
+          fi
+          destination=()
+          if [ "$branch" != production ]; then destination=(--destination-channel production); fi
+          eas update:republish --group "$INPUT_GROUP" "${destination[@]}" --platform ios \
+            --message "Republish $INPUT_GROUP (run $GITHUB_RUN_ID)" --non-interactive --json > republish.json
+          echo "::notice::Channel production on runtime $runtime now serves group $INPUT_GROUP, republished as $(jq -r '.[0].group' republish.json)."
+
       - name: Build and submit to TestFlight
         if: steps.mode.outputs.mode == 'build'
         env:

RELEASE.md

@@ -6,6 +6,9 @@ real workflow at the same time.
 
 Stim Desktop has its own tags and workflow; see
 [apps/desktop/RELEASING.md](./apps/desktop/RELEASING.md).
+The phone app ships with its own workflow, which also rolls back a bad update;
+see [Each release](./apps/mobile/README.md#each-release) and
+[Roll back a bad update](./apps/mobile/README.md#roll-back-a-bad-update).
 
 ## 0. The six packages
 

apps/mobile/README.md

@@ -638,6 +638,20 @@ pnpm test
 `.github/workflows/mobile.yml` runs them for changes under `apps/mobile` and to
 the root `package.json`, `pnpm-lock.yaml` and `pnpm-workspace.yaml`.
 
+`src/app-boot.test.tsx` is the boot test. It renders the whole app, from
+`src/app/_layout.tsx` down to home, in Jest. The phone starts from what the
+previous release stored: a paired machine, that machine's cached status from an
+older `stim`, and notification preferences and state in their old shapes. The
+machine then connects to a fake `stim-server` that serves
+`mock-server/fixtures/status.json`, and the app turns `active` the way iOS does
+after launch. The test fails when React reports an error or anything throws,
+so it catches JS that throws while launching over an earlier release's state,
+on the screens it renders. Reanimated, the drawer and Lottie are stubbed, so
+the test does not cover animations or native code.
+
+When a change stores something new on the phone, or reads a stored value in a
+new shape, add the previous shape to `seedPreviousInstall` in that test.
+
 ## Ship to TestFlight
 
 The app ships to TestFlight with EAS, under the App&Flow Expo account
@@ -732,9 +746,21 @@ or prefix each command with `npx`).
     JS-only; an update for a runtime no build has reaches no one.
   - `build`: builds with the `production` profile and submits to
     TestFlight (`eas build --auto-submit`).
+  - `rollback`: points channel `production` back at the JS embedded in the
+    build (`eas update:roll-back-to-embedded`). By default it uses the runtime
+    of the latest finished production build; the `runtime` input names
+    another one. See [Roll back a bad update](#roll-back-a-bad-update).
+  - `republish`: publishes the update group named by the `group` input again
+    on channel `production` (`eas update:republish`), so installed builds go
+    back to that update.
 - A `mobile-v<version>` tag on `main` always builds and submits. The version
   must equal `version` in `app.config.ts`, so raise it first.
 
+Every mode except `rollback` and `republish` runs the unit tests, the boot test
+included, before the step that loads `EXPO_TOKEN`. A failing test stops the run
+before anything is published or built. `rollback` and `republish` skip them
+because they must still work when `main` is broken.
+
 `eas build` records as the build's runtime the fingerprint computed on the
 machine that starts it, so a build from this workflow and the fingerprint
 `auto` compares are both computed on the Linux runner. A build started by hand
@@ -767,3 +793,69 @@ The build appears in TestFlight after Apple finishes processing it, usually
 within 30 minutes. Add testers under the app's **TestFlight** tab. Raise
 `version` in `app.config.ts` for a new marketing version; build numbers need no
 change.
+
+### Roll back a bad update
+
+An update reaches every installed build on its runtime the next time the app
+launches, and it runs on the launch after that. A bad update shows up as:
+
+- the app closing at launch, or showing an error screen, on phones that were
+  fine before;
+- crashes in `eas update:view <group> --insights` for the newest group.
+
+Find the newest groups and the one that was good before the bad one:
+
+```bash
+cd apps/mobile
+eas update:list --branch production --limit 10
+eas update:view <group> --insights
+```
+
+Roll back to the JS embedded in the build. This works whatever is on the
+channel, and needs no group id:
+
+```bash
+gh workflow run mobile-release.yml --ref main -f mode=rollback
+gh workflow run mobile-release.yml --ref main -f mode=rollback -f runtime=<runtime>
+```
+
+Or go back to a specific update that worked:
+
+```bash
+gh workflow run mobile-release.yml --ref main -f mode=republish -f group=<group>
+```
+
+Both runs wait for approval in the `release` environment. They queue apart
+from the publishing modes, so a build in progress does not hold them up. A
+publishing run that is still in progress or waiting for approval can publish
+after the rollback and undo it, so cancel it first
+(`gh run cancel <run-id>`). Approve them as in
+step 7 of [RELEASE.md](../../RELEASE.md#4-cut-the-release), with the run id
+from `gh run list --workflow mobile-release.yml --limit 1`. The run log's
+notice names the runtime and group it acted on. `republish` warns when the
+group's runtime is not the latest production build's, because only builds on
+that runtime get it. When the group is on a branch other than `production`,
+`republish` publishes it to channel `production` with `--destination-channel`.
+
+Phones pick up the rollback when a launch checks for updates, and run it on the
+launch after that. A phone that crashes at launch may need a few launches.
+
+After a rollback, fix the bug on `main`. The next `auto` run publishes the fix
+as a new update, which replaces the rollback. Build for TestFlight (`mode=build`)
+instead when the fix changes native code or dependencies. Do the same when the
+embedded JS itself is broken, because then a rollback cannot help: `auto` builds
+only when the runtime changes, so pick `build` explicitly.
+
+The workflow has no staging channel. Publishing to a staging branch first and
+promoting it with `republish` would need a build that reads that channel,
+installed on a phone someone checks before promoting. Without that build,
+staging adds a step and verifies nothing. A simulator run of each update on a
+macOS runner, using a production-runtime simulator build per runtime, would
+catch native and render crashes. It would also cost a macOS runner and an EAS
+simulator build for every runtime. The boot test catches JS errors on launch,
+which covers this class of crash, at no extra cost. To check an update by hand
+before publishing it, run a Release build of the same commit on a simulator
+with `stim ios --configuration Release`. Stim puts the current JS into the
+cached build, so this works without Metro. Then read `stim logs --errors`. A
+Release build ignores `.env.local`, so it starts with no paired machine unless
+you pair it.

apps/mobile/package.json

@@ -49,6 +49,7 @@
   },
   "devDependencies": {
     "@react-native/metro-config": "0.88.0-rc.1",
+    "@testing-library/react-native": "14.0.1",
     "@types/jest": "^29.5.14",
     "@types/react": "~19.3.0",
     "@types/ws": "^8.18.1",
@@ -60,10 +61,14 @@
     "oxfmt": "^0.65.0",
     "sf-symbols-typescript": "2.2.0",
     "sharp": "^0.35.4",
+    "test-renderer": "^1.3.0",
     "typescript": "~6.0.3",
     "ws": "^8.21.3"
   },
   "jest": {
+    "moduleNameMapper": {
+      "^@/assets/(.*)$": "<rootDir>/assets/$1"
+    },
     "preset": "jest-expo",
     "testPathIgnorePatterns": [
       "/node_modules/",