infra(runner): keep-warm, CI deploy, and scheduled canary

.github/workflows/canary.yml

@@ -0,0 +1,55 @@
+name: Runner canary
+
+# A scheduled dry-run smoke test against the production FlutterFlow AI runner,
+# so a broken runner is caught the next morning instead of by a user's deploy.
+# Runs daily and can be triggered manually with workflow_dispatch.
+#
+# Additive only: it never deploys and never touches this repository.
+
+on:
+  schedule:
+    # 07:21 UTC daily (off the :00/:30 marks to spread GHA load).
+    - cron: "21 7 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  canary:
+    runs-on: ubuntu-latest
+    steps:
+      # Skip (rather than fail noisily) until the FlutterFlow API key secret
+      # exists, so scheduled runs before one-time setup complete are quiet. The
+      # gate has to be a step: the `secrets` context is not available in a
+      # job-level `if`, and using it there makes the workflow file invalid. The
+      # script itself also fails clearly if the key ever comes back empty.
+      - name: Check for canary credentials
+        id: creds
+        env:
+          CANARY_KEY: ${{ secrets.CANARY_FF_API_KEY }}
+        run: |
+          if [ -n "$CANARY_KEY" ]; then
+            echo "present=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "present=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::CANARY_FF_API_KEY is not set, so the canary did not run. See DEPLOYMENT.md for the one-time setup."
+          fi
+
+      - name: Checkout
+        if: steps.creds.outputs.present == 'true'
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Set up Node.js
+        if: steps.creds.outputs.present == 'true'
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version-file: ".nvmrc"
+
+      - name: Run the dry-run canary
+        if: steps.creds.outputs.present == 'true'
+        run: node scripts/canary-deploy.mjs
+        env:
+          CANARY_FF_API_KEY: ${{ secrets.CANARY_FF_API_KEY }}
+          CANARY_RUNNER_URL: ${{ vars.CANARY_RUNNER_URL || '' }}
+          CANARY_PROJECT_ID: ${{ vars.CANARY_PROJECT_ID || 'automated-test-miedro' }}

.github/workflows/deploy-runner.yml

@@ -0,0 +1,68 @@
+name: Deploy runner
+
+# Deploy the FlutterFlow AI runner to Cloud Run whenever cloud-run/** changes
+# land on main, so a merged runner fix actually ships instead of sitting in the
+# repo. Additive only: it never touches branch protection and never runs on
+# pull requests.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "cloud-run/**"
+
+# The deploy script needs `gcloud` (auth + setup-gcloud below). No write access
+# to this repository itself is required.
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-runner
+  cancel-in-progress: true
+
+jobs:
+  deploy-runner:
+    runs-on: ubuntu-latest
+    steps:
+      # The secret cannot gate the job itself: the `secrets` context is not
+      # available in a job-level `if` (only github/needs/vars/inputs are), and
+      # using it there makes the whole workflow file invalid. It is available to
+      # a step, so the gate is a first step that reports whether the key exists
+      # and every step that needs it skips while it does not. The intent is
+      # unchanged: pushes before one-time setup complete are skipped quietly
+      # rather than failing.
+      - name: Check for deploy credentials
+        id: creds
+        env:
+          SA_KEY_JSON: ${{ secrets.CCC_FFAI_RUNNER_SA_KEY_JSON }}
+        run: |
+          if [ -n "$SA_KEY_JSON" ]; then
+            echo "present=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "present=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::CCC_FFAI_RUNNER_SA_KEY_JSON is not set, so the runner was not deployed. See DEPLOYMENT.md for the one-time setup."
+          fi
+
+      - name: Checkout
+        if: steps.creds.outputs.present == 'true'
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Authenticate to Google Cloud
+        if: steps.creds.outputs.present == 'true'
+        uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2
+        with:
+          # JSON of a service account with roles/run.admin + roles/cloudbuild.builds.editor + roles/iam.serviceAccountUser.
+          credentials_json: ${{ secrets.CCC_FFAI_RUNNER_SA_KEY_JSON }}
+
+      - name: Set up gcloud
+        if: steps.creds.outputs.present == 'true'
+        uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2
+        with:
+          project_id: ${{ vars.CCC_PROJECT_ID || 'low-code-connect' }}
+
+      - name: Deploy the FlutterFlow AI runner
+        if: steps.creds.outputs.present == 'true'
+        run: |
+          PROJECT_ID="${{ vars.CCC_PROJECT_ID || 'low-code-connect' }}" \
+          REGION="${{ vars.CCC_REGION || 'us-west1' }}" \
+          ./scripts/deploy_cloud_run_ffai.sh

DEPLOYMENT.md

@@ -159,6 +159,58 @@ Deploy the runner with:
 PROJECT_ID=low-code-connect REGION=us-west1 ./scripts/deploy_cloud_run_ffai.sh
 ```
 
+Keep one instance warm by default: the script passes `--min-instances 1`, because
+most of a cold start (FlutterFlow AI SDK download and a fresh `flutter pub get`)
+is paid once per container instead of once per deploy. Override the count with
+`MIN_INSTANCES`, e.g. `MIN_INSTANCES=0 ./scripts/deploy_cloud_run_ffai.sh` to
+scale to zero and pay per-request cold starts instead.
+
+#### One-time setup for automated deploys (GitHub Actions)
+
+The workflow `.github/workflows/deploy-runner.yml` deploys the runner whenever
+`cloud-run/**` changes on `main`. It uses a Google Cloud service-account key
+from a repository secret. Until that secret exists the job **silently skips**
+(rather than failing on every push), so set up this once or merged runner
+changes will deploy nothing:
+
+1. Create a service account with at least `roles/run.admin`,
+   `roles/cloudbuild.builds.editor` and `roles/iam.serviceAccountUser`, generate
+   a JSON key, and download it.
+2. In **Settings → Secrets and variables → Actions**, create a repository secret
+   named `CCC_FFAI_RUNNER_SA_KEY_JSON` whose value is the key's full JSON
+   (including the surrounding `{ }`).
+3. The workflow reads `CCC_PROJECT_ID` (default `low-code-connect`) and
+   `CCC_REGION` (default `us-west1`) from repository variables; create those if
+   the defaults are not right. This is per-repo like every secret here, because
+   GitHub does not expose organisation secrets to private repositories on the
+   free plan.
+
+#### Runner canary (scheduled)
+
+The workflow `.github/workflows/canary.yml` runs daily (07:21 UTC) and on demand
+via **Actions → Runner canary → Run workflow**. It sends a `dryRun: true`
+deploy for the throwaway project `automated-test-miedro` to the runner's
+`/deployCustomClasses`, so it exercises workspace init, the compile gate and the
+DSL push path without writing anything to a real project. It fails (non-zero)
+on any non-200 response, on `success:false`, or if the runner rejects the
+`flutter_web_plugins` SDK package, and also reports the runner's `/healthz`
+versions when that endpoint is deployed - without failing when it is not yet.
+
+`GET /healthz` answers `{sha, flutterflowCli, minFlutterflowCli}`. The deploy
+script passes the checkout's short SHA as `RUNNER_GIT_SHA`, so comparing `sha`
+against `git log -1 --format=%h` tells you whether a merged runner fix is
+actually live. That question is what a months-long stall in a merged-but-never-
+deployed fix looked like; it is now one request.
+
+Until the FlutterFlow API key secret exists the canary job skips cleanly, so it
+never fails noisily before setup:
+
+1. Create a repository secret named `CANARY_FF_API_KEY` whose value is a
+   FlutterFlow API key with at least read access to `automated-test-miedro`.
+2. Optionally set a repository variable `CANARY_RUNNER_URL` if the production
+   runner lives somewhere other than the default, and `CANARY_PROJECT_ID` if the
+   throwaway project changes.
+
 ## BuildShip workflow and step prompts
 
 The generation pipeline (Architect → Generator → Review) runs on BuildShip, and

scripts/canary-deploy.mjs

@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+
+// Scheduled canary for the FlutterFlow custom-class runner.
+//
+// Every scheduled run POSTs a *dry-run* deploy to the runner so a regression
+// that would break users' deploys is caught the next morning instead of on
+// their next deploy. dryRun: true makes the runner go through `ai validate`
+// (workspace init, compile gate, DSL validation) but never write to the
+// project, so this is safe to run unattended.
+//
+// It also curls the runner's /healthz when present and reports its versions,
+// but never fails on a missing /healthz - another change is adding it in
+// parallel and it may not be deployed yet.
+
+const RUNNER_DEFAULT_URL = "https://ccc-ffai-runner-y5cyj3473a-uw.a.run.app";
+
+// The throwaway class and dependencies the canary sends. These are real enough
+// to compile: the runner builds a scratch package from the verification
+// manifest and fails the deploy if `flutter analyze` reports an error, so a
+// canary class that does not compile would fail every run for the wrong reason.
+const CANARY_CLASS_NAME = "CanaryProbe";
+const CANARY_SOURCE = `import 'package:flutter/widgets.dart';
+
+class ${CANARY_CLASS_NAME} extends StatelessWidget {
+  const ${CANARY_CLASS_NAME}({super.key});
+
+  @override
+  Widget build(BuildContext context) => const SizedBox.shrink();
+}`;
+
+function fail(message) {
+  console.error(`[canary] FAIL: ${message}`);
+  process.exitCode = 1;
+}
+
+async function main() {
+  const apiKey = process.env.CANARY_FF_API_KEY;
+  if (!apiKey) {
+    fail(
+      "CANARY_FF_API_KEY is not set. Create the repository secret " +
+        "CANARY_FF_API_KEY with a FlutterFlow API key (see DEPLOYMENT.md) " +
+        "and give this job access to it.",
+    );
+    return;
+  }
+
+  const baseUrl = (process.env.CANARY_RUNNER_URL || RUNNER_DEFAULT_URL).replace(
+    /\/+$/,
+    "",
+  );
+  const projectId = process.env.CANARY_PROJECT_ID || "automated-test-miedro";
+  const timeoutMs = Number(process.env.CANARY_TIMEOUT_MS || 60000);
+
+  const payload = {
+    apiKey,
+    projectId,
+    commitMessage: "canary dry-run",
+    dryRun: true,
+    customClasses: [
+      { className: CANARY_CLASS_NAME, content: CANARY_SOURCE },
+    ],
+    verification: {
+      sdkConstraint: ">=3.0.0 <4.0.0",
+      dependencies: {},
+      dependencyOverrides: {},
+      sdkPackages: ["flutter_web_plugins"],
+      // The runner hard-requires this array: without it _normalizeVerification
+      // throws and returns HTTP 400 before anything runs. It is also what the
+      // compile gate compiles - an empty sources list skips _verifyCustomCode
+      // entirely, so this canary would detect nothing.
+      sources: [{ fileName: "canary_probe.dart", content: CANARY_SOURCE }],
+    },
+  };
+
+  // The SDK-package rejection this canary is meant to catch. The runner
+  // currently rejects an invalid SDK name as "Invalid SDK package name: X";
+  // an older/newer wording may say "unsupported". Match both so a regression
+  // where flutter_web_plugins stops being accepted fails the canary.
+  const sdkPackageRejection = /unsupported sdk package|invalid sdk package name/i;
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    let response;
+    try {
+      response = await fetch(`${baseUrl}/deployCustomClasses`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: controller.signal,
+      });
+    } catch (error) {
+      fail(
+        `request to ${baseUrl}/deployCustomClasses failed: ` +
+          `${error?.name === "AbortError" ? "timed out" : error.message}`,
+      );
+      return;
+    }
+
+    const text = await response.text();
+    let body;
+    try {
+      body = JSON.parse(text);
+    } catch {
+      body = { error: text.slice(0, 500) };
+    }
+
+    if (!response.ok) {
+      fail(
+        `runner returned HTTP ${response.status} on a dry-run deploy for ` +
+          `project "${projectId}". ${formatBodyError(body)}`,
+      );
+      return;
+    }
+
+    if (body.success === false) {
+      fail(
+        `runner reported success:false on a dry-run deploy for project ` +
+          `"${projectId}". ${formatBodyError(body)}`,
+      );
+      return;
+    }
+
+    const joined = [
+      body.error,
+      body.details,
+      body.analyzerErrors,
+      ...(Array.isArray(body.analyzerErrors) ? body.analyzerErrors : []),
+    ]
+      .filter(Boolean)
+      .join("\n");
+    if (sdkPackageRejection.test(joined)) {
+      fail(
+        `runner rejected the flutter_web_plugins SDK package on a dry-run ` +
+          `deploy - a dependency regression users would hit: ${joined}`,
+      );
+      return;
+    }
+
+    console.log(
+      `[canary] OK: dry-run deploy for project "${projectId}" accepted at ${baseUrl} ` +
+        `(success:${body.success === undefined ? "true" : body.success})`,
+    );
+  } finally {
+    clearTimeout(timer);
+  }
+
+  // The /healthz route is being added alongside this canary; do not fail the
+  // canary when it is not deployed yet.
+  try {
+    const health = await fetch(`${baseUrl}/healthz`, {
+      signal: AbortSignal.timeout(10000),
+    });
+    if (health.ok) {
+      const healthBody = await health.json().catch(() => ({}));
+      console.log(
+        `[canary] healthz: HTTP ${health.status}` +
+          (healthBody.versions
+            ? ` versions=${JSON.stringify(healthBody.versions)}`
+            : ` ${JSON.stringify(healthBody)}`),
+      );
+    } else {
+      console.log(
+        `[canary] healthz: HTTP ${health.status} (not reported as a failure)`,
+      );
+    }
+  } catch {
+    console.log("[canary] healthz: unavailable (not reported as a failure)");
+  }
+}
+
+function formatBodyError(body) {
+  const detail = [
+    body.error,
+    body.details,
+    ...(Array.isArray(body.analyzerErrors) ? body.analyzerErrors : []),
+  ]
+    .filter(Boolean)
+    .join(" | ")
+    .slice(0, 1000);
+  return detail ? `Details: ${detail}` : "No detail in the response body.";
+}
+
+await main();