diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 0000000..26d1072 --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,55 @@ +name: Runner canary + +# A scheduled dry-run smoke test against the production FlutterFlow AI runner, +# so a broken runner is caught the next morning instead of by a user's deploy. +# Runs daily and can be triggered manually with workflow_dispatch. +# +# Additive only: it never deploys and never touches this repository. + +on: + schedule: + # 07:21 UTC daily (off the :00/:30 marks to spread GHA load). + - cron: "21 7 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + canary: + runs-on: ubuntu-latest + steps: + # Skip (rather than fail noisily) until the FlutterFlow API key secret + # exists, so scheduled runs before one-time setup complete are quiet. The + # gate has to be a step: the `secrets` context is not available in a + # job-level `if`, and using it there makes the workflow file invalid. The + # script itself also fails clearly if the key ever comes back empty. + - name: Check for canary credentials + id: creds + env: + CANARY_KEY: ${{ secrets.CANARY_FF_API_KEY }} + run: | + if [ -n "$CANARY_KEY" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "::notice::CANARY_FF_API_KEY is not set, so the canary did not run. See DEPLOYMENT.md for the one-time setup." + fi + + - name: Checkout + if: steps.creds.outputs.present == 'true' + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Node.js + if: steps.creds.outputs.present == 'true' + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version-file: ".nvmrc" + + - name: Run the dry-run canary + if: steps.creds.outputs.present == 'true' + run: node scripts/canary-deploy.mjs + env: + CANARY_FF_API_KEY: ${{ secrets.CANARY_FF_API_KEY }} + CANARY_RUNNER_URL: ${{ vars.CANARY_RUNNER_URL || '' }} + CANARY_PROJECT_ID: ${{ vars.CANARY_PROJECT_ID || 'automated-test-miedro' }} diff --git a/.github/workflows/deploy-runner.yml b/.github/workflows/deploy-runner.yml new file mode 100644 index 0000000..db86562 --- /dev/null +++ b/.github/workflows/deploy-runner.yml @@ -0,0 +1,68 @@ +name: Deploy runner + +# Deploy the FlutterFlow AI runner to Cloud Run whenever cloud-run/** changes +# land on main, so a merged runner fix actually ships instead of sitting in the +# repo. Additive only: it never touches branch protection and never runs on +# pull requests. + +on: + push: + branches: [main] + paths: + - "cloud-run/**" + +# The deploy script needs `gcloud` (auth + setup-gcloud below). No write access +# to this repository itself is required. +permissions: + contents: read + +concurrency: + group: deploy-runner + cancel-in-progress: true + +jobs: + deploy-runner: + runs-on: ubuntu-latest + steps: + # The secret cannot gate the job itself: the `secrets` context is not + # available in a job-level `if` (only github/needs/vars/inputs are), and + # using it there makes the whole workflow file invalid. It is available to + # a step, so the gate is a first step that reports whether the key exists + # and every step that needs it skips while it does not. The intent is + # unchanged: pushes before one-time setup complete are skipped quietly + # rather than failing. + - name: Check for deploy credentials + id: creds + env: + SA_KEY_JSON: ${{ secrets.CCC_FFAI_RUNNER_SA_KEY_JSON }} + run: | + if [ -n "$SA_KEY_JSON" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "::notice::CCC_FFAI_RUNNER_SA_KEY_JSON is not set, so the runner was not deployed. See DEPLOYMENT.md for the one-time setup." + fi + + - name: Checkout + if: steps.creds.outputs.present == 'true' + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Authenticate to Google Cloud + if: steps.creds.outputs.present == 'true' + uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 + with: + # JSON of a service account with roles/run.admin + roles/cloudbuild.builds.editor + roles/iam.serviceAccountUser. + credentials_json: ${{ secrets.CCC_FFAI_RUNNER_SA_KEY_JSON }} + + - name: Set up gcloud + if: steps.creds.outputs.present == 'true' + uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2 + with: + project_id: ${{ vars.CCC_PROJECT_ID || 'low-code-connect' }} + + - name: Deploy the FlutterFlow AI runner + if: steps.creds.outputs.present == 'true' + run: | + PROJECT_ID="${{ vars.CCC_PROJECT_ID || 'low-code-connect' }}" \ + REGION="${{ vars.CCC_REGION || 'us-west1' }}" \ + ./scripts/deploy_cloud_run_ffai.sh diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 3947986..0ef90b2 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -159,6 +159,58 @@ Deploy the runner with: PROJECT_ID=low-code-connect REGION=us-west1 ./scripts/deploy_cloud_run_ffai.sh ``` +Keep one instance warm by default: the script passes `--min-instances 1`, because +most of a cold start (FlutterFlow AI SDK download and a fresh `flutter pub get`) +is paid once per container instead of once per deploy. Override the count with +`MIN_INSTANCES`, e.g. `MIN_INSTANCES=0 ./scripts/deploy_cloud_run_ffai.sh` to +scale to zero and pay per-request cold starts instead. + +#### One-time setup for automated deploys (GitHub Actions) + +The workflow `.github/workflows/deploy-runner.yml` deploys the runner whenever +`cloud-run/**` changes on `main`. It uses a Google Cloud service-account key +from a repository secret. Until that secret exists the job **silently skips** +(rather than failing on every push), so set up this once or merged runner +changes will deploy nothing: + +1. Create a service account with at least `roles/run.admin`, + `roles/cloudbuild.builds.editor` and `roles/iam.serviceAccountUser`, generate + a JSON key, and download it. +2. In **Settings → Secrets and variables → Actions**, create a repository secret + named `CCC_FFAI_RUNNER_SA_KEY_JSON` whose value is the key's full JSON + (including the surrounding `{ }`). +3. The workflow reads `CCC_PROJECT_ID` (default `low-code-connect`) and + `CCC_REGION` (default `us-west1`) from repository variables; create those if + the defaults are not right. This is per-repo like every secret here, because + GitHub does not expose organisation secrets to private repositories on the + free plan. + +#### Runner canary (scheduled) + +The workflow `.github/workflows/canary.yml` runs daily (07:21 UTC) and on demand +via **Actions → Runner canary → Run workflow**. It sends a `dryRun: true` +deploy for the throwaway project `automated-test-miedro` to the runner's +`/deployCustomClasses`, so it exercises workspace init, the compile gate and the +DSL push path without writing anything to a real project. It fails (non-zero) +on any non-200 response, on `success:false`, or if the runner rejects the +`flutter_web_plugins` SDK package, and also reports the runner's `/healthz` +versions when that endpoint is deployed - without failing when it is not yet. + +`GET /healthz` answers `{sha, flutterflowCli, minFlutterflowCli}`. The deploy +script passes the checkout's short SHA as `RUNNER_GIT_SHA`, so comparing `sha` +against `git log -1 --format=%h` tells you whether a merged runner fix is +actually live. That question is what a months-long stall in a merged-but-never- +deployed fix looked like; it is now one request. + +Until the FlutterFlow API key secret exists the canary job skips cleanly, so it +never fails noisily before setup: + +1. Create a repository secret named `CANARY_FF_API_KEY` whose value is a + FlutterFlow API key with at least read access to `automated-test-miedro`. +2. Optionally set a repository variable `CANARY_RUNNER_URL` if the production + runner lives somewhere other than the default, and `CANARY_PROJECT_ID` if the + throwaway project changes. + ## BuildShip workflow and step prompts The generation pipeline (Architect → Generator → Review) runs on BuildShip, and diff --git a/scripts/canary-deploy.mjs b/scripts/canary-deploy.mjs new file mode 100644 index 0000000..6be1644 --- /dev/null +++ b/scripts/canary-deploy.mjs @@ -0,0 +1,184 @@ +#!/usr/bin/env node + +// Scheduled canary for the FlutterFlow custom-class runner. +// +// Every scheduled run POSTs a *dry-run* deploy to the runner so a regression +// that would break users' deploys is caught the next morning instead of on +// their next deploy. dryRun: true makes the runner go through `ai validate` +// (workspace init, compile gate, DSL validation) but never write to the +// project, so this is safe to run unattended. +// +// It also curls the runner's /healthz when present and reports its versions, +// but never fails on a missing /healthz - another change is adding it in +// parallel and it may not be deployed yet. + +const RUNNER_DEFAULT_URL = "https://ccc-ffai-runner-y5cyj3473a-uw.a.run.app"; + +// The throwaway class and dependencies the canary sends. These are real enough +// to compile: the runner builds a scratch package from the verification +// manifest and fails the deploy if `flutter analyze` reports an error, so a +// canary class that does not compile would fail every run for the wrong reason. +const CANARY_CLASS_NAME = "CanaryProbe"; +const CANARY_SOURCE = `import 'package:flutter/widgets.dart'; + +class ${CANARY_CLASS_NAME} extends StatelessWidget { + const ${CANARY_CLASS_NAME}({super.key}); + + @override + Widget build(BuildContext context) => const SizedBox.shrink(); +}`; + +function fail(message) { + console.error(`[canary] FAIL: ${message}`); + process.exitCode = 1; +} + +async function main() { + const apiKey = process.env.CANARY_FF_API_KEY; + if (!apiKey) { + fail( + "CANARY_FF_API_KEY is not set. Create the repository secret " + + "CANARY_FF_API_KEY with a FlutterFlow API key (see DEPLOYMENT.md) " + + "and give this job access to it.", + ); + return; + } + + const baseUrl = (process.env.CANARY_RUNNER_URL || RUNNER_DEFAULT_URL).replace( + /\/+$/, + "", + ); + const projectId = process.env.CANARY_PROJECT_ID || "automated-test-miedro"; + const timeoutMs = Number(process.env.CANARY_TIMEOUT_MS || 60000); + + const payload = { + apiKey, + projectId, + commitMessage: "canary dry-run", + dryRun: true, + customClasses: [ + { className: CANARY_CLASS_NAME, content: CANARY_SOURCE }, + ], + verification: { + sdkConstraint: ">=3.0.0 <4.0.0", + dependencies: {}, + dependencyOverrides: {}, + sdkPackages: ["flutter_web_plugins"], + // The runner hard-requires this array: without it _normalizeVerification + // throws and returns HTTP 400 before anything runs. It is also what the + // compile gate compiles - an empty sources list skips _verifyCustomCode + // entirely, so this canary would detect nothing. + sources: [{ fileName: "canary_probe.dart", content: CANARY_SOURCE }], + }, + }; + + // The SDK-package rejection this canary is meant to catch. The runner + // currently rejects an invalid SDK name as "Invalid SDK package name: X"; + // an older/newer wording may say "unsupported". Match both so a regression + // where flutter_web_plugins stops being accepted fails the canary. + const sdkPackageRejection = /unsupported sdk package|invalid sdk package name/i; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + let response; + try { + response = await fetch(`${baseUrl}/deployCustomClasses`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + } catch (error) { + fail( + `request to ${baseUrl}/deployCustomClasses failed: ` + + `${error?.name === "AbortError" ? "timed out" : error.message}`, + ); + return; + } + + const text = await response.text(); + let body; + try { + body = JSON.parse(text); + } catch { + body = { error: text.slice(0, 500) }; + } + + if (!response.ok) { + fail( + `runner returned HTTP ${response.status} on a dry-run deploy for ` + + `project "${projectId}". ${formatBodyError(body)}`, + ); + return; + } + + if (body.success === false) { + fail( + `runner reported success:false on a dry-run deploy for project ` + + `"${projectId}". ${formatBodyError(body)}`, + ); + return; + } + + const joined = [ + body.error, + body.details, + body.analyzerErrors, + ...(Array.isArray(body.analyzerErrors) ? body.analyzerErrors : []), + ] + .filter(Boolean) + .join("\n"); + if (sdkPackageRejection.test(joined)) { + fail( + `runner rejected the flutter_web_plugins SDK package on a dry-run ` + + `deploy - a dependency regression users would hit: ${joined}`, + ); + return; + } + + console.log( + `[canary] OK: dry-run deploy for project "${projectId}" accepted at ${baseUrl} ` + + `(success:${body.success === undefined ? "true" : body.success})`, + ); + } finally { + clearTimeout(timer); + } + + // The /healthz route is being added alongside this canary; do not fail the + // canary when it is not deployed yet. + try { + const health = await fetch(`${baseUrl}/healthz`, { + signal: AbortSignal.timeout(10000), + }); + if (health.ok) { + const healthBody = await health.json().catch(() => ({})); + console.log( + `[canary] healthz: HTTP ${health.status}` + + (healthBody.versions + ? ` versions=${JSON.stringify(healthBody.versions)}` + : ` ${JSON.stringify(healthBody)}`), + ); + } else { + console.log( + `[canary] healthz: HTTP ${health.status} (not reported as a failure)`, + ); + } + } catch { + console.log("[canary] healthz: unavailable (not reported as a failure)"); + } +} + +function formatBodyError(body) { + const detail = [ + body.error, + body.details, + ...(Array.isArray(body.analyzerErrors) ? body.analyzerErrors : []), + ] + .filter(Boolean) + .join(" | ") + .slice(0, 1000); + return detail ? `Details: ${detail}` : "No detail in the response body."; +} + +await main(); diff --git a/scripts/deploy_cloud_run_ffai.sh b/scripts/deploy_cloud_run_ffai.sh index 023eb2d..ae5580c 100755 --- a/scripts/deploy_cloud_run_ffai.sh +++ b/scripts/deploy_cloud_run_ffai.sh @@ -11,6 +11,17 @@ CONCURRENCY="${CONCURRENCY:-1}" # covers `flutter pub get` plus `flutter analyze` on top of the DSL run. Cloud # Run's 300s default would cut that off on a cold instance. TIMEOUT="${TIMEOUT:-900}" +# Keep one instance always on. Most of a deploy's 103-215s on a cold start is +# the FlutterFlow AI SDK download and a fresh `flutter pub get` after `ai init`; +# a warm instance does that once at container boot instead of per request. The +# tradeoff is a small always-on instance cost for a much shorter deploy. +MIN_INSTANCES="${MIN_INSTANCES:-1}" +# Reported by the runner's /healthz, so which revision is live - and therefore +# whether a merged fix actually shipped - is answerable in seconds instead of +# guessed at. This is exactly how a merged-but-never-deployed runner fix went +# unnoticed for months. Taken from the build's own checkout; empty (never +# wrong) if git is unavailable. +RUNNER_GIT_SHA="${RUNNER_GIT_SHA:-$(git rev-parse --short HEAD 2>/dev/null || true)}" if [[ -z "$PROJECT_ID" ]]; then PROJECT_ID="$(gcloud config get-value project 2>/dev/null || true)" @@ -28,8 +39,12 @@ gcloud run deploy "$SERVICE" \ --allow-unauthenticated \ --memory "$MEMORY" \ --concurrency "$CONCURRENCY" \ + --min-instances "$MIN_INSTANCES" \ --timeout "$TIMEOUT" \ - --set-env-vars "ALLOWED_ORIGIN=$ALLOWED_ORIGIN" + --set-env-vars "ALLOWED_ORIGIN=$ALLOWED_ORIGIN,RUNNER_GIT_SHA=$RUNNER_GIT_SHA" + +echo +echo "Deployed revision reports RUNNER_GIT_SHA=${RUNNER_GIT_SHA:-}" echo echo "Set VITE_FLUTTERFLOW_DSL_DEPLOY_ENDPOINT to:"