feat(migrate): Alembic and goose output for planned migrations; fix Python quoting (#148)

src/commands/migrate.ts

@@ -10,7 +10,6 @@ import { Baseline, baselineFile, resolveBaseline } from "../lib/plan/baseline";
 import { describeQuestion, optionLabel, splitDefaults, sqlHint } from "../lib/plan/prompts";
 import {
   cliBinPath,
-  migrateUnsupported,
   report,
   run,
   supportsPglite,
@@ -94,8 +93,6 @@ export default class Migrate extends BaseCommand {
     if (this.json) console.info = (...m: unknown[]) => process.stderr.write(m.join(" ") + "\n");
 
     const { apsorc } = readApsorcFile();
-    const unsupported = migrateUnsupported(apsorc.language);
-    if (unsupported) this.fail(unsupported);
     const cliBin = cliBinPath(this.config.root);
 
     if (!flags["skip-generate"]) {
@@ -163,7 +160,9 @@ export default class Migrate extends BaseCommand {
     for (const f of written.findings ?? []) this.say(`  warning ${f.code}: ${f.message}`);
 
     let verified: boolean | "skipped" = "skipped";
-    if (!flags["skip-verify"] && supportsPglite(cwd)) {
+    // Python and Go verify without the project's own database driver.
+    const canVerify = (apsorc.language ?? "typescript") !== "typescript" || supportsPglite(cwd);
+    if (!flags["skip-verify"] && canVerify) {
       this.say("Testing the migration on a local copy of the deployed schema...");
       const result = await verifyMigration(cwd, baselineFile(baseline), written.file!, cliBin, {
         seedFile: flags.seed,

src/commands/migrate/generate.ts

@@ -1,7 +1,7 @@
 import * as path from "path";
 import { Flags } from "@oclif/core";
 import BaseCommand from "../../lib/base-command";
-import { parseApsorc, readApsorcFile } from "../../lib/apsorc-parser";
+import { readApsorcFile } from "../../lib/apsorc-parser";
 import { writePlannedMigration } from "../../lib/ddl/migration-file";
 import { Question } from "../../lib/plan";
 import {
@@ -11,14 +11,13 @@ import {
 } from "../../lib/plan/baseline";
 import {
   cliBinPath,
-  migrateUnsupported,
   generateMigration,
   report,
 } from "../../lib/verify";
 
 export default class MigrateGenerate extends BaseCommand {
   static description =
-    "Write a migration from the deployed schema to this project's .apsorc. Apso renders the SQL from the answered migration plan (apso migrate plan), including data moves and a down() that reverses them. Exits 2 while plan questions are open. --typeorm uses TypeORM's generator on a local PGlite database instead (TypeScript services).";
+    "Write a migration from the deployed schema to this project's .apsorc. Apso renders the SQL from the answered migration plan (apso migrate plan), including data moves and a down() that reverses them. Exits 2 while plan questions are open. The migration is a TypeORM class, an Alembic revision or a goose SQL file, by the service's language. --typeorm uses TypeORM's generator on a local PGlite database instead (TypeScript services).";
 
   static examples = [
     `$ apso migrate plan --answer entity:Company=rename:Organization`,
@@ -52,10 +51,9 @@ export default class MigrateGenerate extends BaseCommand {
 
   async run(): Promise<void> {
     const { flags } = await this.parse(MigrateGenerate);
-    const unsupported = migrateUnsupported(parseApsorc().language);
-    if (unsupported) this.error(unsupported);
-
     const { apsorc } = readApsorcFile();
+    if (flags.typeorm && (apsorc.language ?? "typescript") !== "typescript")
+      this.error("--typeorm works on TypeScript services only.");
     let baseline: Baseline;
     try {
       baseline = resolveBaseline(

src/commands/migrate/verify.ts

@@ -1,18 +1,17 @@
 import * as path from "path";
 import { Flags } from "@oclif/core";
 import BaseCommand from "../../lib/base-command";
-import { parseApsorc, readApsorcFile } from "../../lib/apsorc-parser";
+import { readApsorcFile } from "../../lib/apsorc-parser";
 import { baselineFile, latestPlannedMigration } from "../../lib/plan/baseline";
 import {
   cliBinPath,
-  migrateUnsupported,
   report,
   verifyMigration,
 } from "../../lib/verify";
 
 export default class MigrateVerify extends BaseCommand {
   static description =
-    "Run a migration on a local PGlite copy of the deployed schema, then check the database matches this project's entities (TypeScript services)";
+    "Run a migration on a local PGlite copy of the deployed schema, then check the database matches this project's entities (TypeScript), or a fresh build of its .apsorc (Python and Go, which also run down and check it gives back the deployed schema)";
 
   static examples = [
     `$ apso migrate verify`,
@@ -56,8 +55,6 @@ export default class MigrateVerify extends BaseCommand {
 
   async run(): Promise<void> {
     const { flags } = await this.parse(MigrateVerify);
-    const unsupported = migrateUnsupported(parseApsorc().language);
-    if (unsupported) this.error(unsupported);
     const { apsorc } = readApsorcFile();
     const latest = latestPlannedMigration(process.cwd(), apsorc);
     const migration = flags.migration ?? latest?.file;

src/lib/ddl/migration-file.ts

@@ -1,15 +1,18 @@
 /**
- * Write a TypeORM migration from an answered plan: up() is Apso's change DDL,
- * down() the reverse, and the plan is saved beside the migration so the
- * answers that produced it stay with it.
+ * Write a migration from an answered plan: up is Apso's change DDL, down the
+ * reverse, and the plan is saved beside the migration so the answers that
+ * produced it stay with it. The SQL is the same for every language; only the
+ * wrapper differs: a TypeORM class, an Alembic revision, or a goose SQL file.
  */
 import * as fs from "fs";
 import * as path from "path";
 import { ApsorcType } from "../apsorc-parser";
 import { openQuestions, Plan, planMigration, Question } from "../plan";
-import { pascalCase } from "../utils/casing";
+import { migrationExtension, migrationsDir } from "../plan/baseline";
+import { pascalCase, snakeCase } from "../utils/casing";
 import { reviewMigration, Finding } from "../verify/review";
 import { changeDdl, DownDdl, downDdl } from "./change";
+import { literal } from ".";
 
 const template = (sql: string) =>
   "`" +
@@ -28,11 +31,7 @@ export function typeormMigration(
   const downBody =
     "down" in down
       ? queries(down.down)
-      : `        throw new Error(${JSON.stringify(
-          `This migration can't be reversed automatically: ${down.irreversible.join(
-            "; "
-          )}`
-        )});`;
+      : `        throw new Error(${JSON.stringify(irreversible(down))});`;
   return `import { MigrationInterface, QueryRunner } from "typeorm";
 
 // Generated by apso migrate generate from the answered migration plan saved beside this file.
@@ -51,6 +50,125 @@ ${downBody}
 `;
 }
 
+const irreversible = (down: { irreversible: string[] }) =>
+  `This migration can't be reversed automatically: ${down.irreversible.join(
+    "; "
+  )}`;
+
+/**
+ * A Python string literal for any SQL. A JSON string is one (same escapes),
+ * where """...""" breaks on SQL that ends in a quoted identifier (#148).
+ */
+export const pyString = (s: string): string => JSON.stringify(s);
+
+const executes = (statements: string[]) =>
+  statements.map((s) => `    op.execute(${pyString(s)})`).join("\n");
+
+export function alembicMigration(
+  revision: string,
+  downRevision: string | undefined,
+  up: string[],
+  down: DownDdl
+): string {
+  const downBody =
+    "down" in down
+      ? executes(down.down) || "    pass"
+      : `    raise NotImplementedError(${pyString(irreversible(down))})`;
+  return `"""Generated by apso migrate generate from the answered migration plan saved beside this file."""
+from alembic import op
+
+revision = ${pyString(revision)}
+down_revision = ${downRevision === undefined ? "None" : pyString(downRevision)}
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+${executes(up)}
+
+
+def downgrade() -> None:
+${downBody}
+`;
+}
+
+/**
+ * Statements for a goose SQL file. goose splits on lines ending in ";", so a
+ * statement with a ";" of its own is fenced with StatementBegin/End.
+ */
+export const gooseStatements = (statements: string[]): string =>
+  statements
+    .map((s) =>
+      s.includes(";")
+        ? `-- +goose StatementBegin\n${s};\n-- +goose StatementEnd`
+        : `${s};`
+    )
+    .join("\n");
+
+export function gooseMigration(up: string[], down: DownDdl): string {
+  // goose has no way to refuse a down, so the SQL refuses it instead.
+  const downBody =
+    "down" in down
+      ? gooseStatements(down.down)
+      : gooseStatements([
+          `DO $$ BEGIN RAISE EXCEPTION ${literal(
+            irreversible(down).replace(/%/g, "%%")
+          )}; END $$`,
+        ]);
+  return `-- Generated by apso migrate generate from the answered migration plan saved beside this file.
+-- +goose Up
+${gooseStatements(up)}
+
+-- +goose Down
+${downBody}
+`;
+}
+
+/** The newest Alembic revision: the one no other revision names as its down_revision. */
+export function alembicHead(dir: string): string | undefined {
+  if (!fs.existsSync(dir)) return undefined;
+  const revisions: string[] = [];
+  const downs = new Set<string>();
+  for (const f of fs
+    .readdirSync(dir)
+    .filter((f) => f.endsWith(".py"))
+    .sort()) {
+    const source = fs.readFileSync(path.join(dir, f)).toString();
+    const rev = source.match(/^revision\b[^\n=]*=\s*["']([^"']+)["']/m);
+    if (rev) revisions.push(rev[1]);
+    const down = source.match(/^down_revision\b[^\n=]*=(.*)$/m);
+    for (const d of down?.[1].matchAll(/["']([^"']+)["']/g) ?? [])
+      downs.add(d[1]);
+  }
+  return revisions.filter((r) => !downs.has(r)).pop();
+}
+
+const calls = (body: string) =>
+  [...body.matchAll(/^\s*op\.execute\((".*")\)\s*$/gm)]
+    .map((m) => JSON.parse(m[1]) as string)
+    .join(";\n");
+
+/**
+ * The up and down SQL of an Alembic or goose migration, for running it on a
+ * database without the framework. down is undefined when it refuses to run.
+ */
+export function migrationSql(
+  file: string,
+  source: string
+): { up: string; down?: string } {
+  if (file.endsWith(".sql")) {
+    const [up, down = ""] = source
+      .split(/^-- \+goose Down\b.*$/m)
+      .map((s) => s.replace(/^[\S\s]*?^-- \+goose Up\b.*$/m, ""));
+    return { up, down: /RAISE EXCEPTION/.test(down) ? undefined : down };
+  }
+  const [upBody, downBody = ""] = source.split(/^def downgrade\b/m);
+  return {
+    up: calls(upBody),
+    down: /raise NotImplementedError/.test(downBody) ? undefined : calls(downBody),
+  };
+}
+
 export interface PlannedMigration {
   ok: boolean;
   needed: boolean;
@@ -123,11 +241,23 @@ export function writePlannedMigration(
     return { ok: true, needed: false, steps: [{ name: "render", ok: true }] };
 
   const className = `${pascalCase(name)}Migration${now}`;
-  const dir = path.join(projectDir, after.rootFolder || "src", "migrations");
+  const dir = migrationsDir(projectDir, after);
   fs.mkdirSync(dir, { recursive: true });
-  const base = `${now}-${name}-migration`;
-  const source = typeormMigration(className, up, down);
-  fs.writeFileSync(path.join(dir, `${base}.ts`), source);
+  // goose orders by the number before "_", and its own files use
+  // YYYYMMDDHHMMSS, so Python and Go use that too.
+  const stamp = new Date(now).toISOString().replace(/\D/g, "").slice(0, 14);
+  const base =
+    after.language === "python" || after.language === "go"
+      ? `${stamp}_${snakeCase(name)}`
+      : `${now}-${name}-migration`;
+  const file = base + migrationExtension(after);
+  const source =
+    after.language === "python"
+      ? alembicMigration(stamp, alembicHead(dir), up, down)
+      : after.language === "go"
+      ? gooseMigration(up, down)
+      : typeormMigration(className, up, down);
+  fs.writeFileSync(path.join(dir, file), source);
   fs.writeFileSync(
     path.join(dir, `${base}.plan.json`),
     // The schemas let later runs find the baseline without being told.
@@ -139,9 +269,10 @@ export function writePlannedMigration(
   return {
     ok: true,
     needed: true,
-    file: rel(`${base}.ts`),
+    file: rel(file),
     planFile: rel(`${base}.plan.json`),
-    findings: reviewMigration(source),
+    // The review reads TypeORM's wrapper; the SQL is the same in all three.
+    findings: reviewMigration(typeormMigration(className, up, down)),
     steps: [{ name: "render", ok: true }],
   };
 }

src/lib/generators/go.ts

@@ -22,6 +22,7 @@ import {
 } from "../utils/relationships";
 import { snakeCase, camelCase, pascalCase, kebabCase } from "../utils/casing";
 import { getEventEmittingEntities } from "../events";
+import { gooseStatements } from "../ddl/migration-file";
 import pluralize from "pluralize";
 
 /**
@@ -392,8 +393,8 @@ export class GoGenerator extends BaseGenerator {
     const name = migrationName || "auto";
     const filename = `${timestamp}_${name}.sql`;
 
-    const upStatements = upSql.map((sql) => `${sql};`).join("\n");
-    const downStatements = (downSql || []).map((sql) => `${sql};`).join("\n");
+    const upStatements = gooseStatements(upSql);
+    const downStatements = gooseStatements(downSql || []);
 
     const content = `-- +goose Up
 ${upStatements}

src/lib/generators/python.ts

@@ -22,6 +22,7 @@ import {
 } from "../utils/relationships";
 import { snakeCase, camelCase } from "../utils/casing";
 import { getEventEmittingEntities } from "../events";
+import { pyString } from "../ddl/migration-file";
 import pluralize from "pluralize";
 
 /**
@@ -319,10 +320,10 @@ export class PythonGenerator extends BaseGenerator {
     const filename = `${timestamp}_${name}.py`;
 
     const upStatements = upSql
-      .map((sql) => `    op.execute("""${sql}""")`)
+      .map((sql) => `    op.execute(${pyString(sql)})`)
       .join("\n");
     const downStatements = (downSql || [])
-      .map((sql) => `    op.execute("""${sql}""")`)
+      .map((sql) => `    op.execute(${pyString(sql)})`)
       .join("\n");
 
     const content = `"""${name}"""

src/lib/plan/baseline.ts

@@ -23,8 +23,19 @@ export interface Baseline {
   source: string;
 }
 
+/** Where each language's migrations live, and the extension of one. */
 export const migrationsDir = (projectDir: string, apsorc?: ApsorcType) =>
-  path.join(projectDir, apsorc?.rootFolder || "src", "migrations");
+  apsorc?.language === "python"
+    ? path.join(projectDir, "alembic", "versions")
+    : apsorc?.language === "go"
+    ? path.join(projectDir, "migrations")
+    : path.join(projectDir, apsorc?.rootFolder || "src", "migrations");
+export const migrationExtension = (apsorc?: ApsorcType) =>
+  apsorc?.language === "python"
+    ? ".py"
+    : apsorc?.language === "go"
+    ? ".sql"
+    : ".ts";
 
 /** The newest migration with a saved plan, by its timestamp prefix. */
 export function latestPlannedMigration(
@@ -35,14 +46,14 @@ export function latestPlannedMigration(
   if (!fs.existsSync(dir)) return undefined;
   const plans = fs
     .readdirSync(dir)
-    .filter((f) => /^\d+-.+-migration\.plan\.json$/.test(f))
-    .sort((a, b) => Number(b.split("-")[0]) - Number(a.split("-")[0]));
+    .filter((f) => /^\d+[_-].+\.plan\.json$/.test(f))
+    .sort((a, b) => Number(b.split(/[_-]/)[0]) - Number(a.split(/[_-]/)[0]));
   for (const planFile of plans) {
     const saved = JSON.parse(
       fs.readFileSync(path.join(dir, planFile)).toString()
     );
     if (!saved.before || !saved.after) continue;
-    const file = planFile.replace(/\.plan\.json$/, ".ts");
+    const file = planFile.replace(/\.plan\.json$/, migrationExtension(apsorc));
     return {
       file: path.relative(projectDir, path.join(dir, file)),
       planFile: path.relative(projectDir, path.join(dir, planFile)),