diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts index e047bf8..5e2a01d 100644 --- a/src/commands/migrate.ts +++ b/src/commands/migrate.ts @@ -10,7 +10,6 @@ import { Baseline, baselineFile, resolveBaseline } from "../lib/plan/baseline"; import { describeQuestion, optionLabel, splitDefaults, sqlHint } from "../lib/plan/prompts"; import { cliBinPath, - migrateUnsupported, report, run, supportsPglite, @@ -94,8 +93,6 @@ export default class Migrate extends BaseCommand { if (this.json) console.info = (...m: unknown[]) => process.stderr.write(m.join(" ") + "\n"); const { apsorc } = readApsorcFile(); - const unsupported = migrateUnsupported(apsorc.language); - if (unsupported) this.fail(unsupported); const cliBin = cliBinPath(this.config.root); if (!flags["skip-generate"]) { @@ -163,7 +160,9 @@ export default class Migrate extends BaseCommand { for (const f of written.findings ?? []) this.say(` warning ${f.code}: ${f.message}`); let verified: boolean | "skipped" = "skipped"; - if (!flags["skip-verify"] && supportsPglite(cwd)) { + // Python and Go verify without the project's own database driver. + const canVerify = (apsorc.language ?? "typescript") !== "typescript" || supportsPglite(cwd); + if (!flags["skip-verify"] && canVerify) { this.say("Testing the migration on a local copy of the deployed schema..."); const result = await verifyMigration(cwd, baselineFile(baseline), written.file!, cliBin, { seedFile: flags.seed, diff --git a/src/commands/migrate/generate.ts b/src/commands/migrate/generate.ts index 4d65968..5f20036 100644 --- a/src/commands/migrate/generate.ts +++ b/src/commands/migrate/generate.ts @@ -1,7 +1,7 @@ import * as path from "path"; import { Flags } from "@oclif/core"; import BaseCommand from "../../lib/base-command"; -import { parseApsorc, readApsorcFile } from "../../lib/apsorc-parser"; +import { readApsorcFile } from "../../lib/apsorc-parser"; import { writePlannedMigration } from "../../lib/ddl/migration-file"; import { Question } from "../../lib/plan"; import { @@ -11,14 +11,13 @@ import { } from "../../lib/plan/baseline"; import { cliBinPath, - migrateUnsupported, generateMigration, report, } from "../../lib/verify"; export default class MigrateGenerate extends BaseCommand { static description = - "Write a migration from the deployed schema to this project's .apsorc. Apso renders the SQL from the answered migration plan (apso migrate plan), including data moves and a down() that reverses them. Exits 2 while plan questions are open. --typeorm uses TypeORM's generator on a local PGlite database instead (TypeScript services)."; + "Write a migration from the deployed schema to this project's .apsorc. Apso renders the SQL from the answered migration plan (apso migrate plan), including data moves and a down() that reverses them. Exits 2 while plan questions are open. The migration is a TypeORM class, an Alembic revision or a goose SQL file, by the service's language. --typeorm uses TypeORM's generator on a local PGlite database instead (TypeScript services)."; static examples = [ `$ apso migrate plan --answer entity:Company=rename:Organization`, @@ -52,10 +51,9 @@ export default class MigrateGenerate extends BaseCommand { async run(): Promise { const { flags } = await this.parse(MigrateGenerate); - const unsupported = migrateUnsupported(parseApsorc().language); - if (unsupported) this.error(unsupported); - const { apsorc } = readApsorcFile(); + if (flags.typeorm && (apsorc.language ?? "typescript") !== "typescript") + this.error("--typeorm works on TypeScript services only."); let baseline: Baseline; try { baseline = resolveBaseline( diff --git a/src/commands/migrate/verify.ts b/src/commands/migrate/verify.ts index 7fbb3e0..0e712fb 100644 --- a/src/commands/migrate/verify.ts +++ b/src/commands/migrate/verify.ts @@ -1,18 +1,17 @@ import * as path from "path"; import { Flags } from "@oclif/core"; import BaseCommand from "../../lib/base-command"; -import { parseApsorc, readApsorcFile } from "../../lib/apsorc-parser"; +import { readApsorcFile } from "../../lib/apsorc-parser"; import { baselineFile, latestPlannedMigration } from "../../lib/plan/baseline"; import { cliBinPath, - migrateUnsupported, report, verifyMigration, } from "../../lib/verify"; export default class MigrateVerify extends BaseCommand { static description = - "Run a migration on a local PGlite copy of the deployed schema, then check the database matches this project's entities (TypeScript services)"; + "Run a migration on a local PGlite copy of the deployed schema, then check the database matches this project's entities (TypeScript), or a fresh build of its .apsorc (Python and Go, which also run down and check it gives back the deployed schema)"; static examples = [ `$ apso migrate verify`, @@ -56,8 +55,6 @@ export default class MigrateVerify extends BaseCommand { async run(): Promise { const { flags } = await this.parse(MigrateVerify); - const unsupported = migrateUnsupported(parseApsorc().language); - if (unsupported) this.error(unsupported); const { apsorc } = readApsorcFile(); const latest = latestPlannedMigration(process.cwd(), apsorc); const migration = flags.migration ?? latest?.file; diff --git a/src/lib/ddl/migration-file.ts b/src/lib/ddl/migration-file.ts index 1404b44..015fcc8 100644 --- a/src/lib/ddl/migration-file.ts +++ b/src/lib/ddl/migration-file.ts @@ -1,15 +1,18 @@ /** - * Write a TypeORM migration from an answered plan: up() is Apso's change DDL, - * down() the reverse, and the plan is saved beside the migration so the - * answers that produced it stay with it. + * Write a migration from an answered plan: up is Apso's change DDL, down the + * reverse, and the plan is saved beside the migration so the answers that + * produced it stay with it. The SQL is the same for every language; only the + * wrapper differs: a TypeORM class, an Alembic revision, or a goose SQL file. */ import * as fs from "fs"; import * as path from "path"; import { ApsorcType } from "../apsorc-parser"; import { openQuestions, Plan, planMigration, Question } from "../plan"; -import { pascalCase } from "../utils/casing"; +import { migrationExtension, migrationsDir } from "../plan/baseline"; +import { pascalCase, snakeCase } from "../utils/casing"; import { reviewMigration, Finding } from "../verify/review"; import { changeDdl, DownDdl, downDdl } from "./change"; +import { literal } from "."; const template = (sql: string) => "`" + @@ -28,11 +31,7 @@ export function typeormMigration( const downBody = "down" in down ? queries(down.down) - : ` throw new Error(${JSON.stringify( - `This migration can't be reversed automatically: ${down.irreversible.join( - "; " - )}` - )});`; + : ` throw new Error(${JSON.stringify(irreversible(down))});`; return `import { MigrationInterface, QueryRunner } from "typeorm"; // Generated by apso migrate generate from the answered migration plan saved beside this file. @@ -51,6 +50,125 @@ ${downBody} `; } +const irreversible = (down: { irreversible: string[] }) => + `This migration can't be reversed automatically: ${down.irreversible.join( + "; " + )}`; + +/** + * A Python string literal for any SQL. A JSON string is one (same escapes), + * where """...""" breaks on SQL that ends in a quoted identifier (#148). + */ +export const pyString = (s: string): string => JSON.stringify(s); + +const executes = (statements: string[]) => + statements.map((s) => ` op.execute(${pyString(s)})`).join("\n"); + +export function alembicMigration( + revision: string, + downRevision: string | undefined, + up: string[], + down: DownDdl +): string { + const downBody = + "down" in down + ? executes(down.down) || " pass" + : ` raise NotImplementedError(${pyString(irreversible(down))})`; + return `"""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = ${pyString(revision)} +down_revision = ${downRevision === undefined ? "None" : pyString(downRevision)} +branch_labels = None +depends_on = None + + +def upgrade() -> None: +${executes(up)} + + +def downgrade() -> None: +${downBody} +`; +} + +/** + * Statements for a goose SQL file. goose splits on lines ending in ";", so a + * statement with a ";" of its own is fenced with StatementBegin/End. + */ +export const gooseStatements = (statements: string[]): string => + statements + .map((s) => + s.includes(";") + ? `-- +goose StatementBegin\n${s};\n-- +goose StatementEnd` + : `${s};` + ) + .join("\n"); + +export function gooseMigration(up: string[], down: DownDdl): string { + // goose has no way to refuse a down, so the SQL refuses it instead. + const downBody = + "down" in down + ? gooseStatements(down.down) + : gooseStatements([ + `DO $$ BEGIN RAISE EXCEPTION ${literal( + irreversible(down).replace(/%/g, "%%") + )}; END $$`, + ]); + return `-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +${gooseStatements(up)} + +-- +goose Down +${downBody} +`; +} + +/** The newest Alembic revision: the one no other revision names as its down_revision. */ +export function alembicHead(dir: string): string | undefined { + if (!fs.existsSync(dir)) return undefined; + const revisions: string[] = []; + const downs = new Set(); + for (const f of fs + .readdirSync(dir) + .filter((f) => f.endsWith(".py")) + .sort()) { + const source = fs.readFileSync(path.join(dir, f)).toString(); + const rev = source.match(/^revision\b[^\n=]*=\s*["']([^"']+)["']/m); + if (rev) revisions.push(rev[1]); + const down = source.match(/^down_revision\b[^\n=]*=(.*)$/m); + for (const d of down?.[1].matchAll(/["']([^"']+)["']/g) ?? []) + downs.add(d[1]); + } + return revisions.filter((r) => !downs.has(r)).pop(); +} + +const calls = (body: string) => + [...body.matchAll(/^\s*op\.execute\((".*")\)\s*$/gm)] + .map((m) => JSON.parse(m[1]) as string) + .join(";\n"); + +/** + * The up and down SQL of an Alembic or goose migration, for running it on a + * database without the framework. down is undefined when it refuses to run. + */ +export function migrationSql( + file: string, + source: string +): { up: string; down?: string } { + if (file.endsWith(".sql")) { + const [up, down = ""] = source + .split(/^-- \+goose Down\b.*$/m) + .map((s) => s.replace(/^[\S\s]*?^-- \+goose Up\b.*$/m, "")); + return { up, down: /RAISE EXCEPTION/.test(down) ? undefined : down }; + } + const [upBody, downBody = ""] = source.split(/^def downgrade\b/m); + return { + up: calls(upBody), + down: /raise NotImplementedError/.test(downBody) ? undefined : calls(downBody), + }; +} + export interface PlannedMigration { ok: boolean; needed: boolean; @@ -123,11 +241,23 @@ export function writePlannedMigration( return { ok: true, needed: false, steps: [{ name: "render", ok: true }] }; const className = `${pascalCase(name)}Migration${now}`; - const dir = path.join(projectDir, after.rootFolder || "src", "migrations"); + const dir = migrationsDir(projectDir, after); fs.mkdirSync(dir, { recursive: true }); - const base = `${now}-${name}-migration`; - const source = typeormMigration(className, up, down); - fs.writeFileSync(path.join(dir, `${base}.ts`), source); + // goose orders by the number before "_", and its own files use + // YYYYMMDDHHMMSS, so Python and Go use that too. + const stamp = new Date(now).toISOString().replace(/\D/g, "").slice(0, 14); + const base = + after.language === "python" || after.language === "go" + ? `${stamp}_${snakeCase(name)}` + : `${now}-${name}-migration`; + const file = base + migrationExtension(after); + const source = + after.language === "python" + ? alembicMigration(stamp, alembicHead(dir), up, down) + : after.language === "go" + ? gooseMigration(up, down) + : typeormMigration(className, up, down); + fs.writeFileSync(path.join(dir, file), source); fs.writeFileSync( path.join(dir, `${base}.plan.json`), // The schemas let later runs find the baseline without being told. @@ -139,9 +269,10 @@ export function writePlannedMigration( return { ok: true, needed: true, - file: rel(`${base}.ts`), + file: rel(file), planFile: rel(`${base}.plan.json`), - findings: reviewMigration(source), + // The review reads TypeORM's wrapper; the SQL is the same in all three. + findings: reviewMigration(typeormMigration(className, up, down)), steps: [{ name: "render", ok: true }], }; } diff --git a/src/lib/generators/go.ts b/src/lib/generators/go.ts index 20d32f4..b2c556e 100644 --- a/src/lib/generators/go.ts +++ b/src/lib/generators/go.ts @@ -22,6 +22,7 @@ import { } from "../utils/relationships"; import { snakeCase, camelCase, pascalCase, kebabCase } from "../utils/casing"; import { getEventEmittingEntities } from "../events"; +import { gooseStatements } from "../ddl/migration-file"; import pluralize from "pluralize"; /** @@ -392,8 +393,8 @@ export class GoGenerator extends BaseGenerator { const name = migrationName || "auto"; const filename = `${timestamp}_${name}.sql`; - const upStatements = upSql.map((sql) => `${sql};`).join("\n"); - const downStatements = (downSql || []).map((sql) => `${sql};`).join("\n"); + const upStatements = gooseStatements(upSql); + const downStatements = gooseStatements(downSql || []); const content = `-- +goose Up ${upStatements} diff --git a/src/lib/generators/python.ts b/src/lib/generators/python.ts index 2f92a9e..192f585 100644 --- a/src/lib/generators/python.ts +++ b/src/lib/generators/python.ts @@ -22,6 +22,7 @@ import { } from "../utils/relationships"; import { snakeCase, camelCase } from "../utils/casing"; import { getEventEmittingEntities } from "../events"; +import { pyString } from "../ddl/migration-file"; import pluralize from "pluralize"; /** @@ -319,10 +320,10 @@ export class PythonGenerator extends BaseGenerator { const filename = `${timestamp}_${name}.py`; const upStatements = upSql - .map((sql) => ` op.execute("""${sql}""")`) + .map((sql) => ` op.execute(${pyString(sql)})`) .join("\n"); const downStatements = (downSql || []) - .map((sql) => ` op.execute("""${sql}""")`) + .map((sql) => ` op.execute(${pyString(sql)})`) .join("\n"); const content = `"""${name}""" diff --git a/src/lib/plan/baseline.ts b/src/lib/plan/baseline.ts index 7125b1b..d3319b7 100644 --- a/src/lib/plan/baseline.ts +++ b/src/lib/plan/baseline.ts @@ -23,8 +23,19 @@ export interface Baseline { source: string; } +/** Where each language's migrations live, and the extension of one. */ export const migrationsDir = (projectDir: string, apsorc?: ApsorcType) => - path.join(projectDir, apsorc?.rootFolder || "src", "migrations"); + apsorc?.language === "python" + ? path.join(projectDir, "alembic", "versions") + : apsorc?.language === "go" + ? path.join(projectDir, "migrations") + : path.join(projectDir, apsorc?.rootFolder || "src", "migrations"); +export const migrationExtension = (apsorc?: ApsorcType) => + apsorc?.language === "python" + ? ".py" + : apsorc?.language === "go" + ? ".sql" + : ".ts"; /** The newest migration with a saved plan, by its timestamp prefix. */ export function latestPlannedMigration( @@ -35,14 +46,14 @@ export function latestPlannedMigration( if (!fs.existsSync(dir)) return undefined; const plans = fs .readdirSync(dir) - .filter((f) => /^\d+-.+-migration\.plan\.json$/.test(f)) - .sort((a, b) => Number(b.split("-")[0]) - Number(a.split("-")[0])); + .filter((f) => /^\d+[_-].+\.plan\.json$/.test(f)) + .sort((a, b) => Number(b.split(/[_-]/)[0]) - Number(a.split(/[_-]/)[0])); for (const planFile of plans) { const saved = JSON.parse( fs.readFileSync(path.join(dir, planFile)).toString() ); if (!saved.before || !saved.after) continue; - const file = planFile.replace(/\.plan\.json$/, ".ts"); + const file = planFile.replace(/\.plan\.json$/, migrationExtension(apsorc)); return { file: path.relative(projectDir, path.join(dir, file)), planFile: path.relative(projectDir, path.join(dir, planFile)), diff --git a/src/lib/verify/index.ts b/src/lib/verify/index.ts index 225b940..2b9ebaf 100644 --- a/src/lib/verify/index.ts +++ b/src/lib/verify/index.ts @@ -1,7 +1,7 @@ /** * Pre-deploy checks for a generated service: generate the migration from the - * deployed schema, run it and check for drift (TypeScript only, on PGlite), - * and smoke test the endpoints. The CLI and the in-browser editor (which runs + * deployed schema, run it and check for drift (on PGlite), and smoke test the + * endpoints. The CLI and the in-browser editor (which runs * the CLI in a WebContainer) share these steps. */ @@ -12,6 +12,10 @@ import * as net from "net"; import * as os from "os"; import * as path from "path"; import pluralize from "pluralize"; +import { parseApsorcV2 } from "../apsorc-parser"; +import { schemaDdl } from "../ddl"; +import { catalogDiff, readCatalog } from "../ddl/catalog"; +import { migrationSql, typeormMigration } from "../ddl/migration-file"; import { Entity } from "../types/entity"; import { TargetLanguage } from "../types/generator"; import { RelationshipMap } from "../types/relationship"; @@ -125,21 +129,6 @@ export async function buildBaseline( } } -/** - * Why `apso migrate` can't run for this project's language, or undefined. - * ponytail: Python (Alembic) and Go (goose) have no local scratch database to - * build the baseline on; add when they get one like PGlite for TypeScript. - */ -export const migrateUnsupported = ( - language: TargetLanguage = "typescript" -): string | undefined => { - if (language === "python") - return "apso migrate supports TypeScript services only for now. This is a Python service: use Alembic directly (alembic revision --autogenerate, alembic upgrade head, alembic check)."; - if (language === "go") - return "apso migrate supports TypeScript services only for now. This is a Go service: write goose migrations in migrations/ and run them with `go run cmd/migrate/main.go up`."; - return undefined; -}; - export interface GenerateResult { ok: boolean; needed: boolean; @@ -365,6 +354,12 @@ export async function verifyMigration( acceptLoss = [], }: { skipBuild?: boolean; seedFile?: string; checkFile?: string; acceptLoss?: string[] } = {} ): Promise { + if (/\.(py|sql)$/.test(migrationFile)) + return verifySqlMigration(projectDir, baselineApsorc, migrationFile, cliBin, { + seedFile, + checkFile, + acceptLoss, + }); if (!supportsPglite(projectDir)) return NO_PGLITE; const source = fs.readFileSync(path.resolve(projectDir, migrationFile), "utf8"); const findings = reviewMigration(source); @@ -432,6 +427,109 @@ export async function verifyMigration( } } +const ddlOf = (apsorcFile: string) => + schemaDdl(parseApsorcV2(JSON.parse(fs.readFileSync(apsorcFile).toString()))).join(";\n"); + +/** + * verifyMigration for an Alembic or goose migration. Python and Go services + * have no TypeORM, so Apso's DDL writer builds the deployed schema, the + * migration's SQL runs as written, and drift is a catalog diff against a + * fresh build of the new .apsorc. down then runs and must give back the + * deployed schema. + */ +async function verifySqlMigration( + projectDir: string, + baselineApsorc: string, + migrationFile: string, + cliBin: string, + { seedFile, checkFile, acceptLoss = [] }: { seedFile?: string; checkFile?: string; acceptLoss?: string[] } +): Promise { + const { PGlite } = await import("@electric-sql/pglite"); + const steps: Step[] = []; + const done = (s: Step) => { + steps.push(s); + return s.ok; + }; + let sql: { up: string; down?: string }; + try { + sql = migrationSql(migrationFile, fs.readFileSync(path.resolve(projectDir, migrationFile), "utf8")); + } catch (error) { + done({ name: "read", ok: false, output: (error as Error).message }); + return { ok: false, steps }; + } + const findings = reviewMigration(typeormMigration("Migration", [sql.up], { down: [] })); + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-pglite-")); + // The CLI's own PGlite: the project has no node_modules to load one from. + const cliRoot = path.dirname(path.dirname(cliBin)); + const script = (name: string, code: string, arg?: string) => + run(process.execPath, ["-e", code, ...(arg ? [path.resolve(arg)] : [])], cliRoot, pgliteEnv(dataDir)).then( + (r) => { + done(step(name, r)); + return r; + } + ); + const exec = async (name: string, statements: string) => { + const db = new PGlite(dataDir); + try { + await db.exec(statements); + return done({ name, ok: true }); + } catch (error) { + return done({ name, ok: false, output: (error as Error).message }); + } finally { + await db.close(); + } + }; + const drift = async (name: string, apsorcFile: string) => { + const db = new PGlite(dataDir); + const fresh = new PGlite(); + try { + await fresh.exec(ddlOf(apsorcFile)); + const diff = catalogDiff(await readCatalog(fresh), await readCatalog(db)); + return done({ + name, + ok: diff.length === 0, + output: diff.length > 0 ? `The database differs from ${path.basename(apsorcFile)} (- missing, + extra):\n${diff.join("\n")}` : undefined, + }); + } finally { + await db.close(); + await fresh.close(); + } + }; + try { + if (!(await exec("baseline", ddlOf(baselineApsorc)))) return { ok: false, steps, findings }; + const count = async (name: string, file?: string) => { + const r = await script(name, COUNT_ROWS, file); + return r.ok ? (JSON.parse(r.output.trim().split("\n").pop()!) as Counts) : undefined; + }; + const before = seedFile ? await count("seed", seedFile) : undefined; + if (seedFile && !before) return { ok: false, steps, findings }; + if (!(await exec("migrate", sql.up))) return { ok: false, steps, findings }; + if (before) { + const after = await count("count"); + if (!after) return { ok: false, steps, findings }; + const lost = dataLoss(before, after, sql.up); + const ok = unaccepted(lost, acceptLoss).length === 0; + done({ + name: "data", + ok, + output: lost.length > 0 + ? `The migration lost seeded data:\n${lost.join("\n")}${ok ? "" : "\nFix the migration. If data moved somewhere the check can't follow (e.g. a field split in two), prove it with --check and rerun with --accept-loss ."}` + : undefined, + }); + if (!ok) return { ok: false, steps, findings }; + } + const checked = checkFile ? await script("check", RUN_CHECKS, checkFile) : undefined; + if (checked && !checked.ok) return { ok: false, steps, findings }; + if (!(await drift("drift", path.join(projectDir, ".apsorc")))) return { ok: false, steps, findings }; + if (sql.down === undefined) done({ name: "down", ok: true, output: "down refuses to run (irreversible), so it was not tested." }); + else if (!(await exec("down", sql.down)) || !(await drift("down drift", baselineApsorc))) + return { ok: false, steps, findings }; + return { ok: true, steps, findings }; + } finally { + fs.rmSync(dataDir, { recursive: true, force: true }); + } +} + const freePort = () => new Promise((resolve, reject) => { const s = net.createServer(); diff --git a/test/lib/ddl/__snapshots__/wrappers.test.ts.snap b/test/lib/ddl/__snapshots__/wrappers.test.ts.snap new file mode 100644 index 0000000..ac9fa3e --- /dev/null +++ b/test/lib/ddl/__snapshots__/wrappers.test.ts.snap @@ -0,0 +1,343 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`enum-replace Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TYPE \\"public\\".\\"contact_status_enum\\" RENAME TO \\"contact_status_enum_old\\"") + op.execute("CREATE TYPE \\"public\\".\\"contact_status_enum\\" AS ENUM('lead', 'active', 'closed')") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" DROP DEFAULT") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" TYPE \\"public\\".\\"contact_status_enum\\" USING (CASE \\"status\\"::text WHEN 'archived' THEN 'closed' ELSE \\"status\\"::text END)::\\"public\\".\\"contact_status_enum\\"") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" SET DEFAULT 'lead'") + op.execute("DROP TYPE \\"public\\".\\"contact_status_enum_old\\"") + + +def downgrade() -> None: + op.execute("ALTER TYPE \\"public\\".\\"contact_status_enum\\" RENAME TO \\"contact_status_enum_old\\"") + op.execute("CREATE TYPE \\"public\\".\\"contact_status_enum\\" AS ENUM('lead', 'active', 'archived')") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" DROP DEFAULT") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" TYPE \\"public\\".\\"contact_status_enum\\" USING (CASE \\"status\\"::text WHEN 'closed' THEN 'archived' ELSE \\"status\\"::text END)::\\"public\\".\\"contact_status_enum\\"") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" SET DEFAULT 'lead'") + op.execute("DROP TYPE \\"public\\".\\"contact_status_enum_old\\"") +" +`; + +exports[`enum-replace goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TYPE "public"."contact_status_enum" RENAME TO "contact_status_enum_old"; +CREATE TYPE "public"."contact_status_enum" AS ENUM('lead', 'active', 'closed'); +ALTER TABLE "contact" ALTER COLUMN "status" DROP DEFAULT; +ALTER TABLE "contact" ALTER COLUMN "status" TYPE "public"."contact_status_enum" USING (CASE "status"::text WHEN 'archived' THEN 'closed' ELSE "status"::text END)::"public"."contact_status_enum"; +ALTER TABLE "contact" ALTER COLUMN "status" SET DEFAULT 'lead'; +DROP TYPE "public"."contact_status_enum_old"; + +-- +goose Down +ALTER TYPE "public"."contact_status_enum" RENAME TO "contact_status_enum_old"; +CREATE TYPE "public"."contact_status_enum" AS ENUM('lead', 'active', 'archived'); +ALTER TABLE "contact" ALTER COLUMN "status" DROP DEFAULT; +ALTER TABLE "contact" ALTER COLUMN "status" TYPE "public"."contact_status_enum" USING (CASE "status"::text WHEN 'closed' THEN 'archived' ELSE "status"::text END)::"public"."contact_status_enum"; +ALTER TABLE "contact" ALTER COLUMN "status" SET DEFAULT 'lead'; +DROP TYPE "public"."contact_status_enum_old"; +" +`; + +exports[`extract-entity Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE TYPE \\"public\\".\\"phone_number_label_enum\\" AS ENUM('mobile', 'work', 'home')") + op.execute("CREATE TABLE \\"phone_number\\" (\\"id\\" SERIAL NOT NULL, \\"created_at\\" TIMESTAMP NOT NULL DEFAULT now(), \\"updated_at\\" TIMESTAMP NOT NULL DEFAULT now(), \\"number\\" character varying(40) NOT NULL, \\"label\\" \\"public\\".\\"phone_number_label_enum\\" NOT NULL DEFAULT 'mobile', \\"contactId\\" integer NOT NULL, CONSTRAINT \\"PK_c16f58426537a660b3f2a26e983\\" PRIMARY KEY (\\"id\\"))") + op.execute("INSERT INTO \\"phone_number\\" (\\"number\\", \\"contactId\\") SELECT \\"phone\\", \\"id\\" FROM \\"contact\\" WHERE \\"phone\\" IS NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"phone\\"") + op.execute("CREATE INDEX \\"IDX_069067410ee19d423487a189e4\\" ON \\"phone_number\\" (\\"contactId\\")") + op.execute("ALTER TABLE \\"phone_number\\" ADD CONSTRAINT \\"FK_069067410ee19d423487a189e4d\\" FOREIGN KEY (\\"contactId\\") REFERENCES \\"contact\\"(\\"id\\") ON DELETE CASCADE ON UPDATE NO ACTION") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"phone_number\\" DROP CONSTRAINT \\"FK_069067410ee19d423487a189e4d\\"") + op.execute("ALTER TABLE \\"contact\\" ADD \\"phone\\" character varying(40)") + op.execute("UPDATE \\"contact\\" SET \\"phone\\" = (SELECT \\"number\\" FROM \\"phone_number\\" x WHERE x.\\"contactId\\" = \\"contact\\".\\"id\\" ORDER BY x.\\"id\\" LIMIT 1) WHERE \\"phone\\" IS NULL") + op.execute("DROP TABLE \\"phone_number\\"") + op.execute("DROP TYPE \\"public\\".\\"phone_number_label_enum\\"") +" +`; + +exports[`extract-entity goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +CREATE TYPE "public"."phone_number_label_enum" AS ENUM('mobile', 'work', 'home'); +CREATE TABLE "phone_number" ("id" SERIAL NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), "number" character varying(40) NOT NULL, "label" "public"."phone_number_label_enum" NOT NULL DEFAULT 'mobile', "contactId" integer NOT NULL, CONSTRAINT "PK_c16f58426537a660b3f2a26e983" PRIMARY KEY ("id")); +INSERT INTO "phone_number" ("number", "contactId") SELECT "phone", "id" FROM "contact" WHERE "phone" IS NOT NULL; +ALTER TABLE "contact" DROP COLUMN "phone"; +CREATE INDEX "IDX_069067410ee19d423487a189e4" ON "phone_number" ("contactId"); +ALTER TABLE "phone_number" ADD CONSTRAINT "FK_069067410ee19d423487a189e4d" FOREIGN KEY ("contactId") REFERENCES "contact"("id") ON DELETE CASCADE ON UPDATE NO ACTION; + +-- +goose Down +ALTER TABLE "phone_number" DROP CONSTRAINT "FK_069067410ee19d423487a189e4d"; +ALTER TABLE "contact" ADD "phone" character varying(40); +UPDATE "contact" SET "phone" = (SELECT "number" FROM "phone_number" x WHERE x."contactId" = "contact"."id" ORDER BY x."id" LIMIT 1) WHERE "phone" IS NULL; +DROP TABLE "phone_number"; +DROP TYPE "public"."phone_number_label_enum"; +" +`; + +exports[`many-to-many Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" DROP CONSTRAINT \\"FK_196374b312744cf2c400c962383\\"") + op.execute("DROP INDEX \\"public\\".\\"IDX_196374b312744cf2c400c96238\\"") + op.execute("CREATE TABLE \\"contact_companies_company\\" (\\"contactId\\" integer NOT NULL, \\"companyId\\" integer NOT NULL, CONSTRAINT \\"PK_a240a097bb1e1d698bc213e0c72\\" PRIMARY KEY (\\"contactId\\", \\"companyId\\"))") + op.execute("INSERT INTO \\"contact_companies_company\\" (\\"contactId\\", \\"companyId\\") SELECT \\"id\\", \\"companyId\\" FROM \\"contact\\" WHERE \\"companyId\\" IS NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"companyId\\"") + op.execute("CREATE INDEX \\"IDX_e6e2d85d35645bd6b752226c49\\" ON \\"contact_companies_company\\" (\\"contactId\\")") + op.execute("CREATE INDEX \\"IDX_30d0ef2a176bf879a2d36df096\\" ON \\"contact_companies_company\\" (\\"companyId\\")") + op.execute("ALTER TABLE \\"contact_companies_company\\" ADD CONSTRAINT \\"FK_e6e2d85d35645bd6b752226c49d\\" FOREIGN KEY (\\"contactId\\") REFERENCES \\"contact\\"(\\"id\\") ON DELETE CASCADE ON UPDATE CASCADE") + op.execute("ALTER TABLE \\"contact_companies_company\\" ADD CONSTRAINT \\"FK_30d0ef2a176bf879a2d36df0966\\" FOREIGN KEY (\\"companyId\\") REFERENCES \\"company\\"(\\"id\\") ON DELETE NO ACTION ON UPDATE NO ACTION") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact_companies_company\\" DROP CONSTRAINT \\"FK_e6e2d85d35645bd6b752226c49d\\"") + op.execute("ALTER TABLE \\"contact_companies_company\\" DROP CONSTRAINT \\"FK_30d0ef2a176bf879a2d36df0966\\"") + op.execute("ALTER TABLE \\"contact\\" ADD \\"companyId\\" integer") + op.execute("UPDATE \\"contact\\" SET \\"companyId\\" = (SELECT j.\\"companyId\\" FROM \\"contact_companies_company\\" j WHERE j.\\"contactId\\" = \\"contact\\".\\"id\\" LIMIT 1) WHERE \\"companyId\\" IS NULL") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"companyId\\" SET NOT NULL") + op.execute("DROP TABLE \\"contact_companies_company\\"") + op.execute("CREATE INDEX \\"IDX_196374b312744cf2c400c96238\\" ON \\"contact\\" (\\"companyId\\")") + op.execute("ALTER TABLE \\"contact\\" ADD CONSTRAINT \\"FK_196374b312744cf2c400c962383\\" FOREIGN KEY (\\"companyId\\") REFERENCES \\"company\\"(\\"id\\") ON DELETE NO ACTION ON UPDATE NO ACTION") +" +`; + +exports[`many-to-many goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" DROP CONSTRAINT "FK_196374b312744cf2c400c962383"; +DROP INDEX "public"."IDX_196374b312744cf2c400c96238"; +CREATE TABLE "contact_companies_company" ("contactId" integer NOT NULL, "companyId" integer NOT NULL, CONSTRAINT "PK_a240a097bb1e1d698bc213e0c72" PRIMARY KEY ("contactId", "companyId")); +INSERT INTO "contact_companies_company" ("contactId", "companyId") SELECT "id", "companyId" FROM "contact" WHERE "companyId" IS NOT NULL; +ALTER TABLE "contact" DROP COLUMN "companyId"; +CREATE INDEX "IDX_e6e2d85d35645bd6b752226c49" ON "contact_companies_company" ("contactId"); +CREATE INDEX "IDX_30d0ef2a176bf879a2d36df096" ON "contact_companies_company" ("companyId"); +ALTER TABLE "contact_companies_company" ADD CONSTRAINT "FK_e6e2d85d35645bd6b752226c49d" FOREIGN KEY ("contactId") REFERENCES "contact"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "contact_companies_company" ADD CONSTRAINT "FK_30d0ef2a176bf879a2d36df0966" FOREIGN KEY ("companyId") REFERENCES "company"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- +goose Down +ALTER TABLE "contact_companies_company" DROP CONSTRAINT "FK_e6e2d85d35645bd6b752226c49d"; +ALTER TABLE "contact_companies_company" DROP CONSTRAINT "FK_30d0ef2a176bf879a2d36df0966"; +ALTER TABLE "contact" ADD "companyId" integer; +UPDATE "contact" SET "companyId" = (SELECT j."companyId" FROM "contact_companies_company" j WHERE j."contactId" = "contact"."id" LIMIT 1) WHERE "companyId" IS NULL; +ALTER TABLE "contact" ALTER COLUMN "companyId" SET NOT NULL; +DROP TABLE "contact_companies_company"; +CREATE INDEX "IDX_196374b312744cf2c400c96238" ON "contact" ("companyId"); +ALTER TABLE "contact" ADD CONSTRAINT "FK_196374b312744cf2c400c962383" FOREIGN KEY ("companyId") REFERENCES "company"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; +" +`; + +exports[`rename-entity Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"companyId\\" TO \\"organizationId\\"") + op.execute("ALTER TABLE \\"company\\" RENAME TO \\"organization\\"") + op.execute("ALTER SEQUENCE \\"company_id_seq\\" RENAME TO \\"organization_id_seq\\"") + op.execute("ALTER TABLE \\"organization\\" RENAME CONSTRAINT \\"PK_056f7854a7afdba7cbd6d45fc20\\" TO \\"PK_472c1f99a32def1b0abb219cd67\\"") + op.execute("ALTER TABLE \\"contact\\" RENAME CONSTRAINT \\"FK_196374b312744cf2c400c962383\\" TO \\"FK_7719d73cd16a9f57ecc6ac24b3d\\"") + op.execute("ALTER INDEX \\"public\\".\\"IDX_196374b312744cf2c400c96238\\" RENAME TO \\"IDX_7719d73cd16a9f57ecc6ac24b3\\"") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"organizationId\\" TO \\"companyId\\"") + op.execute("ALTER TABLE \\"organization\\" RENAME TO \\"company\\"") + op.execute("ALTER SEQUENCE \\"organization_id_seq\\" RENAME TO \\"company_id_seq\\"") + op.execute("ALTER TABLE \\"company\\" RENAME CONSTRAINT \\"PK_472c1f99a32def1b0abb219cd67\\" TO \\"PK_056f7854a7afdba7cbd6d45fc20\\"") + op.execute("ALTER TABLE \\"contact\\" RENAME CONSTRAINT \\"FK_7719d73cd16a9f57ecc6ac24b3d\\" TO \\"FK_196374b312744cf2c400c962383\\"") + op.execute("ALTER INDEX \\"public\\".\\"IDX_7719d73cd16a9f57ecc6ac24b3\\" RENAME TO \\"IDX_196374b312744cf2c400c96238\\"") +" +`; + +exports[`rename-entity goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" RENAME COLUMN "companyId" TO "organizationId"; +ALTER TABLE "company" RENAME TO "organization"; +ALTER SEQUENCE "company_id_seq" RENAME TO "organization_id_seq"; +ALTER TABLE "organization" RENAME CONSTRAINT "PK_056f7854a7afdba7cbd6d45fc20" TO "PK_472c1f99a32def1b0abb219cd67"; +ALTER TABLE "contact" RENAME CONSTRAINT "FK_196374b312744cf2c400c962383" TO "FK_7719d73cd16a9f57ecc6ac24b3d"; +ALTER INDEX "public"."IDX_196374b312744cf2c400c96238" RENAME TO "IDX_7719d73cd16a9f57ecc6ac24b3"; + +-- +goose Down +ALTER TABLE "contact" RENAME COLUMN "organizationId" TO "companyId"; +ALTER TABLE "organization" RENAME TO "company"; +ALTER SEQUENCE "organization_id_seq" RENAME TO "company_id_seq"; +ALTER TABLE "company" RENAME CONSTRAINT "PK_472c1f99a32def1b0abb219cd67" TO "PK_056f7854a7afdba7cbd6d45fc20"; +ALTER TABLE "contact" RENAME CONSTRAINT "FK_7719d73cd16a9f57ecc6ac24b3d" TO "FK_196374b312744cf2c400c962383"; +ALTER INDEX "public"."IDX_7719d73cd16a9f57ecc6ac24b3" RENAME TO "IDX_196374b312744cf2c400c96238"; +" +`; + +exports[`rename-field Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"phone\\" TO \\"mobilePhone\\"") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"mobilePhone\\" TO \\"phone\\"") +" +`; + +exports[`rename-field goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" RENAME COLUMN "phone" TO "mobilePhone"; + +-- +goose Down +ALTER TABLE "contact" RENAME COLUMN "mobilePhone" TO "phone"; +" +`; + +exports[`required-field Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE TYPE \\"public\\".\\"company_region_enum\\" AS ENUM('us', 'eu', 'apac')") + op.execute("ALTER TABLE \\"company\\" ADD \\"region\\" \\"public\\".\\"company_region_enum\\" NOT NULL DEFAULT 'us'") + op.execute("UPDATE \\"company\\" SET \\"region\\" = 'us' WHERE \\"region\\" IS NULL") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"company\\" DROP COLUMN \\"region\\"") + op.execute("DROP TYPE \\"public\\".\\"company_region_enum\\"") +" +`; + +exports[`required-field goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +CREATE TYPE "public"."company_region_enum" AS ENUM('us', 'eu', 'apac'); +ALTER TABLE "company" ADD "region" "public"."company_region_enum" NOT NULL DEFAULT 'us'; +UPDATE "company" SET "region" = 'us' WHERE "region" IS NULL; + +-- +goose Down +ALTER TABLE "company" DROP COLUMN "region"; +DROP TYPE "public"."company_region_enum"; +" +`; + +exports[`split-field Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ADD \\"firstName\\" text") + op.execute("ALTER TABLE \\"contact\\" ADD \\"lastName\\" text") + op.execute("UPDATE \\"contact\\" SET \\"firstName\\" = (split_part(btrim(\\"fullName\\"), ' ', 1)), \\"lastName\\" = (NULLIF(btrim(substr(btrim(\\"fullName\\"), length(split_part(btrim(\\"fullName\\"), ' ', 1)) + 1)), ''))") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"firstName\\" SET NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"fullName\\"") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ADD \\"fullName\\" text") + op.execute("UPDATE \\"contact\\" SET \\"fullName\\" = (concat_ws(' ', \\"firstName\\", \\"lastName\\")) WHERE \\"fullName\\" IS NULL") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"fullName\\" SET NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"firstName\\"") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"lastName\\"") +" +`; + +exports[`split-field goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" ADD "firstName" text; +ALTER TABLE "contact" ADD "lastName" text; +UPDATE "contact" SET "firstName" = (split_part(btrim("fullName"), ' ', 1)), "lastName" = (NULLIF(btrim(substr(btrim("fullName"), length(split_part(btrim("fullName"), ' ', 1)) + 1)), '')); +ALTER TABLE "contact" ALTER COLUMN "firstName" SET NOT NULL; +ALTER TABLE "contact" DROP COLUMN "fullName"; + +-- +goose Down +ALTER TABLE "contact" ADD "fullName" text; +UPDATE "contact" SET "fullName" = (concat_ws(' ', "firstName", "lastName")) WHERE "fullName" IS NULL; +ALTER TABLE "contact" ALTER COLUMN "fullName" SET NOT NULL; +ALTER TABLE "contact" DROP COLUMN "firstName"; +ALTER TABLE "contact" DROP COLUMN "lastName"; +" +`; + +exports[`type-change Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"score\\" TYPE integer USING (NULLIF(regexp_replace(\\"score\\", '[^0-9-]', '', 'g'), '')::integer)") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"score\\" TYPE text USING \\"score\\"::text") +" +`; + +exports[`type-change goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" ALTER COLUMN "score" TYPE integer USING (NULLIF(regexp_replace("score", '[^0-9-]', '', 'g'), '')::integer); + +-- +goose Down +ALTER TABLE "contact" ALTER COLUMN "score" TYPE text USING "score"::text; +" +`; diff --git a/test/lib/ddl/answers.ts b/test/lib/ddl/answers.ts new file mode 100644 index 0000000..0894308 --- /dev/null +++ b/test/lib/ddl/answers.ts @@ -0,0 +1,49 @@ +import { Plan, planMigration } from "../../../src/lib/plan"; + +// Each migration eval case, answered the way the eval expects. The change +// must leave the seeded database identical in schema to a fresh build of the +// new .apsorc, with every row check in checks.sql passing. +export const answers: Record< + string, + Record< + string, + string | { answer: string; sql: string | Record } + > +> = { + "rename-entity": { "entity:Company": "rename:Organization" }, + "rename-field": { "field:Contact.phone": "rename:mobilePhone" }, + "split-field": { + "field:Contact.fullName": { + answer: "split:firstName,lastName", + sql: { + firstName: `split_part(btrim("fullName"), ' ', 1)`, + lastName: `NULLIF(btrim(substr(btrim("fullName"), length(split_part(btrim("fullName"), ' ', 1)) + 1)), '')`, + }, + }, + }, + "extract-entity": { "field:Contact.phone": "move:PhoneNumber.number" }, + "many-to-many": { "relation:Contact.Company": "copy-links" }, + "enum-replace": { "enum:Contact.status:archived": "closed" }, + "required-field": { "backfill:Company.region": "us" }, + "type-change": { + "cast:Contact.score": { + answer: "sql", + sql: `NULLIF(regexp_replace("score", '[^0-9-]', '', 'g'), '')::integer`, + }, + }, +}; + +export const answeredPlan = (c: string, before: any, after: any): Plan => { + let plan = planMigration(before, after); + for (let round = 0; round < 3; round++) { + for (const q of plan.questions) { + const a = answers[c][q.id]; + if (a === undefined) continue; + if (typeof a === "string") q.answer = a; + else Object.assign(q, a); + } + plan = planMigration(before, after, plan); + } + return plan; +}; + diff --git a/test/lib/ddl/change.test.ts b/test/lib/ddl/change.test.ts index d096a26..24bc39a 100644 --- a/test/lib/ddl/change.test.ts +++ b/test/lib/ddl/change.test.ts @@ -5,59 +5,13 @@ import { parseApsorcV2 } from "../../../src/lib/apsorc-parser"; import { schemaDdl, schemaModel } from "../../../src/lib/ddl"; import { changeDdl, downDdl } from "../../../src/lib/ddl/change"; import { catalogDiff, readCatalog } from "../../../src/lib/ddl/catalog"; -import { openQuestions, planMigration, Plan } from "../../../src/lib/plan"; - -// Each migration eval case, answered the way the eval expects. The change -// must leave the seeded database identical in schema to a fresh build of the -// new .apsorc, with every row check in checks.sql passing. -const answers: Record< - string, - Record< - string, - string | { answer: string; sql: string | Record } - > -> = { - "rename-entity": { "entity:Company": "rename:Organization" }, - "rename-field": { "field:Contact.phone": "rename:mobilePhone" }, - "split-field": { - "field:Contact.fullName": { - answer: "split:firstName,lastName", - sql: { - firstName: `split_part(btrim("fullName"), ' ', 1)`, - lastName: `NULLIF(btrim(substr(btrim("fullName"), length(split_part(btrim("fullName"), ' ', 1)) + 1)), '')`, - }, - }, - }, - "extract-entity": { "field:Contact.phone": "move:PhoneNumber.number" }, - "many-to-many": { "relation:Contact.Company": "copy-links" }, - "enum-replace": { "enum:Contact.status:archived": "closed" }, - "required-field": { "backfill:Company.region": "us" }, - "type-change": { - "cast:Contact.score": { - answer: "sql", - sql: `NULLIF(regexp_replace("score", '[^0-9-]', '', 'g'), '')::integer`, - }, - }, -}; +import { openQuestions, planMigration } from "../../../src/lib/plan"; +import { answeredPlan, answers } from "./answers"; const dir = (c: string) => path.join(__dirname, "../../fixtures/plan", c); const read = (c: string, f: string) => fs.readFileSync(path.join(dir(c), f)).toString(); -const answeredPlan = (c: string, before: any, after: any): Plan => { - let plan = planMigration(before, after); - for (let round = 0; round < 3; round++) { - for (const q of plan.questions) { - const a = answers[c][q.id]; - if (a === undefined) continue; - if (typeof a === "string") q.answer = a; - else Object.assign(q, a); - } - plan = planMigration(before, after, plan); - } - return plan; -}; - const pglite = async () => { const { PGlite } = await import("@electric-sql/pglite"); return new PGlite(); diff --git a/test/lib/ddl/wrappers.test.ts b/test/lib/ddl/wrappers.test.ts new file mode 100644 index 0000000..d42619e --- /dev/null +++ b/test/lib/ddl/wrappers.test.ts @@ -0,0 +1,206 @@ +import { expect, describe, test } from "@jest/globals"; +import { execFileSync } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { changeDdl, downDdl } from "../../../src/lib/ddl/change"; +import { + alembicHead, + alembicMigration, + gooseMigration, + migrationSql, + writePlannedMigration, +} from "../../../src/lib/ddl/migration-file"; +import { PythonGenerator } from "../../../src/lib/generators/python"; +import { verifyMigration } from "../../../src/lib/verify"; +import { answeredPlan, answers } from "./answers"; + +const fixture = (c: string, f: string) => + path.join(__dirname, "../../fixtures/plan", c, f); +const read = (c: string, f: string) => + JSON.parse(fs.readFileSync(fixture(c, f)).toString()); +const cliBin = path.join(__dirname, "../../../bin/run"); + +let python = true; +try { + execFileSync("python3", ["--version"], { stdio: "ignore" }); +} catch { + python = false; +} +/** Python accepts the file. Without python3, every SQL literal must round-trip. */ +const compiles = (source: string) => { + if (!python) { + for (const m of source.matchAll(/op\.execute\((.*)\)$/gm)) + expect(typeof JSON.parse(m[1])).toBe("string"); + return; + } + const file = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "apso-py-")), + "m.py" + ); + fs.writeFileSync(file, source); + execFileSync("python3", ["-m", "py_compile", file]); +}; + +describe.each(Object.keys(answers))("%s", (c) => { + const before = read(c, "before.apsorc"); + const after = read(c, "after.apsorc"); + const plan = answeredPlan(c, before, after); + const { up } = changeDdl(before, after, plan); + const down = downDdl(before, after, plan); + + test("Alembic revision compiles and gives back its SQL", () => { + const source = alembicMigration("20231114221320", "20231101000000", up, down); + expect(source).toMatchSnapshot(); + compiles(source); + const sql = migrationSql("x.py", source); + expect(sql.up).toBe(up.join(";\n")); + expect(sql.down).toBe((down as { down: string[] }).down.join(";\n")); + }); + + test("goose file matches its snapshot", () => { + expect(gooseMigration(up, down)).toMatchSnapshot(); + }); +}); + +describe("wrappers", () => { + test("SQL ending in a quoted identifier stays valid Python (#148)", async () => { + const [file] = await new PythonGenerator({ + language: "python", + } as any).generateMigration({ + entities: [], + upSql: ['DROP TABLE "user"'], + downSql: ['CREATE TABLE "user" ("id" integer, "note" text DEFAULT \'a\\b """\')'], + } as any); + compiles(file.content); + expect(file.content).toContain('op.execute("DROP TABLE \\"user\\"")'); + }); + + test("an irreversible down refuses to run", () => { + const irreversible = { irreversible: ["backfill:Contact.fullName: 100%"] }; + const py = alembicMigration("2", undefined, ["SELECT 1"], irreversible); + compiles(py); + expect(py).toContain("raise NotImplementedError("); + expect(py).toContain("down_revision = None"); + expect(migrationSql("x.py", py).down).toBeUndefined(); + const sql = gooseMigration(["SELECT 1"], irreversible); + expect(sql).toContain("-- +goose StatementBegin\nDO $$ BEGIN RAISE EXCEPTION"); + expect(sql).toContain("100%%"); + expect(migrationSql("x.sql", sql)).toEqual({ up: "\nSELECT 1;\n\n", down: undefined }); + }); + + test("a statement with its own semicolon is fenced for goose", () => { + expect(gooseMigration(["SELECT ';'", "SELECT 2"], { down: [] })).toContain( + "-- +goose StatementBegin\nSELECT ';';\n-- +goose StatementEnd\nSELECT 2;" + ); + }); + + test("the Alembic head is the revision nothing revises", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-alembic-")); + expect(alembicHead(dir)).toBeUndefined(); + const rev = (f: string, r: string, d: string) => + fs.writeFileSync( + path.join(dir, f), + `revision: str = '${r}'\ndown_revision: Union[str, None] = ${d}\n` + ); + rev("a_init.py", "aaa", "None"); + rev("c_third.py", "ccc", "'bbb'"); + rev("b_second.py", "bbb", "'aaa'"); + expect(alembicHead(dir)).toBe("ccc"); + }); +}); + +describe.each(["python", "go"])("apso migrate for %s", (language) => { + const project = (c: string) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `apso-${language}-`)); + const after = { ...read(c, "after.apsorc"), language }; + fs.writeFileSync(path.join(dir, ".apsorc"), JSON.stringify(after)); + const plan = answeredPlan(c, read(c, "before.apsorc"), after); + const planPath = path.join(dir, ".apso", "migration-plan.json"); + fs.mkdirSync(path.dirname(planPath)); + fs.writeFileSync(planPath, JSON.stringify(plan)); + return { dir, planPath }; + }; + + test("writes the migration where the template keeps them", () => { + const { dir, planPath } = project("rename-field"); + const result = writePlannedMigration( + dir, + read("rename-field", "before.apsorc"), + "SchemaUpdate", + planPath, + 1_700_000_000_000 + ); + expect(result.file).toBe( + language === "python" + ? "alembic/versions/20231114221320_schema_update.py" + : "migrations/20231114221320_schema_update.sql" + ); + expect(result.planFile).toBe( + result.file!.replace(/\.(py|sql)$/, ".plan.json") + ); + expect(result.findings).toEqual([]); + }); + + test.each([ + ["rename-field", []], + ["split-field", ["contact.fullName"]], + ["extract-entity", []], + ])( + "%s verifies with its seed and checks", + async (c, acceptLoss) => { + const { dir, planPath } = project(c); + const written = writePlannedMigration( + dir, + read(c, "before.apsorc"), + "SchemaUpdate", + planPath + ); + const result = await verifyMigration( + dir, + fixture(c, "before.apsorc"), + written.file!, + cliBin, + { + seedFile: fixture(c, "seed.sql"), + checkFile: fixture(c, "checks.sql"), + acceptLoss: acceptLoss as string[], + } + ); + expect(result.steps).toEqual( + ["baseline", "seed", "migrate", "count", "data", "check", "drift", "down", "down drift"].map( + (name) => expect.objectContaining({ name, ok: true }) + ) + ); + expect(result.ok).toBe(true); + }, + 60_000 + ); + + test("drift fails when the migration leaves the schema short", async () => { + const { dir, planPath } = project("rename-field"); + const written = writePlannedMigration( + dir, + read("rename-field", "before.apsorc"), + "SchemaUpdate", + planPath + ); + const file = path.join(dir, written.file!); + // Drop the rename: the deployed column stays "phone". + fs.writeFileSync( + file, + fs + .readFileSync(file) + .toString() + .replace(/^.*RENAME COLUMN \\?"phone\\?" TO .*$/m, "") + ); + const result = await verifyMigration( + dir, + fixture("rename-field", "before.apsorc"), + written.file!, + cliBin + ); + expect(result.ok).toBe(false); + expect(result.steps[result.steps.length - 1]).toMatchObject({ name: "drift", ok: false }); + }, 60_000); +});