From a10423def50cf776997383bf31f51b9569f79a65 Mon Sep 17 00:00:00 2001 From: cultron Date: Wed, 28 Oct 2026 15:01:16 -0500 Subject: [PATCH] feat(migrate): Alembic and goose output for planned migrations; fix Python quoting (#148) apso migrate and apso migrate generate now write the planned migration in the service's own wrapper: a TypeORM class (TypeScript), an Alembic revision in alembic/versions/ (Python, down_revision chained to the current head), or a goose SQL file in migrations/ (Go, YYYYMMDDHHMMSS_.sql so it sorts with goose's own files). The plan JSON is saved beside it as before, and latestPlannedMigration finds it in each language's folder. Verify for Python and Go runs without TypeORM: Apso's DDL writer builds the deployed schema in PGlite, the seed loads, the migration's SQL runs as written, the data-loss and --check steps run as for TypeScript, and drift is a catalogDiff against a fresh build of the new .apsorc. down then runs and must give back the deployed schema. TypeScript verify is unchanged. migrateUnsupported is gone; --typeorm still needs a TypeScript service. literal) instead of """...""", which broke on SQL ending in a quoted identifier. The old sandbox generators get the same fix, and goose output fences any statement with its own ";" in StatementBegin/End. Tests: Alembic and goose snapshots for all 8 plan fixtures, py_compile of each Alembic file, and verify passing for python and go on rename-field, split-field and extract-entity with their seed and checks. --- src/commands/migrate.ts | 7 +- src/commands/migrate/generate.ts | 10 +- src/commands/migrate/verify.ts | 7 +- src/lib/ddl/migration-file.ts | 161 +++++++- src/lib/generators/go.ts | 5 +- src/lib/generators/python.ts | 5 +- src/lib/plan/baseline.ts | 19 +- src/lib/verify/index.ts | 132 ++++++- .../ddl/__snapshots__/wrappers.test.ts.snap | 343 ++++++++++++++++++ test/lib/ddl/answers.ts | 49 +++ test/lib/ddl/change.test.ts | 50 +-- test/lib/ddl/wrappers.test.ts | 206 +++++++++++ 12 files changed, 891 insertions(+), 103 deletions(-) create mode 100644 test/lib/ddl/__snapshots__/wrappers.test.ts.snap create mode 100644 test/lib/ddl/answers.ts create mode 100644 test/lib/ddl/wrappers.test.ts diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts index e047bf8..5e2a01d 100644 --- a/src/commands/migrate.ts +++ b/src/commands/migrate.ts @@ -10,7 +10,6 @@ import { Baseline, baselineFile, resolveBaseline } from "../lib/plan/baseline"; import { describeQuestion, optionLabel, splitDefaults, sqlHint } from "../lib/plan/prompts"; import { cliBinPath, - migrateUnsupported, report, run, supportsPglite, @@ -94,8 +93,6 @@ export default class Migrate extends BaseCommand { if (this.json) console.info = (...m: unknown[]) => process.stderr.write(m.join(" ") + "\n"); const { apsorc } = readApsorcFile(); - const unsupported = migrateUnsupported(apsorc.language); - if (unsupported) this.fail(unsupported); const cliBin = cliBinPath(this.config.root); if (!flags["skip-generate"]) { @@ -163,7 +160,9 @@ export default class Migrate extends BaseCommand { for (const f of written.findings ?? []) this.say(` warning ${f.code}: ${f.message}`); let verified: boolean | "skipped" = "skipped"; - if (!flags["skip-verify"] && supportsPglite(cwd)) { + // Python and Go verify without the project's own database driver. + const canVerify = (apsorc.language ?? "typescript") !== "typescript" || supportsPglite(cwd); + if (!flags["skip-verify"] && canVerify) { this.say("Testing the migration on a local copy of the deployed schema..."); const result = await verifyMigration(cwd, baselineFile(baseline), written.file!, cliBin, { seedFile: flags.seed, diff --git a/src/commands/migrate/generate.ts b/src/commands/migrate/generate.ts index 4d65968..5f20036 100644 --- a/src/commands/migrate/generate.ts +++ b/src/commands/migrate/generate.ts @@ -1,7 +1,7 @@ import * as path from "path"; import { Flags } from "@oclif/core"; import BaseCommand from "../../lib/base-command"; -import { parseApsorc, readApsorcFile } from "../../lib/apsorc-parser"; +import { readApsorcFile } from "../../lib/apsorc-parser"; import { writePlannedMigration } from "../../lib/ddl/migration-file"; import { Question } from "../../lib/plan"; import { @@ -11,14 +11,13 @@ import { } from "../../lib/plan/baseline"; import { cliBinPath, - migrateUnsupported, generateMigration, report, } from "../../lib/verify"; export default class MigrateGenerate extends BaseCommand { static description = - "Write a migration from the deployed schema to this project's .apsorc. Apso renders the SQL from the answered migration plan (apso migrate plan), including data moves and a down() that reverses them. Exits 2 while plan questions are open. --typeorm uses TypeORM's generator on a local PGlite database instead (TypeScript services)."; + "Write a migration from the deployed schema to this project's .apsorc. Apso renders the SQL from the answered migration plan (apso migrate plan), including data moves and a down() that reverses them. Exits 2 while plan questions are open. The migration is a TypeORM class, an Alembic revision or a goose SQL file, by the service's language. --typeorm uses TypeORM's generator on a local PGlite database instead (TypeScript services)."; static examples = [ `$ apso migrate plan --answer entity:Company=rename:Organization`, @@ -52,10 +51,9 @@ export default class MigrateGenerate extends BaseCommand { async run(): Promise { const { flags } = await this.parse(MigrateGenerate); - const unsupported = migrateUnsupported(parseApsorc().language); - if (unsupported) this.error(unsupported); - const { apsorc } = readApsorcFile(); + if (flags.typeorm && (apsorc.language ?? "typescript") !== "typescript") + this.error("--typeorm works on TypeScript services only."); let baseline: Baseline; try { baseline = resolveBaseline( diff --git a/src/commands/migrate/verify.ts b/src/commands/migrate/verify.ts index 7fbb3e0..0e712fb 100644 --- a/src/commands/migrate/verify.ts +++ b/src/commands/migrate/verify.ts @@ -1,18 +1,17 @@ import * as path from "path"; import { Flags } from "@oclif/core"; import BaseCommand from "../../lib/base-command"; -import { parseApsorc, readApsorcFile } from "../../lib/apsorc-parser"; +import { readApsorcFile } from "../../lib/apsorc-parser"; import { baselineFile, latestPlannedMigration } from "../../lib/plan/baseline"; import { cliBinPath, - migrateUnsupported, report, verifyMigration, } from "../../lib/verify"; export default class MigrateVerify extends BaseCommand { static description = - "Run a migration on a local PGlite copy of the deployed schema, then check the database matches this project's entities (TypeScript services)"; + "Run a migration on a local PGlite copy of the deployed schema, then check the database matches this project's entities (TypeScript), or a fresh build of its .apsorc (Python and Go, which also run down and check it gives back the deployed schema)"; static examples = [ `$ apso migrate verify`, @@ -56,8 +55,6 @@ export default class MigrateVerify extends BaseCommand { async run(): Promise { const { flags } = await this.parse(MigrateVerify); - const unsupported = migrateUnsupported(parseApsorc().language); - if (unsupported) this.error(unsupported); const { apsorc } = readApsorcFile(); const latest = latestPlannedMigration(process.cwd(), apsorc); const migration = flags.migration ?? latest?.file; diff --git a/src/lib/ddl/migration-file.ts b/src/lib/ddl/migration-file.ts index 1404b44..015fcc8 100644 --- a/src/lib/ddl/migration-file.ts +++ b/src/lib/ddl/migration-file.ts @@ -1,15 +1,18 @@ /** - * Write a TypeORM migration from an answered plan: up() is Apso's change DDL, - * down() the reverse, and the plan is saved beside the migration so the - * answers that produced it stay with it. + * Write a migration from an answered plan: up is Apso's change DDL, down the + * reverse, and the plan is saved beside the migration so the answers that + * produced it stay with it. The SQL is the same for every language; only the + * wrapper differs: a TypeORM class, an Alembic revision, or a goose SQL file. */ import * as fs from "fs"; import * as path from "path"; import { ApsorcType } from "../apsorc-parser"; import { openQuestions, Plan, planMigration, Question } from "../plan"; -import { pascalCase } from "../utils/casing"; +import { migrationExtension, migrationsDir } from "../plan/baseline"; +import { pascalCase, snakeCase } from "../utils/casing"; import { reviewMigration, Finding } from "../verify/review"; import { changeDdl, DownDdl, downDdl } from "./change"; +import { literal } from "."; const template = (sql: string) => "`" + @@ -28,11 +31,7 @@ export function typeormMigration( const downBody = "down" in down ? queries(down.down) - : ` throw new Error(${JSON.stringify( - `This migration can't be reversed automatically: ${down.irreversible.join( - "; " - )}` - )});`; + : ` throw new Error(${JSON.stringify(irreversible(down))});`; return `import { MigrationInterface, QueryRunner } from "typeorm"; // Generated by apso migrate generate from the answered migration plan saved beside this file. @@ -51,6 +50,125 @@ ${downBody} `; } +const irreversible = (down: { irreversible: string[] }) => + `This migration can't be reversed automatically: ${down.irreversible.join( + "; " + )}`; + +/** + * A Python string literal for any SQL. A JSON string is one (same escapes), + * where """...""" breaks on SQL that ends in a quoted identifier (#148). + */ +export const pyString = (s: string): string => JSON.stringify(s); + +const executes = (statements: string[]) => + statements.map((s) => ` op.execute(${pyString(s)})`).join("\n"); + +export function alembicMigration( + revision: string, + downRevision: string | undefined, + up: string[], + down: DownDdl +): string { + const downBody = + "down" in down + ? executes(down.down) || " pass" + : ` raise NotImplementedError(${pyString(irreversible(down))})`; + return `"""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = ${pyString(revision)} +down_revision = ${downRevision === undefined ? "None" : pyString(downRevision)} +branch_labels = None +depends_on = None + + +def upgrade() -> None: +${executes(up)} + + +def downgrade() -> None: +${downBody} +`; +} + +/** + * Statements for a goose SQL file. goose splits on lines ending in ";", so a + * statement with a ";" of its own is fenced with StatementBegin/End. + */ +export const gooseStatements = (statements: string[]): string => + statements + .map((s) => + s.includes(";") + ? `-- +goose StatementBegin\n${s};\n-- +goose StatementEnd` + : `${s};` + ) + .join("\n"); + +export function gooseMigration(up: string[], down: DownDdl): string { + // goose has no way to refuse a down, so the SQL refuses it instead. + const downBody = + "down" in down + ? gooseStatements(down.down) + : gooseStatements([ + `DO $$ BEGIN RAISE EXCEPTION ${literal( + irreversible(down).replace(/%/g, "%%") + )}; END $$`, + ]); + return `-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +${gooseStatements(up)} + +-- +goose Down +${downBody} +`; +} + +/** The newest Alembic revision: the one no other revision names as its down_revision. */ +export function alembicHead(dir: string): string | undefined { + if (!fs.existsSync(dir)) return undefined; + const revisions: string[] = []; + const downs = new Set(); + for (const f of fs + .readdirSync(dir) + .filter((f) => f.endsWith(".py")) + .sort()) { + const source = fs.readFileSync(path.join(dir, f)).toString(); + const rev = source.match(/^revision\b[^\n=]*=\s*["']([^"']+)["']/m); + if (rev) revisions.push(rev[1]); + const down = source.match(/^down_revision\b[^\n=]*=(.*)$/m); + for (const d of down?.[1].matchAll(/["']([^"']+)["']/g) ?? []) + downs.add(d[1]); + } + return revisions.filter((r) => !downs.has(r)).pop(); +} + +const calls = (body: string) => + [...body.matchAll(/^\s*op\.execute\((".*")\)\s*$/gm)] + .map((m) => JSON.parse(m[1]) as string) + .join(";\n"); + +/** + * The up and down SQL of an Alembic or goose migration, for running it on a + * database without the framework. down is undefined when it refuses to run. + */ +export function migrationSql( + file: string, + source: string +): { up: string; down?: string } { + if (file.endsWith(".sql")) { + const [up, down = ""] = source + .split(/^-- \+goose Down\b.*$/m) + .map((s) => s.replace(/^[\S\s]*?^-- \+goose Up\b.*$/m, "")); + return { up, down: /RAISE EXCEPTION/.test(down) ? undefined : down }; + } + const [upBody, downBody = ""] = source.split(/^def downgrade\b/m); + return { + up: calls(upBody), + down: /raise NotImplementedError/.test(downBody) ? undefined : calls(downBody), + }; +} + export interface PlannedMigration { ok: boolean; needed: boolean; @@ -123,11 +241,23 @@ export function writePlannedMigration( return { ok: true, needed: false, steps: [{ name: "render", ok: true }] }; const className = `${pascalCase(name)}Migration${now}`; - const dir = path.join(projectDir, after.rootFolder || "src", "migrations"); + const dir = migrationsDir(projectDir, after); fs.mkdirSync(dir, { recursive: true }); - const base = `${now}-${name}-migration`; - const source = typeormMigration(className, up, down); - fs.writeFileSync(path.join(dir, `${base}.ts`), source); + // goose orders by the number before "_", and its own files use + // YYYYMMDDHHMMSS, so Python and Go use that too. + const stamp = new Date(now).toISOString().replace(/\D/g, "").slice(0, 14); + const base = + after.language === "python" || after.language === "go" + ? `${stamp}_${snakeCase(name)}` + : `${now}-${name}-migration`; + const file = base + migrationExtension(after); + const source = + after.language === "python" + ? alembicMigration(stamp, alembicHead(dir), up, down) + : after.language === "go" + ? gooseMigration(up, down) + : typeormMigration(className, up, down); + fs.writeFileSync(path.join(dir, file), source); fs.writeFileSync( path.join(dir, `${base}.plan.json`), // The schemas let later runs find the baseline without being told. @@ -139,9 +269,10 @@ export function writePlannedMigration( return { ok: true, needed: true, - file: rel(`${base}.ts`), + file: rel(file), planFile: rel(`${base}.plan.json`), - findings: reviewMigration(source), + // The review reads TypeORM's wrapper; the SQL is the same in all three. + findings: reviewMigration(typeormMigration(className, up, down)), steps: [{ name: "render", ok: true }], }; } diff --git a/src/lib/generators/go.ts b/src/lib/generators/go.ts index 20d32f4..b2c556e 100644 --- a/src/lib/generators/go.ts +++ b/src/lib/generators/go.ts @@ -22,6 +22,7 @@ import { } from "../utils/relationships"; import { snakeCase, camelCase, pascalCase, kebabCase } from "../utils/casing"; import { getEventEmittingEntities } from "../events"; +import { gooseStatements } from "../ddl/migration-file"; import pluralize from "pluralize"; /** @@ -392,8 +393,8 @@ export class GoGenerator extends BaseGenerator { const name = migrationName || "auto"; const filename = `${timestamp}_${name}.sql`; - const upStatements = upSql.map((sql) => `${sql};`).join("\n"); - const downStatements = (downSql || []).map((sql) => `${sql};`).join("\n"); + const upStatements = gooseStatements(upSql); + const downStatements = gooseStatements(downSql || []); const content = `-- +goose Up ${upStatements} diff --git a/src/lib/generators/python.ts b/src/lib/generators/python.ts index 2f92a9e..192f585 100644 --- a/src/lib/generators/python.ts +++ b/src/lib/generators/python.ts @@ -22,6 +22,7 @@ import { } from "../utils/relationships"; import { snakeCase, camelCase } from "../utils/casing"; import { getEventEmittingEntities } from "../events"; +import { pyString } from "../ddl/migration-file"; import pluralize from "pluralize"; /** @@ -319,10 +320,10 @@ export class PythonGenerator extends BaseGenerator { const filename = `${timestamp}_${name}.py`; const upStatements = upSql - .map((sql) => ` op.execute("""${sql}""")`) + .map((sql) => ` op.execute(${pyString(sql)})`) .join("\n"); const downStatements = (downSql || []) - .map((sql) => ` op.execute("""${sql}""")`) + .map((sql) => ` op.execute(${pyString(sql)})`) .join("\n"); const content = `"""${name}""" diff --git a/src/lib/plan/baseline.ts b/src/lib/plan/baseline.ts index 7125b1b..d3319b7 100644 --- a/src/lib/plan/baseline.ts +++ b/src/lib/plan/baseline.ts @@ -23,8 +23,19 @@ export interface Baseline { source: string; } +/** Where each language's migrations live, and the extension of one. */ export const migrationsDir = (projectDir: string, apsorc?: ApsorcType) => - path.join(projectDir, apsorc?.rootFolder || "src", "migrations"); + apsorc?.language === "python" + ? path.join(projectDir, "alembic", "versions") + : apsorc?.language === "go" + ? path.join(projectDir, "migrations") + : path.join(projectDir, apsorc?.rootFolder || "src", "migrations"); +export const migrationExtension = (apsorc?: ApsorcType) => + apsorc?.language === "python" + ? ".py" + : apsorc?.language === "go" + ? ".sql" + : ".ts"; /** The newest migration with a saved plan, by its timestamp prefix. */ export function latestPlannedMigration( @@ -35,14 +46,14 @@ export function latestPlannedMigration( if (!fs.existsSync(dir)) return undefined; const plans = fs .readdirSync(dir) - .filter((f) => /^\d+-.+-migration\.plan\.json$/.test(f)) - .sort((a, b) => Number(b.split("-")[0]) - Number(a.split("-")[0])); + .filter((f) => /^\d+[_-].+\.plan\.json$/.test(f)) + .sort((a, b) => Number(b.split(/[_-]/)[0]) - Number(a.split(/[_-]/)[0])); for (const planFile of plans) { const saved = JSON.parse( fs.readFileSync(path.join(dir, planFile)).toString() ); if (!saved.before || !saved.after) continue; - const file = planFile.replace(/\.plan\.json$/, ".ts"); + const file = planFile.replace(/\.plan\.json$/, migrationExtension(apsorc)); return { file: path.relative(projectDir, path.join(dir, file)), planFile: path.relative(projectDir, path.join(dir, planFile)), diff --git a/src/lib/verify/index.ts b/src/lib/verify/index.ts index 225b940..2b9ebaf 100644 --- a/src/lib/verify/index.ts +++ b/src/lib/verify/index.ts @@ -1,7 +1,7 @@ /** * Pre-deploy checks for a generated service: generate the migration from the - * deployed schema, run it and check for drift (TypeScript only, on PGlite), - * and smoke test the endpoints. The CLI and the in-browser editor (which runs + * deployed schema, run it and check for drift (on PGlite), and smoke test the + * endpoints. The CLI and the in-browser editor (which runs * the CLI in a WebContainer) share these steps. */ @@ -12,6 +12,10 @@ import * as net from "net"; import * as os from "os"; import * as path from "path"; import pluralize from "pluralize"; +import { parseApsorcV2 } from "../apsorc-parser"; +import { schemaDdl } from "../ddl"; +import { catalogDiff, readCatalog } from "../ddl/catalog"; +import { migrationSql, typeormMigration } from "../ddl/migration-file"; import { Entity } from "../types/entity"; import { TargetLanguage } from "../types/generator"; import { RelationshipMap } from "../types/relationship"; @@ -125,21 +129,6 @@ export async function buildBaseline( } } -/** - * Why `apso migrate` can't run for this project's language, or undefined. - * ponytail: Python (Alembic) and Go (goose) have no local scratch database to - * build the baseline on; add when they get one like PGlite for TypeScript. - */ -export const migrateUnsupported = ( - language: TargetLanguage = "typescript" -): string | undefined => { - if (language === "python") - return "apso migrate supports TypeScript services only for now. This is a Python service: use Alembic directly (alembic revision --autogenerate, alembic upgrade head, alembic check)."; - if (language === "go") - return "apso migrate supports TypeScript services only for now. This is a Go service: write goose migrations in migrations/ and run them with `go run cmd/migrate/main.go up`."; - return undefined; -}; - export interface GenerateResult { ok: boolean; needed: boolean; @@ -365,6 +354,12 @@ export async function verifyMigration( acceptLoss = [], }: { skipBuild?: boolean; seedFile?: string; checkFile?: string; acceptLoss?: string[] } = {} ): Promise { + if (/\.(py|sql)$/.test(migrationFile)) + return verifySqlMigration(projectDir, baselineApsorc, migrationFile, cliBin, { + seedFile, + checkFile, + acceptLoss, + }); if (!supportsPglite(projectDir)) return NO_PGLITE; const source = fs.readFileSync(path.resolve(projectDir, migrationFile), "utf8"); const findings = reviewMigration(source); @@ -432,6 +427,109 @@ export async function verifyMigration( } } +const ddlOf = (apsorcFile: string) => + schemaDdl(parseApsorcV2(JSON.parse(fs.readFileSync(apsorcFile).toString()))).join(";\n"); + +/** + * verifyMigration for an Alembic or goose migration. Python and Go services + * have no TypeORM, so Apso's DDL writer builds the deployed schema, the + * migration's SQL runs as written, and drift is a catalog diff against a + * fresh build of the new .apsorc. down then runs and must give back the + * deployed schema. + */ +async function verifySqlMigration( + projectDir: string, + baselineApsorc: string, + migrationFile: string, + cliBin: string, + { seedFile, checkFile, acceptLoss = [] }: { seedFile?: string; checkFile?: string; acceptLoss?: string[] } +): Promise { + const { PGlite } = await import("@electric-sql/pglite"); + const steps: Step[] = []; + const done = (s: Step) => { + steps.push(s); + return s.ok; + }; + let sql: { up: string; down?: string }; + try { + sql = migrationSql(migrationFile, fs.readFileSync(path.resolve(projectDir, migrationFile), "utf8")); + } catch (error) { + done({ name: "read", ok: false, output: (error as Error).message }); + return { ok: false, steps }; + } + const findings = reviewMigration(typeormMigration("Migration", [sql.up], { down: [] })); + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-pglite-")); + // The CLI's own PGlite: the project has no node_modules to load one from. + const cliRoot = path.dirname(path.dirname(cliBin)); + const script = (name: string, code: string, arg?: string) => + run(process.execPath, ["-e", code, ...(arg ? [path.resolve(arg)] : [])], cliRoot, pgliteEnv(dataDir)).then( + (r) => { + done(step(name, r)); + return r; + } + ); + const exec = async (name: string, statements: string) => { + const db = new PGlite(dataDir); + try { + await db.exec(statements); + return done({ name, ok: true }); + } catch (error) { + return done({ name, ok: false, output: (error as Error).message }); + } finally { + await db.close(); + } + }; + const drift = async (name: string, apsorcFile: string) => { + const db = new PGlite(dataDir); + const fresh = new PGlite(); + try { + await fresh.exec(ddlOf(apsorcFile)); + const diff = catalogDiff(await readCatalog(fresh), await readCatalog(db)); + return done({ + name, + ok: diff.length === 0, + output: diff.length > 0 ? `The database differs from ${path.basename(apsorcFile)} (- missing, + extra):\n${diff.join("\n")}` : undefined, + }); + } finally { + await db.close(); + await fresh.close(); + } + }; + try { + if (!(await exec("baseline", ddlOf(baselineApsorc)))) return { ok: false, steps, findings }; + const count = async (name: string, file?: string) => { + const r = await script(name, COUNT_ROWS, file); + return r.ok ? (JSON.parse(r.output.trim().split("\n").pop()!) as Counts) : undefined; + }; + const before = seedFile ? await count("seed", seedFile) : undefined; + if (seedFile && !before) return { ok: false, steps, findings }; + if (!(await exec("migrate", sql.up))) return { ok: false, steps, findings }; + if (before) { + const after = await count("count"); + if (!after) return { ok: false, steps, findings }; + const lost = dataLoss(before, after, sql.up); + const ok = unaccepted(lost, acceptLoss).length === 0; + done({ + name: "data", + ok, + output: lost.length > 0 + ? `The migration lost seeded data:\n${lost.join("\n")}${ok ? "" : "\nFix the migration. If data moved somewhere the check can't follow (e.g. a field split in two), prove it with --check and rerun with --accept-loss ."}` + : undefined, + }); + if (!ok) return { ok: false, steps, findings }; + } + const checked = checkFile ? await script("check", RUN_CHECKS, checkFile) : undefined; + if (checked && !checked.ok) return { ok: false, steps, findings }; + if (!(await drift("drift", path.join(projectDir, ".apsorc")))) return { ok: false, steps, findings }; + if (sql.down === undefined) done({ name: "down", ok: true, output: "down refuses to run (irreversible), so it was not tested." }); + else if (!(await exec("down", sql.down)) || !(await drift("down drift", baselineApsorc))) + return { ok: false, steps, findings }; + return { ok: true, steps, findings }; + } finally { + fs.rmSync(dataDir, { recursive: true, force: true }); + } +} + const freePort = () => new Promise((resolve, reject) => { const s = net.createServer(); diff --git a/test/lib/ddl/__snapshots__/wrappers.test.ts.snap b/test/lib/ddl/__snapshots__/wrappers.test.ts.snap new file mode 100644 index 0000000..ac9fa3e --- /dev/null +++ b/test/lib/ddl/__snapshots__/wrappers.test.ts.snap @@ -0,0 +1,343 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`enum-replace Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TYPE \\"public\\".\\"contact_status_enum\\" RENAME TO \\"contact_status_enum_old\\"") + op.execute("CREATE TYPE \\"public\\".\\"contact_status_enum\\" AS ENUM('lead', 'active', 'closed')") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" DROP DEFAULT") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" TYPE \\"public\\".\\"contact_status_enum\\" USING (CASE \\"status\\"::text WHEN 'archived' THEN 'closed' ELSE \\"status\\"::text END)::\\"public\\".\\"contact_status_enum\\"") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" SET DEFAULT 'lead'") + op.execute("DROP TYPE \\"public\\".\\"contact_status_enum_old\\"") + + +def downgrade() -> None: + op.execute("ALTER TYPE \\"public\\".\\"contact_status_enum\\" RENAME TO \\"contact_status_enum_old\\"") + op.execute("CREATE TYPE \\"public\\".\\"contact_status_enum\\" AS ENUM('lead', 'active', 'archived')") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" DROP DEFAULT") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" TYPE \\"public\\".\\"contact_status_enum\\" USING (CASE \\"status\\"::text WHEN 'closed' THEN 'archived' ELSE \\"status\\"::text END)::\\"public\\".\\"contact_status_enum\\"") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"status\\" SET DEFAULT 'lead'") + op.execute("DROP TYPE \\"public\\".\\"contact_status_enum_old\\"") +" +`; + +exports[`enum-replace goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TYPE "public"."contact_status_enum" RENAME TO "contact_status_enum_old"; +CREATE TYPE "public"."contact_status_enum" AS ENUM('lead', 'active', 'closed'); +ALTER TABLE "contact" ALTER COLUMN "status" DROP DEFAULT; +ALTER TABLE "contact" ALTER COLUMN "status" TYPE "public"."contact_status_enum" USING (CASE "status"::text WHEN 'archived' THEN 'closed' ELSE "status"::text END)::"public"."contact_status_enum"; +ALTER TABLE "contact" ALTER COLUMN "status" SET DEFAULT 'lead'; +DROP TYPE "public"."contact_status_enum_old"; + +-- +goose Down +ALTER TYPE "public"."contact_status_enum" RENAME TO "contact_status_enum_old"; +CREATE TYPE "public"."contact_status_enum" AS ENUM('lead', 'active', 'archived'); +ALTER TABLE "contact" ALTER COLUMN "status" DROP DEFAULT; +ALTER TABLE "contact" ALTER COLUMN "status" TYPE "public"."contact_status_enum" USING (CASE "status"::text WHEN 'closed' THEN 'archived' ELSE "status"::text END)::"public"."contact_status_enum"; +ALTER TABLE "contact" ALTER COLUMN "status" SET DEFAULT 'lead'; +DROP TYPE "public"."contact_status_enum_old"; +" +`; + +exports[`extract-entity Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE TYPE \\"public\\".\\"phone_number_label_enum\\" AS ENUM('mobile', 'work', 'home')") + op.execute("CREATE TABLE \\"phone_number\\" (\\"id\\" SERIAL NOT NULL, \\"created_at\\" TIMESTAMP NOT NULL DEFAULT now(), \\"updated_at\\" TIMESTAMP NOT NULL DEFAULT now(), \\"number\\" character varying(40) NOT NULL, \\"label\\" \\"public\\".\\"phone_number_label_enum\\" NOT NULL DEFAULT 'mobile', \\"contactId\\" integer NOT NULL, CONSTRAINT \\"PK_c16f58426537a660b3f2a26e983\\" PRIMARY KEY (\\"id\\"))") + op.execute("INSERT INTO \\"phone_number\\" (\\"number\\", \\"contactId\\") SELECT \\"phone\\", \\"id\\" FROM \\"contact\\" WHERE \\"phone\\" IS NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"phone\\"") + op.execute("CREATE INDEX \\"IDX_069067410ee19d423487a189e4\\" ON \\"phone_number\\" (\\"contactId\\")") + op.execute("ALTER TABLE \\"phone_number\\" ADD CONSTRAINT \\"FK_069067410ee19d423487a189e4d\\" FOREIGN KEY (\\"contactId\\") REFERENCES \\"contact\\"(\\"id\\") ON DELETE CASCADE ON UPDATE NO ACTION") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"phone_number\\" DROP CONSTRAINT \\"FK_069067410ee19d423487a189e4d\\"") + op.execute("ALTER TABLE \\"contact\\" ADD \\"phone\\" character varying(40)") + op.execute("UPDATE \\"contact\\" SET \\"phone\\" = (SELECT \\"number\\" FROM \\"phone_number\\" x WHERE x.\\"contactId\\" = \\"contact\\".\\"id\\" ORDER BY x.\\"id\\" LIMIT 1) WHERE \\"phone\\" IS NULL") + op.execute("DROP TABLE \\"phone_number\\"") + op.execute("DROP TYPE \\"public\\".\\"phone_number_label_enum\\"") +" +`; + +exports[`extract-entity goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +CREATE TYPE "public"."phone_number_label_enum" AS ENUM('mobile', 'work', 'home'); +CREATE TABLE "phone_number" ("id" SERIAL NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), "number" character varying(40) NOT NULL, "label" "public"."phone_number_label_enum" NOT NULL DEFAULT 'mobile', "contactId" integer NOT NULL, CONSTRAINT "PK_c16f58426537a660b3f2a26e983" PRIMARY KEY ("id")); +INSERT INTO "phone_number" ("number", "contactId") SELECT "phone", "id" FROM "contact" WHERE "phone" IS NOT NULL; +ALTER TABLE "contact" DROP COLUMN "phone"; +CREATE INDEX "IDX_069067410ee19d423487a189e4" ON "phone_number" ("contactId"); +ALTER TABLE "phone_number" ADD CONSTRAINT "FK_069067410ee19d423487a189e4d" FOREIGN KEY ("contactId") REFERENCES "contact"("id") ON DELETE CASCADE ON UPDATE NO ACTION; + +-- +goose Down +ALTER TABLE "phone_number" DROP CONSTRAINT "FK_069067410ee19d423487a189e4d"; +ALTER TABLE "contact" ADD "phone" character varying(40); +UPDATE "contact" SET "phone" = (SELECT "number" FROM "phone_number" x WHERE x."contactId" = "contact"."id" ORDER BY x."id" LIMIT 1) WHERE "phone" IS NULL; +DROP TABLE "phone_number"; +DROP TYPE "public"."phone_number_label_enum"; +" +`; + +exports[`many-to-many Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" DROP CONSTRAINT \\"FK_196374b312744cf2c400c962383\\"") + op.execute("DROP INDEX \\"public\\".\\"IDX_196374b312744cf2c400c96238\\"") + op.execute("CREATE TABLE \\"contact_companies_company\\" (\\"contactId\\" integer NOT NULL, \\"companyId\\" integer NOT NULL, CONSTRAINT \\"PK_a240a097bb1e1d698bc213e0c72\\" PRIMARY KEY (\\"contactId\\", \\"companyId\\"))") + op.execute("INSERT INTO \\"contact_companies_company\\" (\\"contactId\\", \\"companyId\\") SELECT \\"id\\", \\"companyId\\" FROM \\"contact\\" WHERE \\"companyId\\" IS NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"companyId\\"") + op.execute("CREATE INDEX \\"IDX_e6e2d85d35645bd6b752226c49\\" ON \\"contact_companies_company\\" (\\"contactId\\")") + op.execute("CREATE INDEX \\"IDX_30d0ef2a176bf879a2d36df096\\" ON \\"contact_companies_company\\" (\\"companyId\\")") + op.execute("ALTER TABLE \\"contact_companies_company\\" ADD CONSTRAINT \\"FK_e6e2d85d35645bd6b752226c49d\\" FOREIGN KEY (\\"contactId\\") REFERENCES \\"contact\\"(\\"id\\") ON DELETE CASCADE ON UPDATE CASCADE") + op.execute("ALTER TABLE \\"contact_companies_company\\" ADD CONSTRAINT \\"FK_30d0ef2a176bf879a2d36df0966\\" FOREIGN KEY (\\"companyId\\") REFERENCES \\"company\\"(\\"id\\") ON DELETE NO ACTION ON UPDATE NO ACTION") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact_companies_company\\" DROP CONSTRAINT \\"FK_e6e2d85d35645bd6b752226c49d\\"") + op.execute("ALTER TABLE \\"contact_companies_company\\" DROP CONSTRAINT \\"FK_30d0ef2a176bf879a2d36df0966\\"") + op.execute("ALTER TABLE \\"contact\\" ADD \\"companyId\\" integer") + op.execute("UPDATE \\"contact\\" SET \\"companyId\\" = (SELECT j.\\"companyId\\" FROM \\"contact_companies_company\\" j WHERE j.\\"contactId\\" = \\"contact\\".\\"id\\" LIMIT 1) WHERE \\"companyId\\" IS NULL") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"companyId\\" SET NOT NULL") + op.execute("DROP TABLE \\"contact_companies_company\\"") + op.execute("CREATE INDEX \\"IDX_196374b312744cf2c400c96238\\" ON \\"contact\\" (\\"companyId\\")") + op.execute("ALTER TABLE \\"contact\\" ADD CONSTRAINT \\"FK_196374b312744cf2c400c962383\\" FOREIGN KEY (\\"companyId\\") REFERENCES \\"company\\"(\\"id\\") ON DELETE NO ACTION ON UPDATE NO ACTION") +" +`; + +exports[`many-to-many goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" DROP CONSTRAINT "FK_196374b312744cf2c400c962383"; +DROP INDEX "public"."IDX_196374b312744cf2c400c96238"; +CREATE TABLE "contact_companies_company" ("contactId" integer NOT NULL, "companyId" integer NOT NULL, CONSTRAINT "PK_a240a097bb1e1d698bc213e0c72" PRIMARY KEY ("contactId", "companyId")); +INSERT INTO "contact_companies_company" ("contactId", "companyId") SELECT "id", "companyId" FROM "contact" WHERE "companyId" IS NOT NULL; +ALTER TABLE "contact" DROP COLUMN "companyId"; +CREATE INDEX "IDX_e6e2d85d35645bd6b752226c49" ON "contact_companies_company" ("contactId"); +CREATE INDEX "IDX_30d0ef2a176bf879a2d36df096" ON "contact_companies_company" ("companyId"); +ALTER TABLE "contact_companies_company" ADD CONSTRAINT "FK_e6e2d85d35645bd6b752226c49d" FOREIGN KEY ("contactId") REFERENCES "contact"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "contact_companies_company" ADD CONSTRAINT "FK_30d0ef2a176bf879a2d36df0966" FOREIGN KEY ("companyId") REFERENCES "company"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- +goose Down +ALTER TABLE "contact_companies_company" DROP CONSTRAINT "FK_e6e2d85d35645bd6b752226c49d"; +ALTER TABLE "contact_companies_company" DROP CONSTRAINT "FK_30d0ef2a176bf879a2d36df0966"; +ALTER TABLE "contact" ADD "companyId" integer; +UPDATE "contact" SET "companyId" = (SELECT j."companyId" FROM "contact_companies_company" j WHERE j."contactId" = "contact"."id" LIMIT 1) WHERE "companyId" IS NULL; +ALTER TABLE "contact" ALTER COLUMN "companyId" SET NOT NULL; +DROP TABLE "contact_companies_company"; +CREATE INDEX "IDX_196374b312744cf2c400c96238" ON "contact" ("companyId"); +ALTER TABLE "contact" ADD CONSTRAINT "FK_196374b312744cf2c400c962383" FOREIGN KEY ("companyId") REFERENCES "company"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; +" +`; + +exports[`rename-entity Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"companyId\\" TO \\"organizationId\\"") + op.execute("ALTER TABLE \\"company\\" RENAME TO \\"organization\\"") + op.execute("ALTER SEQUENCE \\"company_id_seq\\" RENAME TO \\"organization_id_seq\\"") + op.execute("ALTER TABLE \\"organization\\" RENAME CONSTRAINT \\"PK_056f7854a7afdba7cbd6d45fc20\\" TO \\"PK_472c1f99a32def1b0abb219cd67\\"") + op.execute("ALTER TABLE \\"contact\\" RENAME CONSTRAINT \\"FK_196374b312744cf2c400c962383\\" TO \\"FK_7719d73cd16a9f57ecc6ac24b3d\\"") + op.execute("ALTER INDEX \\"public\\".\\"IDX_196374b312744cf2c400c96238\\" RENAME TO \\"IDX_7719d73cd16a9f57ecc6ac24b3\\"") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"organizationId\\" TO \\"companyId\\"") + op.execute("ALTER TABLE \\"organization\\" RENAME TO \\"company\\"") + op.execute("ALTER SEQUENCE \\"organization_id_seq\\" RENAME TO \\"company_id_seq\\"") + op.execute("ALTER TABLE \\"company\\" RENAME CONSTRAINT \\"PK_472c1f99a32def1b0abb219cd67\\" TO \\"PK_056f7854a7afdba7cbd6d45fc20\\"") + op.execute("ALTER TABLE \\"contact\\" RENAME CONSTRAINT \\"FK_7719d73cd16a9f57ecc6ac24b3d\\" TO \\"FK_196374b312744cf2c400c962383\\"") + op.execute("ALTER INDEX \\"public\\".\\"IDX_7719d73cd16a9f57ecc6ac24b3\\" RENAME TO \\"IDX_196374b312744cf2c400c96238\\"") +" +`; + +exports[`rename-entity goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" RENAME COLUMN "companyId" TO "organizationId"; +ALTER TABLE "company" RENAME TO "organization"; +ALTER SEQUENCE "company_id_seq" RENAME TO "organization_id_seq"; +ALTER TABLE "organization" RENAME CONSTRAINT "PK_056f7854a7afdba7cbd6d45fc20" TO "PK_472c1f99a32def1b0abb219cd67"; +ALTER TABLE "contact" RENAME CONSTRAINT "FK_196374b312744cf2c400c962383" TO "FK_7719d73cd16a9f57ecc6ac24b3d"; +ALTER INDEX "public"."IDX_196374b312744cf2c400c96238" RENAME TO "IDX_7719d73cd16a9f57ecc6ac24b3"; + +-- +goose Down +ALTER TABLE "contact" RENAME COLUMN "organizationId" TO "companyId"; +ALTER TABLE "organization" RENAME TO "company"; +ALTER SEQUENCE "organization_id_seq" RENAME TO "company_id_seq"; +ALTER TABLE "company" RENAME CONSTRAINT "PK_472c1f99a32def1b0abb219cd67" TO "PK_056f7854a7afdba7cbd6d45fc20"; +ALTER TABLE "contact" RENAME CONSTRAINT "FK_7719d73cd16a9f57ecc6ac24b3d" TO "FK_196374b312744cf2c400c962383"; +ALTER INDEX "public"."IDX_7719d73cd16a9f57ecc6ac24b3" RENAME TO "IDX_196374b312744cf2c400c96238"; +" +`; + +exports[`rename-field Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"phone\\" TO \\"mobilePhone\\"") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" RENAME COLUMN \\"mobilePhone\\" TO \\"phone\\"") +" +`; + +exports[`rename-field goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" RENAME COLUMN "phone" TO "mobilePhone"; + +-- +goose Down +ALTER TABLE "contact" RENAME COLUMN "mobilePhone" TO "phone"; +" +`; + +exports[`required-field Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE TYPE \\"public\\".\\"company_region_enum\\" AS ENUM('us', 'eu', 'apac')") + op.execute("ALTER TABLE \\"company\\" ADD \\"region\\" \\"public\\".\\"company_region_enum\\" NOT NULL DEFAULT 'us'") + op.execute("UPDATE \\"company\\" SET \\"region\\" = 'us' WHERE \\"region\\" IS NULL") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"company\\" DROP COLUMN \\"region\\"") + op.execute("DROP TYPE \\"public\\".\\"company_region_enum\\"") +" +`; + +exports[`required-field goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +CREATE TYPE "public"."company_region_enum" AS ENUM('us', 'eu', 'apac'); +ALTER TABLE "company" ADD "region" "public"."company_region_enum" NOT NULL DEFAULT 'us'; +UPDATE "company" SET "region" = 'us' WHERE "region" IS NULL; + +-- +goose Down +ALTER TABLE "company" DROP COLUMN "region"; +DROP TYPE "public"."company_region_enum"; +" +`; + +exports[`split-field Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ADD \\"firstName\\" text") + op.execute("ALTER TABLE \\"contact\\" ADD \\"lastName\\" text") + op.execute("UPDATE \\"contact\\" SET \\"firstName\\" = (split_part(btrim(\\"fullName\\"), ' ', 1)), \\"lastName\\" = (NULLIF(btrim(substr(btrim(\\"fullName\\"), length(split_part(btrim(\\"fullName\\"), ' ', 1)) + 1)), ''))") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"firstName\\" SET NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"fullName\\"") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ADD \\"fullName\\" text") + op.execute("UPDATE \\"contact\\" SET \\"fullName\\" = (concat_ws(' ', \\"firstName\\", \\"lastName\\")) WHERE \\"fullName\\" IS NULL") + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"fullName\\" SET NOT NULL") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"firstName\\"") + op.execute("ALTER TABLE \\"contact\\" DROP COLUMN \\"lastName\\"") +" +`; + +exports[`split-field goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" ADD "firstName" text; +ALTER TABLE "contact" ADD "lastName" text; +UPDATE "contact" SET "firstName" = (split_part(btrim("fullName"), ' ', 1)), "lastName" = (NULLIF(btrim(substr(btrim("fullName"), length(split_part(btrim("fullName"), ' ', 1)) + 1)), '')); +ALTER TABLE "contact" ALTER COLUMN "firstName" SET NOT NULL; +ALTER TABLE "contact" DROP COLUMN "fullName"; + +-- +goose Down +ALTER TABLE "contact" ADD "fullName" text; +UPDATE "contact" SET "fullName" = (concat_ws(' ', "firstName", "lastName")) WHERE "fullName" IS NULL; +ALTER TABLE "contact" ALTER COLUMN "fullName" SET NOT NULL; +ALTER TABLE "contact" DROP COLUMN "firstName"; +ALTER TABLE "contact" DROP COLUMN "lastName"; +" +`; + +exports[`type-change Alembic revision compiles and gives back its SQL 1`] = ` +""""Generated by apso migrate generate from the answered migration plan saved beside this file.""" +from alembic import op + +revision = "20231114221320" +down_revision = "20231101000000" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"score\\" TYPE integer USING (NULLIF(regexp_replace(\\"score\\", '[^0-9-]', '', 'g'), '')::integer)") + + +def downgrade() -> None: + op.execute("ALTER TABLE \\"contact\\" ALTER COLUMN \\"score\\" TYPE text USING \\"score\\"::text") +" +`; + +exports[`type-change goose file matches its snapshot 1`] = ` +"-- Generated by apso migrate generate from the answered migration plan saved beside this file. +-- +goose Up +ALTER TABLE "contact" ALTER COLUMN "score" TYPE integer USING (NULLIF(regexp_replace("score", '[^0-9-]', '', 'g'), '')::integer); + +-- +goose Down +ALTER TABLE "contact" ALTER COLUMN "score" TYPE text USING "score"::text; +" +`; diff --git a/test/lib/ddl/answers.ts b/test/lib/ddl/answers.ts new file mode 100644 index 0000000..0894308 --- /dev/null +++ b/test/lib/ddl/answers.ts @@ -0,0 +1,49 @@ +import { Plan, planMigration } from "../../../src/lib/plan"; + +// Each migration eval case, answered the way the eval expects. The change +// must leave the seeded database identical in schema to a fresh build of the +// new .apsorc, with every row check in checks.sql passing. +export const answers: Record< + string, + Record< + string, + string | { answer: string; sql: string | Record } + > +> = { + "rename-entity": { "entity:Company": "rename:Organization" }, + "rename-field": { "field:Contact.phone": "rename:mobilePhone" }, + "split-field": { + "field:Contact.fullName": { + answer: "split:firstName,lastName", + sql: { + firstName: `split_part(btrim("fullName"), ' ', 1)`, + lastName: `NULLIF(btrim(substr(btrim("fullName"), length(split_part(btrim("fullName"), ' ', 1)) + 1)), '')`, + }, + }, + }, + "extract-entity": { "field:Contact.phone": "move:PhoneNumber.number" }, + "many-to-many": { "relation:Contact.Company": "copy-links" }, + "enum-replace": { "enum:Contact.status:archived": "closed" }, + "required-field": { "backfill:Company.region": "us" }, + "type-change": { + "cast:Contact.score": { + answer: "sql", + sql: `NULLIF(regexp_replace("score", '[^0-9-]', '', 'g'), '')::integer`, + }, + }, +}; + +export const answeredPlan = (c: string, before: any, after: any): Plan => { + let plan = planMigration(before, after); + for (let round = 0; round < 3; round++) { + for (const q of plan.questions) { + const a = answers[c][q.id]; + if (a === undefined) continue; + if (typeof a === "string") q.answer = a; + else Object.assign(q, a); + } + plan = planMigration(before, after, plan); + } + return plan; +}; + diff --git a/test/lib/ddl/change.test.ts b/test/lib/ddl/change.test.ts index d096a26..24bc39a 100644 --- a/test/lib/ddl/change.test.ts +++ b/test/lib/ddl/change.test.ts @@ -5,59 +5,13 @@ import { parseApsorcV2 } from "../../../src/lib/apsorc-parser"; import { schemaDdl, schemaModel } from "../../../src/lib/ddl"; import { changeDdl, downDdl } from "../../../src/lib/ddl/change"; import { catalogDiff, readCatalog } from "../../../src/lib/ddl/catalog"; -import { openQuestions, planMigration, Plan } from "../../../src/lib/plan"; - -// Each migration eval case, answered the way the eval expects. The change -// must leave the seeded database identical in schema to a fresh build of the -// new .apsorc, with every row check in checks.sql passing. -const answers: Record< - string, - Record< - string, - string | { answer: string; sql: string | Record } - > -> = { - "rename-entity": { "entity:Company": "rename:Organization" }, - "rename-field": { "field:Contact.phone": "rename:mobilePhone" }, - "split-field": { - "field:Contact.fullName": { - answer: "split:firstName,lastName", - sql: { - firstName: `split_part(btrim("fullName"), ' ', 1)`, - lastName: `NULLIF(btrim(substr(btrim("fullName"), length(split_part(btrim("fullName"), ' ', 1)) + 1)), '')`, - }, - }, - }, - "extract-entity": { "field:Contact.phone": "move:PhoneNumber.number" }, - "many-to-many": { "relation:Contact.Company": "copy-links" }, - "enum-replace": { "enum:Contact.status:archived": "closed" }, - "required-field": { "backfill:Company.region": "us" }, - "type-change": { - "cast:Contact.score": { - answer: "sql", - sql: `NULLIF(regexp_replace("score", '[^0-9-]', '', 'g'), '')::integer`, - }, - }, -}; +import { openQuestions, planMigration } from "../../../src/lib/plan"; +import { answeredPlan, answers } from "./answers"; const dir = (c: string) => path.join(__dirname, "../../fixtures/plan", c); const read = (c: string, f: string) => fs.readFileSync(path.join(dir(c), f)).toString(); -const answeredPlan = (c: string, before: any, after: any): Plan => { - let plan = planMigration(before, after); - for (let round = 0; round < 3; round++) { - for (const q of plan.questions) { - const a = answers[c][q.id]; - if (a === undefined) continue; - if (typeof a === "string") q.answer = a; - else Object.assign(q, a); - } - plan = planMigration(before, after, plan); - } - return plan; -}; - const pglite = async () => { const { PGlite } = await import("@electric-sql/pglite"); return new PGlite(); diff --git a/test/lib/ddl/wrappers.test.ts b/test/lib/ddl/wrappers.test.ts new file mode 100644 index 0000000..d42619e --- /dev/null +++ b/test/lib/ddl/wrappers.test.ts @@ -0,0 +1,206 @@ +import { expect, describe, test } from "@jest/globals"; +import { execFileSync } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { changeDdl, downDdl } from "../../../src/lib/ddl/change"; +import { + alembicHead, + alembicMigration, + gooseMigration, + migrationSql, + writePlannedMigration, +} from "../../../src/lib/ddl/migration-file"; +import { PythonGenerator } from "../../../src/lib/generators/python"; +import { verifyMigration } from "../../../src/lib/verify"; +import { answeredPlan, answers } from "./answers"; + +const fixture = (c: string, f: string) => + path.join(__dirname, "../../fixtures/plan", c, f); +const read = (c: string, f: string) => + JSON.parse(fs.readFileSync(fixture(c, f)).toString()); +const cliBin = path.join(__dirname, "../../../bin/run"); + +let python = true; +try { + execFileSync("python3", ["--version"], { stdio: "ignore" }); +} catch { + python = false; +} +/** Python accepts the file. Without python3, every SQL literal must round-trip. */ +const compiles = (source: string) => { + if (!python) { + for (const m of source.matchAll(/op\.execute\((.*)\)$/gm)) + expect(typeof JSON.parse(m[1])).toBe("string"); + return; + } + const file = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "apso-py-")), + "m.py" + ); + fs.writeFileSync(file, source); + execFileSync("python3", ["-m", "py_compile", file]); +}; + +describe.each(Object.keys(answers))("%s", (c) => { + const before = read(c, "before.apsorc"); + const after = read(c, "after.apsorc"); + const plan = answeredPlan(c, before, after); + const { up } = changeDdl(before, after, plan); + const down = downDdl(before, after, plan); + + test("Alembic revision compiles and gives back its SQL", () => { + const source = alembicMigration("20231114221320", "20231101000000", up, down); + expect(source).toMatchSnapshot(); + compiles(source); + const sql = migrationSql("x.py", source); + expect(sql.up).toBe(up.join(";\n")); + expect(sql.down).toBe((down as { down: string[] }).down.join(";\n")); + }); + + test("goose file matches its snapshot", () => { + expect(gooseMigration(up, down)).toMatchSnapshot(); + }); +}); + +describe("wrappers", () => { + test("SQL ending in a quoted identifier stays valid Python (#148)", async () => { + const [file] = await new PythonGenerator({ + language: "python", + } as any).generateMigration({ + entities: [], + upSql: ['DROP TABLE "user"'], + downSql: ['CREATE TABLE "user" ("id" integer, "note" text DEFAULT \'a\\b """\')'], + } as any); + compiles(file.content); + expect(file.content).toContain('op.execute("DROP TABLE \\"user\\"")'); + }); + + test("an irreversible down refuses to run", () => { + const irreversible = { irreversible: ["backfill:Contact.fullName: 100%"] }; + const py = alembicMigration("2", undefined, ["SELECT 1"], irreversible); + compiles(py); + expect(py).toContain("raise NotImplementedError("); + expect(py).toContain("down_revision = None"); + expect(migrationSql("x.py", py).down).toBeUndefined(); + const sql = gooseMigration(["SELECT 1"], irreversible); + expect(sql).toContain("-- +goose StatementBegin\nDO $$ BEGIN RAISE EXCEPTION"); + expect(sql).toContain("100%%"); + expect(migrationSql("x.sql", sql)).toEqual({ up: "\nSELECT 1;\n\n", down: undefined }); + }); + + test("a statement with its own semicolon is fenced for goose", () => { + expect(gooseMigration(["SELECT ';'", "SELECT 2"], { down: [] })).toContain( + "-- +goose StatementBegin\nSELECT ';';\n-- +goose StatementEnd\nSELECT 2;" + ); + }); + + test("the Alembic head is the revision nothing revises", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-alembic-")); + expect(alembicHead(dir)).toBeUndefined(); + const rev = (f: string, r: string, d: string) => + fs.writeFileSync( + path.join(dir, f), + `revision: str = '${r}'\ndown_revision: Union[str, None] = ${d}\n` + ); + rev("a_init.py", "aaa", "None"); + rev("c_third.py", "ccc", "'bbb'"); + rev("b_second.py", "bbb", "'aaa'"); + expect(alembicHead(dir)).toBe("ccc"); + }); +}); + +describe.each(["python", "go"])("apso migrate for %s", (language) => { + const project = (c: string) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `apso-${language}-`)); + const after = { ...read(c, "after.apsorc"), language }; + fs.writeFileSync(path.join(dir, ".apsorc"), JSON.stringify(after)); + const plan = answeredPlan(c, read(c, "before.apsorc"), after); + const planPath = path.join(dir, ".apso", "migration-plan.json"); + fs.mkdirSync(path.dirname(planPath)); + fs.writeFileSync(planPath, JSON.stringify(plan)); + return { dir, planPath }; + }; + + test("writes the migration where the template keeps them", () => { + const { dir, planPath } = project("rename-field"); + const result = writePlannedMigration( + dir, + read("rename-field", "before.apsorc"), + "SchemaUpdate", + planPath, + 1_700_000_000_000 + ); + expect(result.file).toBe( + language === "python" + ? "alembic/versions/20231114221320_schema_update.py" + : "migrations/20231114221320_schema_update.sql" + ); + expect(result.planFile).toBe( + result.file!.replace(/\.(py|sql)$/, ".plan.json") + ); + expect(result.findings).toEqual([]); + }); + + test.each([ + ["rename-field", []], + ["split-field", ["contact.fullName"]], + ["extract-entity", []], + ])( + "%s verifies with its seed and checks", + async (c, acceptLoss) => { + const { dir, planPath } = project(c); + const written = writePlannedMigration( + dir, + read(c, "before.apsorc"), + "SchemaUpdate", + planPath + ); + const result = await verifyMigration( + dir, + fixture(c, "before.apsorc"), + written.file!, + cliBin, + { + seedFile: fixture(c, "seed.sql"), + checkFile: fixture(c, "checks.sql"), + acceptLoss: acceptLoss as string[], + } + ); + expect(result.steps).toEqual( + ["baseline", "seed", "migrate", "count", "data", "check", "drift", "down", "down drift"].map( + (name) => expect.objectContaining({ name, ok: true }) + ) + ); + expect(result.ok).toBe(true); + }, + 60_000 + ); + + test("drift fails when the migration leaves the schema short", async () => { + const { dir, planPath } = project("rename-field"); + const written = writePlannedMigration( + dir, + read("rename-field", "before.apsorc"), + "SchemaUpdate", + planPath + ); + const file = path.join(dir, written.file!); + // Drop the rename: the deployed column stays "phone". + fs.writeFileSync( + file, + fs + .readFileSync(file) + .toString() + .replace(/^.*RENAME COLUMN \\?"phone\\?" TO .*$/m, "") + ); + const result = await verifyMigration( + dir, + fixture("rename-field", "before.apsorc"), + written.file!, + cliBin + ); + expect(result.ok).toBe(false); + expect(result.steps[result.steps.length - 1]).toMatchObject({ name: "drift", ok: false }); + }, 60_000); +});