diff --git a/.changeset/cross-project-deploy.md b/.changeset/cross-project-deploy.md new file mode 100644 index 000000000..5cba6b305 --- /dev/null +++ b/.changeset/cross-project-deploy.md @@ -0,0 +1,6 @@ +--- +'@openfn/cli': minor +'@openfn/project': patch +--- + +Deploying a workspace to a different project now makes that project match it diff --git a/integration-tests/cli/test/sync/cross-project.test.ts b/integration-tests/cli/test/sync/cross-project.test.ts new file mode 100644 index 000000000..d70a2c573 --- /dev/null +++ b/integration-tests/cli/test/sync/cross-project.test.ts @@ -0,0 +1,226 @@ +// Deploying a workspace to a different project (eg, updating prod to match a project from +// the cloud, or replicating a project onto another server), against mock Lightnings: +// the target is made to look exactly like the workspace + +import test from 'ava'; +import { generateProject } from '@openfn/project'; +import createLightningServer from '@openfn/lightning-mock'; +import { setup } from './helpers'; + +const { lightning, initWorkspace, gen } = setup(test, 'cross-project', 5357); + +// a second server, for deploying across servers +const OTHER_PORT = 5358; +const otherEndpoint = `http://localhost:${OTHER_PORT}/api/provision`; +const other: any = createLightningServer({ port: OTHER_PORT }); + +// a project's workflows on a server, as { name: code } +const code = (server: any, project: any) => + Object.fromEntries( + ( + Object.values( + server.state.projects[project.openfn.uuid].workflows + ) as any[] + ).map((wf) => [ + wf.name, + Object.values(wf.jobs).find((j) => j.body).body, + ]) + ); + +// a project's workflows on a server, as { name: uuid } +const uuids = (server: any, project: any) => + Object.fromEntries( + ( + Object.values( + server.state.projects[project.openfn.uuid].workflows + ) as any[] + ).map((wf) => [wf.name, wf.id]) + ); + +// The workspace is pulled from cloud (Alpha and Beta). Prod has its own version of +// Alpha, and a workflow of its own (Gamma) +const genProjects = () => ({ + cloud: gen('cloud', [ + '@name Alpha trigger-job(expression="fn(a)")', + '@name Beta trigger-job(expression="fn(b)")', + ]), + prod: gen('prod', [ + '@name Alpha trigger-job(expression="fn(old)")', + '@name Gamma trigger-job(expression="fn(g)")', + ]), +}); + +test('deploy to a different project to make it match the workspace', async (t) => { + const { read, run } = initWorkspace(t); + const { cloud, prod } = genProjects(); + const prodAlpha = uuids(lightning, prod).Alpha; + + await run(`openfn project pull ${cloud.openfn.uuid}`); + + const { stdout } = await run( + `openfn project deploy ${prod.openfn.uuid} --force` + ); + t.regex(stdout, /Gamma: deleted/); + t.regex(stdout, /Alpha: changed/); + t.regex(stdout, /Beta: added/); + + // prod has exactly the workspace's workflows, and none of its own + t.deepEqual(code(lightning, prod), { Alpha: 'fn(a)', Beta: 'fn(b)' }); + + // its existing workflow was updated in place (it keeps its own uuid) + t.is(uuids(lightning, prod).Alpha, prodAlpha); + + // and cloud is untouched + t.deepEqual(code(lightning, cloud), { Alpha: 'fn(a)', Beta: 'fn(b)' }); + t.regex(read('workflows/alpha/alpha.yaml'), /name: Alpha/); +}); + +test('workflows deployed to a different project get new uuids', async (t) => { + const { run } = initWorkspace(t); + const { cloud, prod } = genProjects(); + + await run(`openfn project pull ${cloud.openfn.uuid}`); + await run(`openfn project deploy ${prod.openfn.uuid} --force`); + + const prodUuids = uuids(lightning, prod); + const cloudUuids = uuids(lightning, cloud); + + // Ensure prod's workflows have their own UUIDs + t.truthy(prodUuids.Alpha); + t.truthy(prodUuids.Beta); + t.not(prodUuids.Beta, cloudUuids.Beta); + t.not(prodUuids.Alpha, cloudUuids.Alpha); +}); + +test('deploy to a different project without a prompt will not remove workflows unless forced', async (t) => { + const { run } = initWorkspace(t); + const { cloud, prod } = genProjects(); + + await run(`openfn project pull ${cloud.openfn.uuid}`); + + // (the test runner never prompts, so this is a deploy with --no-confirm) + const { stdout, stderr, err } = await run( + `openfn project deploy ${prod.openfn.uuid}` + ); + t.truthy(err); + t.regex(stdout + stderr, /would remove workflows/); + t.regex(stdout + stderr, /gamma/); + + // prod is untouched + t.deepEqual(code(lightning, prod), { Alpha: 'fn(old)', Gamma: 'fn(g)' }); +}); + +test('a dry run to a different project shows what would change, and changes nothing', async (t) => { + const { run } = initWorkspace(t); + const cloud = gen('cloud', [ + '@name Alpha trigger-job(expression="fn(a)")', + '@name Beta trigger-job(expression="fn(b)")', + ]); + // a project with only a workflow of its own, which the deploy would remove + const prod = gen('prod', ['@name Gamma trigger-job(expression="fn(g)")']); + + await run(`openfn project pull ${cloud.openfn.uuid}`); + + // (without --force: a dry run isn't blocked by the removal guard) + const { stdout, err } = await run( + `openfn project deploy ${prod.openfn.uuid} --dry-run` + ); + t.falsy(err); + t.regex(stdout, /Gamma: deleted/); + t.regex(stdout, /Alpha: added/); + t.regex(stdout, /Beta: added/); + t.regex(stdout, /dryRun option set/); + + t.deepEqual(code(lightning, prod), { Gamma: 'fn(g)' }); +}); + +test('deploy to a different project that was edited elsewhere needs --force', async (t) => { + const { run } = initWorkspace(t); + const cloud = gen('cloud', [ + '@name Alpha trigger-job(expression="fn(a)")', + '@name Beta trigger-job(expression="fn(b)")', + ]); + // the same workflows, with different code (so no workflows are removed) + const prod = gen('prod', [ + '@name Alpha trigger-job(expression="fn(old-a)")', + '@name Beta trigger-job(expression="fn(old-b)")', + ]); + + await run(`openfn project pull ${cloud.openfn.uuid}`); + + const { stdout, stderr, err } = await run( + `openfn project deploy ${prod.openfn.uuid}` + ); + t.truthy(err); + t.regex(stdout + stderr, /not the project this workspace was synced from/); + t.deepEqual(code(lightning, prod), { + Alpha: 'fn(old-a)', + Beta: 'fn(old-b)', + }); + + await run(`openfn project deploy ${prod.openfn.uuid} --force`); + t.deepEqual(code(lightning, prod), { Alpha: 'fn(a)', Beta: 'fn(b)' }); +}); + +test('deploy to a different project by its local alias', async (t) => { + const { run } = initWorkspace(t); + const { cloud, prod } = genProjects(); + + await run(`openfn project pull ${cloud.openfn.uuid}`); + await run(`openfn project fetch --alias prod ${prod.openfn.uuid}`); + + await run('openfn project deploy prod --force'); + + t.deepEqual(code(lightning, prod), { Alpha: 'fn(a)', Beta: 'fn(b)' }); + t.deepEqual(code(lightning, cloud), { Alpha: 'fn(a)', Beta: 'fn(b)' }); +}); + +test('deploy a project yaml file to a different project', async (t) => { + const { workspace, run } = initWorkspace(t); + const { cloud, prod } = genProjects(); + + // save cloud to a file, rather than expanding it into a workspace + await run( + `openfn project fetch --output ${workspace}/cloud.yaml ${cloud.openfn.uuid}` + ); + + const { stdout } = await run( + `openfn project deploy cloud.yaml ${prod.openfn.uuid} --force` + ); + t.regex(stdout, /Gamma: deleted/); + t.regex(stdout, /Alpha: changed/); + t.regex(stdout, /Beta: added/); + + // prod matches the file, and cloud is untouched + t.deepEqual(code(lightning, prod), { Alpha: 'fn(a)', Beta: 'fn(b)' }); + t.deepEqual(code(lightning, cloud), { Alpha: 'fn(a)', Beta: 'fn(b)' }); +}); + +test('deploy a workspace to a project on a different server', async (t) => { + const { run } = initWorkspace(t); + const cloud = gen('cloud', [ + '@name Alpha trigger-job(expression="fn(a)")', + '@name Beta trigger-job(expression="fn(b)")', + ]); + + // a replica of cloud, on another server, with older code + const replica = generateProject( + 'replica', + ['@name Alpha trigger-job(expression="fn(old)")'], + { openfnUuid: true } + ); + other.addProject(replica.serialize('state', { format: 'json' })); + + await run(`openfn project pull ${cloud.openfn.uuid}`); + await run( + `openfn project deploy ${replica.openfn.uuid} --endpoint ${otherEndpoint} --force` + ); + + // the replica matches the workspace, and cloud is untouched + t.deepEqual(code(other, replica), { Alpha: 'fn(a)', Beta: 'fn(b)' }); + t.deepEqual(code(lightning, cloud), { Alpha: 'fn(a)', Beta: 'fn(b)' }); +}); + +test.todo( + 'after deploying to a different project, the workspace still deploys to the project it came from' +); diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index 892b53b55..45014beb1 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -6,9 +6,10 @@ import Project, { RESOURCES_FILE, toResourceChannels, versionsEqual, + Workflow, Workspace, } from '@openfn/project'; -import { isEqual } from 'lodash-es'; +import { isEqual, uniq } from 'lodash-es'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; @@ -220,7 +221,8 @@ export type SyncResult = { // It returns a merged state object // // skipLocallyChangedCheck: merge every workflow in the source, not just those -// edited since its own last snapshot (that check is meaningless for a file) +// edited since its own last snapshot (that check is meaningless for a file, or +// when deploying to a different project) const syncProjects = async ( options: DeployOptions, config: Required, @@ -261,6 +263,8 @@ const syncProjects = async ( // this will actually happen later } + const isCrossProjectDeploy = localProject.uuid !== remoteProject.uuid; + let mergeCandidates: string[]; if (options.workflow?.length) { const missing = options.workflow.filter( @@ -277,6 +281,14 @@ const syncProjects = async ( } else if (skipLocallyChangedCheck) { // the source is the spec, so every workflow in it is a candidate mergeCandidates = localProject.workflows.map((w) => w.id); + if (isCrossProjectDeploy) { + // a different project should end up looking exactly like the source, + // so its own workflows are candidates too (to be removed) + mergeCandidates = uniq([ + ...mergeCandidates, + ...remoteProject.workflows.map((w) => w.id), + ]); + } } else { mergeCandidates = await findLocallyChangedWorkflows(ws, localProject); } @@ -292,6 +304,26 @@ const syncProjects = async ( return null; } + const removedWorkflows = isCrossProjectDeploy + ? workflowDiffs.filter((d) => d.type === 'removed').map((d) => d.id) + : []; + // Without a prompt nobody reviews the removals + if ( + removedWorkflows.length && + !options.confirm && + !options.force && + !options.dryRun + ) { + logger.error(`Error: Deploying would remove workflows from ${ + remoteProject.uuid + }: ${removedWorkflows.join(', ')} + + The target is a different project, so it will be made to look exactly like the local project. + + Pass --force to remove them anyway, or deploy without --no-confirm to review the changes first.`); + throw new Error('REMOVES_WORKFLOWS'); + } + // Ensure there's no divergence // Skip divergence testing if the remote has no history in its workflows @@ -317,9 +349,13 @@ const syncProjects = async ( `The following workflows have diverged: ${divergentWorkflows}` ); if (!options.force) { + const reason = + localProject.uuid === remoteProject.uuid + ? 'The remote project has been edited since the local project was branched.' + : `The target project ${remoteProject.uuid} is not the project this workspace was synced from (${localProject.uuid}), so its version history can't be compared.`; logger.error(`Error: Projects have diverged! - The remote project has been edited since the local project was branched. Changes may be lost. + ${reason} Changes may be lost. Pass --force to override this error and deploy anyway.`); const e: any = new Error('PROJECTS_DIVERGED'); @@ -355,6 +391,15 @@ const syncProjects = async ( } const merged = Project.merge(localProject, remoteProject!, mergeOptions); + // Flag target-only workflows as deleted: just leaving them out of the + // payload would not remove them from the app + if (removedWorkflows.length) { + const ids = new Set(removedWorkflows); + merged.workflows = merged.workflows.map((wf) => + ids.has(wf.id) ? new Workflow(wf.toJSON() as any).remove() : wf + ); + } + return { merged, remoteProject, locallyChangedWorkflows: mergeCandidates }; }; @@ -547,8 +592,9 @@ export async function handler(options: DeployOptions, logger: Logger) { localProject, { uuid: targetUuid!, endpoint }, logger, - // a file's own snapshot tells us nothing about the target - !!filePath + // a file's own snapshot, or a snapshot of a different project, tells us + // nothing about the target + !!filePath || localProject.uuid !== targetUuid ); if (!syncResult) { return; diff --git a/packages/cli/test/projects/deploy.test.ts b/packages/cli/test/projects/deploy.test.ts index ebe3a2ae5..14bc45c1e 100644 --- a/packages/cli/test/projects/deploy.test.ts +++ b/packages/cli/test/projects/deploy.test.ts @@ -348,6 +348,8 @@ test.serial( project: '/ws/.projects/main@localhost.yaml', target: 'dev', confirm: false, + // dev's own workflows get removed, which needs --force without a prompt + force: true, } as any, logger ); @@ -1610,6 +1612,109 @@ test.serial( } ); +const findWorkflow = (projectId: string, name: string) => + Object.values(server.state.projects[projectId].workflows).find( + (wf: any) => wf.name === name + ) as any; + +test.serial( + 'deploy: makes a different project look exactly like the local project', + async (t) => { + await server.addProject(two_workflows_yaml); + await setup(projectYaml); + + await deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + project: TWO_WORKFLOWS_UUID, + checkout: false, + confirm: false, + force: true, + } as any, + logger + ); + + const workflows = Object.values( + server.state.projects[TWO_WORKFLOWS_UUID].workflows + ) as any[]; + t.deepEqual( + workflows.map((wf) => wf.name), + ['My Workflow'] + ); + t.is(workflows[0].jobs['transform-data'].body, 'fn()'); + } +); + +test.serial( + 'deploy: refuses to remove workflows from a different project with --no-confirm, unless --force', + async (t) => { + await server.addProject(two_workflows_yaml); + await setup(projectYaml); + + await t.throwsAsync( + () => + deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + project: TWO_WORKFLOWS_UUID, + checkout: false, + confirm: false, + } as any, + logger + ), + { message: 'REMOVES_WORKFLOWS' } + ); + + // the target is untouched + t.truthy(findWorkflow(TWO_WORKFLOWS_UUID, 'Workflow A')); + t.truthy(findWorkflow(TWO_WORKFLOWS_UUID, 'Workflow B')); + t.falsy(findWorkflow(TWO_WORKFLOWS_UUID, 'My Workflow')); + } +); + +test.serial( + 'deploy: errors when a different target project has the same workflow, unless --force', + async (t) => { + const OTHER_UUID = 'a1b2c3d4-0000-4000-8000-000000000001'; + server.addProject({ ...myProject_v1, id: OTHER_UUID }); + const modified = JSON.parse( + JSON.stringify(server.state.projects[OTHER_UUID].workflows['my-workflow']) + ); + modified.jobs['transform-data'].body = 'each()'; + server.updateWorkflow(OTHER_UUID, modified); + + await setup(projectYaml); + + const options = { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + project: OTHER_UUID, + checkout: false, + confirm: false, + } as any; + + await t.throwsAsync(() => deploy(options, logger), { + message: /PROJECTS_DIVERGED/, + }); + t.truthy( + logger._find('error', /not the project this workspace was synced from/) + ); + + await deploy({ ...options, force: true }, logger); + t.is( + server.state.projects[OTHER_UUID].workflows['my-workflow'].jobs[ + 'transform-data' + ].body, + 'fn()' + ); + } +); + test.serial( 'deploy: an explicit endpoint overrides the one in the project file', async (t) => { diff --git a/packages/project/src/index.ts b/packages/project/src/index.ts index 81081a5ad..6e2ed120c 100644 --- a/packages/project/src/index.ts +++ b/packages/project/src/index.ts @@ -1,10 +1,17 @@ import { Project, MultipleMatchingWorkflowsError } from './Project'; import { Workspace } from './Workspace'; +import Workflow from './Workflow'; import { yamlToJson, jsonToYaml } from './util/yaml'; export default Project; -export { Workspace, yamlToJson, jsonToYaml, MultipleMatchingWorkflowsError }; +export { + Workspace, + Workflow, + yamlToJson, + jsonToYaml, + MultipleMatchingWorkflowsError, +}; export { generateWorkflow, generateProject } from './gen/generator';