diff --git a/.changeset/soft-flowers-lay.md b/.changeset/soft-flowers-lay.md new file mode 100644 index 000000000..cf199183a --- /dev/null +++ b/.changeset/soft-flowers-lay.md @@ -0,0 +1,7 @@ +--- +'@openfn/cli': minor +--- + +Add a fetch command, which will download a project from an app but not check it out. This will throw if the local project version has diverged from the remote version. + +Rebased `pull --beta` to simply be fetch & checkout diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index 7e8c1cb9c..295ea83d9 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -24,7 +24,6 @@ options: allow_support_access: false requires_mfa: false retention_policy: retain_all - version_history: [] workflows: - name: Hello Workflow steps: @@ -80,8 +79,6 @@ options: allow_support_access: false requires_mfa: false retention_policy: retain_all - version_history: - - 7b0f5af558f5 workflows: - name: Hello Workflow steps: @@ -149,8 +146,7 @@ test.serial('Checkout a project', async (t) => { workflowYaml, `id: hello-workflow name: Hello Workflow -options: - history: [] +options: {} steps: - id: trigger type: webhook diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 3427c2c7e..dc9541c9d 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -17,6 +17,7 @@ import testCommand from './test/command'; import projectsCommand from './projects/command'; import mergeCommand from './projects/merge'; import checkoutCommand from './projects/checkout'; +import fetchCommand from './projects/fetch'; const env = loadDotEnv(); if (env) { @@ -44,6 +45,7 @@ export const cmd = y .command(projectsCommand) .command(checkoutCommand) .command(mergeCommand) + .command(fetchCommand as any) .command({ command: 'version', describe: diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 892cfbc8f..cbded7e4f 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -19,6 +19,7 @@ import mapAdaptorsToMonorepo, { import printVersions from './util/print-versions'; import abort from './util/abort'; import { report } from './env'; +import { CLIError } from './errors'; export type CommandList = | 'apollo' @@ -42,6 +43,7 @@ export type CommandList = | 'project-version' | 'project-merge' | 'project-checkout' + | 'project-fetch' | 'test' | 'version'; @@ -68,6 +70,7 @@ const handlers = { ['project-version']: projects.version, ['project-merge']: projects.merge, ['project-checkout']: projects.checkout, + ['project-fetch']: projects.fetch, version: async (opts: Opts, logger: Logger) => printVersions(logger, opts, true), }; @@ -132,7 +135,11 @@ const parse = async (options: Opts, log?: Logger) => { // This is unexpected error and we should try to log something logger.break(); logger.error('Command failed!'); - logger.error(e); + if (e instanceof CLIError) { + logger.error(e.message); + } else { + logger.error(e); + } } } }; diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts new file mode 100644 index 000000000..a673f0cb0 --- /dev/null +++ b/packages/cli/src/errors.ts @@ -0,0 +1,5 @@ +export class CLIError extends Error { + constructor(message: string) { + super(message); + } +} diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index bcde2dee5..652ce4af0 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -10,16 +10,16 @@ import * as o from '../options'; import type { Opts } from '../options'; -export type CheckoutOptions = Required< - Pick -> & - Pick; +export type CheckoutOptions = Pick< + Opts, + 'command' | 'projectId' | 'workspace' | 'log' +>; const options = [o.projectId, o.workspace, o.log]; const command: yargs.CommandModule = { command: 'checkout ', - describe: 'Switch to a different openfn project in the same workspace', + describe: 'Switch to a different OpenFn project in the same workspace', handler: ensure('project-checkout', options), builder: (yargs) => build(options, yargs), }; @@ -27,12 +27,9 @@ const command: yargs.CommandModule = { export default command; export const handler = async (options: CheckoutOptions, logger: Logger) => { - const commandPath = options.workspace; - const workspace = new Workspace(commandPath); - if (!workspace.valid) { - logger.error('Command was run in an invalid openfn workspace'); - return; - } + const projectId = options.projectId!; + const workspacePath = options.workspace ?? process.cwd(); + const workspace = new Workspace(workspacePath, logger); // get the config // TODO: try to retain the endpoint for the projects @@ -40,37 +37,35 @@ export const handler = async (options: CheckoutOptions, logger: Logger) => { // get the project let switchProject; - if (/\.(yaml|json)$/.test(options.projectId)) { + if (/\.(yaml|json)$/.test(projectId)) { // TODO: should we allow checkout into an arbitrary folder? - const filePath = options.projectId.startsWith('/') - ? options.projectId - : path.join(commandPath, options.projectId); + const filePath = projectId.startsWith('/') + ? projectId + : path.join(workspacePath, projectId); logger.debug('Loading project from path ', filePath); switchProject = await Project.from('path', filePath, config); } else { - switchProject = workspace.get(options.projectId); + switchProject = workspace.get(projectId); } if (!switchProject) { - throw new Error( - `Project with id ${options.projectId} not found in the workspace` - ); + throw new Error(`Project with id ${projectId} not found in the workspace`); } // delete workflow dir before expanding project - await rimraf(path.join(commandPath, config.workflowRoot ?? 'workflows')); + await rimraf(path.join(workspacePath, config.workflowRoot ?? 'workflows')); // expand project into directory const files: any = switchProject.serialize('fs'); for (const f in files) { if (files[f]) { - fs.mkdirSync(path.join(commandPath, path.dirname(f)), { + fs.mkdirSync(path.join(workspacePath, path.dirname(f)), { recursive: true, }); - fs.writeFileSync(path.join(commandPath, f), files[f]); + fs.writeFileSync(path.join(workspacePath, f), files[f]); } else { logger.warn('WARNING! No content for file', f); } } - logger.success(`Expanded project to ${commandPath}`); + logger.success(`Expanded project to ${workspacePath}`); }; diff --git a/packages/cli/src/projects/command.ts b/packages/cli/src/projects/command.ts index 5ccdbfec3..fe666789b 100644 --- a/packages/cli/src/projects/command.ts +++ b/packages/cli/src/projects/command.ts @@ -2,6 +2,7 @@ import list from './list'; import version from './version'; import merge from './merge'; import checkout from './checkout'; +import fetch from './fetch'; import type yargs from 'yargs'; @@ -16,6 +17,7 @@ export const projectsCommand = { .command(version) .command(merge) .command(checkout) + .command(fetch as any) .example('project', 'list all projects in the workspace') .example('project list', 'list all projects in the workspace') .example( diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts new file mode 100644 index 000000000..9cbbbcf8c --- /dev/null +++ b/packages/cli/src/projects/fetch.ts @@ -0,0 +1,144 @@ +import yargs from 'yargs'; +import path from 'node:path'; +import Project, { Workspace } from '@openfn/project'; + +import { build, ensure, override } from '../util/command-builders'; +import type { Logger } from '../util/logger'; +import * as o from '../options'; + +import type { Opts } from '../options'; +import { serialize, getProject, loadAppAuthConfig } from './util'; + +// TODO need to implement these +// type Config = { +// requireConfirmation?: boolean; // alias to y maybe +// dryRun?: boolean; +// }; + +export type FetchOptions = Pick< + Opts, + | 'apiKey' + | 'command' + | 'endpoint' + | 'env' + | 'force' + | 'log' + | 'logJson' + | 'outputPath' + | 'projectId' + | 'workspace' +>; + +const options = [ + o.apikey, + o.configPath, + o.endpoint, + o.env, + o.log, + override(o.outputPath, { + description: 'Path to output the fetched project to', + }), + o.logJson, + o.workspace, + o.snapshots, + o.statePath, + override(o.force, { + description: 'Overwrite local file contents with the fetched contents', + }), +]; + +const command: yargs.CommandModule = { + command: 'fetch [projectId]', + describe: `Fetch a project's state and spec from a Lightning Instance to the local state file without expanding to the filesystem.`, + builder: (yargs: yargs.Argv) => + build(options, yargs) + .positional('projectId', { + describe: + 'The id of the project that should be fetched, should be a UUID', + demandOption: true, + }) + .example( + 'fetch 57862287-23e6-4650-8d79-e1dd88b24b1c', + 'Fetch an updated copy of a the above spec and state from a Lightning Instance' + ), + handler: ensure('project-fetch', options), +}; + +export default command; + +export const handler = async (options: FetchOptions, logger: Logger) => { + const workspacePath = path.resolve(options.workspace ?? process.cwd()); + const workspace = new Workspace(workspacePath); + const { projectId, outputPath } = options; + + const config = loadAppAuthConfig(options, logger); + + const { data } = await getProject(logger, config, projectId!); + + const project = await Project.from( + 'state', + data!, + { + endpoint: config.endpoint, + env: options.env || 'project', + }, + workspace.getConfig() + ); + + // Work out where and how to serialize the project + const outputRoot = path.resolve(outputPath || workspacePath); + const projectFileName = project.getIdentifier(); + const projectsDir = project.config.dirs.projects ?? '.projects'; + + const finalOutputPath = + outputPath ?? `${outputRoot}/${projectsDir}/${projectFileName}`; + let format: undefined | 'json' | 'yaml' = undefined; + + if (outputPath) { + // If the user gave us a path for output, we need to respect the format we've been given + const ext = path.extname(outputPath!).substring(1) as any; + if (ext.length) { + format = ext; + } + } + + // See if a project already exists there + const finalOutput = await serialize( + project, + finalOutputPath!, + format, + true // dry run - this won't trigger an actual write! + ); + + // If a project already exists at the output path, make sure it's compatible + let current: Project | null = null; + try { + current = await Project.from('path', finalOutput); + } catch (e) { + // Do nothing - project doesn't exist + } + + const hasAnyHistory = project.workflows.find( + (w) => w.workflow.history?.length + ); + + // Skip version checking if: + const skipVersionCheck = + options.force || // The user forced the checkout + !current || // there is no project on disk + !hasAnyHistory; // the remote project has no history (can happen in old apps) + + if (!skipVersionCheck && !project.canMergeInto(current!)) { + // TODO allow rename + throw new Error('Error! An incompatible project exists at this location'); + } + + // TODO report whether we've updated or not + + // finally, write it! + await serialize(project, finalOutputPath!, format as any); + + logger.success(`Fetched project file to ${finalOutput}`); + + return project; +}; diff --git a/packages/cli/src/projects/handler.ts b/packages/cli/src/projects/handler.ts index fa56a3d9a..b8f1e4cc0 100644 --- a/packages/cli/src/projects/handler.ts +++ b/packages/cli/src/projects/handler.ts @@ -2,3 +2,4 @@ export { handler as list } from './list'; export { handler as version } from './version'; export { handler as merge } from './merge'; export { handler as checkout } from './checkout'; +export { handler as fetch } from './fetch'; diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts new file mode 100644 index 000000000..07f8ae75c --- /dev/null +++ b/packages/cli/src/projects/pull.ts @@ -0,0 +1,33 @@ +import { handler as fetch } from './fetch'; +import { handler as checkout } from './checkout'; + +import type { Logger } from '../util/logger'; +import type { Opts } from '../options'; + +export type PullOptions = Pick< + Opts, + | 'apiKey' + | 'endpoint' + | 'env' + | 'force' + | 'log' + | 'logJson' + | 'projectId' + | 'workspace' +>; + +export async function handler(options: PullOptions, logger: Logger) { + const project = await fetch(options, logger); + logger.success(`Downloaded latest project version`); + + await checkout( + { + ...options, + projectId: project.id, + }, + logger + ); + logger.success(`Checked out project locally`); +} + +export default handler; diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index e69de29bb..3763c9f69 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -0,0 +1,141 @@ +import path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { Provisioner } from '@openfn/lexicon/lightning'; + +import type { Opts } from '../options'; +import type { Logger } from '@openfn/logger'; +import type Project from '@openfn/project'; +import { CLIError } from '../errors'; + +type AuthOptions = Pick; + +export const loadAppAuthConfig = ( + options: AuthOptions, + logger: Logger +): Required => { + const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; + + const config: AuthOptions = { + apiKey: options.apiKey, + endpoint: options.endpoint, + }; + + if (!options.apiKey && OPENFN_API_KEY) { + logger.info('Using OPENFN_API_KEY environment variable'); + config.apiKey = OPENFN_API_KEY; + } + + if (!options.endpoint && OPENFN_ENDPOINT) { + logger.info('Using OPENFN_ENDPOINT environment variable'); + config.endpoint = OPENFN_ENDPOINT; + } + + // TODO probably need to throw + + return config as Required; +}; + +const ensureExt = (filePath: string, ext: string) => { + if (!filePath.endsWith(ext)) { + return `${filePath}.${ext}`; + } + return filePath; +}; + +export const serialize = async ( + project: Project, + outputPath: string, + formatOverride?: 'yaml' | 'json', + dryRun = false +) => { + const root = path.dirname(outputPath); + await mkdir(root, { recursive: true }); + + const format = formatOverride ?? project.config?.formats.project; + const output = project?.serialize('project', { format }); + + const maybeWriteFile = (filePath: string, output: string) => { + if (!dryRun) { + return writeFile(filePath, output); + } + }; + + let finalPath; + if (format === 'yaml') { + finalPath = ensureExt(outputPath, 'yaml'); + await maybeWriteFile(finalPath, output as string); + } else { + finalPath = ensureExt(outputPath, 'json'); + await maybeWriteFile(finalPath, JSON.stringify(output, null, 2)); + } + + return finalPath; +}; + +export const getLightningUrl = ( + config: AuthOptions, + path: string = '', + snapshots?: string[] +) => { + const params = new URLSearchParams(); + snapshots?.forEach((snapshot) => params.append('snapshots[]', snapshot)); + return new URL( + `/api/provision/${path}?${params.toString()}`, + config.endpoint + ); +}; + +export async function getProject( + logger: Logger, + config: AuthOptions, + projectId: string, + snapshots?: string[] +): Promise<{ data: Provisioner.Project | null }> { + const url = getLightningUrl(config, projectId, snapshots); + logger.info(`Checking ${url} for existing project`); + + try { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${config.apiKey}`, + Accept: 'application/json', + }, + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new CLIError( + `Failed to authorize request with endpoint ${config.endpoint}, got ${response.status} ${response.statusText}` + ); + } + if (response.status === 404) { + throw new CLIError(`Project not found: ${projectId}`); + } + + throw new CLIError( + `Failed to fetch project ${projectId}: ${response.statusText}` + ); + } + + logger.info('Project found'); + return response.json(); + } catch (error: any) { + handleCommonErrors(config, error); + + throw error; + } +} + +function handleCommonErrors(config: AuthOptions, error: any) { + if (error.cause?.code === 'ECONNREFUSED') { + throw new DeployError( + `Failed to connect to endpoint ${config.endpoint}, got ECONNREFUSED.` + ); + } +} + +class DeployError extends Error { + constructor(message: string) { + super(message); + } +} diff --git a/packages/cli/src/pull/beta.ts b/packages/cli/src/pull/beta.ts deleted file mode 100644 index 5e0948905..000000000 --- a/packages/cli/src/pull/beta.ts +++ /dev/null @@ -1,138 +0,0 @@ -// beta v2 version of CLI pull -import path from 'node:path'; -import fs from 'node:fs/promises'; -import { rimraf } from 'rimraf'; -import { confirm } from '@inquirer/prompts'; - -import { DeployConfig, getProject } from '@openfn/deploy'; -import Project, { Workspace } from '@openfn/project'; -import { Provisioner } from '@openfn/lexicon/lightning'; - -import type { Logger } from '../util/logger'; -import type { Opts } from '../options'; - -// new config -type Config = { - endpoint: string; - apiKey: string | null; - - // maybe keep? - requireConfirmation?: boolean; - dryRun?: boolean; -}; - -export type PullOptionsBeta = Required< - Pick< - Opts, - | 'apiKey' - | 'beta' - | 'command' - | 'confirm' - | 'endpoint' - | 'env' - | 'log' - | 'logJson' - | 'path' - | 'projectId' - > ->; - -export async function handler(options: PullOptionsBeta, logger: Logger) { - const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; - - const cfg: Partial = { - apiKey: options.apiKey, - endpoint: options.endpoint, - }; - - if (!options.apiKey && OPENFN_API_KEY) { - logger.info('Using OPENFN_API_KEY environment variable'); - cfg.apiKey = OPENFN_API_KEY; - } - - if (!options.endpoint && OPENFN_ENDPOINT) { - logger.info('Using OPENFN_ENDPOINT environment variable'); - cfg.endpoint = OPENFN_ENDPOINT; - } - - // TODO `path` or `output` ? - // I don't think I want to model this as output. deploy is really - // designed to run from the working folder - // could be projectPath or repoPath too - const outputRoot = path.resolve(options.path || '.'); - - // TODO is outputRoot the right dir for this? - const workspace = new Workspace(outputRoot); - const config = workspace.getConfig(); - - // download the state.json from lightning - const { data } = await getProject(cfg as DeployConfig, options.projectId); - - // TODO if the user doesn't specify an env name, prompt for one - const name = options.env || 'project'; - - const project = await Project.from( - 'state', - data as Provisioner.Project, - { - endpoint: cfg.endpoint, - env: name, - - // TODO this is NOT an openfn metadata key - // (it should not be sent back to lighting) - // should add it to the local or meta objects instead - fetched_at: new Date().toISOString(), - }, - config - ); - - const projectFileName = project.getIdentifier(); - - await fs.mkdir(`${outputRoot}/.projects`, { recursive: true }); - const stateOutputPath = `${outputRoot}/.projects/${projectFileName}`; - - const workflowsRoot = path.resolve( - outputRoot, - project.config.dirs.workflows ?? 'workflows' - ); - // Prompt before deleting - // TODO this is actually the wrong path - if ( - !(await confirm({ - message: `This will remove all files in ${path.resolve( - workflowsRoot - )}. Are you sure you wish to proceed? -`, - default: true, - })) - ) { - logger.always('Cancelled'); - return false; - } - await rimraf(workflowsRoot); - - const projFile = project?.serialize('project'); - - if (typeof projFile === 'string') { - await fs.writeFile(`${stateOutputPath}.yaml`, projFile); - } else { - await fs.writeFile( - `${stateOutputPath}.json`, - JSON.stringify(projFile, null, 2) - ); - } - logger.success(`Saved project file to ${stateOutputPath}`); - - const files = project?.serialize('fs') as any; - for (const f in files) { - if (files[f]) { - await fs.mkdir(path.join(outputRoot, path.dirname(f)), { - recursive: true, - }); - await fs.writeFile(path.join(outputRoot, f), files[f]); - } else { - console.log('WARNING! No content for file', f); - } - } - logger.success(`Expanded project to ${outputRoot}`); -} diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index 0d81af776..2ddb8c1f3 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -35,6 +35,11 @@ const options = [ o.snapshots, o.statePath, o.path, + + // These are hidden commands used only by beta + // The need to be declared here to be initialised and defaulted properly + override(o.force, { hidden: true }), + override(o.workspace, { hidden: true }), ]; const pullCommand: yargs.CommandModule = { diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index 4bf9ca5cf..7309a33ba 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -10,11 +10,11 @@ import { } from '@openfn/deploy'; import type { Logger } from '../util/logger'; import { PullOptions } from '../pull/command'; -import * as beta from './beta'; +import beta from '../projects/pull'; async function pullHandler(options: PullOptions, logger: Logger) { if (options.beta) { - return beta.handler(options as any, logger); + return beta(options as any, logger); } try { diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts new file mode 100644 index 000000000..fc880f42e --- /dev/null +++ b/packages/cli/test/projects/fetch.test.ts @@ -0,0 +1,280 @@ +import { readFile } from 'node:fs/promises'; +import test from 'ava'; +import mock from 'mock-fs'; +import { MockAgent, setGlobalDispatcher } from 'undici'; +import { createMockLogger } from '@openfn/logger'; + +import { handler as fetchHandler } from '../../src/projects/fetch'; +import { myProject_v1, myProject_yaml } from './fixtures'; + +const logger = createMockLogger('', { level: 'debug' }); + +const ENDPOINT = 'https://app.openfn.org'; +const PROJECT_ID = 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00'; + +let mockAgent = new MockAgent(); +mockAgent.disableNetConnect(); +setGlobalDispatcher(mockAgent); + +test.before(() => { + const mockPool = mockAgent.get(ENDPOINT); + mockPool + .intercept({ + path: `/api/provision/${PROJECT_ID}?`, + method: 'GET', + }) + .reply(200, { + data: myProject_v1, + }) + .persist(); +}); + +test.beforeEach(() => { + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + }); + logger._reset(); +}); + +test.afterEach(() => { + mock.restore(); +}); + +test.serial('fetch from lightning and save as v2 yaml file', async (t) => { + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + } as any, + logger + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + const yaml = myProject_yaml; + + t.is(fileContent.trim(), yaml); + + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); +}); + +test.serial('save to a custom location', async (t) => { + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws/out.yaml', + } as any, + logger + ); + + const filePath = '/ws/out.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + const yaml = myProject_yaml; + + t.is(fileContent.trim(), yaml); + + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); +}); + +test.serial( + 'save JSON to a custom location, overriding project defaults', + async (t) => { + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws/out.json', + } as any, + logger + ); + + const filePath = '/ws/out.json'; + const fileContent = await readFile(filePath, 'utf-8'); + + const json = { + id: 'my-project', + name: 'My Project', + version: 2, + description: 'my lovely project', + collections: [], + credentials: [], + openfn: { + uuid: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00', + endpoint: 'https://app.openfn.org', + env: 'project', + inserted_at: '2025-04-23T11:15:59Z', + updated_at: '2025-04-23T11:15:59Z', + }, + options: { + allow_support_access: false, + requires_mfa: false, + retention_policy: 'retain_all', + }, + workflows: [ + { + name: 'My Workflow', + steps: [ + { + id: 'transform-data', + name: 'Transform data', + expression: 'fn()', + adaptor: '@openfn/language-common@latest', + openfn: { + uuid: '66add020-e6eb-4eec-836b-20008afca816', + }, + }, + { + id: 'trigger', + type: 'webhook', + openfn: { + enabled: true, + uuid: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + }, + next: { + 'transform-data': { + disabled: false, + condition: true, + openfn: { + uuid: 'a9a3adef-b394-4405-814d-3ac4323f4b4b', + }, + }, + }, + }, + ], + openfn: { + uuid: '72ca3eb0-042c-47a0-a2a1-a545ed4a8406', + inserted_at: '2025-04-23T11:19:32Z', + updated_at: '2025-04-23T11:19:32Z', + lock_version: 1, + }, + id: 'my-workflow', + history: ['a'], + }, + ], + }; + + t.deepEqual(JSON.parse(fileContent), json); + + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); + } +); + +test.serial('Override a compatible project', async (t) => { + // Change project.yaml + const modified = myProject_yaml.replace('my lovely project', 'renamed'); + + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + '/ws/.projects/project@app.openfn.org.yaml': modified, + }); + + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + } as any, + logger + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + // This should overwrite the renamed value back to the default + t.regex(fileContent, /my lovely project/); +}); + +// In this test, the file on disk has diverged from the remove +// This means changes could be lost, so we throw! +test.serial('throw for an incompatible project', async (t) => { + // Change project.yaml + const modified = myProject_yaml + .replace('fn()', 'fn(x)') // arbitrary edit so that we can track the change + .replace(' - a', ' - z'); // change the local history to be incompatible + + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + '/ws/.projects/project@app.openfn.org.yaml': modified, + }); + + await t.throwsAsync( + () => + fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + } as any, + logger + ), + { + message: /incompatible project/, + } + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + // The file should NOT be overwritten + t.regex(fileContent, /fn\(x\)/); +}); + +test.serial('force merge an incompatible project', async (t) => { + // Change project.yaml + const modified = myProject_yaml.replace('fn()', 'fn(x)'); + + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + '/ws/.projects/project@app.openfn.org.yaml': modified, + }); + + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + force: true, + } as any, + logger + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + // The file should be overwritten + t.regex(fileContent, /fn\(\)/); +}); diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts new file mode 100644 index 000000000..de1f06896 --- /dev/null +++ b/packages/cli/test/projects/fixtures.ts @@ -0,0 +1,101 @@ +import type { Provisioner } from '@openfn/lexicon/lightning'; + +export const myProject_v1: Provisioner.Project = { + id: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00', + name: 'My Project', + description: 'my lovely project', + concurrency: null, + inserted_at: '2025-04-23T11:15:59Z', + collections: [], + workflows: [ + { + id: '72ca3eb0-042c-47a0-a2a1-a545ed4a8406', + name: 'My Workflow', + edges: [ + { + enabled: true, + id: 'a9a3adef-b394-4405-814d-3ac4323f4b4b', + source_trigger_id: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + condition_type: 'always', + target_job_id: '66add020-e6eb-4eec-836b-20008afca816', + }, + ], + concurrency: null, + inserted_at: '2025-04-23T11:19:32Z', + updated_at: '2025-04-23T11:19:32Z', + jobs: [ + { + id: '66add020-e6eb-4eec-836b-20008afca816', + name: 'Transform data', + body: 'fn()', + adaptor: '@openfn/language-common@latest', + project_credential_id: null, + keychain_credential_id: null, + }, + ], + triggers: [ + { + enabled: true, // TODO enabled: false is a bit interesting + id: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + type: 'webhook', + }, + ], + lock_version: 1, + deleted_at: null, + version_history: ['a'], // TODO not implemented yet? needed for tests + }, + ], + updated_at: '2025-04-23T11:15:59Z', + project_credentials: [], + scheduled_deletion: null, + allow_support_access: false, + requires_mfa: false, + retention_policy: 'retain_all', + history_retention_period: null, + dataclip_retention_period: null, +}; + +export const myProject_yaml = `id: my-project +name: My Project +version: 2 +description: my lovely project +collections: [] +credentials: [] +openfn: + uuid: e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00 + endpoint: https://app.openfn.org + env: project + inserted_at: 2025-04-23T11:15:59Z + updated_at: 2025-04-23T11:15:59Z +options: + allow_support_access: false + requires_mfa: false + retention_policy: retain_all +workflows: + - name: My Workflow + steps: + - id: transform-data + name: Transform data + expression: fn() + adaptor: "@openfn/language-common@latest" + openfn: + uuid: 66add020-e6eb-4eec-836b-20008afca816 + - id: trigger + type: webhook + openfn: + enabled: true + uuid: 4a06289c-15aa-4662-8dc6-f0aaacd8a058 + next: + transform-data: + disabled: false + condition: true + openfn: + uuid: a9a3adef-b394-4405-814d-3ac4323f4b4b + history: + - a + openfn: + uuid: 72ca3eb0-042c-47a0-a2a1-a545ed4a8406 + inserted_at: 2025-04-23T11:19:32Z + updated_at: 2025-04-23T11:19:32Z + lock_version: 1 + id: my-workflow`; diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts index ce4d7cb7c..4e1b50259 100644 --- a/packages/lexicon/lightning.d.ts +++ b/packages/lexicon/lightning.d.ts @@ -253,6 +253,8 @@ export namespace Provisioner { delete?: boolean; project_id?: string; + version_history?: string[]; + concurrency?: null; // TODO lock_version?: number; inserted_at?: string; diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 9780ad340..6555143b3 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -6,7 +6,7 @@ import fromAppState, { fromAppStateConfig } from './parse/from-app-state'; import fromPath, { FromPathConfig } from './parse/from-path'; // TODO this naming clearly isn't right import { parseProject as fromFs, FromFsConfig } from './parse/from-fs'; -import fromProject from './parse/from-project'; +import fromProject, { SerializedProject } from './parse/from-project'; import getIdentifier from './util/get-identifier'; import slugify from './util/slugify'; import { getUuidForEdge, getUuidForStep } from './util/uuid'; @@ -150,6 +150,9 @@ export class Project { this.config = buildConfig(config); } + serialize(type: 'project', options?: any): SerializedProject | string; + serialize(type: 'state', options?: any): Provisioner.Project | string; + serialize(type: 'fs', options?: any): Record; serialize(type: 'project' | 'fs' | 'state' = 'project', options?: any) { if (type in serializers) { // @ts-ignore @@ -199,6 +202,22 @@ export class Project { } return result; } + + canMergeInto(target: Project) { + const potentialConflicts: Record = {}; + for (const sourceWorkflow of this.workflows) { + // TODO mapping needs work + const targetId = sourceWorkflow.id; + const targetWorkflow = target.getWorkflow(targetId); + if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) { + potentialConflicts[sourceWorkflow.id] = targetWorkflow?.id; + } + } + if (Object.keys(potentialConflicts).length) { + return false; + } + return true; + } } export default Project; diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 68d30d4c1..98820651d 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -30,7 +30,7 @@ class Workflow { // history needs to be on workflow object. this.workflow.history = workflow.history?.length ? workflow.history : []; - const { id, name, openfn, steps, ...options } = workflow; + const { id, name, openfn, steps, history, ...options } = workflow; if (!(id || name)) { throw new Error('A Workflow MUST have a name or id'); } diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index e55658cf9..7e48d16d6 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -1,4 +1,5 @@ import * as l from '@openfn/lexicon'; +import createLogger from '@openfn/logger'; import path from 'node:path'; import fs from 'node:fs'; @@ -10,6 +11,7 @@ import { findWorkspaceFile, } from './util/config'; import fromProject from './parse/from-project'; +import type { Logger } from '@openfn/logger'; export class Workspace { // @ts-ignore config not definitely assigned - it sure is @@ -19,27 +21,27 @@ export class Workspace { private projects: Project[] = []; private projectPaths = new Map(); private isValid: boolean = false; + private logger: Logger; - constructor(workspacePath: string) { - let context; + constructor(workspacePath: string, logger?: Logger) { + this.logger = logger ?? createLogger('Workspace', { level: 'info' }); + + let context = { workspace: undefined, project: undefined }; try { const { type, content } = findWorkspaceFile(workspacePath); context = loadWorkspaceFile(content, type as any); this.isValid = true; } catch (e) { - // TODO use logger - // TODO maybe we should throw here? - // TODO showing hte direct error here is rarely useful - console.error(e); - // invalid workspace - return; + this.logger.warn( + `Could not find openfn.yaml at ${workspacePath}. Using default values.` + ); } this.config = buildConfig(context.workspace); this.activeProject = context.project; const projectsPath = path.join(workspacePath, this.config.dirs.projects); // dealing with projects - if (this.isValid && pathExists(projectsPath, 'directory')) { + if (pathExists(projectsPath, 'directory')) { const ext = `.${this.config.formats.project}`; const stateFiles = fs .readdirSync(projectsPath) @@ -65,6 +67,10 @@ export class Workspace { } }) .filter((s) => s) as Project[]; + } else { + this.logger.warn( + `No projects found: directory at ${projectsPath} does not exist` + ); } } diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index 64bf96b61..ff8b01c5d 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -83,10 +83,12 @@ const mapTriggerEdgeCondition = (edge: Provisioner.Edge) => { // map a project workflow to a local cli workflow // TODO this probably gets easier if I index everything by name export const mapWorkflow = (workflow: Provisioner.Workflow) => { - const { jobs, edges, triggers, name, ...remoteProps } = workflow; + const { jobs, edges, triggers, name, version_history, ...remoteProps } = + workflow; const mapped: l.Workflow = { name: workflow.name, steps: [], + history: workflow.version_history ?? [], openfn: renameKeys(remoteProps, { id: 'uuid' }), }; if (workflow.name) { diff --git a/packages/project/src/parse/from-project.ts b/packages/project/src/parse/from-project.ts index 9e4bcbfee..5ac129b6a 100644 --- a/packages/project/src/parse/from-project.ts +++ b/packages/project/src/parse/from-project.ts @@ -4,6 +4,7 @@ import Project from '../Project'; import ensureJson from '../util/ensure-json'; import { Provisioner } from '@openfn/lexicon/lightning'; import fromAppState from './from-app-state'; +import { WithMeta } from '../Workflow'; // Load a project from any JSON or yaml representation // This is backwards-compatible with v1 state.json files @@ -19,7 +20,7 @@ export type SerializedWorkflow = { id: string; name: string; - steps: l.Step[]; + steps: WithMeta; openfn?: l.ProjectMeta; }; diff --git a/packages/project/src/serialize/to-project.ts b/packages/project/src/serialize/to-project.ts index 6c2833a16..4463e3e93 100644 --- a/packages/project/src/serialize/to-project.ts +++ b/packages/project/src/serialize/to-project.ts @@ -1,3 +1,4 @@ +import * as l from '@openfn/lexicon'; import { omitBy, isNil } from 'lodash-es'; import type { @@ -6,6 +7,7 @@ import type { } from '../parse/from-project'; import Project from '../Project'; import { jsonToYaml } from '../util/yaml'; +import { WithMeta } from '../Workflow'; const SERIALIZE_VERSION = 2; @@ -29,14 +31,33 @@ export default (project: Project, options: ToProjectOptions = {}) => { collections: project.collections, credentials: project.credentials, - openfn: project.openfn, + openfn: omitBy(project.openfn, isNil), meta: project.meta, options: omitBy(project.options, isNil), - //workflows: project.workflows.map(mapWorkflow) as SerializedWorkflow[], - workflows: project.workflows.map((w) => - w.toJSON() - ) as SerializedWorkflow[], + workflows: project.workflows.map((w) => { + const obj = w.toJSON() as SerializedWorkflow; + if (obj.openfn) { + obj.openfn = omitBy(obj.openfn, isNil); + } + if (obj.steps) { + obj.steps = obj.steps.sort((a: any, b: any) => { + return a.id < b.id ? -1 : a.id > b.id ? 1 : 0; + }); + obj.steps.forEach((s: WithMeta) => { + s.openfn = omitBy(s.openfn, isNil); + if (s.next && typeof s.next !== 'string') { + for (const id in s.next) { + const edge = s.next[id] as any; + if (edge.openfn) { + edge.openfn = omitBy(edge.openfn, isNil); + } + } + } + }); + } + return obj; + }), }, isNil ) as SerializedProject; diff --git a/packages/project/test/fixtures/sample-v2-project.ts b/packages/project/test/fixtures/sample-v2-project.ts index 2ae20f223..8a471ac0a 100644 --- a/packages/project/test/fixtures/sample-v2-project.ts +++ b/packages/project/test/fixtures/sample-v2-project.ts @@ -8,44 +8,29 @@ export const json: SerializedProject = { name: 'My Project', version: 2, description: 'my lovely project', - openfn: { - uuid: '1234', - endpoint: 'https://app.openfn.org', - }, - options: { - allow_support_access: false, - }, + openfn: { uuid: '1234', endpoint: 'https://app.openfn.org' }, + options: { allow_support_access: false }, workflows: [ { steps: [ - { - name: 'a', - id: 'a', - openfn: { - uuid: 2, - }, - next: { - b: { - openfn: { - uuid: 4, - }, - }, - }, - }, { name: 'b', id: 'b', - openfn: { - uuid: 3, - }, + openfn: { uuid: 3 }, + expression: 'fn()', + adaptor: 'common', + }, + { + id: 'trigger', + openfn: { uuid: 2 }, + type: 'webhook', + next: { b: { openfn: { uuid: 4 } } }, }, ], name: 'Workflow', id: 'workflow', + openfn: { uuid: 1 }, history: [], - openfn: { - uuid: 1, - }, }, ], }; @@ -61,18 +46,20 @@ options: allow_support_access: false workflows: - steps: - - name: a - id: a + - name: b + id: b + openfn: + uuid: 3 + expression: fn() + adaptor: common + - id: trigger openfn: uuid: 2 + type: webhook next: b: openfn: uuid: 4 - - name: b - id: b - openfn: - uuid: 3 name: Workflow id: workflow openfn: diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts index 993985641..abda93472 100644 --- a/packages/project/test/parse/from-project.test.ts +++ b/packages/project/test/parse/from-project.test.ts @@ -87,8 +87,17 @@ test('import from a v2 project as JSON', async (t) => { history: [], steps: [ { - name: 'a', - id: 'a', + name: 'b', + id: 'b', + expression: 'fn()', + adaptor: 'common', + openfn: { + uuid: 3, + }, + }, + { + id: 'trigger', + type: 'webhook', openfn: { uuid: 2, }, @@ -100,13 +109,6 @@ test('import from a v2 project as JSON', async (t) => { }, }, }, - { - name: 'b', - id: 'b', - openfn: { - uuid: 3, - }, - }, ], }); }); @@ -130,8 +132,17 @@ test('import from a v2 project as YAML', async (t) => { history: [], steps: [ { - name: 'a', - id: 'a', + name: 'b', + id: 'b', + expression: 'fn()', + adaptor: 'common', + openfn: { + uuid: 3, + }, + }, + { + id: 'trigger', + type: 'webhook', openfn: { uuid: 2, }, @@ -143,13 +154,6 @@ test('import from a v2 project as YAML', async (t) => { }, }, }, - { - name: 'b', - id: 'b', - openfn: { - uuid: 3, - }, - }, ], }); }); diff --git a/packages/project/test/serialize/to-project.test.ts b/packages/project/test/serialize/to-project.test.ts index a9d48dfbe..6ad7313d5 100644 --- a/packages/project/test/serialize/to-project.test.ts +++ b/packages/project/test/serialize/to-project.test.ts @@ -4,27 +4,42 @@ import generateWorkflow, { generateProject } from '../../src/gen/generator'; import * as v2 from '../fixtures/sample-v2-project'; -const proj = new Project({ - id: 'my-project', - name: 'My Project', - description: 'my lovely project', - openfn: { - uuid: '1234', - endpoint: 'https://app.openfn.org', - }, - options: { - allow_support_access: false, - }, - workflows: [generateWorkflow('a-b', { uuidSeed: 1, openfnUuid: true })], -}); +const createProject = () => { + const proj = new Project({ + id: 'my-project', + name: 'My Project', + description: 'my lovely project', + openfn: { + uuid: '1234', + endpoint: 'https://app.openfn.org', + }, + options: { + allow_support_access: false, + }, + workflows: [ + generateWorkflow( + 'trigger(type=webhook)-b(expression="fn()",adaptor=common)', + { + uuidSeed: 1, + openfnUuid: true, + } + ), + ], + }); + // hack + delete proj.workflows[0].steps[0].name; + return proj; +}; test('should serialize to YAML format v2 project by default', (t) => { + const proj = createProject(); const yaml = proj.serialize('project'); t.deepEqual(yaml, v2.yaml); }); test('should explicitly serialize to YAML format', (t) => { + const proj = createProject(); const yaml = proj.serialize('project', { format: 'yaml' }); t.deepEqual(yaml, v2.yaml); @@ -34,9 +49,20 @@ test('should explicitly serialize to YAML format', (t) => { // so workflow b-a should look exactly the same as this // That should probably happen in workflow.toJSON test('should serialize to JSON format v2 project', (t) => { + const proj = createProject(); const json = proj.serialize('project', { format: 'json' }); - t.deepEqual(json, v2.json); }); // should load a project and serialize it back + +test('should exclude null values in yaml', (t) => { + const proj = createProject(); + + // force some null values into the workflow structure + proj.workflows[0].openfn.concurrency = null; + proj.workflows[0].steps[1].openfn.project_credential_id = null; + + const yaml = proj.serialize('project', { format: 'yaml' }); + t.deepEqual(yaml, v2.yaml); +}); diff --git a/packages/project/test/workspace.test.ts b/packages/project/test/workspace.test.ts index ff46226f3..93b0f61b3 100644 --- a/packages/project/test/workspace.test.ts +++ b/packages/project/test/workspace.test.ts @@ -183,11 +183,18 @@ mock({ test('workspace-path: valid workspace path', (t) => { const ws = new Workspace('/ws'); t.is(ws.valid, true); + + t.truthy(ws.config); }); test('workspace-path: invalid workspace path', (t) => { const ws = new Workspace('/invalid'); t.is(ws.valid, false); + + // should still have config + t.truthy(ws.config); + t.truthy(ws.config.dirs); + t.truthy(ws.config.formats); }); test('workspace-list: list projects in the workspace', (t) => {