From 05c4eeb70295b56e130cba345d47eba7f557d5a8 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 22 Oct 2025 12:25:45 +0000 Subject: [PATCH 01/28] feat: init fetch command --- packages/cli/src/cli.ts | 2 + packages/cli/src/commands.ts | 3 ++ packages/cli/src/fetch/command.ts | 57 ++++++++++++++++++++++ packages/cli/src/fetch/handler.ts | 80 +++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 packages/cli/src/fetch/command.ts create mode 100644 packages/cli/src/fetch/handler.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 3427c2c7e..32cbf60c0 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -17,6 +17,7 @@ import testCommand from './test/command'; import projectsCommand from './projects/command'; import mergeCommand from './projects/merge'; import checkoutCommand from './projects/checkout'; +import fetchCommand from './fetch/command'; const env = loadDotEnv(); if (env) { @@ -44,6 +45,7 @@ export const cmd = y .command(projectsCommand) .command(checkoutCommand) .command(mergeCommand) + .command(fetchCommand as any) .command({ command: 'version', describe: diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 892cfbc8f..6bc880a64 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -11,6 +11,7 @@ import metadata from './metadata/handler'; import pull from './pull/handler'; import * as projects from './projects/handler'; import * as repo from './repo/handler'; +import fetchHandler from './fetch/handler'; import createLogger, { CLI, Logger } from './util/logger'; import mapAdaptorsToMonorepo, { @@ -32,6 +33,7 @@ export type CommandList = | 'execute' | 'metadata' | 'pull' + | 'fetch' | 'projects' | 'project' | 'repo-clean' @@ -55,6 +57,7 @@ const handlers = { docs, metadata, pull, + fetch: fetchHandler, projects, project: projects, ['collections-get']: collections.get, diff --git a/packages/cli/src/fetch/command.ts b/packages/cli/src/fetch/command.ts new file mode 100644 index 000000000..3113a7123 --- /dev/null +++ b/packages/cli/src/fetch/command.ts @@ -0,0 +1,57 @@ +import yargs from 'yargs'; +import { build, ensure, override } from '../util/command-builders'; +import { Opts } from '../options'; +import * as o from '../options'; + +export type FetchOptions = Required< + Pick< + Opts, + | 'beta' + | 'command' + | 'log' + | 'logJson' + | 'statePath' + | 'projectPath' + | 'configPath' + | 'projectId' + | 'confirm' + | 'snapshots' + > +>; + +const options = [ + o.apikey, + o.beta, + o.beta, + o.configPath, + o.endpoint, + o.env, + o.log, + override(o.path, { + description: 'path to output the project to', + }), + o.logJson, + o.projectPath, + o.snapshots, + o.statePath, + o.path, +]; + +const fetchCommand: yargs.CommandModule = { + command: 'fetch [projectId]', + describe: `Fetch a project's state and spec from a Lightning Instance to the local state file without expanding to the filesystem.`, + builder: (yargs: yargs.Argv) => + build(options, yargs) + .positional('projectId', { + describe: + 'The id of the project that should be fetched, should be a UUID', + demandOption: true, + }) + .example( + 'fetch 57862287-23e6-4650-8d79-e1dd88b24b1c', + 'Fetch an updated copy of a the above spec and state from a Lightning Instance' + ), + handler: ensure('fetch', options), +}; + +export default fetchCommand; diff --git a/packages/cli/src/fetch/handler.ts b/packages/cli/src/fetch/handler.ts new file mode 100644 index 000000000..3e2d8d7ec --- /dev/null +++ b/packages/cli/src/fetch/handler.ts @@ -0,0 +1,80 @@ +import { DeployConfig, getProject } from '@openfn/deploy'; +import Project from '@openfn/project'; +import fs from 'node:fs/promises'; +import path from 'path'; +import { Opts } from '../options'; +import type { Logger } from '../util/logger'; + +type Config = { + endpoint: string; + apiKey: string | null; + + requireConfirmation?: boolean; + dryRun?: boolean; +}; + +export type FetchOptions = Required< + Pick< + Opts, + | 'apiKey' + | 'beta' + | 'command' + | 'confirm' + | 'endpoint' + | 'env' + | 'log' + | 'logJson' + | 'path' + | 'projectId' + > +>; + +export default async function fetchHandler( + options: FetchOptions, + logger: Logger +) { + const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; + + const config: Partial = { + apiKey: options.apiKey, + endpoint: options.endpoint, + }; + + if (!options.apiKey && OPENFN_API_KEY) { + logger.info('Using OPENFN_API_KEY environment variable'); + config.apiKey = OPENFN_API_KEY; + } + + if (!options.endpoint && OPENFN_ENDPOINT) { + logger.info('Using OPENFN_ENDPOINT environment variable'); + config.endpoint = OPENFN_ENDPOINT; + } + + // download the state.json from lightning + const { data } = await getProject(config as DeployConfig, options.projectId); + const name = options.env || 'project'; + + const project = Project.from('state', data, { + endpoint: config.endpoint, + env: name, + fetched_at: new Date().toISOString(), + }); + + const outputRoot = path.resolve(options.path || '.'); + + const projectFileName = project.getIdentifier(); + + await fs.mkdir(`${outputRoot}/.projects`, { recursive: true }); + let stateOutputPath = `${outputRoot}/.projects/${projectFileName}`; + + const state = project?.serialize('state'); + if (project.repo?.formats.project === 'yaml') { + await fs.writeFile(`${stateOutputPath}.yaml`, state); + } else { + await fs.writeFile( + `${stateOutputPath}.json`, + JSON.stringify(state, null, 2) + ); + } + logger.success(`Fetched project file to ${stateOutputPath}`); +} From 321e867aca148fde699171349f4b0fffd71a6121 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 28 Nov 2025 14:26:07 +0000 Subject: [PATCH 02/28] fooling around --- packages/cli/src/fetch/handler.ts | 28 ++++++++++++++++++---------- packages/project/src/Project.ts | 5 ++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/fetch/handler.ts b/packages/cli/src/fetch/handler.ts index 3e2d8d7ec..8ec155940 100644 --- a/packages/cli/src/fetch/handler.ts +++ b/packages/cli/src/fetch/handler.ts @@ -1,5 +1,5 @@ import { DeployConfig, getProject } from '@openfn/deploy'; -import Project from '@openfn/project'; +import Project, { Workspace } from '@openfn/project'; import fs from 'node:fs/promises'; import path from 'path'; import { Opts } from '../options'; @@ -26,6 +26,7 @@ export type FetchOptions = Required< | 'logJson' | 'path' | 'projectId' + | 'projectPath' > >; @@ -33,6 +34,9 @@ export default async function fetchHandler( options: FetchOptions, logger: Logger ) { + const commandPath = path.resolve(options.projectPath ?? '.'); + const workspace = new Workspace(commandPath); + const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; const config: Partial = { @@ -54,11 +58,15 @@ export default async function fetchHandler( const { data } = await getProject(config as DeployConfig, options.projectId); const name = options.env || 'project'; - const project = Project.from('state', data, { - endpoint: config.endpoint, - env: name, - fetched_at: new Date().toISOString(), - }); + const project = await Project.from( + 'state', + data, + { + endpoint: config.endpoint, + env: name, + }, + workspace.getConfig() + ); const outputRoot = path.resolve(options.path || '.'); @@ -67,13 +75,13 @@ export default async function fetchHandler( await fs.mkdir(`${outputRoot}/.projects`, { recursive: true }); let stateOutputPath = `${outputRoot}/.projects/${projectFileName}`; - const state = project?.serialize('state'); - if (project.repo?.formats.project === 'yaml') { - await fs.writeFile(`${stateOutputPath}.yaml`, state); + const output = project?.serialize('project'); + if (project.config?.formats.project === 'yaml') { + await fs.writeFile(`${stateOutputPath}.yaml`, output as string); } else { await fs.writeFile( `${stateOutputPath}.json`, - JSON.stringify(state, null, 2) + JSON.stringify(output, null, 2) ); } logger.success(`Fetched project file to ${stateOutputPath}`); diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 9780ad340..835dca74c 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -6,7 +6,7 @@ import fromAppState, { fromAppStateConfig } from './parse/from-app-state'; import fromPath, { FromPathConfig } from './parse/from-path'; // TODO this naming clearly isn't right import { parseProject as fromFs, FromFsConfig } from './parse/from-fs'; -import fromProject from './parse/from-project'; +import fromProject, { SerializedProject } from './parse/from-project'; import getIdentifier from './util/get-identifier'; import slugify from './util/slugify'; import { getUuidForEdge, getUuidForStep } from './util/uuid'; @@ -150,6 +150,9 @@ export class Project { this.config = buildConfig(config); } + serialize(type: 'project', options?: any): SerializedProject | string; + serialize(type: 'state', options?: any): Provisioner.Project | string; + serialize(type: 'fs', options?: any): Record; serialize(type: 'project' | 'fs' | 'state' = 'project', options?: any) { if (type in serializers) { // @ts-ignore From 05e4798bb94eb59025519da1a28a60362df5516d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 28 Nov 2025 17:48:16 +0000 Subject: [PATCH 03/28] refactor --- packages/cli/src/cli.ts | 2 +- packages/cli/src/commands.ts | 5 +- packages/cli/src/fetch/command.ts | 57 ------------------ packages/cli/src/projects/command.ts | 2 + .../{fetch/handler.ts => projects/fetch.ts} | 58 +++++++++++++++---- packages/cli/src/projects/handler.ts | 1 + 6 files changed, 54 insertions(+), 71 deletions(-) delete mode 100644 packages/cli/src/fetch/command.ts rename packages/cli/src/{fetch/handler.ts => projects/fetch.ts} (58%) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 32cbf60c0..dc9541c9d 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -17,7 +17,7 @@ import testCommand from './test/command'; import projectsCommand from './projects/command'; import mergeCommand from './projects/merge'; import checkoutCommand from './projects/checkout'; -import fetchCommand from './fetch/command'; +import fetchCommand from './projects/fetch'; const env = loadDotEnv(); if (env) { diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 6bc880a64..51f49c431 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -11,7 +11,6 @@ import metadata from './metadata/handler'; import pull from './pull/handler'; import * as projects from './projects/handler'; import * as repo from './repo/handler'; -import fetchHandler from './fetch/handler'; import createLogger, { CLI, Logger } from './util/logger'; import mapAdaptorsToMonorepo, { @@ -33,7 +32,6 @@ export type CommandList = | 'execute' | 'metadata' | 'pull' - | 'fetch' | 'projects' | 'project' | 'repo-clean' @@ -44,6 +42,7 @@ export type CommandList = | 'project-version' | 'project-merge' | 'project-checkout' + | 'project-fetch' | 'test' | 'version'; @@ -57,7 +56,6 @@ const handlers = { docs, metadata, pull, - fetch: fetchHandler, projects, project: projects, ['collections-get']: collections.get, @@ -71,6 +69,7 @@ const handlers = { ['project-version']: projects.version, ['project-merge']: projects.merge, ['project-checkout']: projects.checkout, + ['project-fetch']: projects.fetch, version: async (opts: Opts, logger: Logger) => printVersions(logger, opts, true), }; diff --git a/packages/cli/src/fetch/command.ts b/packages/cli/src/fetch/command.ts deleted file mode 100644 index 3113a7123..000000000 --- a/packages/cli/src/fetch/command.ts +++ /dev/null @@ -1,57 +0,0 @@ -import yargs from 'yargs'; -import { build, ensure, override } from '../util/command-builders'; -import { Opts } from '../options'; -import * as o from '../options'; - -export type FetchOptions = Required< - Pick< - Opts, - | 'beta' - | 'command' - | 'log' - | 'logJson' - | 'statePath' - | 'projectPath' - | 'configPath' - | 'projectId' - | 'confirm' - | 'snapshots' - > ->; - -const options = [ - o.apikey, - o.beta, - o.beta, - o.configPath, - o.endpoint, - o.env, - o.log, - override(o.path, { - description: 'path to output the project to', - }), - o.logJson, - o.projectPath, - o.snapshots, - o.statePath, - o.path, -]; - -const fetchCommand: yargs.CommandModule = { - command: 'fetch [projectId]', - describe: `Fetch a project's state and spec from a Lightning Instance to the local state file without expanding to the filesystem.`, - builder: (yargs: yargs.Argv) => - build(options, yargs) - .positional('projectId', { - describe: - 'The id of the project that should be fetched, should be a UUID', - demandOption: true, - }) - .example( - 'fetch 57862287-23e6-4650-8d79-e1dd88b24b1c', - 'Fetch an updated copy of a the above spec and state from a Lightning Instance' - ), - handler: ensure('fetch', options), -}; - -export default fetchCommand; diff --git a/packages/cli/src/projects/command.ts b/packages/cli/src/projects/command.ts index 5ccdbfec3..fe666789b 100644 --- a/packages/cli/src/projects/command.ts +++ b/packages/cli/src/projects/command.ts @@ -2,6 +2,7 @@ import list from './list'; import version from './version'; import merge from './merge'; import checkout from './checkout'; +import fetch from './fetch'; import type yargs from 'yargs'; @@ -16,6 +17,7 @@ export const projectsCommand = { .command(version) .command(merge) .command(checkout) + .command(fetch as any) .example('project', 'list all projects in the workspace') .example('project list', 'list all projects in the workspace') .example( diff --git a/packages/cli/src/fetch/handler.ts b/packages/cli/src/projects/fetch.ts similarity index 58% rename from packages/cli/src/fetch/handler.ts rename to packages/cli/src/projects/fetch.ts index 8ec155940..c6ac2e817 100644 --- a/packages/cli/src/fetch/handler.ts +++ b/packages/cli/src/projects/fetch.ts @@ -1,9 +1,14 @@ +import yargs from 'yargs'; import { DeployConfig, getProject } from '@openfn/deploy'; import Project, { Workspace } from '@openfn/project'; import fs from 'node:fs/promises'; import path from 'path'; -import { Opts } from '../options'; + +import { build, ensure, override } from '../util/command-builders'; import type { Logger } from '../util/logger'; +import * as o from '../options'; + +import type { Opts } from '../options'; type Config = { endpoint: string; @@ -24,17 +29,50 @@ export type FetchOptions = Required< | 'env' | 'log' | 'logJson' - | 'path' + | 'outputPath' | 'projectId' - | 'projectPath' + | 'workspace' > >; -export default async function fetchHandler( - options: FetchOptions, - logger: Logger -) { - const commandPath = path.resolve(options.projectPath ?? '.'); +const options = [ + o.apikey, + o.beta, + o.beta, + o.configPath, + o.endpoint, + o.env, + o.log, + override(o.outputPath, { + description: 'Path to output the fetched project to', + }), + o.logJson, + o.workspace, + o.snapshots, + o.statePath, +]; + +const command: yargs.CommandModule = { + command: 'fetch [projectId]', + describe: `Fetch a project's state and spec from a Lightning Instance to the local state file without expanding to the filesystem.`, + builder: (yargs: yargs.Argv) => + build(options, yargs) + .positional('projectId', { + describe: + 'The id of the project that should be fetched, should be a UUID', + demandOption: true, + }) + .example( + 'fetch 57862287-23e6-4650-8d79-e1dd88b24b1c', + 'Fetch an updated copy of a the above spec and state from a Lightning Instance' + ), + handler: ensure('project-fetch', options), +}; + +export default command; + +export const handler = async (options: FetchOptions, logger: Logger) => { + const commandPath = path.resolve(options.workspace ?? '.'); const workspace = new Workspace(commandPath); const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; @@ -68,7 +106,7 @@ export default async function fetchHandler( workspace.getConfig() ); - const outputRoot = path.resolve(options.path || '.'); + const outputRoot = path.resolve(options.outputPath || '.'); const projectFileName = project.getIdentifier(); @@ -85,4 +123,4 @@ export default async function fetchHandler( ); } logger.success(`Fetched project file to ${stateOutputPath}`); -} +}; diff --git a/packages/cli/src/projects/handler.ts b/packages/cli/src/projects/handler.ts index fa56a3d9a..b8f1e4cc0 100644 --- a/packages/cli/src/projects/handler.ts +++ b/packages/cli/src/projects/handler.ts @@ -2,3 +2,4 @@ export { handler as list } from './list'; export { handler as version } from './version'; export { handler as merge } from './merge'; export { handler as checkout } from './checkout'; +export { handler as fetch } from './fetch'; From 94d4405fa5cf72ee64c7dbe7d66f0e5e32577177 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 11:20:50 +0000 Subject: [PATCH 04/28] little refactor of auth logic for a better life --- packages/cli/src/projects/fetch.ts | 28 ++++++---------------------- packages/cli/src/projects/util.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index c6ac2e817..88d865ac3 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -1,20 +1,19 @@ import yargs from 'yargs'; +import path from 'node:path'; +import fs from 'node:fs/promises'; import { DeployConfig, getProject } from '@openfn/deploy'; import Project, { Workspace } from '@openfn/project'; -import fs from 'node:fs/promises'; -import path from 'path'; import { build, ensure, override } from '../util/command-builders'; import type { Logger } from '../util/logger'; import * as o from '../options'; import type { Opts } from '../options'; +import { loadAppAuthConfig } from './util'; +// TODO need to implement these type Config = { - endpoint: string; - apiKey: string | null; - - requireConfirmation?: boolean; + requireConfirmation?: boolean; // alias to y maybe dryRun?: boolean; }; @@ -75,22 +74,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const commandPath = path.resolve(options.workspace ?? '.'); const workspace = new Workspace(commandPath); - const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; - - const config: Partial = { - apiKey: options.apiKey, - endpoint: options.endpoint, - }; - - if (!options.apiKey && OPENFN_API_KEY) { - logger.info('Using OPENFN_API_KEY environment variable'); - config.apiKey = OPENFN_API_KEY; - } - - if (!options.endpoint && OPENFN_ENDPOINT) { - logger.info('Using OPENFN_ENDPOINT environment variable'); - config.endpoint = OPENFN_ENDPOINT; - } + const config = loadAppAuthConfig(options, logger); // download the state.json from lightning const { data } = await getProject(config as DeployConfig, options.projectId); diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index e69de29bb..04ea85c3f 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -0,0 +1,30 @@ +import type { Opts } from '../options'; +import type { Logger } from '@openfn/logger'; + +type AuthOptions = Pick; + +export const loadAppAuthConfig = ( + options: AuthOptions, + logger: Logger +): Required => { + const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; + + const config: AuthOptions = { + apiKey: options.apiKey, + endpoint: options.endpoint, + }; + + if (!options.apiKey && OPENFN_API_KEY) { + logger.info('Using OPENFN_API_KEY environment variable'); + config.apiKey = OPENFN_API_KEY; + } + + if (!options.endpoint && OPENFN_ENDPOINT) { + logger.info('Using OPENFN_ENDPOINT environment variable'); + config.endpoint = OPENFN_ENDPOINT; + } + + // TODO probably need to throw + + return config as Required; +}; From dbeac095f3f7b213c6212a28a825a9d1483bc509 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 12:31:49 +0000 Subject: [PATCH 05/28] add a fetch test --- packages/cli/src/projects/fetch.ts | 5 +- packages/cli/src/projects/util.ts | 72 ++++++++++++++++ packages/cli/test/projects/fetch.test.ts | 75 +++++++++++++++++ packages/cli/test/projects/fixtures.ts | 103 +++++++++++++++++++++++ 4 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 packages/cli/test/projects/fetch.test.ts create mode 100644 packages/cli/test/projects/fixtures.ts diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 88d865ac3..959c4d5d7 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -1,7 +1,6 @@ import yargs from 'yargs'; import path from 'node:path'; import fs from 'node:fs/promises'; -import { DeployConfig, getProject } from '@openfn/deploy'; import Project, { Workspace } from '@openfn/project'; import { build, ensure, override } from '../util/command-builders'; @@ -9,7 +8,7 @@ import type { Logger } from '../util/logger'; import * as o from '../options'; import type { Opts } from '../options'; -import { loadAppAuthConfig } from './util'; +import { getProject, loadAppAuthConfig } from './util'; // TODO need to implement these type Config = { @@ -77,7 +76,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const config = loadAppAuthConfig(options, logger); // download the state.json from lightning - const { data } = await getProject(config as DeployConfig, options.projectId); + const { data } = await getProject(logger, config, options.projectId); const name = options.env || 'project'; const project = await Project.from( diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 04ea85c3f..a6552ddae 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -1,3 +1,4 @@ +import { Provisioner } from '@openfn/lexicon/lightning'; import type { Opts } from '../options'; import type { Logger } from '@openfn/logger'; @@ -28,3 +29,74 @@ export const loadAppAuthConfig = ( return config as Required; }; + +export const getLightningUrl = ( + config: AuthOptions, + path: string = '', + snapshots?: string[] +) => { + const params = new URLSearchParams(); + snapshots?.forEach((snapshot) => params.append('snapshots[]', snapshot)); + return new URL( + `/api/provision/${path}?${params.toString()}`, + config.endpoint + ); +}; + +export async function getProject( + logger: Logger, + config: AuthOptions, + projectId: string, + snapshots?: string[] +): Promise<{ data: Provisioner.Project | null }> { + const url = getLightningUrl(config, projectId, snapshots); + logger.info(`Checking ${url} for existing project`); + + try { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${config.apiKey}`, + Accept: 'application/json', + }, + }); + + // A 404 response means the project doesn't exist yet + if (response.status === 404) { + logger.info('No project found'); + return { data: null }; + } + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new DeployError( + `Failed to authorize request with endpoint ${config.endpoint}, got ${response.status} ${response.statusText}` + ); + } + + throw new Error( + `Failed to fetch project ${projectId}: ${response.statusText}` + ); + } + + logger.info('Project found'); + return response.json(); + } catch (error: any) { + handleCommonErrors(config, error); + + throw error; + } +} + +function handleCommonErrors(config: AuthOptions, error: any) { + if (error.cause?.code === 'ECONNREFUSED') { + throw new DeployError( + `Failed to connect to endpoint ${config.endpoint}, got ECONNREFUSED.` + ); + } +} + +class DeployError extends Error { + constructor(message: string) { + super(message); + } +} diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts new file mode 100644 index 000000000..e5eb43392 --- /dev/null +++ b/packages/cli/test/projects/fetch.test.ts @@ -0,0 +1,75 @@ +import { readFile } from 'node:fs/promises'; +import test from 'ava'; +import mock from 'mock-fs'; +import { MockAgent, setGlobalDispatcher } from 'undici'; +import { createMockLogger } from '@openfn/logger'; + +import { handler as fetchHandler } from '../../src/projects/fetch'; +import { myProject_v1, myProject_yaml } from './fixtures'; + +const logger = createMockLogger('', { level: 'debug' }); + +const ENDPOINT = 'https://app.openfn.org'; +const PROJECT_ID = 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00'; + +let mockAgent = new MockAgent(); +mockAgent.disableNetConnect(); +setGlobalDispatcher(mockAgent); + +test.beforeEach(() => { + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': ` +project: + id: test-project + name: Test Project +workspace: + formats: + project: yaml +`, + }); + logger._reset(); +}); + +test.afterEach(() => { + mock.restore(); +}); + +test.serial( + 'fetch v1 app state and write to disk as project.yaml', + async (t) => { + const mockPool = mockAgent.get(ENDPOINT); + mockPool + .intercept({ + path: `/api/provision/${PROJECT_ID}?`, + method: 'GET', + }) + .reply(200, { + data: myProject_v1, + }); + + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws', + } as any, + logger + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + const yaml = myProject_yaml; + + t.is(fileContent.trim(), yaml); + + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); + } +); diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts new file mode 100644 index 000000000..7e88be846 --- /dev/null +++ b/packages/cli/test/projects/fixtures.ts @@ -0,0 +1,103 @@ +import type { Provisioner } from '@openfn/lexicon/lightning'; + +export const myProject_v1: Provisioner.Project = { + id: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00', + name: 'My Project', + description: 'my lovely project', + concurrency: null, + inserted_at: '2025-04-23T11:15:59Z', + collections: [], + workflows: [ + { + id: '72ca3eb0-042c-47a0-a2a1-a545ed4a8406', + name: 'My Workflow', + edges: [ + { + enabled: true, + id: 'a9a3adef-b394-4405-814d-3ac4323f4b4b', + source_trigger_id: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + condition_type: 'always', + target_job_id: '66add020-e6eb-4eec-836b-20008afca816', + }, + ], + concurrency: null, + inserted_at: '2025-04-23T11:19:32Z', + updated_at: '2025-04-23T11:19:32Z', + jobs: [ + { + id: '66add020-e6eb-4eec-836b-20008afca816', + name: 'Transform data', + body: 'fn()', + adaptor: '@openfn/language-common@latest', + project_credential_id: null, + keychain_credential_id: null, + }, + ], + triggers: [ + { + enabled: true, // TODO enabled: false is a bit interesting + id: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + type: 'webhook', + }, + ], + lock_version: 1, + deleted_at: null, + }, + ], + updated_at: '2025-04-23T11:15:59Z', + project_credentials: [], + scheduled_deletion: null, + allow_support_access: false, + requires_mfa: false, + retention_policy: 'retain_all', + history_retention_period: null, + dataclip_retention_period: null, +}; + +export const myProject_yaml = `id: my-project +name: My Project +version: 2 +description: my lovely project +collections: [] +credentials: [] +openfn: + uuid: e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00 + endpoint: https://app.openfn.org + env: project + inserted_at: 2025-04-23T11:15:59Z + updated_at: 2025-04-23T11:15:59Z +options: + allow_support_access: false + requires_mfa: false + retention_policy: retain_all +workflows: + - name: My Workflow + steps: + - id: trigger + type: webhook + openfn: + enabled: true + uuid: 4a06289c-15aa-4662-8dc6-f0aaacd8a058 + next: + transform-data: + disabled: false + condition: true + openfn: + uuid: a9a3adef-b394-4405-814d-3ac4323f4b4b + - id: transform-data + name: Transform data + expression: fn() + adaptor: "@openfn/language-common@latest" + openfn: + uuid: 66add020-e6eb-4eec-836b-20008afca816 + project_credential_id: null + keychain_credential_id: null + openfn: + uuid: 72ca3eb0-042c-47a0-a2a1-a545ed4a8406 + concurrency: null + inserted_at: 2025-04-23T11:19:32Z + updated_at: 2025-04-23T11:19:32Z + lock_version: 1 + deleted_at: null + id: my-workflow + history: []`; From 162e0eabce233f8a4a111a944f5f73ba382a82f3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 12:33:23 +0000 Subject: [PATCH 06/28] changeset --- .changeset/soft-flowers-lay.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/soft-flowers-lay.md diff --git a/.changeset/soft-flowers-lay.md b/.changeset/soft-flowers-lay.md new file mode 100644 index 000000000..221926edf --- /dev/null +++ b/.changeset/soft-flowers-lay.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': minor +--- + +Add a fetch command, which will download a project from an app but not check it out From 25e6f7a5b67bf1cc6c363d07decfcf6a6610d13e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 16:32:37 +0000 Subject: [PATCH 07/28] better project v2 serialization Sorting, strip nulls --- packages/project/src/serialize/to-project.ts | 28 ++++++++-- .../test/fixtures/sample-v2-project.ts | 53 +++++++----------- .../project/test/parse/from-project.test.ts | 40 +++++++------- .../project/test/serialize/to-project.test.ts | 54 ++++++++++++++----- 4 files changed, 106 insertions(+), 69 deletions(-) diff --git a/packages/project/src/serialize/to-project.ts b/packages/project/src/serialize/to-project.ts index 6c2833a16..04d879907 100644 --- a/packages/project/src/serialize/to-project.ts +++ b/packages/project/src/serialize/to-project.ts @@ -29,14 +29,34 @@ export default (project: Project, options: ToProjectOptions = {}) => { collections: project.collections, credentials: project.credentials, - openfn: project.openfn, + openfn: omitBy(project.openfn, isNil), meta: project.meta, options: omitBy(project.options, isNil), //workflows: project.workflows.map(mapWorkflow) as SerializedWorkflow[], - workflows: project.workflows.map((w) => - w.toJSON() - ) as SerializedWorkflow[], + workflows: project.workflows.map((w) => { + const obj: any = w.toJSON(); + if (obj.openfn) { + obj.openfn = omitBy(obj.openfn, isNil); + } + if (obj.steps) { + obj.steps = obj.steps.sort((a: any, b: any) => { + return a.id < b.id ? -1 : a.id > b.id ? 1 : 0; + }); + obj.steps.forEach((s) => { + s.openfn = omitBy(s.openfn, isNil); + if (s.next) { + for (const id in s.next) { + const edge = s.next[id]; + if (edge.openfn) { + edge.openfn = omitBy(edge.openfn, isNil); + } + } + } + }); + } + return obj; + }) as SerializedWorkflow[], }, isNil ) as SerializedProject; diff --git a/packages/project/test/fixtures/sample-v2-project.ts b/packages/project/test/fixtures/sample-v2-project.ts index 2ae20f223..8a471ac0a 100644 --- a/packages/project/test/fixtures/sample-v2-project.ts +++ b/packages/project/test/fixtures/sample-v2-project.ts @@ -8,44 +8,29 @@ export const json: SerializedProject = { name: 'My Project', version: 2, description: 'my lovely project', - openfn: { - uuid: '1234', - endpoint: 'https://app.openfn.org', - }, - options: { - allow_support_access: false, - }, + openfn: { uuid: '1234', endpoint: 'https://app.openfn.org' }, + options: { allow_support_access: false }, workflows: [ { steps: [ - { - name: 'a', - id: 'a', - openfn: { - uuid: 2, - }, - next: { - b: { - openfn: { - uuid: 4, - }, - }, - }, - }, { name: 'b', id: 'b', - openfn: { - uuid: 3, - }, + openfn: { uuid: 3 }, + expression: 'fn()', + adaptor: 'common', + }, + { + id: 'trigger', + openfn: { uuid: 2 }, + type: 'webhook', + next: { b: { openfn: { uuid: 4 } } }, }, ], name: 'Workflow', id: 'workflow', + openfn: { uuid: 1 }, history: [], - openfn: { - uuid: 1, - }, }, ], }; @@ -61,18 +46,20 @@ options: allow_support_access: false workflows: - steps: - - name: a - id: a + - name: b + id: b + openfn: + uuid: 3 + expression: fn() + adaptor: common + - id: trigger openfn: uuid: 2 + type: webhook next: b: openfn: uuid: 4 - - name: b - id: b - openfn: - uuid: 3 name: Workflow id: workflow openfn: diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts index 993985641..abda93472 100644 --- a/packages/project/test/parse/from-project.test.ts +++ b/packages/project/test/parse/from-project.test.ts @@ -87,8 +87,17 @@ test('import from a v2 project as JSON', async (t) => { history: [], steps: [ { - name: 'a', - id: 'a', + name: 'b', + id: 'b', + expression: 'fn()', + adaptor: 'common', + openfn: { + uuid: 3, + }, + }, + { + id: 'trigger', + type: 'webhook', openfn: { uuid: 2, }, @@ -100,13 +109,6 @@ test('import from a v2 project as JSON', async (t) => { }, }, }, - { - name: 'b', - id: 'b', - openfn: { - uuid: 3, - }, - }, ], }); }); @@ -130,8 +132,17 @@ test('import from a v2 project as YAML', async (t) => { history: [], steps: [ { - name: 'a', - id: 'a', + name: 'b', + id: 'b', + expression: 'fn()', + adaptor: 'common', + openfn: { + uuid: 3, + }, + }, + { + id: 'trigger', + type: 'webhook', openfn: { uuid: 2, }, @@ -143,13 +154,6 @@ test('import from a v2 project as YAML', async (t) => { }, }, }, - { - name: 'b', - id: 'b', - openfn: { - uuid: 3, - }, - }, ], }); }); diff --git a/packages/project/test/serialize/to-project.test.ts b/packages/project/test/serialize/to-project.test.ts index a9d48dfbe..6ad7313d5 100644 --- a/packages/project/test/serialize/to-project.test.ts +++ b/packages/project/test/serialize/to-project.test.ts @@ -4,27 +4,42 @@ import generateWorkflow, { generateProject } from '../../src/gen/generator'; import * as v2 from '../fixtures/sample-v2-project'; -const proj = new Project({ - id: 'my-project', - name: 'My Project', - description: 'my lovely project', - openfn: { - uuid: '1234', - endpoint: 'https://app.openfn.org', - }, - options: { - allow_support_access: false, - }, - workflows: [generateWorkflow('a-b', { uuidSeed: 1, openfnUuid: true })], -}); +const createProject = () => { + const proj = new Project({ + id: 'my-project', + name: 'My Project', + description: 'my lovely project', + openfn: { + uuid: '1234', + endpoint: 'https://app.openfn.org', + }, + options: { + allow_support_access: false, + }, + workflows: [ + generateWorkflow( + 'trigger(type=webhook)-b(expression="fn()",adaptor=common)', + { + uuidSeed: 1, + openfnUuid: true, + } + ), + ], + }); + // hack + delete proj.workflows[0].steps[0].name; + return proj; +}; test('should serialize to YAML format v2 project by default', (t) => { + const proj = createProject(); const yaml = proj.serialize('project'); t.deepEqual(yaml, v2.yaml); }); test('should explicitly serialize to YAML format', (t) => { + const proj = createProject(); const yaml = proj.serialize('project', { format: 'yaml' }); t.deepEqual(yaml, v2.yaml); @@ -34,9 +49,20 @@ test('should explicitly serialize to YAML format', (t) => { // so workflow b-a should look exactly the same as this // That should probably happen in workflow.toJSON test('should serialize to JSON format v2 project', (t) => { + const proj = createProject(); const json = proj.serialize('project', { format: 'json' }); - t.deepEqual(json, v2.json); }); // should load a project and serialize it back + +test('should exclude null values in yaml', (t) => { + const proj = createProject(); + + // force some null values into the workflow structure + proj.workflows[0].openfn.concurrency = null; + proj.workflows[0].steps[1].openfn.project_credential_id = null; + + const yaml = proj.serialize('project', { format: 'yaml' }); + t.deepEqual(yaml, v2.yaml); +}); From 7b0f6d018ce2ec3517eff20010fc364ce28d6b03 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 16:34:42 +0000 Subject: [PATCH 08/28] add basic fetch test --- packages/cli/test/projects/fetch.test.ts | 61 +++++++++++------------- packages/cli/test/projects/fixtures.ts | 16 +++---- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index e5eb43392..2dd785561 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -35,41 +35,38 @@ test.afterEach(() => { mock.restore(); }); -test.serial( - 'fetch v1 app state and write to disk as project.yaml', - async (t) => { - const mockPool = mockAgent.get(ENDPOINT); - mockPool - .intercept({ - path: `/api/provision/${PROJECT_ID}?`, - method: 'GET', - }) - .reply(200, { - data: myProject_v1, - }); +test.serial('fetch from lightning save as v2 yaml file', async (t) => { + const mockPool = mockAgent.get(ENDPOINT); + mockPool + .intercept({ + path: `/api/provision/${PROJECT_ID}?`, + method: 'GET', + }) + .reply(200, { + data: myProject_v1, + }); - await fetchHandler( - { - projectId: PROJECT_ID, - endpoint: ENDPOINT, - apiKey: 'test-api-key', + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', - workspace: '/ws', - env: 'project', - outputPath: '/ws', - } as any, - logger - ); + workspace: '/ws', + env: 'project', + outputPath: '/ws', + } as any, + logger + ); - const filePath = '/ws/.projects/project@app.openfn.org.yaml'; - const fileContent = await readFile(filePath, 'utf-8'); + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); - const yaml = myProject_yaml; + const yaml = myProject_yaml; - t.is(fileContent.trim(), yaml); + t.is(fileContent.trim(), yaml); - const { message, level } = logger._parse(logger._last); - t.is(level, 'success'); - t.regex(message, /Fetched project file to/); - } -); + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); +}); diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts index 7e88be846..db5c25a24 100644 --- a/packages/cli/test/projects/fixtures.ts +++ b/packages/cli/test/projects/fixtures.ts @@ -73,6 +73,12 @@ options: workflows: - name: My Workflow steps: + - id: transform-data + name: Transform data + expression: fn() + adaptor: "@openfn/language-common@latest" + openfn: + uuid: 66add020-e6eb-4eec-836b-20008afca816 - id: trigger type: webhook openfn: @@ -84,20 +90,10 @@ workflows: condition: true openfn: uuid: a9a3adef-b394-4405-814d-3ac4323f4b4b - - id: transform-data - name: Transform data - expression: fn() - adaptor: "@openfn/language-common@latest" - openfn: - uuid: 66add020-e6eb-4eec-836b-20008afca816 - project_credential_id: null - keychain_credential_id: null openfn: uuid: 72ca3eb0-042c-47a0-a2a1-a545ed4a8406 - concurrency: null inserted_at: 2025-04-23T11:19:32Z updated_at: 2025-04-23T11:19:32Z lock_version: 1 - deleted_at: null id: my-workflow history: []`; From 4fe5b48b6566098bc0762fdd234eb51267605767 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 17:05:04 +0000 Subject: [PATCH 09/28] refactoring --- packages/cli/src/projects/fetch.ts | 27 +++++++++---------- packages/cli/src/projects/util.ts | 33 ++++++++++++++++++++++++ packages/cli/test/projects/fetch.test.ts | 9 +------ 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 959c4d5d7..987b1dfbd 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -8,7 +8,7 @@ import type { Logger } from '../util/logger'; import * as o from '../options'; import type { Opts } from '../options'; -import { getProject, loadAppAuthConfig } from './util'; +import { serialize, getProject, loadAppAuthConfig } from './util'; // TODO need to implement these type Config = { @@ -75,7 +75,6 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const config = loadAppAuthConfig(options, logger); - // download the state.json from lightning const { data } = await getProject(logger, config, options.projectId); const name = options.env || 'project'; @@ -89,21 +88,19 @@ export const handler = async (options: FetchOptions, logger: Logger) => { workspace.getConfig() ); + // TODO load whatever is on disk and compare it + // If they have diverged, throw an error + // Also, log whether we are updating or doing nothing + const outputRoot = path.resolve(options.outputPath || '.'); const projectFileName = project.getIdentifier(); - await fs.mkdir(`${outputRoot}/.projects`, { recursive: true }); - let stateOutputPath = `${outputRoot}/.projects/${projectFileName}`; - - const output = project?.serialize('project'); - if (project.config?.formats.project === 'yaml') { - await fs.writeFile(`${stateOutputPath}.yaml`, output as string); - } else { - await fs.writeFile( - `${stateOutputPath}.json`, - JSON.stringify(output, null, 2) - ); - } - logger.success(`Fetched project file to ${stateOutputPath}`); + const projectsDir = project.config.dirs.projects ?? '.projects'; + + // TODO allow this to be overridden by argument + const stateOutputPath = `${outputRoot}/${projectsDir}/${projectFileName}`; + const finalOutput = await serialize(project, stateOutputPath); + + logger.success(`Fetched project file to ${finalOutput}`); }; diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index a6552ddae..ad0270c80 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -1,6 +1,10 @@ +import path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; import { Provisioner } from '@openfn/lexicon/lightning'; + import type { Opts } from '../options'; import type { Logger } from '@openfn/logger'; +import type Project from '@openfn/project'; type AuthOptions = Pick; @@ -30,6 +34,35 @@ export const loadAppAuthConfig = ( return config as Required; }; +const ensureExt = (filePath: string, ext: string) => { + if (!filePath.endsWith(ext)) { + return `${filePath}.${ext}`; + } + return filePath; +}; + +export const serialize = async ( + project: Project, + outputPath: string, + formatOverride?: 'yaml' | 'json' +) => { + const root = path.dirname(outputPath); + await mkdir(root, { recursive: true }); + + const output = project?.serialize('project'); + + const format = formatOverride ?? project.config?.formats.project; + let finalPath; + if (format === 'yaml') { + finalPath = ensureExt(outputPath, 'yaml'); + await writeFile(finalPath, output as string); + } else { + finalPath = ensureExt(outputPath, 'json'); + await writeFile(finalPath, JSON.stringify(output, null, 2)); + } + return finalPath; +}; + export const getLightningUrl = ( config: AuthOptions, path: string = '', diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index 2dd785561..2e9d11391 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -19,14 +19,7 @@ setGlobalDispatcher(mockAgent); test.beforeEach(() => { mock({ '/ws/.projects': {}, - '/ws/openfn.yaml': ` -project: - id: test-project - name: Test Project -workspace: - formats: - project: yaml -`, + '/ws/openfn.yaml': '', }); logger._reset(); }); From 2523d0044f93703ddfca3c68357e9d3b51655a3e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 17:21:19 +0000 Subject: [PATCH 10/28] custom output --- packages/cli/src/projects/fetch.ts | 10 +- packages/cli/src/projects/util.ts | 5 +- packages/cli/test/projects/fetch.test.ts | 141 +++++++++++++++++++++-- 3 files changed, 141 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 987b1dfbd..a3c14b251 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -98,9 +98,13 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const projectsDir = project.config.dirs.projects ?? '.projects'; - // TODO allow this to be overridden by argument - const stateOutputPath = `${outputRoot}/${projectsDir}/${projectFileName}`; - const finalOutput = await serialize(project, stateOutputPath); + const ext = path.extname(options.outputPath).substring(1) || undefined; + + const stateOutputPath = ext + ? options.outputPath + : `${outputRoot}/${projectsDir}/${projectFileName}`; + + const finalOutput = await serialize(project, stateOutputPath, ext as any); logger.success(`Fetched project file to ${finalOutput}`); }; diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index ad0270c80..62a50e828 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -49,9 +49,9 @@ export const serialize = async ( const root = path.dirname(outputPath); await mkdir(root, { recursive: true }); - const output = project?.serialize('project'); - const format = formatOverride ?? project.config?.formats.project; + const output = project?.serialize('project', { format }); + let finalPath; if (format === 'yaml') { finalPath = ensureExt(outputPath, 'yaml'); @@ -60,6 +60,7 @@ export const serialize = async ( finalPath = ensureExt(outputPath, 'json'); await writeFile(finalPath, JSON.stringify(output, null, 2)); } + return finalPath; }; diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index 2e9d11391..9a396d5b6 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -16,6 +16,19 @@ let mockAgent = new MockAgent(); mockAgent.disableNetConnect(); setGlobalDispatcher(mockAgent); +test.before(() => { + const mockPool = mockAgent.get(ENDPOINT); + mockPool + .intercept({ + path: `/api/provision/${PROJECT_ID}?`, + method: 'GET', + }) + .reply(200, { + data: myProject_v1, + }) + .persist(); +}); + test.beforeEach(() => { mock({ '/ws/.projects': {}, @@ -29,16 +42,6 @@ test.afterEach(() => { }); test.serial('fetch from lightning save as v2 yaml file', async (t) => { - const mockPool = mockAgent.get(ENDPOINT); - mockPool - .intercept({ - path: `/api/provision/${PROJECT_ID}?`, - method: 'GET', - }) - .reply(200, { - data: myProject_v1, - }); - await fetchHandler( { projectId: PROJECT_ID, @@ -63,3 +66,121 @@ test.serial('fetch from lightning save as v2 yaml file', async (t) => { t.is(level, 'success'); t.regex(message, /Fetched project file to/); }); + +test.serial('save to a custom location', async (t) => { + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws/out.yaml', + } as any, + logger + ); + + const filePath = '/ws/out.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + const yaml = myProject_yaml; + + t.is(fileContent.trim(), yaml); + + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); +}); + +test.serial( + 'save JSON to a custom location, overriding project defaults', + async (t) => { + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws/out.json', + } as any, + logger + ); + + const filePath = '/ws/out.json'; + const fileContent = await readFile(filePath, 'utf-8'); + + const json = `{ + "id": "my-project", + "name": "My Project", + "version": 2, + "description": "my lovely project", + "collections": [], + "credentials": [], + "openfn": { + "uuid": "e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00", + "endpoint": "https://app.openfn.org", + "env": "project", + "inserted_at": "2025-04-23T11:15:59Z", + "updated_at": "2025-04-23T11:15:59Z" + }, + "options": { + "allow_support_access": false, + "requires_mfa": false, + "retention_policy": "retain_all" + }, + "workflows": [ + { + "name": "My Workflow", + "steps": [ + { + "id": "transform-data", + "name": "Transform data", + "expression": "fn()", + "adaptor": "@openfn/language-common@latest", + "openfn": { + "uuid": "66add020-e6eb-4eec-836b-20008afca816" + } + }, + { + "id": "trigger", + "type": "webhook", + "openfn": { + "enabled": true, + "uuid": "4a06289c-15aa-4662-8dc6-f0aaacd8a058" + }, + "next": { + "transform-data": { + "disabled": false, + "condition": true, + "openfn": { + "uuid": "a9a3adef-b394-4405-814d-3ac4323f4b4b" + } + } + } + } + ], + "openfn": { + "uuid": "72ca3eb0-042c-47a0-a2a1-a545ed4a8406", + "inserted_at": "2025-04-23T11:19:32Z", + "updated_at": "2025-04-23T11:19:32Z", + "lock_version": 1 + }, + "id": "my-workflow", + "history": [] + } + ] +}`; + + t.is(fileContent.trim(), json); + + // should not throw + JSON.parse(fileContent); + + const { message, level } = logger._parse(logger._last); + t.is(level, 'success'); + t.regex(message, /Fetched project file to/); + } +); From 80ecaaebb68e77087e2eb7a8f7e946b83b082457 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 1 Dec 2025 18:06:45 +0000 Subject: [PATCH 11/28] warn if projects have diverged on fetch Still some tests to update --- packages/cli/src/projects/fetch.ts | 35 +++++++--- packages/cli/src/projects/util.ts | 13 +++- packages/cli/test/projects/fetch.test.ts | 70 +++++++++++++++++++- packages/cli/test/projects/fixtures.ts | 3 +- packages/lexicon/lightning.d.ts | 2 + packages/project/src/Project.ts | 16 +++++ packages/project/src/Workflow.ts | 2 +- packages/project/src/parse/from-app-state.ts | 1 + 8 files changed, 127 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index a3c14b251..20f5f04c0 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -88,23 +88,40 @@ export const handler = async (options: FetchOptions, logger: Logger) => { workspace.getConfig() ); - // TODO load whatever is on disk and compare it - // If they have diverged, throw an error - // Also, log whether we are updating or doing nothing - + // Work out where and how to serialize the project const outputRoot = path.resolve(options.outputPath || '.'); - const projectFileName = project.getIdentifier(); - const projectsDir = project.config.dirs.projects ?? '.projects'; - const ext = path.extname(options.outputPath).substring(1) || undefined; - const stateOutputPath = ext ? options.outputPath : `${outputRoot}/${projectsDir}/${projectFileName}`; - const finalOutput = await serialize(project, stateOutputPath, ext as any); + // See if a project already exists there + const finalOutput = await serialize( + project, + stateOutputPath, + ext as any, + true // dry run - this won't trigger an actual write! + ); + + // If a project already exists at the output path, make sure it's compatible + let current: Project | null = null; + try { + current = await Project.from('path', finalOutput); + } catch (e) { + // Do nothing - project doesn't exist + } + + if (current && !project.canMergeInto(current)) { + // TODO allow force or rename + throw new Error('Error! An incompatible project exists at this location'); + } + + // TODO report whether we've updated or not + + // finally, write it! + await serialize(project, stateOutputPath, ext as any); logger.success(`Fetched project file to ${finalOutput}`); }; diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 62a50e828..45d821673 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -44,7 +44,8 @@ const ensureExt = (filePath: string, ext: string) => { export const serialize = async ( project: Project, outputPath: string, - formatOverride?: 'yaml' | 'json' + formatOverride?: 'yaml' | 'json', + dryRun = false ) => { const root = path.dirname(outputPath); await mkdir(root, { recursive: true }); @@ -52,13 +53,19 @@ export const serialize = async ( const format = formatOverride ?? project.config?.formats.project; const output = project?.serialize('project', { format }); + const maybeWriteFile = (filePath: string, output: string) => { + if (!dryRun) { + return writeFile(filePath, output); + } + }; + let finalPath; if (format === 'yaml') { finalPath = ensureExt(outputPath, 'yaml'); - await writeFile(finalPath, output as string); + await maybeWriteFile(finalPath, output as string); } else { finalPath = ensureExt(outputPath, 'json'); - await writeFile(finalPath, JSON.stringify(output, null, 2)); + await maybeWriteFile(finalPath, JSON.stringify(output, null, 2)); } return finalPath; diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index 9a396d5b6..dc4c02e21 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -169,7 +169,7 @@ test.serial( "lock_version": 1 }, "id": "my-workflow", - "history": [] + "history": [a] } ] }`; @@ -184,3 +184,71 @@ test.serial( t.regex(message, /Fetched project file to/); } ); + +test.serial('Override a compatible project', async (t) => { + // Change project.yaml + const modified = myProject_yaml.replace('my lovely project', 'renamed'); + + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + '/ws/.projects/project@app.openfn.org.yaml': modified, + }); + + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws', + } as any, + logger + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + // This should overwrite the renamed value back to the default + t.regex(fileContent, /my lovely project/); +}); + +// In this test, the file on disk has diverged from the remove +// This means changes could be lost, so we throw! +test.serial('throw for an incompatible project', async (t) => { + // Change project.yaml + const modified = myProject_yaml.replace('fn()', 'fn(x)'); + + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + '/ws/.projects/project@app.openfn.org.yaml': modified, + }); + + await t.throwsAsync( + () => + fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws', + } as any, + logger + ), + { + message: /incompatible project/, + } + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + // The file should NOT be overwritte + t.regex(fileContent, /fn()/); +}); diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts index db5c25a24..2b3bbfd10 100644 --- a/packages/cli/test/projects/fixtures.ts +++ b/packages/cli/test/projects/fixtures.ts @@ -42,6 +42,7 @@ export const myProject_v1: Provisioner.Project = { ], lock_version: 1, deleted_at: null, + version_history: ['a'], // TODO not implemented yet? needed for tests }, ], updated_at: '2025-04-23T11:15:59Z', @@ -96,4 +97,4 @@ workflows: updated_at: 2025-04-23T11:19:32Z lock_version: 1 id: my-workflow - history: []`; + history: [a]`; diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts index ce4d7cb7c..4e1b50259 100644 --- a/packages/lexicon/lightning.d.ts +++ b/packages/lexicon/lightning.d.ts @@ -253,6 +253,8 @@ export namespace Provisioner { delete?: boolean; project_id?: string; + version_history?: string[]; + concurrency?: null; // TODO lock_version?: number; inserted_at?: string; diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 835dca74c..6555143b3 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -202,6 +202,22 @@ export class Project { } return result; } + + canMergeInto(target: Project) { + const potentialConflicts: Record = {}; + for (const sourceWorkflow of this.workflows) { + // TODO mapping needs work + const targetId = sourceWorkflow.id; + const targetWorkflow = target.getWorkflow(targetId); + if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) { + potentialConflicts[sourceWorkflow.id] = targetWorkflow?.id; + } + } + if (Object.keys(potentialConflicts).length) { + return false; + } + return true; + } } export default Project; diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 68d30d4c1..98820651d 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -30,7 +30,7 @@ class Workflow { // history needs to be on workflow object. this.workflow.history = workflow.history?.length ? workflow.history : []; - const { id, name, openfn, steps, ...options } = workflow; + const { id, name, openfn, steps, history, ...options } = workflow; if (!(id || name)) { throw new Error('A Workflow MUST have a name or id'); } diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index 64bf96b61..c1ce10a0c 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -87,6 +87,7 @@ export const mapWorkflow = (workflow: Provisioner.Workflow) => { const mapped: l.Workflow = { name: workflow.name, steps: [], + history: workflow.version_history ?? [], openfn: renameKeys(remoteProps, { id: 'uuid' }), }; if (workflow.name) { From 7f66e4b6905dc97efab341a7baf25cb78fa5bab2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 09:29:23 +0000 Subject: [PATCH 12/28] fix tests --- packages/cli/test/projects/fetch.test.ts | 121 +++++++++---------- packages/cli/test/projects/fixtures.ts | 5 +- packages/project/src/parse/from-app-state.ts | 3 +- 3 files changed, 64 insertions(+), 65 deletions(-) diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index dc4c02e21..2bd5d070c 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -112,72 +112,69 @@ test.serial( const filePath = '/ws/out.json'; const fileContent = await readFile(filePath, 'utf-8'); - const json = `{ - "id": "my-project", - "name": "My Project", - "version": 2, - "description": "my lovely project", - "collections": [], - "credentials": [], - "openfn": { - "uuid": "e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00", - "endpoint": "https://app.openfn.org", - "env": "project", - "inserted_at": "2025-04-23T11:15:59Z", - "updated_at": "2025-04-23T11:15:59Z" - }, - "options": { - "allow_support_access": false, - "requires_mfa": false, - "retention_policy": "retain_all" - }, - "workflows": [ - { - "name": "My Workflow", - "steps": [ - { - "id": "transform-data", - "name": "Transform data", - "expression": "fn()", - "adaptor": "@openfn/language-common@latest", - "openfn": { - "uuid": "66add020-e6eb-4eec-836b-20008afca816" - } - }, + const json = { + id: 'my-project', + name: 'My Project', + version: 2, + description: 'my lovely project', + collections: [], + credentials: [], + openfn: { + uuid: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00', + endpoint: 'https://app.openfn.org', + env: 'project', + inserted_at: '2025-04-23T11:15:59Z', + updated_at: '2025-04-23T11:15:59Z', + }, + options: { + allow_support_access: false, + requires_mfa: false, + retention_policy: 'retain_all', + }, + workflows: [ { - "id": "trigger", - "type": "webhook", - "openfn": { - "enabled": true, - "uuid": "4a06289c-15aa-4662-8dc6-f0aaacd8a058" + name: 'My Workflow', + steps: [ + { + id: 'transform-data', + name: 'Transform data', + expression: 'fn()', + adaptor: '@openfn/language-common@latest', + openfn: { + uuid: '66add020-e6eb-4eec-836b-20008afca816', + }, + }, + { + id: 'trigger', + type: 'webhook', + openfn: { + enabled: true, + uuid: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + }, + next: { + 'transform-data': { + disabled: false, + condition: true, + openfn: { + uuid: 'a9a3adef-b394-4405-814d-3ac4323f4b4b', + }, + }, + }, + }, + ], + openfn: { + uuid: '72ca3eb0-042c-47a0-a2a1-a545ed4a8406', + inserted_at: '2025-04-23T11:19:32Z', + updated_at: '2025-04-23T11:19:32Z', + lock_version: 1, }, - "next": { - "transform-data": { - "disabled": false, - "condition": true, - "openfn": { - "uuid": "a9a3adef-b394-4405-814d-3ac4323f4b4b" - } - } - } - } + id: 'my-workflow', + history: ['a'], + }, ], - "openfn": { - "uuid": "72ca3eb0-042c-47a0-a2a1-a545ed4a8406", - "inserted_at": "2025-04-23T11:19:32Z", - "updated_at": "2025-04-23T11:19:32Z", - "lock_version": 1 - }, - "id": "my-workflow", - "history": [a] - } - ] -}`; - - t.is(fileContent.trim(), json); + }; - // should not throw - JSON.parse(fileContent); + t.deepEqual(JSON.parse(fileContent), json); const { message, level } = logger._parse(logger._last); t.is(level, 'success'); diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts index 2b3bbfd10..de1f06896 100644 --- a/packages/cli/test/projects/fixtures.ts +++ b/packages/cli/test/projects/fixtures.ts @@ -91,10 +91,11 @@ workflows: condition: true openfn: uuid: a9a3adef-b394-4405-814d-3ac4323f4b4b + history: + - a openfn: uuid: 72ca3eb0-042c-47a0-a2a1-a545ed4a8406 inserted_at: 2025-04-23T11:19:32Z updated_at: 2025-04-23T11:19:32Z lock_version: 1 - id: my-workflow - history: [a]`; + id: my-workflow`; diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index c1ce10a0c..ff8b01c5d 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -83,7 +83,8 @@ const mapTriggerEdgeCondition = (edge: Provisioner.Edge) => { // map a project workflow to a local cli workflow // TODO this probably gets easier if I index everything by name export const mapWorkflow = (workflow: Provisioner.Workflow) => { - const { jobs, edges, triggers, name, ...remoteProps } = workflow; + const { jobs, edges, triggers, name, version_history, ...remoteProps } = + workflow; const mapped: l.Workflow = { name: workflow.name, steps: [], From c31362bba1fd0b57f13d18c112a879ce0ddc3053 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 09:30:42 +0000 Subject: [PATCH 13/28] types --- packages/cli/src/projects/fetch.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 20f5f04c0..b36b1f1db 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -1,6 +1,5 @@ import yargs from 'yargs'; import path from 'node:path'; -import fs from 'node:fs/promises'; import Project, { Workspace } from '@openfn/project'; import { build, ensure, override } from '../util/command-builders'; @@ -11,10 +10,10 @@ import type { Opts } from '../options'; import { serialize, getProject, loadAppAuthConfig } from './util'; // TODO need to implement these -type Config = { - requireConfirmation?: boolean; // alias to y maybe - dryRun?: boolean; -}; +// type Config = { +// requireConfirmation?: boolean; // alias to y maybe +// dryRun?: boolean; +// }; export type FetchOptions = Required< Pick< @@ -80,7 +79,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const project = await Project.from( 'state', - data, + data!, { endpoint: config.endpoint, env: name, From 795f11abcb4006973f850ab2a44dbd384db69b78 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 09:34:52 +0000 Subject: [PATCH 14/28] add force flag --- packages/cli/src/projects/fetch.ts | 6 ++++- packages/cli/test/projects/fetch.test.ts | 33 +++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index b36b1f1db..33468f937 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -24,6 +24,7 @@ export type FetchOptions = Required< | 'confirm' | 'endpoint' | 'env' + | 'force' | 'log' | 'logJson' | 'outputPath' @@ -47,6 +48,9 @@ const options = [ o.workspace, o.snapshots, o.statePath, + override(o.force, { + description: 'Overwrite local file contents with the fetched contents', + }), ]; const command: yargs.CommandModule = { @@ -112,7 +116,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { // Do nothing - project doesn't exist } - if (current && !project.canMergeInto(current)) { + if (current && !project.canMergeInto(current) && !options.force) { // TODO allow force or rename throw new Error('Error! An incompatible project exists at this location'); } diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index 2bd5d070c..da7be65b4 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -246,6 +246,37 @@ test.serial('throw for an incompatible project', async (t) => { const filePath = '/ws/.projects/project@app.openfn.org.yaml'; const fileContent = await readFile(filePath, 'utf-8'); - // The file should NOT be overwritte + // The file should NOT be overwritten + t.regex(fileContent, /fn()/); +}); + +test.serial('force merge an incompatible project', async (t) => { + // Change project.yaml + const modified = myProject_yaml.replace('fn()', 'fn(x)'); + + mock({ + '/ws/.projects': {}, + '/ws/openfn.yaml': '', + '/ws/.projects/project@app.openfn.org.yaml': modified, + }); + + await fetchHandler( + { + projectId: PROJECT_ID, + endpoint: ENDPOINT, + apiKey: 'test-api-key', + + workspace: '/ws', + env: 'project', + outputPath: '/ws', + force: true, + } as any, + logger + ); + + const filePath = '/ws/.projects/project@app.openfn.org.yaml'; + const fileContent = await readFile(filePath, 'utf-8'); + + // The file should be overwritten t.regex(fileContent, /fn()/); }); From 30501100445b986f4b4705b6fd186ee761508ec8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 17:42:56 +0000 Subject: [PATCH 15/28] simplify beta pull handler --- packages/cli/src/projects/checkout.ts | 38 ++++--- packages/cli/src/projects/fetch.ts | 52 +++++----- packages/cli/src/projects/pull.ts | 35 +++++++ packages/cli/src/pull/beta.ts | 138 -------------------------- packages/cli/src/pull/command.ts | 5 + packages/cli/src/pull/handler.ts | 4 +- packages/project/src/Workspace.ts | 2 +- 7 files changed, 87 insertions(+), 187 deletions(-) create mode 100644 packages/cli/src/projects/pull.ts delete mode 100644 packages/cli/src/pull/beta.ts diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index bcde2dee5..677704b39 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -10,10 +10,10 @@ import * as o from '../options'; import type { Opts } from '../options'; -export type CheckoutOptions = Required< - Pick -> & - Pick; +export type CheckoutOptions = Pick< + Opts, + 'command' | 'projectId' | 'workspace' | 'log' +>; const options = [o.projectId, o.workspace, o.log]; @@ -27,11 +27,11 @@ const command: yargs.CommandModule = { export default command; export const handler = async (options: CheckoutOptions, logger: Logger) => { - const commandPath = options.workspace; - const workspace = new Workspace(commandPath); + const projectId = options.projectId!; + const workspacePath = options.workspace ?? process.cwd(); + const workspace = new Workspace(workspacePath); if (!workspace.valid) { - logger.error('Command was run in an invalid openfn workspace'); - return; + throw new Error('Command was run in an invalid openfn workspace'); } // get the config @@ -40,37 +40,35 @@ export const handler = async (options: CheckoutOptions, logger: Logger) => { // get the project let switchProject; - if (/\.(yaml|json)$/.test(options.projectId)) { + if (/\.(yaml|json)$/.test(projectId)) { // TODO: should we allow checkout into an arbitrary folder? - const filePath = options.projectId.startsWith('/') - ? options.projectId - : path.join(commandPath, options.projectId); + const filePath = projectId.startsWith('/') + ? projectId + : path.join(workspacePath, projectId); logger.debug('Loading project from path ', filePath); switchProject = await Project.from('path', filePath, config); } else { - switchProject = workspace.get(options.projectId); + switchProject = workspace.get(projectId); } if (!switchProject) { - throw new Error( - `Project with id ${options.projectId} not found in the workspace` - ); + throw new Error(`Project with id ${projectId} not found in the workspace`); } // delete workflow dir before expanding project - await rimraf(path.join(commandPath, config.workflowRoot ?? 'workflows')); + await rimraf(path.join(workspacePath, config.workflowRoot ?? 'workflows')); // expand project into directory const files: any = switchProject.serialize('fs'); for (const f in files) { if (files[f]) { - fs.mkdirSync(path.join(commandPath, path.dirname(f)), { + fs.mkdirSync(path.join(workspacePath, path.dirname(f)), { recursive: true, }); - fs.writeFileSync(path.join(commandPath, f), files[f]); + fs.writeFileSync(path.join(workspacePath, f), files[f]); } else { logger.warn('WARNING! No content for file', f); } } - logger.success(`Expanded project to ${commandPath}`); + logger.success(`Expanded project to ${workspacePath}`); }; diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 33468f937..14986c4f9 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -15,28 +15,22 @@ import { serialize, getProject, loadAppAuthConfig } from './util'; // dryRun?: boolean; // }; -export type FetchOptions = Required< - Pick< - Opts, - | 'apiKey' - | 'beta' - | 'command' - | 'confirm' - | 'endpoint' - | 'env' - | 'force' - | 'log' - | 'logJson' - | 'outputPath' - | 'projectId' - | 'workspace' - > +export type FetchOptions = Pick< + Opts, + | 'apiKey' + | 'command' + | 'endpoint' + | 'env' + | 'force' + | 'log' + | 'logJson' + | 'outputPath' + | 'projectId' + | 'workspace' >; const options = [ o.apikey, - o.beta, - o.beta, o.configPath, o.endpoint, o.env, @@ -73,12 +67,15 @@ const command: yargs.CommandModule = { export default command; export const handler = async (options: FetchOptions, logger: Logger) => { - const commandPath = path.resolve(options.workspace ?? '.'); - const workspace = new Workspace(commandPath); + const workspacePath = path.resolve(options.workspace ?? '.'); + const workspace = new Workspace(workspacePath); + const projectId = options.projectId!; + + const outputPath = options.outputPath; const config = loadAppAuthConfig(options, logger); - const { data } = await getProject(logger, config, options.projectId); + const { data } = await getProject(logger, config, projectId); const name = options.env || 'project'; const project = await Project.from( @@ -92,18 +89,19 @@ export const handler = async (options: FetchOptions, logger: Logger) => { ); // Work out where and how to serialize the project - const outputRoot = path.resolve(options.outputPath || '.'); + const outputRoot = path.resolve(options.outputPath || workspacePath); const projectFileName = project.getIdentifier(); const projectsDir = project.config.dirs.projects ?? '.projects'; - const ext = path.extname(options.outputPath).substring(1) || undefined; + const ext = outputPath ? path.extname(outputPath!).substring(1) : undefined; + const stateOutputPath = ext - ? options.outputPath + ? outputPath : `${outputRoot}/${projectsDir}/${projectFileName}`; // See if a project already exists there const finalOutput = await serialize( project, - stateOutputPath, + stateOutputPath!, ext as any, true // dry run - this won't trigger an actual write! ); @@ -124,7 +122,9 @@ export const handler = async (options: FetchOptions, logger: Logger) => { // TODO report whether we've updated or not // finally, write it! - await serialize(project, stateOutputPath, ext as any); + await serialize(project, stateOutputPath!, ext as any); logger.success(`Fetched project file to ${finalOutput}`); + + return project; }; diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts new file mode 100644 index 000000000..73848b997 --- /dev/null +++ b/packages/cli/src/projects/pull.ts @@ -0,0 +1,35 @@ +import { handler as fetch } from './fetch'; +import { handler as checkout } from './checkout'; + +import type { Logger } from '../util/logger'; +import type { Opts } from '../options'; + +export type PullOptionsBeta = Pick< + Opts, + | 'apiKey' + | 'command' + | 'endpoint' + | 'env' + | 'force' + | 'log' + | 'logJson' + | 'path' + | 'projectId' + | 'workspace' +>; + +export async function handler(options: PullOptionsBeta, logger: Logger) { + const project = await fetch(options, logger); + logger.success(`Downloaded latest project version`); + + await checkout( + { + ...options, + projectId: project.id, + }, + logger + ); + logger.success(`Checked out project locally`); +} + +export default handler; diff --git a/packages/cli/src/pull/beta.ts b/packages/cli/src/pull/beta.ts deleted file mode 100644 index 5e0948905..000000000 --- a/packages/cli/src/pull/beta.ts +++ /dev/null @@ -1,138 +0,0 @@ -// beta v2 version of CLI pull -import path from 'node:path'; -import fs from 'node:fs/promises'; -import { rimraf } from 'rimraf'; -import { confirm } from '@inquirer/prompts'; - -import { DeployConfig, getProject } from '@openfn/deploy'; -import Project, { Workspace } from '@openfn/project'; -import { Provisioner } from '@openfn/lexicon/lightning'; - -import type { Logger } from '../util/logger'; -import type { Opts } from '../options'; - -// new config -type Config = { - endpoint: string; - apiKey: string | null; - - // maybe keep? - requireConfirmation?: boolean; - dryRun?: boolean; -}; - -export type PullOptionsBeta = Required< - Pick< - Opts, - | 'apiKey' - | 'beta' - | 'command' - | 'confirm' - | 'endpoint' - | 'env' - | 'log' - | 'logJson' - | 'path' - | 'projectId' - > ->; - -export async function handler(options: PullOptionsBeta, logger: Logger) { - const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env; - - const cfg: Partial = { - apiKey: options.apiKey, - endpoint: options.endpoint, - }; - - if (!options.apiKey && OPENFN_API_KEY) { - logger.info('Using OPENFN_API_KEY environment variable'); - cfg.apiKey = OPENFN_API_KEY; - } - - if (!options.endpoint && OPENFN_ENDPOINT) { - logger.info('Using OPENFN_ENDPOINT environment variable'); - cfg.endpoint = OPENFN_ENDPOINT; - } - - // TODO `path` or `output` ? - // I don't think I want to model this as output. deploy is really - // designed to run from the working folder - // could be projectPath or repoPath too - const outputRoot = path.resolve(options.path || '.'); - - // TODO is outputRoot the right dir for this? - const workspace = new Workspace(outputRoot); - const config = workspace.getConfig(); - - // download the state.json from lightning - const { data } = await getProject(cfg as DeployConfig, options.projectId); - - // TODO if the user doesn't specify an env name, prompt for one - const name = options.env || 'project'; - - const project = await Project.from( - 'state', - data as Provisioner.Project, - { - endpoint: cfg.endpoint, - env: name, - - // TODO this is NOT an openfn metadata key - // (it should not be sent back to lighting) - // should add it to the local or meta objects instead - fetched_at: new Date().toISOString(), - }, - config - ); - - const projectFileName = project.getIdentifier(); - - await fs.mkdir(`${outputRoot}/.projects`, { recursive: true }); - const stateOutputPath = `${outputRoot}/.projects/${projectFileName}`; - - const workflowsRoot = path.resolve( - outputRoot, - project.config.dirs.workflows ?? 'workflows' - ); - // Prompt before deleting - // TODO this is actually the wrong path - if ( - !(await confirm({ - message: `This will remove all files in ${path.resolve( - workflowsRoot - )}. Are you sure you wish to proceed? -`, - default: true, - })) - ) { - logger.always('Cancelled'); - return false; - } - await rimraf(workflowsRoot); - - const projFile = project?.serialize('project'); - - if (typeof projFile === 'string') { - await fs.writeFile(`${stateOutputPath}.yaml`, projFile); - } else { - await fs.writeFile( - `${stateOutputPath}.json`, - JSON.stringify(projFile, null, 2) - ); - } - logger.success(`Saved project file to ${stateOutputPath}`); - - const files = project?.serialize('fs') as any; - for (const f in files) { - if (files[f]) { - await fs.mkdir(path.join(outputRoot, path.dirname(f)), { - recursive: true, - }); - await fs.writeFile(path.join(outputRoot, f), files[f]); - } else { - console.log('WARNING! No content for file', f); - } - } - logger.success(`Expanded project to ${outputRoot}`); -} diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index 0d81af776..2ddb8c1f3 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -35,6 +35,11 @@ const options = [ o.snapshots, o.statePath, o.path, + + // These are hidden commands used only by beta + // The need to be declared here to be initialised and defaulted properly + override(o.force, { hidden: true }), + override(o.workspace, { hidden: true }), ]; const pullCommand: yargs.CommandModule = { diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index 4bf9ca5cf..7309a33ba 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -10,11 +10,11 @@ import { } from '@openfn/deploy'; import type { Logger } from '../util/logger'; import { PullOptions } from '../pull/command'; -import * as beta from './beta'; +import beta from '../projects/pull'; async function pullHandler(options: PullOptions, logger: Logger) { if (options.beta) { - return beta.handler(options as any, logger); + return beta(options as any, logger); } try { diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index e55658cf9..bffa644ea 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -30,7 +30,7 @@ export class Workspace { // TODO use logger // TODO maybe we should throw here? // TODO showing hte direct error here is rarely useful - console.error(e); + // console.error(e); // invalid workspace return; } From c6c1bcf96d9b87a60cb9df18e77056a82b38c799 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 17:51:49 +0000 Subject: [PATCH 16/28] fix test --- packages/cli/src/projects/fetch.ts | 10 +++++++--- packages/cli/test/projects/fetch.test.ts | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 14986c4f9..e1e24a270 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -92,7 +92,11 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const outputRoot = path.resolve(options.outputPath || workspacePath); const projectFileName = project.getIdentifier(); const projectsDir = project.config.dirs.projects ?? '.projects'; - const ext = outputPath ? path.extname(outputPath!).substring(1) : undefined; + let ext = path.extname(outputPath!).substring(1); + let format: undefined | 'json' | 'yaml' = undefined; + if (ext.length) { + format = ext as any; + } const stateOutputPath = ext ? outputPath @@ -102,7 +106,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { const finalOutput = await serialize( project, stateOutputPath!, - ext as any, + format, true // dry run - this won't trigger an actual write! ); @@ -122,7 +126,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { // TODO report whether we've updated or not // finally, write it! - await serialize(project, stateOutputPath!, ext as any); + await serialize(project, stateOutputPath!, format as any); logger.success(`Fetched project file to ${finalOutput}`); diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index da7be65b4..c112d43a5 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -41,7 +41,7 @@ test.afterEach(() => { mock.restore(); }); -test.serial('fetch from lightning save as v2 yaml file', async (t) => { +test.serial('fetch from lightning and save as v2 yaml file', async (t) => { await fetchHandler( { projectId: PROJECT_ID, From 57fe6192f0506073b6d0ae3c9ca93c1ea543ac8e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 17:52:49 +0000 Subject: [PATCH 17/28] types refactor --- packages/cli/src/projects/pull.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index 73848b997..f9f6e14b1 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -4,7 +4,7 @@ import { handler as checkout } from './checkout'; import type { Logger } from '../util/logger'; import type { Opts } from '../options'; -export type PullOptionsBeta = Pick< +export type PullOptions = Pick< Opts, | 'apiKey' | 'command' @@ -18,7 +18,7 @@ export type PullOptionsBeta = Pick< | 'workspace' >; -export async function handler(options: PullOptionsBeta, logger: Logger) { +export async function handler(options: PullOptions, logger: Logger) { const project = await fetch(options, logger); logger.success(`Downloaded latest project version`); From 9f6e51b1b67c556e8f90e72fd8edb361d4423fcd Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 2 Dec 2025 17:53:21 +0000 Subject: [PATCH 18/28] clean up pull types --- packages/cli/src/projects/pull.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index f9f6e14b1..07f8ae75c 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -7,13 +7,11 @@ import type { Opts } from '../options'; export type PullOptions = Pick< Opts, | 'apiKey' - | 'command' | 'endpoint' | 'env' | 'force' | 'log' | 'logJson' - | 'path' | 'projectId' | 'workspace' >; From a6a488fdf4528a831a0943f3a33db7ae7d8b1de0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 09:44:42 +0000 Subject: [PATCH 19/28] typings --- packages/project/src/parse/from-project.ts | 3 ++- packages/project/src/serialize/to-project.ts | 13 +++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/project/src/parse/from-project.ts b/packages/project/src/parse/from-project.ts index 9e4bcbfee..5ac129b6a 100644 --- a/packages/project/src/parse/from-project.ts +++ b/packages/project/src/parse/from-project.ts @@ -4,6 +4,7 @@ import Project from '../Project'; import ensureJson from '../util/ensure-json'; import { Provisioner } from '@openfn/lexicon/lightning'; import fromAppState from './from-app-state'; +import { WithMeta } from '../Workflow'; // Load a project from any JSON or yaml representation // This is backwards-compatible with v1 state.json files @@ -19,7 +20,7 @@ export type SerializedWorkflow = { id: string; name: string; - steps: l.Step[]; + steps: WithMeta; openfn?: l.ProjectMeta; }; diff --git a/packages/project/src/serialize/to-project.ts b/packages/project/src/serialize/to-project.ts index 04d879907..4463e3e93 100644 --- a/packages/project/src/serialize/to-project.ts +++ b/packages/project/src/serialize/to-project.ts @@ -1,3 +1,4 @@ +import * as l from '@openfn/lexicon'; import { omitBy, isNil } from 'lodash-es'; import type { @@ -6,6 +7,7 @@ import type { } from '../parse/from-project'; import Project from '../Project'; import { jsonToYaml } from '../util/yaml'; +import { WithMeta } from '../Workflow'; const SERIALIZE_VERSION = 2; @@ -33,9 +35,8 @@ export default (project: Project, options: ToProjectOptions = {}) => { meta: project.meta, options: omitBy(project.options, isNil), - //workflows: project.workflows.map(mapWorkflow) as SerializedWorkflow[], workflows: project.workflows.map((w) => { - const obj: any = w.toJSON(); + const obj = w.toJSON() as SerializedWorkflow; if (obj.openfn) { obj.openfn = omitBy(obj.openfn, isNil); } @@ -43,11 +44,11 @@ export default (project: Project, options: ToProjectOptions = {}) => { obj.steps = obj.steps.sort((a: any, b: any) => { return a.id < b.id ? -1 : a.id > b.id ? 1 : 0; }); - obj.steps.forEach((s) => { + obj.steps.forEach((s: WithMeta) => { s.openfn = omitBy(s.openfn, isNil); - if (s.next) { + if (s.next && typeof s.next !== 'string') { for (const id in s.next) { - const edge = s.next[id]; + const edge = s.next[id] as any; if (edge.openfn) { edge.openfn = omitBy(edge.openfn, isNil); } @@ -56,7 +57,7 @@ export default (project: Project, options: ToProjectOptions = {}) => { }); } return obj; - }) as SerializedWorkflow[], + }), }, isNil ) as SerializedProject; From 71135acd334f38800036ce488cdc3da3aca312a9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 09:53:53 +0000 Subject: [PATCH 20/28] fix integration tests --- integration-tests/cli/test/project-v2.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index 7e8c1cb9c..295ea83d9 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -24,7 +24,6 @@ options: allow_support_access: false requires_mfa: false retention_policy: retain_all - version_history: [] workflows: - name: Hello Workflow steps: @@ -80,8 +79,6 @@ options: allow_support_access: false requires_mfa: false retention_policy: retain_all - version_history: - - 7b0f5af558f5 workflows: - name: Hello Workflow steps: @@ -149,8 +146,7 @@ test.serial('Checkout a project', async (t) => { workflowYaml, `id: hello-workflow name: Hello Workflow -options: - history: [] +options: {} steps: - id: trigger type: webhook From cedbb4335cd1a1eb11e56e8a4e6692dec43ab16b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 10:35:46 +0000 Subject: [PATCH 21/28] if a project has no history, fetch without error --- packages/cli/src/projects/fetch.ts | 12 +++++++++++- packages/cli/test/projects/fetch.test.ts | 8 +++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index e1e24a270..34a6269b4 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -118,7 +118,17 @@ export const handler = async (options: FetchOptions, logger: Logger) => { // Do nothing - project doesn't exist } - if (current && !project.canMergeInto(current) && !options.force) { + const hasAnyHistory = project.workflows.find( + (w) => w.workflow.history?.length + ); + + // Skip version checking if: + const skipVersionCheck = + options.force || // The user forced the checkout + !current || // there is no project on disk + !hasAnyHistory; // the remote project has no history (can happen in old apps) + + if (!skipVersionCheck && !project.canMergeInto(current!)) { // TODO allow force or rename throw new Error('Error! An incompatible project exists at this location'); } diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index c112d43a5..1538d1077 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -216,7 +216,9 @@ test.serial('Override a compatible project', async (t) => { // This means changes could be lost, so we throw! test.serial('throw for an incompatible project', async (t) => { // Change project.yaml - const modified = myProject_yaml.replace('fn()', 'fn(x)'); + const modified = myProject_yaml + .replace('fn()', 'fn(x)') // arbitrary edit so that we can track the change + .replace(' - a', ' - z'); // change the local history to be incompatible mock({ '/ws/.projects': {}, @@ -247,7 +249,7 @@ test.serial('throw for an incompatible project', async (t) => { const fileContent = await readFile(filePath, 'utf-8'); // The file should NOT be overwritten - t.regex(fileContent, /fn()/); + t.regex(fileContent, /fn\(x\)/); }); test.serial('force merge an incompatible project', async (t) => { @@ -278,5 +280,5 @@ test.serial('force merge an incompatible project', async (t) => { const fileContent = await readFile(filePath, 'utf-8'); // The file should be overwritten - t.regex(fileContent, /fn()/); + t.regex(fileContent, /fn\(\)/); }); From 1ed86c8939081d83581bdd7eaf7ff3cd31097984 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 10:49:43 +0000 Subject: [PATCH 22/28] comment --- packages/cli/src/projects/fetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 34a6269b4..bc762f4dd 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -129,7 +129,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { !hasAnyHistory; // the remote project has no history (can happen in old apps) if (!skipVersionCheck && !project.canMergeInto(current!)) { - // TODO allow force or rename + // TODO allow rename throw new Error('Error! An incompatible project exists at this location'); } From ec1fd33f096c2d518848ceeae361dd4b1333222e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 10:50:50 +0000 Subject: [PATCH 23/28] changeset --- .changeset/soft-flowers-lay.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.changeset/soft-flowers-lay.md b/.changeset/soft-flowers-lay.md index 221926edf..cf199183a 100644 --- a/.changeset/soft-flowers-lay.md +++ b/.changeset/soft-flowers-lay.md @@ -2,4 +2,6 @@ '@openfn/cli': minor --- -Add a fetch command, which will download a project from an app but not check it out +Add a fetch command, which will download a project from an app but not check it out. This will throw if the local project version has diverged from the remote version. + +Rebased `pull --beta` to simply be fetch & checkout From 39623c7fe20ec7207cf04802b5c2ba03d37671ca Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 11:40:27 +0000 Subject: [PATCH 24/28] clean up output path --- packages/cli/src/projects/fetch.ts | 34 ++++++++++++------------ packages/cli/test/projects/fetch.test.ts | 4 --- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index bc762f4dd..9cbbbcf8c 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -67,45 +67,45 @@ const command: yargs.CommandModule = { export default command; export const handler = async (options: FetchOptions, logger: Logger) => { - const workspacePath = path.resolve(options.workspace ?? '.'); + const workspacePath = path.resolve(options.workspace ?? process.cwd()); const workspace = new Workspace(workspacePath); - const projectId = options.projectId!; - - const outputPath = options.outputPath; + const { projectId, outputPath } = options; const config = loadAppAuthConfig(options, logger); - const { data } = await getProject(logger, config, projectId); - const name = options.env || 'project'; + const { data } = await getProject(logger, config, projectId!); const project = await Project.from( 'state', data!, { endpoint: config.endpoint, - env: name, + env: options.env || 'project', }, workspace.getConfig() ); // Work out where and how to serialize the project - const outputRoot = path.resolve(options.outputPath || workspacePath); + const outputRoot = path.resolve(outputPath || workspacePath); const projectFileName = project.getIdentifier(); const projectsDir = project.config.dirs.projects ?? '.projects'; - let ext = path.extname(outputPath!).substring(1); + + const finalOutputPath = + outputPath ?? `${outputRoot}/${projectsDir}/${projectFileName}`; let format: undefined | 'json' | 'yaml' = undefined; - if (ext.length) { - format = ext as any; - } - const stateOutputPath = ext - ? outputPath - : `${outputRoot}/${projectsDir}/${projectFileName}`; + if (outputPath) { + // If the user gave us a path for output, we need to respect the format we've been given + const ext = path.extname(outputPath!).substring(1) as any; + if (ext.length) { + format = ext; + } + } // See if a project already exists there const finalOutput = await serialize( project, - stateOutputPath!, + finalOutputPath!, format, true // dry run - this won't trigger an actual write! ); @@ -136,7 +136,7 @@ export const handler = async (options: FetchOptions, logger: Logger) => { // TODO report whether we've updated or not // finally, write it! - await serialize(project, stateOutputPath!, format as any); + await serialize(project, finalOutputPath!, format as any); logger.success(`Fetched project file to ${finalOutput}`); diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index 1538d1077..fc880f42e 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -50,7 +50,6 @@ test.serial('fetch from lightning and save as v2 yaml file', async (t) => { workspace: '/ws', env: 'project', - outputPath: '/ws', } as any, logger ); @@ -200,7 +199,6 @@ test.serial('Override a compatible project', async (t) => { workspace: '/ws', env: 'project', - outputPath: '/ws', } as any, logger ); @@ -236,7 +234,6 @@ test.serial('throw for an incompatible project', async (t) => { workspace: '/ws', env: 'project', - outputPath: '/ws', } as any, logger ), @@ -270,7 +267,6 @@ test.serial('force merge an incompatible project', async (t) => { workspace: '/ws', env: 'project', - outputPath: '/ws', force: true, } as any, logger From d25f4d53e82942dc1fddac0837a4dde96e8d07f1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 15:24:36 +0000 Subject: [PATCH 25/28] workspace: should still work if no openfn.yaml is present --- packages/project/src/Workspace.ts | 24 +++++++++++++++--------- packages/project/test/workspace.test.ts | 7 +++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index bffa644ea..7e48d16d6 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -1,4 +1,5 @@ import * as l from '@openfn/lexicon'; +import createLogger from '@openfn/logger'; import path from 'node:path'; import fs from 'node:fs'; @@ -10,6 +11,7 @@ import { findWorkspaceFile, } from './util/config'; import fromProject from './parse/from-project'; +import type { Logger } from '@openfn/logger'; export class Workspace { // @ts-ignore config not definitely assigned - it sure is @@ -19,27 +21,27 @@ export class Workspace { private projects: Project[] = []; private projectPaths = new Map(); private isValid: boolean = false; + private logger: Logger; - constructor(workspacePath: string) { - let context; + constructor(workspacePath: string, logger?: Logger) { + this.logger = logger ?? createLogger('Workspace', { level: 'info' }); + + let context = { workspace: undefined, project: undefined }; try { const { type, content } = findWorkspaceFile(workspacePath); context = loadWorkspaceFile(content, type as any); this.isValid = true; } catch (e) { - // TODO use logger - // TODO maybe we should throw here? - // TODO showing hte direct error here is rarely useful - // console.error(e); - // invalid workspace - return; + this.logger.warn( + `Could not find openfn.yaml at ${workspacePath}. Using default values.` + ); } this.config = buildConfig(context.workspace); this.activeProject = context.project; const projectsPath = path.join(workspacePath, this.config.dirs.projects); // dealing with projects - if (this.isValid && pathExists(projectsPath, 'directory')) { + if (pathExists(projectsPath, 'directory')) { const ext = `.${this.config.formats.project}`; const stateFiles = fs .readdirSync(projectsPath) @@ -65,6 +67,10 @@ export class Workspace { } }) .filter((s) => s) as Project[]; + } else { + this.logger.warn( + `No projects found: directory at ${projectsPath} does not exist` + ); } } diff --git a/packages/project/test/workspace.test.ts b/packages/project/test/workspace.test.ts index ff46226f3..93b0f61b3 100644 --- a/packages/project/test/workspace.test.ts +++ b/packages/project/test/workspace.test.ts @@ -183,11 +183,18 @@ mock({ test('workspace-path: valid workspace path', (t) => { const ws = new Workspace('/ws'); t.is(ws.valid, true); + + t.truthy(ws.config); }); test('workspace-path: invalid workspace path', (t) => { const ws = new Workspace('/invalid'); t.is(ws.valid, false); + + // should still have config + t.truthy(ws.config); + t.truthy(ws.config.dirs); + t.truthy(ws.config.formats); }); test('workspace-list: list projects in the workspace', (t) => { From b4659fd94c21ea88faed222a7c0036e1a8e92868 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 15:53:54 +0000 Subject: [PATCH 26/28] relax checkout to work with an invalid workspace --- packages/cli/src/projects/checkout.ts | 7 ++----- packages/cli/src/projects/util.ts | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index 677704b39..652ce4af0 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -19,7 +19,7 @@ const options = [o.projectId, o.workspace, o.log]; const command: yargs.CommandModule = { command: 'checkout ', - describe: 'Switch to a different openfn project in the same workspace', + describe: 'Switch to a different OpenFn project in the same workspace', handler: ensure('project-checkout', options), builder: (yargs) => build(options, yargs), }; @@ -29,10 +29,7 @@ export default command; export const handler = async (options: CheckoutOptions, logger: Logger) => { const projectId = options.projectId!; const workspacePath = options.workspace ?? process.cwd(); - const workspace = new Workspace(workspacePath); - if (!workspace.valid) { - throw new Error('Command was run in an invalid openfn workspace'); - } + const workspace = new Workspace(workspacePath, logger); // get the config // TODO: try to retain the endpoint for the projects diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 45d821673..77f787c4c 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -8,6 +8,8 @@ import type Project from '@openfn/project'; type AuthOptions = Pick; +export const init = (dir: string) => {}; + export const loadAppAuthConfig = ( options: AuthOptions, logger: Logger From a45d1ab4e043e8515ca493165c179d3f2c7dc7b7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 16:38:43 +0000 Subject: [PATCH 27/28] CLI: nicer handling of expected CLI errors --- packages/cli/src/commands.ts | 7 ++++++- packages/cli/src/errors.ts | 5 +++++ packages/cli/src/projects/util.ts | 14 ++++++-------- 3 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 packages/cli/src/errors.ts diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 51f49c431..cbded7e4f 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -19,6 +19,7 @@ import mapAdaptorsToMonorepo, { import printVersions from './util/print-versions'; import abort from './util/abort'; import { report } from './env'; +import { CLIError } from './errors'; export type CommandList = | 'apollo' @@ -134,7 +135,11 @@ const parse = async (options: Opts, log?: Logger) => { // This is unexpected error and we should try to log something logger.break(); logger.error('Command failed!'); - logger.error(e); + if (e instanceof CLIError) { + logger.error(e.message); + } else { + logger.error(e); + } } } }; diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts new file mode 100644 index 000000000..a673f0cb0 --- /dev/null +++ b/packages/cli/src/errors.ts @@ -0,0 +1,5 @@ +export class CLIError extends Error { + constructor(message: string) { + super(message); + } +} diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 77f787c4c..6dc85df9f 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -5,6 +5,7 @@ import { Provisioner } from '@openfn/lexicon/lightning'; import type { Opts } from '../options'; import type { Logger } from '@openfn/logger'; import type Project from '@openfn/project'; +import { CLIError } from '../errors'; type AuthOptions = Pick; @@ -103,20 +104,17 @@ export async function getProject( }, }); - // A 404 response means the project doesn't exist yet - if (response.status === 404) { - logger.info('No project found'); - return { data: null }; - } - if (!response.ok) { if (response.status === 401 || response.status === 403) { - throw new DeployError( + throw new CLIError( `Failed to authorize request with endpoint ${config.endpoint}, got ${response.status} ${response.statusText}` ); } + if (response.status === 404) { + throw new CLIError(`Project not found: ${projectId}`); + } - throw new Error( + throw new CLIError( `Failed to fetch project ${projectId}: ${response.statusText}` ); } From e13610822469efbb1952723848728ef769c6b48d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 3 Dec 2025 16:39:34 +0000 Subject: [PATCH 28/28] typings --- packages/cli/src/projects/util.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 6dc85df9f..3763c9f69 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -9,8 +9,6 @@ import { CLIError } from '../errors'; type AuthOptions = Pick; -export const init = (dir: string) => {}; - export const loadAppAuthConfig = ( options: AuthOptions, logger: Logger