Fetch command (rebased and rebuilt)

.changeset/soft-flowers-lay.md

@@ -0,0 +1,7 @@
+---
+'@openfn/cli': minor
+---
+
+Add a fetch command, which will download a project from an app but not check it out. This will throw if the local project version has diverged from the remote version.
+
+Rebased `pull --beta` to simply be fetch & checkout

integration-tests/cli/test/project-v2.test.ts

@@ -24,7 +24,6 @@ options:
   allow_support_access: false
   requires_mfa: false
   retention_policy: retain_all
-  version_history: []
 workflows:
   - name: Hello Workflow
     steps:
@@ -80,8 +79,6 @@ options:
   allow_support_access: false
   requires_mfa: false
   retention_policy: retain_all
-  version_history:
-    - 7b0f5af558f5
 workflows:
   - name: Hello Workflow
     steps:
@@ -149,8 +146,7 @@ test.serial('Checkout a project', async (t) => {
     workflowYaml,
     `id: hello-workflow
 name: Hello Workflow
-options:
-  history: []
+options: {}
 steps:
   - id: trigger
     type: webhook

packages/cli/src/cli.ts

@@ -17,6 +17,7 @@ import testCommand from './test/command';
 import projectsCommand from './projects/command';
 import mergeCommand from './projects/merge';
 import checkoutCommand from './projects/checkout';
+import fetchCommand from './projects/fetch';
 
 const env = loadDotEnv();
 if (env) {
@@ -44,6 +45,7 @@ export const cmd = y
   .command(projectsCommand)
   .command(checkoutCommand)
   .command(mergeCommand)
+  .command(fetchCommand as any)
   .command({
     command: 'version',
     describe:

packages/cli/src/commands.ts

@@ -19,6 +19,7 @@ import mapAdaptorsToMonorepo, {
 import printVersions from './util/print-versions';
 import abort from './util/abort';
 import { report } from './env';
+import { CLIError } from './errors';
 
 export type CommandList =
   | 'apollo'
@@ -42,6 +43,7 @@ export type CommandList =
   | 'project-version'
   | 'project-merge'
   | 'project-checkout'
+  | 'project-fetch'
   | 'test'
   | 'version';
 
@@ -68,6 +70,7 @@ const handlers = {
   ['project-version']: projects.version,
   ['project-merge']: projects.merge,
   ['project-checkout']: projects.checkout,
+  ['project-fetch']: projects.fetch,
   version: async (opts: Opts, logger: Logger) =>
     printVersions(logger, opts, true),
 };
@@ -132,7 +135,11 @@ const parse = async (options: Opts, log?: Logger) => {
       // This is unexpected error and we should try to log something
       logger.break();
       logger.error('Command failed!');
-      logger.error(e);
+      if (e instanceof CLIError) {
+        logger.error(e.message);
+      } else {
+        logger.error(e);
+      }
     }
   }
 };

packages/cli/src/errors.ts

@@ -0,0 +1,5 @@
+export class CLIError extends Error {
+  constructor(message: string) {
+    super(message);
+  }
+}

packages/cli/src/projects/checkout.ts

@@ -10,67 +10,62 @@ import * as o from '../options';
 
 import type { Opts } from '../options';
 
-export type CheckoutOptions = Required<
-  Pick<Opts, 'command' | 'projectId' | 'workspace'>
-> &
-  Pick<Opts, 'log'>;
+export type CheckoutOptions = Pick<
+  Opts,
+  'command' | 'projectId' | 'workspace' | 'log'
+>;
 
 const options = [o.projectId, o.workspace, o.log];
 
 const command: yargs.CommandModule = {
   command: 'checkout <project-id>',
-  describe: 'Switch to a different openfn project in the same workspace',
+  describe: 'Switch to a different OpenFn project in the same workspace',
   handler: ensure('project-checkout', options),
   builder: (yargs) => build(options, yargs),
 };
 
 export default command;
 
 export const handler = async (options: CheckoutOptions, logger: Logger) => {
-  const commandPath = options.workspace;
-  const workspace = new Workspace(commandPath);
-  if (!workspace.valid) {
-    logger.error('Command was run in an invalid openfn workspace');
-    return;
-  }
+  const projectId = options.projectId!;
+  const workspacePath = options.workspace ?? process.cwd();
+  const workspace = new Workspace(workspacePath, logger);
 
   // get the config
   // TODO: try to retain the endpoint for the projects
   const { project: _, ...config } = workspace.getConfig() as any;
 
   // get the project
   let switchProject;
-  if (/\.(yaml|json)$/.test(options.projectId)) {
+  if (/\.(yaml|json)$/.test(projectId)) {
     // TODO: should we allow checkout into an arbitrary folder?
-    const filePath = options.projectId.startsWith('/')
-      ? options.projectId
-      : path.join(commandPath, options.projectId);
+    const filePath = projectId.startsWith('/')
+      ? projectId
+      : path.join(workspacePath, projectId);
     logger.debug('Loading project from path ', filePath);
     switchProject = await Project.from('path', filePath, config);
   } else {
-    switchProject = workspace.get(options.projectId);
+    switchProject = workspace.get(projectId);
   }
 
   if (!switchProject) {
-    throw new Error(
-      `Project with id ${options.projectId} not found in the workspace`
-    );
+    throw new Error(`Project with id ${projectId} not found in the workspace`);
   }
 
   // delete workflow dir before expanding project
-  await rimraf(path.join(commandPath, config.workflowRoot ?? 'workflows'));
+  await rimraf(path.join(workspacePath, config.workflowRoot ?? 'workflows'));
 
   // expand project into directory
   const files: any = switchProject.serialize('fs');
   for (const f in files) {
     if (files[f]) {
-      fs.mkdirSync(path.join(commandPath, path.dirname(f)), {
+      fs.mkdirSync(path.join(workspacePath, path.dirname(f)), {
         recursive: true,
       });
-      fs.writeFileSync(path.join(commandPath, f), files[f]);
+      fs.writeFileSync(path.join(workspacePath, f), files[f]);
     } else {
       logger.warn('WARNING! No content for file', f);
     }
   }
-  logger.success(`Expanded project to ${commandPath}`);
+  logger.success(`Expanded project to ${workspacePath}`);
 };

packages/cli/src/projects/command.ts

@@ -2,6 +2,7 @@ import list from './list';
 import version from './version';
 import merge from './merge';
 import checkout from './checkout';
+import fetch from './fetch';
 
 import type yargs from 'yargs';
 
@@ -16,6 +17,7 @@ export const projectsCommand = {
       .command(version)
       .command(merge)
       .command(checkout)
+      .command(fetch as any)
       .example('project', 'list all projects in the workspace')
       .example('project list', 'list all projects in the workspace')
       .example(

packages/cli/src/projects/fetch.ts

@@ -0,0 +1,144 @@
+import yargs from 'yargs';
+import path from 'node:path';
+import Project, { Workspace } from '@openfn/project';
+
+import { build, ensure, override } from '../util/command-builders';
+import type { Logger } from '../util/logger';
+import * as o from '../options';
+
+import type { Opts } from '../options';
+import { serialize, getProject, loadAppAuthConfig } from './util';
+
+// TODO need to implement these
+// type Config = {
+//   requireConfirmation?: boolean; // alias to y maybe
+//   dryRun?: boolean;
+// };
+
+export type FetchOptions = Pick<
+  Opts,
+  | 'apiKey'
+  | 'command'
+  | 'endpoint'
+  | 'env'
+  | 'force'
+  | 'log'
+  | 'logJson'
+  | 'outputPath'
+  | 'projectId'
+  | 'workspace'
+>;
+
+const options = [
+  o.apikey,
+  o.configPath,
+  o.endpoint,
+  o.env,
+  o.log,
+  override(o.outputPath, {
+    description: 'Path to output the fetched project to',
+  }),
+  o.logJson,
+  o.workspace,
+  o.snapshots,
+  o.statePath,
+  override(o.force, {
+    description: 'Overwrite local file contents with the fetched contents',
+  }),
+];
+
+const command: yargs.CommandModule<FetchOptions> = {
+  command: 'fetch [projectId]',
+  describe: `Fetch a project's state and spec from a Lightning Instance to the local state file without expanding to the filesystem.`,
+  builder: (yargs: yargs.Argv<FetchOptions>) =>
+    build(options, yargs)
+      .positional('projectId', {
+        describe:
+          'The id of the project that should be fetched, should be a UUID',
+        demandOption: true,
+      })
+      .example(
+        'fetch 57862287-23e6-4650-8d79-e1dd88b24b1c',
+        'Fetch an updated copy of a the above spec and state from a Lightning Instance'
+      ),
+  handler: ensure('project-fetch', options),
+};
+
+export default command;
+
+export const handler = async (options: FetchOptions, logger: Logger) => {
+  const workspacePath = path.resolve(options.workspace ?? process.cwd());
+  const workspace = new Workspace(workspacePath);
+  const { projectId, outputPath } = options;
+
+  const config = loadAppAuthConfig(options, logger);
+
+  const { data } = await getProject(logger, config, projectId!);
+
+  const project = await Project.from(
+    'state',
+    data!,
+    {
+      endpoint: config.endpoint,
+      env: options.env || 'project',
+    },
+    workspace.getConfig()
+  );
+
+  // Work out where and how to serialize the project
+  const outputRoot = path.resolve(outputPath || workspacePath);
+  const projectFileName = project.getIdentifier();
+  const projectsDir = project.config.dirs.projects ?? '.projects';
+
+  const finalOutputPath =
+    outputPath ?? `${outputRoot}/${projectsDir}/${projectFileName}`;
+  let format: undefined | 'json' | 'yaml' = undefined;
+
+  if (outputPath) {
+    // If the user gave us a path for output, we need to respect the format we've been given
+    const ext = path.extname(outputPath!).substring(1) as any;
+    if (ext.length) {
+      format = ext;
+    }
+  }
+
+  // See if a project already exists there
+  const finalOutput = await serialize(
+    project,
+    finalOutputPath!,
+    format,
+    true // dry run - this won't trigger an actual write!
+  );
+
+  // If a project already exists at the output path, make sure it's compatible
+  let current: Project | null = null;
+  try {
+    current = await Project.from('path', finalOutput);
+  } catch (e) {
+    // Do nothing - project doesn't exist
+  }
+
+  const hasAnyHistory = project.workflows.find(
+    (w) => w.workflow.history?.length
+  );
+
+  // Skip version checking if:
+  const skipVersionCheck =
+    options.force || // The user forced the checkout
+    !current || // there is no project on disk
+    !hasAnyHistory; // the remote project has no history (can happen in old apps)
+
+  if (!skipVersionCheck && !project.canMergeInto(current!)) {
+    // TODO allow rename
+    throw new Error('Error! An incompatible project exists at this location');
+  }
+
+  // TODO report whether we've updated or not
+
+  // finally, write it!
+  await serialize(project, finalOutputPath!, format as any);
+
+  logger.success(`Fetched project file to ${finalOutput}`);
+
+  return project;
+};

packages/cli/src/projects/handler.ts

@@ -2,3 +2,4 @@ export { handler as list } from './list';
 export { handler as version } from './version';
 export { handler as merge } from './merge';
 export { handler as checkout } from './checkout';
+export { handler as fetch } from './fetch';

packages/cli/src/projects/pull.ts

@@ -0,0 +1,33 @@
+import { handler as fetch } from './fetch';
+import { handler as checkout } from './checkout';
+
+import type { Logger } from '../util/logger';
+import type { Opts } from '../options';
+
+export type PullOptions = Pick<
+  Opts,
+  | 'apiKey'
+  | 'endpoint'
+  | 'env'
+  | 'force'
+  | 'log'
+  | 'logJson'
+  | 'projectId'
+  | 'workspace'
+>;
+
+export async function handler(options: PullOptions, logger: Logger) {
+  const project = await fetch(options, logger);
+  logger.success(`Downloaded latest project version`);
+
+  await checkout(
+    {
+      ...options,
+      projectId: project.id,
+    },
+    logger
+  );
+  logger.success(`Checked out project locally`);
+}
+
+export default handler;