diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8c52ff9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index b6df596..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "eslint:recommended", - "parserOptions": { - "ecmaVersion": "latest" - }, - "env": { - "es2023": true, - "node": true - }, - "rules": { - "no-console": "off" - } -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..122e23a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +permissions: + contents: read + +jobs: + test: + name: Node ${{ matrix.node }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + node: [24, 26] + steps: + - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + cache: npm + - run: npm ci --ignore-scripts + - run: npm run lint + - run: npm test + - run: npm pack --dry-run diff --git a/.gitignore b/.gitignore index 3a02206..b5a4218 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .DS_Store *.key +*.tgz node_modules .pushapi-config.json +.pushapi.buffer.* diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index e4c8bf8..0000000 --- a/.jshintrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "esversion": 9, - "node": true -} diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..b6f27f1 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..a45fd52 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/README.md b/README.md index 5351c18..868000a 100644 --- a/README.md +++ b/README.md @@ -1,78 +1,132 @@ # PushAPI-node.js -## Description +[![CI](https://github.com/coveooss/pushapi-nodejs/actions/workflows/ci.yml/badge.svg)](https://github.com/coveooss/pushapi-nodejs/actions/workflows/ci.yml) +[![npm](https://img.shields.io/npm/v/coveo-pushapi-cli)](https://www.npmjs.com/package/coveo-pushapi-cli) -This project is a simple wrapper to help sending JSON payloads using Coveo Cloud V2 PushAPI. +Command line tool to push JSON payloads to a Coveo [Push source](https://docs.coveo.com/en/68/cloud-v2-developers/push-api) (or a Catalog source, through the Stream API). -## Available documentation +## Requirements -The code in this project uses the Push API from the Coveo Cloud V2 Platform. You can use the following resource for more information and get started: +- Node.js **24** or newer (see `.nvmrc`). +- No runtime dependencies: HTTP calls use Node's built-in `fetch`, and arguments are parsed with `node:util.parseArgs`. -- Push API documentation: [Use the Push API](https://docs.coveo.com/en/68/cloud-v2-developers/push-api) +## Install -## Contributing +```bash +npm install --global coveo-pushapi-cli +``` -- Branch or Fork -- Pull Request -- And... that's pretty much it! +## Usage -## How-to build +The first time you run `pushapi` in a folder, it helps you create the config file `.pushapi-config.json` by asking for your source ID, organization ID, platform and API key. The file is created with `0600` permissions since it holds your API key. Don't commit it. -1. `git clone` this project. -1. `npm install` to get the dependencies (request, fs). +```bash +# Push one JSON payload file. Don't forget to set DocumentId in the payloads. +pushapi file.json -## CLI command +# Push one file, then delete every item of the source that is older than 24 hours. +# Typical when you create/update all valid items and want the leftovers gone. +pushapi file.json -d 24 -Install it with NPM: +# Push every .json file of a folder, batched into as few requests as possible. +pushapi folder -```bash -npm i coveo-pushapi-cli --global +# Dry run: build the batch files (.pushapi.buffer.N) locally, without pushing anything. +pushapi folder --dry-run + +pushapi --help ``` -## How-to run +| Option | Description | +| --- | --- | +| `-d, --deleteOlderThan ` | Delete source items older than `` before pushing. | +| `-D, --dry-run` | Create the batch files without pushing them. | +| `--help`, `--version` | Show help / version. | -The first time you run in one folder, it will help you to set up your config `.pushapi-config.json` by asking about your `sourceId` and your `API key`. +Uploads accept a mix of these JSON formats, including within the same folder: -- Push one JSON payload file: +- an array of items +- a Push API payload: `{ "AddOrUpdate": [ ... ] }` +- a single item object - `pushapi file.json` +See the `samples/` folder for examples. - where `file.json` is the path to a Push API payload (JSON) file. Don't forget to set `DocumentId` in the payloads. +### Config file -- Push one JSON payload file, but delete source item older than 24 hours: +```json +{ + "org": "your-org-id", + "source": "your-source-id", + "apiKey": "xx--your-api-key--abc", + "platform": "push.cloud.coveo.com", + "useStreamApi": false, + "debug": false +} +``` - `pushapi file.json -d 24` +| Key | Description | +| --- | --- | +| `org` | Coveo organization ID. | +| `source` | Push (or Catalog) source ID. | +| `apiKey` | API key with the `Edit` privilege on the source. | +| `platform` | Push API host. Defaults to `push.cloud.coveo.com`. Use `pushdev.`, `pushstg.` or `pushhipaa.` for the other platforms. | +| `useStreamApi` | `true` for a Catalog source: payloads go through the [Stream API](https://docs.coveo.com/en/p4eb0129/) instead of the Push API. | +| `debug` | `true` to log every request and its response code. | - to use typically when you create/update all valid items, you want to delete all other items that are older. +## What does it do? -- Push all JSON payload files from one folder: +The tool does a minimal validation of the payload (every document needs a `DocumentId`), then runs these API calls: - `pushapi folder` +1. Change the source state to `REBUILD` +2. Get a file container +3. Upload the payload to the file container +4. Send the batch command to process the file container +5. Change the source state back to `IDLE` (even if a previous step failed) - where `folder` is the path to a folder containing multiple Push API JSON payload files. - Folder uploads accept a mix of these JSON formats in the same folder: - - an array of items - - a Push API payload with `AddOrUpdate` - - a single item object +With `useStreamApi`, it instead opens a stream, uploads one chunk per batch, and closes the stream. -## What does it do? +### Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Success. | +| `1` | Missing config file, missing input file, or invalid payload. | +| `2` to `5` | Invalid config: missing config / `apiKey` / `org` / `source`. | +| `10` | A request to the platform failed. The response body is printed. | -The helper does a minimal validation on the payload, then executes these API calls: +## Programmatic use -1. Change source state to REBUILD -2. Get a File container -3. Upload the payload to the File container -4. Sends the batch command to process the File container -5. Change the source state back to IDLE +The package is an ES module. It can also be loaded with `require()` on Node 24+. -> **Note** -> You can add this attribute `"debug": true` in your `.pushapi-config.json` to see these requests and their response code. +```js +import { main } from 'coveo-pushapi-cli'; -## Dependencies +// Reads .pushapi-config.json from the current folder. +await main('folder-with-payloads', { deleteOlderThan: 24, dryRun: false }); +``` + +## Development + +```bash +git clone https://github.com/coveooss/pushapi-nodejs.git +cd pushapi-nodejs +npm install +npm test # node --test +npm run lint # eslint +``` -- Node.js -- Node modules: `fs`, `request` +Tests run against a mocked `fetch`; nothing is pushed to Coveo. CI runs lint and tests on every Node release line declared in `engines`. + +## Contributing + +- Branch or fork +- Pull request +- And... that's pretty much it! ## Authors - Jérôme Devost ([GitHub profile](https://github.com/jdevost)) + +## License + +[Apache-2.0](LICENSE) diff --git a/client-global.js b/client-global.js index b3cf662..f41ff75 100755 --- a/client-global.js +++ b/client-global.js @@ -1,30 +1,93 @@ #!/usr/bin/env node -const { main, Config } = require('./push.js'); -const fs = require('fs'); -const yargs = require('yargs/yargs'); -const { hideBin } = require('yargs/helpers'); - -const argv = yargs(hideBin(process.argv)) - .usage('\nUsage: $0 [options]') - .example('$0 file1.json', 'Upload a single file to a Push Source') - .example('$0 folder2', 'Upload all .json files from a folder to a Push Source') - .example('$0 folder3 -d 2', 'Sends a deleteOlderThan 2 hours before pushing new data') - .alias('d', 'deleteOlderThan') - .default('d', null) - .describe('d', 'Set the deleteOlderThan delay (in hours)') - .alias('D', 'dry-run') - .boolean('D') - .describe('D', 'Dry run - creates the batch files, without pushing them') - .demandCommand(1, 'You need to specify a FILE or a FOLDER\n') - .help() - .parse(); - -const FILE_OR_FOLDER = argv._[0]; - -let configFile = `${process.cwd()}/.pushapi-config.json`; -if (!fs.existsSync(configFile)) { - console.warn(`\n\tCouldn't load ${configFile} file`); - Config.createConfig(configFile, () => main(FILE_OR_FOLDER, argv)); -} else { - return main(FILE_OR_FOLDER, argv); +import { existsSync } from 'node:fs'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; + +import pkg from './package.json' with { type: 'json' }; +import { main, reportError, Config } from './push.js'; +import { CONFIG_FILE_NAME } from './src/Config.js'; + +const HELP = ` +Usage: pushapi [options] + +Options: + -d, --deleteOlderThan Set the deleteOlderThan delay (in hours) + -D, --dry-run Dry run - creates the batch files, without pushing them + -h, --help Show help + -v, --version Show version number + +Examples: + pushapi file1.json Upload a single file to a Push Source + pushapi folder2 Upload all .json files from a folder to a Push Source + pushapi folder3 -d 2 Sends a deleteOlderThan 2 hours before pushing new data +`; + +/** Prints the usage and the error message to stderr, then exits with code 1. */ +function usageError(message) { + console.error(HELP); + console.error(`${message}\n`); + process.exit(1); +} + +/** + * Parses the command line into `{ fileOrFolder, deleteOlderThan, dryRun }`. + * Exits the process on --help, --version, or invalid arguments. + */ +function parseCommandLine(args) { + let parsed; + try { + parsed = parseArgs({ + args, + allowPositionals: true, + options: { + deleteOlderThan: { type: 'string', short: 'd' }, + 'dry-run': { type: 'boolean', short: 'D', default: false }, + help: { type: 'boolean', short: 'h', default: false }, + version: { type: 'boolean', short: 'v', default: false }, + }, + }); + } catch (error) { + return usageError(error.message); + } + + const { values, positionals } = parsed; + if (values.help) { + console.log(HELP); + process.exit(0); + } + if (values.version) { + console.log(pkg.version); + process.exit(0); + } + + if (positionals.length !== 1) { + return usageError(positionals.length === 0 ? 'You need to specify a FILE or a FOLDER' : `Too many arguments: ${positionals.join(' ')}`); + } + + let deleteOlderThan = null; + if (values.deleteOlderThan !== undefined) { + deleteOlderThan = Number(values.deleteOlderThan); + if (values.deleteOlderThan.trim() === '' || !Number.isFinite(deleteOlderThan) || deleteOlderThan < 0) { + return usageError(`--deleteOlderThan expects a number of hours, got: ${values.deleteOlderThan}`); + } + } + + return { fileOrFolder: positionals[0], deleteOlderThan, dryRun: values['dry-run'] }; +} + +const { fileOrFolder, ...options } = parseCommandLine(process.argv.slice(2)); + +try { + const configFile = path.join(process.cwd(), CONFIG_FILE_NAME); + if (!existsSync(configFile)) { + console.warn(`\n\tCouldn't load ${configFile} file`); + const created = await Config.createConfig(configFile); + if (!created) { + process.exit(0); + } + } + await main(fileOrFolder, options); +} catch (error) { + reportError(error); + process.exitCode = typeof error?.exitCode === 'number' ? error.exitCode : 10; } diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..4729ea3 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,23 @@ +import js from '@eslint/js'; +import { defineConfig, globalIgnores } from 'eslint/config'; +import globals from 'globals'; + +export default defineConfig([ + globalIgnores(['node_modules/', 'samples/', '*.tgz']), + { + files: ['**/*.js'], + plugins: { js }, + extends: ['js/recommended'], + languageOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + globals: globals.node, + }, + rules: { + 'no-console': 'off', + 'prefer-const': 'error', + 'no-var': 'error', + eqeqeq: ['error', 'always'], + }, + }, +]); diff --git a/package-lock.json b/package-lock.json index cad9573..9b33c61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,90 +1,658 @@ { "name": "coveo-pushapi-cli", - "version": "2.0.0", + "version": "3.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "coveo-pushapi-cli", - "version": "2.0.0", + "version": "3.0.1", "license": "Apache-2.0", + "bin": { + "pushapi": "client-global.js" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "eslint": "^10.13.0", + "globals": "^17.13.0" + }, + "engines": { + "node": ">=24" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", + "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.5.0", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz", + "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", "dependencies": { - "yargs": "^18.0.0" + "eslint-visitor-keys": "^3.4.3" }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", + "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz", + "integrity": "sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", "bin": { - "pushapi": "client-global.js" + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/cacheable": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", + "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.2.0", + "@cacheable/utils": "^2.5.0", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" }, - "devDependencies": {}, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, "license": "MIT", "engines": { - "node": ">=12" + "node": ">=10" }, "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "node_modules/eslint": { + "version": "10.13.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.13.0.tgz", + "integrity": "sha512-UohDjD4zSzEwhf58M3O7vc/ycc8O1I7dNlhtD0WxE8Px8EHNJKtUwzBqrzeGhisRuv2ag4sAYswszCJRgKXYUg==", + "dev": true, "license": "MIT", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.7.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.3", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "11.1.5 || >11.1.6 <12", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, "engines": { - "node": ">=12" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } } }, - "node_modules/cliui": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", - "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", - "license": "ISC", + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", "dependencies": { - "string-width": "^7.2.0", - "strip-ansi": "^7.1.0", - "wrap-ansi": "^9.0.0" + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, - "node_modules/emoji-regex": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", - "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, "license": "MIT" }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/file-entry-cache": { + "version": "11.1.5", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz", + "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^6.1.23" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "6.1.23", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz", + "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cacheable": "^2.5.0", + "flatted": "^3.4.2", + "hookified": "^1.15.0" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, "engines": { - "node": "6.* || 8.* || >= 10.*" + "node": ">=10.13.0" } }, - "node_modules/get-east-asian-width": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", - "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", + "node_modules/globals": { + "version": "17.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.13.0.tgz", + "integrity": "sha512-RwMTC61u7hrG3ZJMkZQOK4JLJrKS8QtoTii63EmWvFXHqycY9FObBJQCbsLxSO8kjKhWE5kV1GC+Vb1g3RI0Zg==", + "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -93,88 +661,353 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "dev": true, "license": "MIT", "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" + "hookified": "^1.15.0" }, "engines": { - "node": ">=18" + "node": ">=20" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "license": "MIT", + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", "dependencies": { - "ansi-regex": "^6.2.2" + "brace-expansion": "^5.0.8" }, "engines": { - "node": ">=12" + "node": "18 || 20 || >=22" }, "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/wrap-ansi": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", - "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, "license": "MIT", "dependencies": { - "ansi-styles": "^6.2.1", - "string-width": "^7.0.0", - "strip-ansi": "^7.1.0" + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" }, "engines": { - "node": ">=18" + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" }, "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "dev": true, + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" } }, - "node_modules/yargs": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", - "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, "license": "MIT", "dependencies": { - "cliui": "^9.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "string-width": "^7.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^22.0.0" + "prelude-ls": "^1.2.1" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" + "node": ">= 0.8.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" } }, - "node_modules/yargs-parser": { - "version": "22.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", - "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } } } diff --git a/package.json b/package.json index 5b7dc85..1609076 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,28 @@ { "name": "coveo-pushapi-cli", - "version": "2.0.0", + "version": "3.0.1", "description": "Command line tool to push JSON payloads to the Coveo Cloud using the PushAPI", + "type": "module", "main": "push.js", + "exports": "./push.js", + "bin": { + "pushapi": "client-global.js" + }, + "files": [ + "client-global.js", + "push.js", + "src/*.js" + ], + "engines": { + "node": ">=24" + }, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test": "node --test", + "test:watch": "node --test --watch", + "prepack": "npm run lint && npm test" + }, "keywords": [ "coveo", "cloud", @@ -10,14 +30,11 @@ "pushapi", "json" ], - "dependencies": { - "yargs": "^18.0.0" + "devDependencies": { + "@eslint/js": "^10.0.1", + "eslint": "^10.13.0", + "globals": "^17.13.0" }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" - }, - "devDependencies": {}, - "scripts": {}, "repository": { "type": "git", "url": "git+https://github.com/coveooss/pushapi-nodejs.git" @@ -27,12 +44,5 @@ "bugs": { "url": "https://github.com/coveooss/pushapi-nodejs/issues" }, - "files": [ - "push.js", - "src/*.js" - ], - "bin": { - "pushapi": "client-global.js" - }, "homepage": "https://github.com/coveooss/pushapi-nodejs#readme" } diff --git a/push.js b/push.js index 5d64429..8eb9f18 100755 --- a/push.js +++ b/push.js @@ -1,73 +1,65 @@ -const fs = require('fs'); - -const Config = require('./src/Config'); -const JsonBuffer = require('./src/JsonBuffer'); -const PushApi = require('./src/PushApi'); -const StreamApi = require('./src/StreamApi'); - - +import { readdir, stat, unlink } from 'node:fs/promises'; +import path from 'node:path'; +import { styleText } from 'node:util'; + +import Config from './src/Config.js'; +import { CliError, HttpError } from './src/errors.js'; +import JsonBuffer, { BUFFER_FILE_PREFIX, loadJsonFile } from './src/JsonBuffer.js'; +import { countPayloadItems, looksLikePushPayload } from './src/payload.js'; +import PushApi from './src/PushApi.js'; +import StreamApi from './src/StreamApi.js'; + +const highlight = (text) => styleText(['yellow', 'bold'], String(text)); + +/** + * Runs `work()` with the source in REBUILD status, and always puts it back to IDLE afterwards. + */ async function runWithSourceInRebuild(pushApiHelper, work) { - let statusChanged = false; - let result; - let workError = null; - await pushApiHelper.changeStatus('REBUILD'); - statusChanged = true; + let result; + let workError = null; try { result = await work(); } catch (error) { workError = error; } - if (statusChanged) { - try { - await pushApiHelper.changeStatus('IDLE'); - } catch (statusError) { - if (workError) { - statusError.cause = workError; - } - throw statusError; + try { + await pushApiHelper.changeStatus('IDLE'); + } catch (statusError) { + if (workError) { + statusError.cause = workError; } + throw statusError; } if (workError) { throw workError; } - return result; } -function reportError(error) { - if (error && error.statusCode) { +/** + * Prints an error to stderr in a readable way. + * @param {unknown} error + */ +export function reportError(error) { + if (error instanceof HttpError) { console.error(error.message); if (error.body) { console.error(error.body); } return; } - - console.error(error); -} - -function countPayloadItems(payload) { - if (!payload) { - return 0; - } - - if (payload instanceof Array) { - return payload.length; - } - - if (payload.AddOrUpdate instanceof Array) { - return payload.AddOrUpdate.length; - } - - if (payload.addOrUpdate instanceof Array) { - return payload.addOrUpdate.length; + if (error instanceof CliError) { + console.error(`\n\t${error.message}`); + if (error.cause && process.env.DEBUG) { + console.error(error.cause); + } + return; } - - return 1; + console.error(error); } async function pushFile(config, file, dryRun = false) { @@ -77,122 +69,106 @@ async function pushFile(config, file, dryRun = false) { return; } - let payload; - try { - payload = JSON.parse(await fs.promises.readFile(file, 'utf8')); - } catch (error) { - if (error instanceof SyntaxError) { - error.message = `Invalid JSON in ${file}: ${error.message}`; - } - throw error; - } + const payload = await loadJsonFile(file); const itemCount = countPayloadItems(payload); - // quick validation of the payload - if (!payload || (!(payload instanceof Array) && !payload.AddOrUpdate && !payload.addOrUpdate)) { - console.warn(`\n\t !! Your payload seems to be in a wrong format !!\n\n\tMissing \x1b[33m\x1b[1m{ "AddOrUpdate": [] }\x1b[0m around your data?\n\n`); + if (!looksLikePushPayload(payload)) { + console.warn(`\n\t !! Your payload seems to be in a wrong format !!\n\n\tMissing ${highlight('{ "AddOrUpdate": [] }')} around your data?\n\n`); } if (config.useStreamApi) { const streamHelper = new StreamApi(config); await streamHelper.pushFile(payload); } else { - console.log(`\nPushing one file to source: \x1b[33m\x1b[1m${config.source}\x1b[0m`); + console.log(`\nPushing one file to source: ${highlight(config.source)}`); const pushApiHelper = new PushApi(config); await runWithSourceInRebuild(pushApiHelper, () => pushApiHelper.pushFile(payload)); } console.log(`Processed items: ${itemCount}`); - console.log(`\nDone\n`); + console.log('\nDone\n'); } -function deleteBuffers() { - let buffers = fs.readdirSync('.').filter(fileName => fileName.startsWith('.pushapi.buffer.')); - buffers.forEach(fileName => { - console.log('deleting buffer: ', fileName); - fs.unlinkSync(fileName); - }); - console.log(''); -} +async function pushFolder(config, folder, dryRun = false) { + const folderPath = path.resolve(folder); + console.log(`Loading folder: ${folderPath}`); -async function main(FILE_OR_FOLDER, argv = { deleteOlderThan: null }) { + const useStreamApi = Boolean(config.useStreamApi); + const apiHelper = useStreamApi ? new StreamApi(config) : new PushApi(config); - try { - const dryRun = argv['dry-run'] ? true : false; + const processFiles = async () => { + if (useStreamApi && !dryRun) { + await apiHelper.openStream(); + } + try { + const pushApiBuffer = new JsonBuffer(apiHelper, config, dryRun); + // process every .json file in the folder, batched into as few requests as possible. + const files = (await readdir(folderPath)).filter((fileName) => fileName.endsWith('.json')).sort(); + for (const fileName of files) { + await pushApiBuffer.addJsonFile(path.join(folderPath, fileName)); + } + await pushApiBuffer.sendBuffer(); + console.log(`Processed items: ${pushApiBuffer.totalItemCount}`); + } finally { + if (useStreamApi && !dryRun) { + await apiHelper.closeStream(); + } + } + }; - const config = new Config(); - const pushApiHelper = new PushApi(config, dryRun); + if (useStreamApi || dryRun) { + await processFiles(); + } else { + console.log(`\nUpdate status for source: ${highlight(config.source)}`); + await runWithSourceInRebuild(apiHelper, processFiles); + } - if (argv.deleteOlderThan !== null) { - const orderingId = Date.now() - (argv.deleteOlderThan * 60 * 60 * 1000) - 1; - console.log(`Deleting items older than ${argv.deleteOlderThan} hours (${orderingId}).`); - await pushApiHelper.deleteOlderThan(orderingId); - } + console.log('\nDone\n'); +} - if (dryRun) { - deleteBuffers(); - } +async function deleteBuffers(dir = process.cwd()) { + const buffers = (await readdir(dir)).filter((fileName) => fileName.startsWith(BUFFER_FILE_PREFIX)); + for (const fileName of buffers) { + console.log('deleting buffer: ', fileName); + await unlink(path.join(dir, fileName)); + } + console.log(''); +} - let stats = fs.statSync(FILE_OR_FOLDER); - if (stats.isDirectory()) { - - let _dir = process.cwd(); - let folderName = FILE_OR_FOLDER; - - // process every .json files in the folder as separate batch requests. - console.log(`Loading folder: ${_dir}/${folderName}`); - - console.log('\nUpdate status for source: \x1b[33m \x1b[1m', config.source, '\x1b[0m'); - const pushFolder = async () => { - let apiHelper = null; - if (config.useStreamApi) { - apiHelper = new StreamApi(config); - await apiHelper.openStream(); - } - else { - apiHelper = new PushApi(config); - } - - try { - let pushApiBuffer = new JsonBuffer(apiHelper, config, dryRun); - let files = fs.readdirSync(`${_dir}/${folderName}`); - - // consider only .json files - files = files.filter(fileName => (/\.json$/.test(fileName))); - for (let fileName of files) { - await pushApiBuffer.addJsonFile(`${_dir}/${folderName}/${fileName}`); - } - await pushApiBuffer.sendBuffer(); - console.log(`Processed items: ${pushApiBuffer.totalItemCount}`); - } finally { - if (config.useStreamApi && apiHelper) { - await apiHelper.closeStream(); - } - } - }; - - if (config.useStreamApi) { - await pushFolder(); - } else { - await runWithSourceInRebuild(pushApiHelper, pushFolder); - } +/** + * Pushes a JSON payload file, or every .json file of a folder, to the source configured in `.pushapi-config.json`. + * + * @param {string} fileOrFolder + * @param {{ deleteOlderThan?: number | null, dryRun?: boolean }} [options] + */ +export async function main(fileOrFolder, { deleteOlderThan = null, dryRun = false } = {}) { + const config = await Config.load(); + const pushApiHelper = new PushApi(config, dryRun); + + if (deleteOlderThan !== null && deleteOlderThan !== undefined) { + const orderingId = Date.now() - deleteOlderThan * 60 * 60 * 1000 - 1; + console.log(`Deleting items older than ${deleteOlderThan} hours (${orderingId}).`); + await pushApiHelper.deleteOlderThan(orderingId); + } - console.log(`\nDone\n`); + if (dryRun) { + await deleteBuffers(); + } - } else if (stats.isFile()) { - await pushFile(config, FILE_OR_FOLDER, argv['dry-run'] ? true : false); - } else if (argv.help) { - argv.help(); - } + let stats; + try { + stats = await stat(fileOrFolder); + } catch (cause) { + throw new CliError(`Couldn't find file or folder: ${fileOrFolder}`, { cause }); + } - } catch (e) { - reportError(e); - process.exit(10); + if (stats.isDirectory()) { + await pushFolder(config, fileOrFolder, dryRun); + } else if (stats.isFile()) { + await pushFile(config, fileOrFolder, dryRun); + } else { + throw new CliError(`Not a file or a folder: ${fileOrFolder}`); } } - -exports.main = main; -exports.Config = Config; -exports.PushApi = PushApi; -exports.StreamApi = StreamApi; +export { Config, PushApi, StreamApi, CliError, HttpError }; diff --git a/renovate.json5 b/renovate.json5 new file mode 100644 index 0000000..cd4b4b8 --- /dev/null +++ b/renovate.json5 @@ -0,0 +1,8 @@ +{ + $schema: "https://docs.renovatebot.com/renovate-schema.json", + extends: [ + "github>coveooss/renovate-presets", + "schedule:earlyMondays", + "group:allNonMajor", + ], +} diff --git a/src/Config.js b/src/Config.js index 68c3572..2d39bf5 100644 --- a/src/Config.js +++ b/src/Config.js @@ -1,106 +1,131 @@ -const fs = require("fs"); - -class Config { - constructor() { - this._dir = process.cwd(); - +import { chmod, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { stdin, stdout } from 'node:process'; +import { createInterface } from 'node:readline/promises'; + +import { CliError } from './errors.js'; + +export const CONFIG_FILE_NAME = '.pushapi-config.json'; +export const DEFAULT_PLATFORM = 'push.cloud.coveo.com'; + +const PLACEHOLDERS = { + apiKey: 'xx--your-api-key--abc', + org: 'your-org-id', + source: 'your-source-id', +}; + +const EXIT_CODES = { + missingFile: 1, + missingConfig: 2, + apiKey: 3, + org: 4, + source: 5, +}; + +const isYes = (answer) => /^y(es)?$/i.test(answer); + +export default class Config { + /** + * Loads and validates `.pushapi-config.json` from `dir`. + * @param {string} [dir] defaults to the current working directory + * @returns {Promise} the validated config + */ + static async load(dir = process.cwd()) { + const configFile = path.join(dir, CONFIG_FILE_NAME); + let config; try { - this.config = require(`${this._dir}/.pushapi-config`); - } catch (e) { - Config.throwError(`Couldn't load .pushapi-config.json file from ${this._dir}`); + config = JSON.parse(await readFile(configFile, 'utf8')); + } catch (cause) { + throw new CliError(`Couldn't load ${CONFIG_FILE_NAME} file from ${dir}`, { exitCode: EXIT_CODES.missingFile, cause }); } - - this.validateConfig(); - return this.config; + return Config.validate(config); } - validateConfig() { - if (!this.config) { - Config.throwError("Missing config (.pushapi-config.json)", 2); - } - if (!this.config.platform) { - this.config.platform = "push.cloud.coveo.com"; + /** + * Validates a config object, filling in defaults. + * @param {object} config + * @returns {object} the same object, with defaults applied + */ + static validate(config) { + if (!config || typeof config !== 'object') { + throw new CliError(`Missing config (${CONFIG_FILE_NAME})`, { exitCode: EXIT_CODES.missingConfig }); } + config.platform ||= DEFAULT_PLATFORM; - if (!this.config.apiKey || this.config.apiKey === "xx--your-api-key--abc") { - Config.throwError("Missing apiKey in .pushapi-config.json", 3); - } - if (!this.config.org || this.config.org === "your-org-id") { - Config.throwError("Missing org in .pushapi-config.json", 4); - } - if (!this.config.source || this.config.source === "your-source-id") { - Config.throwError("Missing source in .pushapi-config.json", 5); + for (const field of ['apiKey', 'org', 'source']) { + if (!config[field] || config[field] === PLACEHOLDERS[field]) { + throw new CliError(`Missing ${field} in ${CONFIG_FILE_NAME}`, { exitCode: EXIT_CODES[field] }); + } } + return config; } - static async ask(rl, message) { - return new Promise((resolve) => { - rl.question(message, (answer) => { - resolve((answer || "").trim()); - }); + /** + * Interactively creates the config file. + * @param {string} configFilePath + * @returns {Promise} true when the file was created, false when the user declined + */ + static async createConfig(configFilePath) { + const rl = createInterface({ input: stdin, output: stdout }); + + // Lines received while no question is pending (e.g. piped stdin) are kept for the next question. + const pendingLines = []; + rl.on('line', (line) => pendingLines.push(line)); + // If stdin closes before every question is answered, bail out instead of hanging forever. + const closed = new Promise((_, reject) => { + rl.once('close', () => reject(new CliError('Input closed before the config was complete. No config file created.'))); }); - } + closed.catch(() => {}); - static async createConfig(configFilePath, callback) { - const readline = require("readline"); - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); + const ask = async (message) => { + if (pendingLines.length > 0) { + const answer = pendingLines.shift(); + stdout.write(`${message}${answer}\n`); + return answer.trim(); + } + const answer = await Promise.race([rl.question(message), closed]); + return answer.trim(); + }; - const shouldCreate = await Config.ask(rl, "\nWould you like to create the config file now? [Y/n] "); - if (!shouldCreate || /^y(es)?$/i.test(shouldCreate)) { - let source = await Config.ask(rl, "Source ID: "); + try { + const shouldCreate = await ask('\nWould you like to create the config file now? [Y/n] '); + if (shouldCreate && !isYes(shouldCreate)) { + return false; + } - let isCatalogSource = await Config.ask(rl, "Is it a Catalog Source? [y/N] "); - isCatalogSource = /^y(es)?$/i.test(isCatalogSource); + const source = await ask('Source ID: '); + const isCatalogSource = isYes(await ask('Is it a Catalog Source? [y/N] ')); - const orgFromSourceId = source.split("-")[0]; - let org = await Config.ask(rl, `Org ID: [${orgFromSourceId}]`); - if (!org && orgFromSourceId) { - org = orgFromSourceId; - } + const orgFromSourceId = source.split('-')[0]; + const org = (await ask(`Org ID: [${orgFromSourceId}] `)) || orgFromSourceId; - let platform = await Config.ask(rl, "Platform: (empty for prod) [dev, stg, hipaa]"); - platform = platform.toLowerCase(); - // validate platform input + let platform = (await ask('Platform (empty for prod) [dev, stg, hipaa]: ')).toLowerCase(); if (platform && !/^(dev|stg|hipaa|prod)$/.test(platform)) { - console.warn("Invalid platform, defaulting to prod"); - platform = ""; + console.warn('Invalid platform, defaulting to prod'); + platform = ''; } - if (platform === "prod") { - platform = ""; + if (platform === 'prod') { + platform = ''; } - const platformHost = `push${platform || ""}.cloud.coveo.com`; - const apiKey = await Config.ask(rl, `API key: `); + const apiKey = await ask('API key: '); - rl.close(); - console.log("creating file: ", configFilePath); - let payload = { + const payload = { org, source, apiKey, - platform: platformHost, + platform: `push${platform}.cloud.coveo.com`, }; if (isCatalogSource) { payload.useStreamApi = true; } - fs.writeFileSync(configFilePath, JSON.stringify(payload, null, 2)); - fs.chmodSync(configFilePath, 0o600); - - await callback(); - } else { + console.log('creating file: ', configFilePath); + await writeFile(configFilePath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); + await chmod(configFilePath, 0o600); + return true; + } finally { rl.close(); - process.exit(); } } - - static throwError(msg, code) { - console.warn(`\n\t${msg}`); - process.exit(code || 1); - } } - -module.exports = Config; diff --git a/src/JsonBuffer.js b/src/JsonBuffer.js index c35605d..8df90a1 100644 --- a/src/JsonBuffer.js +++ b/src/JsonBuffer.js @@ -1,85 +1,71 @@ -const fs = require('fs'); - -const MAX_BUFFER_SIZE = 256000000; // 256 MB is max for Push payloads. +import { readFile, stat, writeFile } from 'node:fs/promises'; + +import { CliError } from './errors.js'; +import { normalizePayloadItems } from './payload.js'; + +export const MAX_BUFFER_SIZE = 256_000_000; // 256 MB is max for Push payloads. +export const BUFFER_FILE_PREFIX = '.pushapi.buffer.'; + +/** + * Reads and parses a JSON file, with a helpful error message when the JSON is invalid. + * @param {string} pathToJson + */ +export async function loadJsonFile(pathToJson) { + const data = await readFile(pathToJson, 'utf8'); + try { + return JSON.parse(data); + } catch (cause) { + throw new CliError(`Invalid JSON in ${pathToJson}: ${cause.message}`, { cause }); + } +} -class JsonBuffer { +/** + * Accumulates the items of many JSON files, and sends them in batches of at most MAX_BUFFER_SIZE bytes. + */ +export default class JsonBuffer { constructor(apiHelper, config, dryRun = false) { this._dryRun = dryRun; this.apiHelper = apiHelper; + this.config = config; this.buffer = []; this.bufferSize = 0; this.bufferCount = 1; this.totalItemCount = 0; - this.config = config; } - _debug() { + _debug(...args) { if (this.config.debug) { - console.debug.apply(console, arguments); + console.debug(...args); } } normalizePayloadItems(payload, pathToJson) { - if (payload instanceof Array) { - return payload; - } - - if (payload && typeof payload === 'object') { - const addOrUpdateKey = Object.keys(payload).find(key => key.toLowerCase() === 'addorupdate'); - if (addOrUpdateKey) { - if (!(payload[addOrUpdateKey] instanceof Array)) { - throw new Error(`Invalid payload in ${pathToJson}: AddOrUpdate must be an array.`); - } - return payload[addOrUpdateKey]; - } - } - - return [payload]; + return normalizePayloadItems(payload, pathToJson); } async addJsonFile(pathToJson) { - // add to Buffer - let { - size: fileSize - } = fs.statSync(pathToJson); + const { size: fileSize } = await stat(pathToJson); if (fileSize > MAX_BUFFER_SIZE) { console.warn('\n File is bigger than maximum size. You need to break it up.\nSkipping this file: ', pathToJson); - } else if ((this.bufferSize + fileSize) > MAX_BUFFER_SIZE) { + return; + } + + if (this.bufferSize + fileSize > MAX_BUFFER_SIZE) { console.log('\n ------ BATCH ------ \n', this.bufferSize); await this.sendBuffer(); - await this.addJsonFile(pathToJson); - } else { - this._debug('Loading file: ', pathToJson); - let payload = await this.loadFile(pathToJson); - const items = this.normalizePayloadItems(payload, pathToJson); - this.bufferSize += fileSize; - - const len = items.length; - this.totalItemCount += len; - // Need to use for(){} here, - // because this.buffer.push(...items); fails for large files - for (let i = 0; i < len; i++) { - this.buffer.push(items[i]); - } } - } - async loadFile(pathToJson) { - return new Promise((resolve, reject) => { - fs.readFile(pathToJson, 'utf8', (err, data) => { - if (err) { - reject(err); - return; - } - try { - resolve(JSON.parse(data)); - } catch (error) { - error.message = `Invalid JSON in ${pathToJson}: ${error.message}`; - reject(error); - } - }); - }); + this._debug('Loading file: ', pathToJson); + const payload = await loadJsonFile(pathToJson); + const items = this.normalizePayloadItems(payload, pathToJson); + this.bufferSize += fileSize; + this.totalItemCount += items.length; + + // `this.buffer.push(...items)` overflows the call stack for large files. + for (const item of items) { + this.buffer.push(item); + } } async sendBuffer() { @@ -87,29 +73,20 @@ class JsonBuffer { return; } - const bufferName = `.pushapi.buffer.${this.bufferCount}`; - this._debug('Buffer full, sending ', bufferName); + const bufferName = `${BUFFER_FILE_PREFIX}${this.bufferCount}`; + const payload = { AddOrUpdate: this.buffer }; if (this._dryRun) { - console.log(`Created buffer file (not pushing): `, bufferName); - fs.writeFileSync(bufferName, JSON.stringify({ - AddOrUpdate: this.buffer - })); - - this.buffer = []; - this.bufferSize = 0; - this.bufferCount++; - return; + console.log('Created buffer file (not pushing): ', bufferName); + await writeFile(bufferName, JSON.stringify(payload)); + } else { + this._debug('Buffer full, sending ', bufferName); + await this.apiHelper.pushJsonPayload(payload); + console.log('UPLOAD done ', bufferName); } - await this.apiHelper.pushJsonPayload({ - AddOrUpdate: this.buffer - }); - console.log('UPLOAD done ', bufferName); this.buffer = []; this.bufferSize = 0; this.bufferCount++; } } - -module.exports = JsonBuffer; diff --git a/src/PlatformRequestsHelper.js b/src/PlatformRequestsHelper.js index a377bfa..e40eb28 100644 --- a/src/PlatformRequestsHelper.js +++ b/src/PlatformRequestsHelper.js @@ -1,201 +1,106 @@ -'use strict'; -const https = require('https'); +import { styleText } from 'node:util'; -class PlatformRequestsHelper { +import { HttpError } from './errors.js'; +import { validatePayload } from './payload.js'; +export default class PlatformRequestsHelper { constructor(config, dryRun = false) { this._dryRun = dryRun; this.config = config; - this._debug('\nconfig = ', this.config); + this._debug('\nconfig = ', { ...config, apiKey: '***' }); } - _debug() { + _debug(...args) { if (this.config.debug) { - console.debug.apply(console, arguments); + console.debug(...args); } } - _log() { - console.log.apply(console, arguments); + _log(...args) { + console.log(...args); } _now() { - let time = new Date().toLocaleTimeString('en-US', { - hour12: false - }); - return `\x1b[94m ${time} \x1b[0m`; + const time = new Date().toLocaleTimeString('en-US', { hour12: false }); + return styleText('blueBright', ` ${time} `); } - /** - * Utility function to check if a key in a JSON object. The JSON payload for the Push Api is case-insensitive, so DocumentId is the same as documentid. - * @param {string} key - * @param {Object} obj - */ - _isKeyMissingInObject(key, obj) { - const keys = Object.keys(obj).map(k => k.toLowerCase()); - return !(keys.includes(key.toLowerCase())); + _highlight(text) { + return styleText(['yellow', 'bold'], String(text)); } - _buildHttpError(response, body, context) { - const error = new Error(`${context} failed with status ${response.statusCode}${response.statusMessage ? ` ${response.statusMessage}` : ''}`); - error.statusCode = response.statusCode; - error.statusMessage = response.statusMessage; - error.body = body; - error.req = response.req; - return error; + /** + * Builds the URL for a Push API action. + * `action` is either a path relative to the source (e.g. `status?statusType=IDLE`) or an absolute URL. + */ + _buildUrl(action) { + if (/^https?:\/\//.test(action)) { + return new URL(action); + } + const { platform, org, source } = this.config; + return new URL(`https://${platform}/v1/organizations/${org}/sources/${source}/${action}`); } - async _sendRequest(method, action) { - + async _sendRequest(method, action, body) { this._debug('_sendRequest::', method, action); - const config = this.config; - let hostname = config.platform; - let path = `/v1/organizations/${config.org}/sources/${config.source}/${action}`; - - if (/^http/.test(action)) { - const url = new URL(action); - hostname = url.hostname; - path = `${url.pathname}${url.search}`; - } + const url = this._buildUrl(action); + const context = `${method} ${url.host}${url.pathname}`; if (this._dryRun) { - console.log('DRY-RUN: skip '); - return Promise.resolve(); + console.log('DRY-RUN: skip ', context); + return undefined; } - return new Promise((resolve, reject) => { - const options = { - method: method, - hostname, - path, - port: 443, - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${config.apiKey}`, - }, - }; - - const req = https.request(options, - (resp) => { - this._debug('statusCode:', resp.statusCode); - this._debug('headers:', resp.headers); - - let data = ""; - resp.on("data", chunk => { - data += chunk; - }); - resp.on("end", () => { - this._debug('\nREQUEST: ', method, resp.statusCode, data); - if (resp.statusCode >= 400) { - reject(this._buildHttpError(resp, data, `${method} ${hostname}${path}`)); - } else { - resolve(data); - } - }); - }); - req.on('error', (err) => { - console.log('ERROR: ', err); - reject(err); - }); - req.end(); + const response = await fetch(url, { + method, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.config.apiKey}`, + }, + body: body === undefined ? undefined : JSON.stringify(body), }); + + const data = await response.text(); + this._debug('\nREQUEST: ', method, response.status, data); + + if (!response.ok) { + throw new HttpError(context, response, data); + } + return data; + } + + async _sendJsonRequest(method, action, body) { + const data = await this._sendRequest(method, action, body); + return data ? JSON.parse(data) : data; } async uploadFileToAws(uploadUri, body) { const url = new URL(uploadUri); - const path = `${url.pathname}${url.search}`; - - return new Promise((resolve, reject) => { - const postData = JSON.stringify(body); - const options = { - method: 'PUT', - hostname: url.hostname, - path, - port: 443, - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Length': Buffer.byteLength(postData), - 'x-amz-server-side-encryption': 'AES256' - }, - }; - this._debug('\n\nuploadFileToAws:\n', options); - - const req = https.request(options, (response) => { - let data = ""; - response.on('data', chunk => { - data += chunk; - }); - response.on('end', () => { - this._debug(response.statusCode, response.statusMessage); - if (response.statusCode >= 200 && response.statusCode < 300) { - console.log('File uploaded to AWS. ', this._now()); - resolve(response); - } else { - reject(this._buildHttpError(response, data, `PUT ${url.hostname}${path}`)); - } - }); - }); - - req.on('error', (error) => { - console.log('ERROR 1: ', error, uploadUri); - reject(error); - }); - req.write(postData); - req.end(); + const postData = JSON.stringify(body); + this._debug('\n\nuploadFileToAws:\n', `PUT ${url.host}${url.pathname}`, `${Buffer.byteLength(postData)} bytes`); + + const response = await fetch(url, { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + 'x-amz-server-side-encryption': 'AES256', + }, + body: postData, }); - } - validatePayload(data) { - // validate payload first - if (!data) { - console.warn('Invalid payload: not defined.'); - } - if (data instanceof Array) { - data = { - AddOrUpdate: data - }; // need to wrap arrays of documents into AddOrUpdate - } + const data = await response.text(); + this._debug(response.status, response.statusText); - if (this._isKeyMissingInObject('AddOrUpdate', data)) { - // wrap payload into {"AddOrUpdate": [data]} - data = { - AddOrUpdate: [ - data - ] - }; - } else if (!data.AddOrUpdate) { - // AddOrUpdate is present, but using a different case. - - // find the key using the different case - let key = Object.keys(data).filter(k => k.match(/AddOrUpdate/i))[0]; - - // replacing key by 'AddOrUpdate' - data.AddOrUpdate = data[key]; - delete data[key]; + if (!response.ok) { + throw new HttpError(`PUT ${url.host}${url.pathname}`, response, data); } - - let fileExtensionWarning = false; - // validate each document has a DocumentId - data.AddOrUpdate.forEach(d => { - if (this._isKeyMissingInObject('DocumentId', d)) { - console.warn(`Missing DocumentId in some documents in the payload. Stopping.`); - throw new Error('No DocumentId.'); - } - if (!fileExtensionWarning && this._isKeyMissingInObject('FileExtension', d)) { - console.log(`Missing FileExtension in some documents. It's good practice to provide them.`); - fileExtensionWarning = true; - } - }); - - return data; + console.log('File uploaded to AWS. ', this._now()); + return response; } - static throwError(msg, code) { - console.warn(`\n\t${msg}`); - process.exit(code || 1); + validatePayload(data) { + return validatePayload(data); } } - -module.exports = PlatformRequestsHelper; diff --git a/src/PushApi.js b/src/PushApi.js index 6274069..c620c91 100644 --- a/src/PushApi.js +++ b/src/PushApi.js @@ -1,42 +1,34 @@ -'use strict'; - -const PlatformRequestsHelper = require('./PlatformRequestsHelper'); - -class PushApi extends PlatformRequestsHelper { +import PlatformRequestsHelper from './PlatformRequestsHelper.js'; +export default class PushApi extends PlatformRequestsHelper { async changeStatus(state) { - console.log(`Change source status to \x1b[33m\x1b[1m${state}\x1b[0m`, this._now()); - return this._sendRequest(`POST`, `status?statusType=${state}`); + console.log(`Change source status to ${this._highlight(state)}`, this._now()); + return this._sendRequest('POST', `status?statusType=${encodeURIComponent(state)}`); } async deleteOlderThan(orderingId) { if (orderingId < Date.now()) { - return this._sendRequest(`DELETE`, `documents/olderthan?orderingId=${orderingId}`); + return this._sendRequest('DELETE', `documents/olderthan?orderingId=${orderingId}`); } + return undefined; } async getLargeFileContainer() { - let config = this.config; - return this._sendRequest(`POST`, `https://${config.platform}/v1/organizations/${config.org}/files`).then( - body => { - console.log('getLargeFileContainer'); - let resp = (typeof body === 'string') ? JSON.parse(body) : body; + const { platform, org } = this.config; + const resp = await this._sendJsonRequest('POST', `https://${platform}/v1/organizations/${org}/files`); + console.log('getLargeFileContainer'); - this.uploadUri = resp.uploadUri; - this.fileId = resp.fileId; + this.uploadUri = resp.uploadUri; + this.fileId = resp.fileId; - this._debug('uploadUri: ', resp.uploadUri); - this._log('fileId: ', resp.fileId); + this._debug('uploadUri: ', resp.uploadUri); + this._log('fileId: ', resp.fileId); - return resp; - } - ); + return resp; } async pushFile(data) { - data = this.validatePayload(data); - - return await this.pushJsonPayload(data); + return this.pushJsonPayload(this.validatePayload(data)); } async pushJsonPayload(data) { @@ -50,10 +42,7 @@ class PushApi extends PlatformRequestsHelper { await this.sendBatchRequest(); } - async sendBatchRequest(fileId) { - return this._sendRequest(`PUT`, `documents/batch?fileId=${fileId || this.fileId}`); + async sendBatchRequest(fileId = this.fileId) { + return this._sendRequest('PUT', `documents/batch?fileId=${encodeURIComponent(fileId)}`); } - } - -module.exports = PushApi; diff --git a/src/StreamApi.js b/src/StreamApi.js index 57cf064..a8b7d72 100644 --- a/src/StreamApi.js +++ b/src/StreamApi.js @@ -1,53 +1,40 @@ -'use strict'; - -const PlatformRequestsHelper = require('./PlatformRequestsHelper'); - -class StreamApi extends PlatformRequestsHelper { +import PlatformRequestsHelper from './PlatformRequestsHelper.js'; +export default class StreamApi extends PlatformRequestsHelper { getApiEndpoint() { return this.config.platform.replace(/^push/, 'api'); } + _streamUrl(suffix) { + const { org, source } = this.config; + return `https://${this.getApiEndpoint()}/push/v1/organizations/${org}/sources/${source}/stream/${suffix}`; + } + async openStream() { - let config = this.config; - return this._sendRequest(`POST`, `https://${this.getApiEndpoint()}/push/v1/organizations/${config.org}/sources/${config.source}/stream/open`).then( - body => { - console.log('\nOpen stream to source: \x1b[33m \x1b[1m', this.config.source, '\x1b[0m', this._now()); - const resp = (typeof body === 'string') ? JSON.parse(body) : body; + const resp = await this._sendJsonRequest('POST', this._streamUrl('open')); + console.log(`\nOpen stream to source: ${this._highlight(this.config.source)}`, this._now()); - this._debug('uploadUri: ', resp.uploadUri); - this._log('streamId: ', resp.streamId); + this._debug('uploadUri: ', resp.uploadUri); + this._log('streamId: ', resp.streamId); - this._last_uploadUri = resp.uploadUri; - this._last_streamId = resp.streamId; + this._last_uploadUri = resp.uploadUri; + this._last_streamId = resp.streamId; - return resp; - } - ); + return resp; } - async getChunk(streamInfo) { - let config = this.config; - return this._sendRequest(`POST`, `https://${this.getApiEndpoint()}/push/v1/organizations/${config.org}/sources/${config.source}/stream/${this._last_streamId}/chunk`).then( - body => { - console.log('\nGet chunk for stream: \x1b[33m \x1b[1m', this.config.source, '\x1b[0m', this._now()); - const resp = (typeof body === 'string') ? JSON.parse(body) : body; + async getChunk() { + const resp = await this._sendJsonRequest('POST', this._streamUrl(`${this._last_streamId}/chunk`)); + console.log(`\nGet chunk for stream: ${this._highlight(this.config.source)}`, this._now()); - this._debug('uploadUri: ', resp.uploadUri); - return resp; - } - ); + this._debug('uploadUri: ', resp.uploadUri); + return resp; } - async closeStream(streamInfo) { - let config = this.config; - return this._sendRequest(`POST`, `https://${this.getApiEndpoint()}/push/v1/organizations/${config.org}/sources/${config.source}/stream/${this._last_streamId}/close`).then( - body => { - console.log('Close stream.', this._now()); - let resp = (typeof body === 'string') ? JSON.parse(body) : body; - return resp; - } - ); + async closeStream() { + const resp = await this._sendJsonRequest('POST', this._streamUrl(`${this._last_streamId}/close`)); + console.log('Close stream.', this._now()); + return resp; } async pushFile(data) { @@ -57,15 +44,10 @@ class StreamApi extends PlatformRequestsHelper { try { await this.uploadFileToAws(streamInfo.uploadUri, data); } finally { - await this.closeStream(streamInfo); + await this.closeStream(); } } - static throwError(msg, code) { - console.warn(`\n\t${msg}`); - process.exit(code || 1); - } - async pushJsonPayload(data) { if (this._dryRun) { console.log('DRY-RUN: not pushing.'); @@ -81,7 +63,4 @@ class StreamApi extends PlatformRequestsHelper { await this.uploadFileToAws(uploadUri, data); this._last_uploadUri = null; } - } - -module.exports = StreamApi; diff --git a/src/errors.js b/src/errors.js new file mode 100644 index 0000000..18d5cc0 --- /dev/null +++ b/src/errors.js @@ -0,0 +1,25 @@ +/** + * Error raised for user-facing failures (bad config, bad payload...). + * `exitCode` is used by the CLI entry point as the process exit code. + */ +export class CliError extends Error { + constructor(message, { exitCode = 1, cause } = {}) { + super(message, cause ? { cause } : undefined); + this.name = 'CliError'; + this.exitCode = exitCode; + } +} + +/** + * Error raised when an HTTP call to the Coveo platform (or to the AWS file container) fails. + */ +export class HttpError extends Error { + constructor(context, response, body) { + const statusText = response.statusText ? ` ${response.statusText}` : ''; + super(`${context} failed with status ${response.status}${statusText}`); + this.name = 'HttpError'; + this.statusCode = response.status; + this.statusMessage = response.statusText; + this.body = body; + } +} diff --git a/src/payload.js b/src/payload.js new file mode 100644 index 0000000..e149842 --- /dev/null +++ b/src/payload.js @@ -0,0 +1,117 @@ +import { CliError } from './errors.js'; + +/** + * The Push API JSON payload keys are case-insensitive (DocumentId === documentid). + * @param {object} obj + * @param {string} key + * @returns {string | undefined} the actual key found in `obj`, if any + */ +export function findKeyIgnoringCase(obj, key) { + const wanted = key.toLowerCase(); + return Object.keys(obj).find((k) => k.toLowerCase() === wanted); +} + +export function hasKeyIgnoringCase(obj, key) { + return findKeyIgnoringCase(obj, key) !== undefined; +} + +function isPlainObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * Turns any supported payload shape into a flat array of items. + * Supported shapes: an array of items, `{ "AddOrUpdate": [...] }` (any case), or a single item. + * @param {unknown} payload + * @param {string} source a label used in error messages (typically the file path) + * @returns {object[]} + */ +export function normalizePayloadItems(payload, source = 'payload') { + if (Array.isArray(payload)) { + return payload; + } + + if (isPlainObject(payload)) { + const addOrUpdateKey = findKeyIgnoringCase(payload, 'AddOrUpdate'); + if (addOrUpdateKey) { + if (!Array.isArray(payload[addOrUpdateKey])) { + throw new CliError(`Invalid payload in ${source}: AddOrUpdate must be an array.`); + } + return payload[addOrUpdateKey]; + } + return [payload]; + } + + throw new CliError(`Invalid payload in ${source}: expected an array or an object.`); +} + +/** + * Counts the items in a payload without validating it. + * @param {unknown} payload + * @returns {number} + */ +export function countPayloadItems(payload) { + if (!payload) { + return 0; + } + try { + return normalizePayloadItems(payload).length; + } catch { + return 1; + } +} + +/** + * Tells whether the payload looks like something the Push API will accept. + * @param {unknown} payload + */ +export function looksLikePushPayload(payload) { + return Array.isArray(payload) || (isPlainObject(payload) && hasKeyIgnoringCase(payload, 'AddOrUpdate')); +} + +/** + * Wraps the payload into `{ "AddOrUpdate": [...] }` and checks each document has a DocumentId. + * @param {unknown} payload + * @param {{ warn?: (message: string) => void }} [options] + * @returns {{ AddOrUpdate: object[] } & Record} + */ +export function validatePayload(payload, { warn = console.warn } = {}) { + if (!payload) { + throw new CliError('Invalid payload: not defined.'); + } + + let data; + if (Array.isArray(payload)) { + data = { AddOrUpdate: payload }; + } else if (!isPlainObject(payload)) { + throw new CliError('Invalid payload: expected an array or an object.'); + } else { + const addOrUpdateKey = findKeyIgnoringCase(payload, 'AddOrUpdate'); + if (!addOrUpdateKey) { + data = { AddOrUpdate: [payload] }; + } else if (addOrUpdateKey === 'AddOrUpdate') { + data = payload; + } else { + // AddOrUpdate is present, but with a different case: normalize the key. + const { [addOrUpdateKey]: items, ...rest } = payload; + data = { ...rest, AddOrUpdate: items }; + } + } + + if (!Array.isArray(data.AddOrUpdate)) { + throw new CliError('Invalid payload: AddOrUpdate must be an array.'); + } + + let fileExtensionWarned = false; + for (const document of data.AddOrUpdate) { + if (!isPlainObject(document) || !hasKeyIgnoringCase(document, 'DocumentId')) { + throw new CliError('Missing DocumentId in some documents in the payload. Stopping.'); + } + if (!fileExtensionWarned && !hasKeyIgnoringCase(document, 'FileExtension')) { + warn("Missing FileExtension in some documents. It's good practice to provide them."); + fileExtensionWarned = true; + } + } + + return data; +} diff --git a/test/Config.test.js b/test/Config.test.js new file mode 100644 index 0000000..a5dd919 --- /dev/null +++ b/test/Config.test.js @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, it } from 'node:test'; + +import Config, { CONFIG_FILE_NAME, DEFAULT_PLATFORM } from '../src/Config.js'; + +const validConfig = () => ({ org: 'myorg', source: 'myorg-abc123', apiKey: 'xx-secret' }); + +describe('Config.validate', () => { + it('fills in the default platform', () => { + const config = Config.validate(validConfig()); + assert.equal(config.platform, DEFAULT_PLATFORM); + }); + + it('keeps a custom platform', () => { + const config = Config.validate({ ...validConfig(), platform: 'pushdev.cloud.coveo.com' }); + assert.equal(config.platform, 'pushdev.cloud.coveo.com'); + }); + + it('rejects a missing config with exit code 2', () => { + assert.throws(() => Config.validate(null), { name: 'CliError', exitCode: 2 }); + }); + + it('rejects missing or placeholder fields with their own exit code', () => { + const cases = [ + ['apiKey', 3], + ['org', 4], + ['source', 5], + ]; + for (const [field, exitCode] of cases) { + const missing = validConfig(); + delete missing[field]; + assert.throws(() => Config.validate(missing), { exitCode, message: new RegExp(`Missing ${field}`) }, `missing ${field}`); + + const placeholder = validConfig(); + placeholder[field] = { apiKey: 'xx--your-api-key--abc', org: 'your-org-id', source: 'your-source-id' }[field]; + assert.throws(() => Config.validate(placeholder), { exitCode }, `placeholder ${field}`); + } + }); +}); + +describe('Config.load', () => { + it('loads and validates the config file from a folder', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pushapi-')); + await writeFile(path.join(dir, CONFIG_FILE_NAME), JSON.stringify(validConfig())); + + const config = await Config.load(dir); + assert.equal(config.org, 'myorg'); + assert.equal(config.platform, DEFAULT_PLATFORM); + }); + + it('fails with exit code 1 when the file is missing', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pushapi-')); + await assert.rejects(Config.load(dir), { name: 'CliError', exitCode: 1, message: /Couldn't load/ }); + }); + + it('fails with exit code 1 when the file is not valid JSON', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pushapi-')); + await writeFile(path.join(dir, CONFIG_FILE_NAME), '{ not json'); + await assert.rejects(Config.load(dir), { name: 'CliError', exitCode: 1 }); + }); +}); diff --git a/test/JsonBuffer.test.js b/test/JsonBuffer.test.js new file mode 100644 index 0000000..2e90199 --- /dev/null +++ b/test/JsonBuffer.test.js @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, it, mock } from 'node:test'; + +import JsonBuffer, { loadJsonFile } from '../src/JsonBuffer.js'; + +async function tempFolderWith(files) { + const dir = await mkdtemp(path.join(tmpdir(), 'pushapi-buffer-')); + for (const [name, content] of Object.entries(files)) { + await writeFile(path.join(dir, name), typeof content === 'string' ? content : JSON.stringify(content)); + } + return dir; +} + +describe('loadJsonFile', () => { + it('reports the file name on invalid JSON', async () => { + const dir = await tempFolderWith({ 'bad.json': '{ nope' }); + await assert.rejects(loadJsonFile(path.join(dir, 'bad.json')), { name: 'CliError', message: /Invalid JSON in .*bad\.json/ }); + }); +}); + +describe('JsonBuffer', () => { + beforeEach(() => { + mock.method(console, 'log', () => {}); + }); + afterEach(() => { + mock.restoreAll(); + }); + + it('merges files of mixed shapes into a single AddOrUpdate payload', async () => { + const dir = await tempFolderWith({ + 'a.json': [{ DocumentId: 'a1' }, { DocumentId: 'a2' }], + 'b.json': { addOrUpdate: [{ DocumentId: 'b1' }] }, + 'c.json': { DocumentId: 'c1' }, + }); + const pushed = []; + const apiHelper = { pushJsonPayload: async (payload) => pushed.push(payload) }; + + const buffer = new JsonBuffer(apiHelper, {}); + for (const name of ['a.json', 'b.json', 'c.json']) { + await buffer.addJsonFile(path.join(dir, name)); + } + await buffer.sendBuffer(); + + assert.equal(buffer.totalItemCount, 4); + assert.equal(pushed.length, 1); + assert.deepEqual( + pushed[0].AddOrUpdate.map((d) => d.DocumentId), + ['a1', 'a2', 'b1', 'c1'], + ); + }); + + it('does nothing when the buffer is empty', async () => { + const pushed = []; + const buffer = new JsonBuffer({ pushJsonPayload: async (p) => pushed.push(p) }, {}); + await buffer.sendBuffer(); + assert.equal(pushed.length, 0); + }); + + it('writes a buffer file instead of pushing in dry-run mode', async () => { + const dir = await tempFolderWith({ 'a.json': [{ DocumentId: 'a1' }] }); + const cwd = process.cwd(); + process.chdir(dir); + try { + const buffer = new JsonBuffer({ pushJsonPayload: async () => assert.fail('should not push') }, {}, true); + await buffer.addJsonFile(path.join(dir, 'a.json')); + await buffer.sendBuffer(); + + const written = JSON.parse(await readFile(path.join(dir, '.pushapi.buffer.1'), 'utf8')); + assert.deepEqual(written, { AddOrUpdate: [{ DocumentId: 'a1' }] }); + assert.equal(buffer.bufferCount, 2); + } finally { + process.chdir(cwd); + } + }); +}); diff --git a/test/PushApi.test.js b/test/PushApi.test.js new file mode 100644 index 0000000..f77d241 --- /dev/null +++ b/test/PushApi.test.js @@ -0,0 +1,146 @@ +import assert from 'node:assert/strict'; +import { afterEach, beforeEach, describe, it, mock } from 'node:test'; + +import { HttpError } from '../src/errors.js'; +import PushApi from '../src/PushApi.js'; +import StreamApi from '../src/StreamApi.js'; + +const config = { org: 'myorg', source: 'myorg-src', apiKey: 'xx-secret', platform: 'push.cloud.coveo.com' }; + +/** Replaces global fetch with a mock that answers from a queue of responses. */ +function mockFetch(responses) { + const fetchMock = mock.method(globalThis, 'fetch', async () => { + const next = responses.shift() ?? { status: 200, body: '' }; + return new Response(next.body ?? '', { status: next.status ?? 200, statusText: next.statusText ?? '' }); + }); + return { + calls: () => fetchMock.mock.calls.map(({ arguments: [url, init] }) => ({ url: String(url), ...init })), + }; +} + +describe('PushApi', () => { + let log; + beforeEach(() => { + log = mock.method(console, 'log', () => {}); + }); + afterEach(() => { + mock.restoreAll(); + }); + + it('changes the source status with the API key as bearer token', async () => { + const fetched = mockFetch([{ status: 200, body: '' }]); + + await new PushApi(config).changeStatus('REBUILD'); + + const [call] = fetched.calls(); + assert.equal(call.url, 'https://push.cloud.coveo.com/v1/organizations/myorg/sources/myorg-src/status?statusType=REBUILD'); + assert.equal(call.method, 'POST'); + assert.equal(call.headers.Authorization, 'Bearer xx-secret'); + }); + + it('pushes a payload: file container, AWS upload, batch request', async () => { + const fetched = mockFetch([ + { status: 200, body: JSON.stringify({ uploadUri: 'https://s3.example.com/upload?sig=1', fileId: 'file-1' }) }, + { status: 200, body: '' }, + { status: 202, body: '' }, + ]); + + await new PushApi(config).pushFile([{ DocumentId: 'doc-1', FileExtension: '.txt' }]); + + const calls = fetched.calls(); + assert.equal(calls.length, 3); + assert.equal(calls[0].url, 'https://push.cloud.coveo.com/v1/organizations/myorg/files'); + assert.equal(calls[1].url, 'https://s3.example.com/upload?sig=1'); + assert.equal(calls[1].method, 'PUT'); + assert.equal(calls[1].headers['x-amz-server-side-encryption'], 'AES256'); + assert.deepEqual(JSON.parse(calls[1].body), { AddOrUpdate: [{ DocumentId: 'doc-1', FileExtension: '.txt' }] }); + assert.equal(calls[2].url, 'https://push.cloud.coveo.com/v1/organizations/myorg/sources/myorg-src/documents/batch?fileId=file-1'); + assert.equal(calls[2].method, 'PUT'); + }); + + it('throws an HttpError with the response body on failure', async () => { + mockFetch([{ status: 401, statusText: 'Unauthorized', body: '{"message":"bad key"}' }]); + + await assert.rejects(new PushApi(config).changeStatus('IDLE'), (error) => { + assert.ok(error instanceof HttpError); + assert.equal(error.statusCode, 401); + assert.equal(error.body, '{"message":"bad key"}'); + assert.match(error.message, /POST push\.cloud\.coveo\.com.*failed with status 401 Unauthorized/); + return true; + }); + }); + + it('does not call the network in dry-run mode', async () => { + const fetched = mockFetch([]); + + const api = new PushApi(config, true); + await api.changeStatus('REBUILD'); + await api.pushFile([{ DocumentId: 'doc-1' }]); + + assert.equal(fetched.calls().length, 0); + assert.ok(log.mock.calls.some(({ arguments: args }) => String(args[0]).startsWith('DRY-RUN'))); + }); + + it('skips deleteOlderThan for a future orderingId', async () => { + const fetched = mockFetch([]); + await new PushApi(config).deleteOlderThan(Date.now() + 60_000); + assert.equal(fetched.calls().length, 0); + }); +}); + +describe('StreamApi', () => { + beforeEach(() => { + mock.method(console, 'log', () => {}); + }); + afterEach(() => { + mock.restoreAll(); + }); + + it('targets the api host and opens, uploads, then closes the stream', async () => { + const fetched = mockFetch([ + { status: 200, body: JSON.stringify({ uploadUri: 'https://s3.example.com/chunk-1', streamId: 'stream-1' }) }, + { status: 200, body: '' }, + { status: 200, body: '{}' }, + ]); + + await new StreamApi({ ...config, platform: 'pushdev.cloud.coveo.com' }).pushFile({ DocumentId: 'doc-1', FileExtension: '.txt' }); + + const calls = fetched.calls(); + assert.equal(calls[0].url, 'https://apidev.cloud.coveo.com/push/v1/organizations/myorg/sources/myorg-src/stream/open'); + assert.equal(calls[1].url, 'https://s3.example.com/chunk-1'); + assert.equal(calls[2].url, 'https://apidev.cloud.coveo.com/push/v1/organizations/myorg/sources/myorg-src/stream/stream-1/close'); + }); + + it('closes the stream even when the upload fails', async () => { + const fetched = mockFetch([ + { status: 200, body: JSON.stringify({ uploadUri: 'https://s3.example.com/chunk-1', streamId: 'stream-1' }) }, + { status: 500, statusText: 'Internal Server Error', body: 'boom' }, + { status: 200, body: '{}' }, + ]); + + await assert.rejects(new StreamApi(config).pushFile([{ DocumentId: 'doc-1' }]), HttpError); + assert.match(fetched.calls().at(-1).url, /stream\/stream-1\/close$/); + }); + + it('reuses the open-stream upload URI for the first chunk, then asks for new chunks', async () => { + const fetched = mockFetch([ + { status: 200, body: JSON.stringify({ uploadUri: 'https://s3.example.com/chunk-1', streamId: 'stream-1' }) }, + { status: 200, body: '' }, + { status: 200, body: JSON.stringify({ uploadUri: 'https://s3.example.com/chunk-2' }) }, + { status: 200, body: '' }, + ]); + + const api = new StreamApi(config); + await api.openStream(); + await api.pushJsonPayload({ AddOrUpdate: [] }); + await api.pushJsonPayload({ AddOrUpdate: [] }); + + const urls = fetched.calls().map((c) => c.url); + assert.deepEqual(urls, [ + 'https://api.cloud.coveo.com/push/v1/organizations/myorg/sources/myorg-src/stream/open', + 'https://s3.example.com/chunk-1', + 'https://api.cloud.coveo.com/push/v1/organizations/myorg/sources/myorg-src/stream/stream-1/chunk', + 'https://s3.example.com/chunk-2', + ]); + }); +}); diff --git a/test/cli.test.js b/test/cli.test.js new file mode 100644 index 0000000..ddf65f3 --- /dev/null +++ b/test/cli.test.js @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { cp, mkdtemp, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const CLI = path.join(ROOT, 'client-global.js'); + +/** Runs the CLI in `cwd`, feeding `input` on stdin. Never rejects: the caller inspects the exit code. */ +function runCli(args, { cwd, input = '' } = {}) { + return new Promise((resolve) => { + const child = execFile(process.execPath, [CLI, ...args], { cwd, env: { ...process.env, NO_COLOR: '1' } }, (error, stdout, stderr) => { + resolve({ code: error?.code ?? 0, stdout, stderr }); + }); + child.stdin.end(input); + }); +} + +async function workdir() { + const dir = await mkdtemp(path.join(tmpdir(), 'pushapi-cli-')); + await cp(path.join(ROOT, 'samples'), path.join(dir, 'samples'), { recursive: true }); + return dir; +} + +describe('pushapi CLI', () => { + it('prints usage with --help', async () => { + const { code, stdout } = await runCli(['--help']); + assert.equal(code, 0); + assert.match(stdout, /Usage: pushapi \[options\]/); + assert.match(stdout, /--dry-run/); + }); + + it('fails when no file or folder is given', async () => { + const { code, stderr } = await runCli([]); + assert.equal(code, 1); + assert.match(stderr, /You need to specify a FILE or a FOLDER/); + }); + + it('prints the version with --version', async () => { + const { code, stdout } = await runCli(['--version']); + assert.equal(code, 0); + assert.match(stdout.trim(), /^\d+\.\d+\.\d+/); + }); + + it('rejects unknown options, extra positionals, and a non-numeric --deleteOlderThan', async () => { + const cases = [ + [['samples', '--bogus'], /Unknown option '--bogus'/], + [['a', 'b'], /Too many arguments: a b/], + [['samples', '-d', 'abc'], /--deleteOlderThan expects a number of hours, got: abc/], + [['samples', '-d'], /argument missing/], + ]; + for (const [args, expected] of cases) { + const { code, stderr } = await runCli(args); + assert.equal(code, 1, args.join(' ')); + assert.match(stderr, expected); + } + }); + + it('offers to create the config, and exits cleanly when declined', async () => { + const cwd = await workdir(); + const { code, stdout, stderr } = await runCli(['samples'], { cwd, input: 'n\n' }); + assert.equal(code, 0); + assert.match(stderr, /Couldn't load .*\.pushapi-config\.json/); + assert.match(stdout, /Would you like to create the config file now\?/); + }); + + it('fails instead of hanging when stdin closes during the config prompts', async () => { + const cwd = await workdir(); + const { code, stderr } = await runCli(['samples'], { cwd, input: 'y\n' }); + assert.equal(code, 1); + assert.match(stderr, /Input closed before the config was complete/); + }); + + it('creates the config from piped answers, then does a dry run of a folder', async () => { + const cwd = await workdir(); + const answers = ['y', 'myorg-abc123', 'n', '', 'dev', 'xx-key', ''].join('\n'); + const { code, stdout } = await runCli(['samples', '--dry-run'], { cwd, input: answers }); + assert.equal(code, 0, stdout); + + const config = JSON.parse(await readFile(path.join(cwd, '.pushapi-config.json'), 'utf8')); + assert.deepEqual(config, { org: 'myorg', source: 'myorg-abc123', apiKey: 'xx-key', platform: 'pushdev.cloud.coveo.com' }); + + assert.match(stdout, /Created buffer file \(not pushing\)/); + assert.match(stdout, /Processed items: 2/); + const buffer = JSON.parse(await readFile(path.join(cwd, '.pushapi.buffer.1'), 'utf8')); + assert.equal(buffer.AddOrUpdate.length, 2); + }); + + it('reports a missing payload path with exit code 1', async () => { + const cwd = await workdir(); + const answers = ['y', 'myorg-abc123', 'n', '', '', 'xx-key', ''].join('\n'); + const { code, stderr } = await runCli(['nope.json'], { cwd, input: answers }); + assert.equal(code, 1); + assert.match(stderr, /Couldn't find file or folder: nope\.json/); + }); +}); diff --git a/test/payload.test.js b/test/payload.test.js new file mode 100644 index 0000000..0639f2b --- /dev/null +++ b/test/payload.test.js @@ -0,0 +1,116 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CliError } from '../src/errors.js'; +import { + countPayloadItems, + findKeyIgnoringCase, + looksLikePushPayload, + normalizePayloadItems, + validatePayload, +} from '../src/payload.js'; + +describe('findKeyIgnoringCase', () => { + it('finds a key regardless of case', () => { + assert.equal(findKeyIgnoringCase({ documentid: 1 }, 'DocumentId'), 'documentid'); + assert.equal(findKeyIgnoringCase({ DocumentId: 1 }, 'documentid'), 'DocumentId'); + assert.equal(findKeyIgnoringCase({ other: 1 }, 'DocumentId'), undefined); + }); +}); + +describe('normalizePayloadItems', () => { + it('returns arrays as-is', () => { + const items = [{ DocumentId: 'a' }, { DocumentId: 'b' }]; + assert.equal(normalizePayloadItems(items), items); + }); + + it('unwraps AddOrUpdate, whatever the case of the key', () => { + const items = [{ DocumentId: 'a' }]; + assert.equal(normalizePayloadItems({ AddOrUpdate: items }), items); + assert.equal(normalizePayloadItems({ addOrUpdate: items }), items); + assert.equal(normalizePayloadItems({ ADDORUPDATE: items }), items); + }); + + it('wraps a single item into an array', () => { + const item = { DocumentId: 'a' }; + assert.deepEqual(normalizePayloadItems(item), [item]); + }); + + it('rejects AddOrUpdate that is not an array', () => { + assert.throws(() => normalizePayloadItems({ AddOrUpdate: {} }, 'file.json'), { + name: 'CliError', + message: /Invalid payload in file\.json: AddOrUpdate must be an array/, + }); + }); + + it('rejects scalars', () => { + assert.throws(() => normalizePayloadItems('nope', 'file.json'), CliError); + assert.throws(() => normalizePayloadItems(null, 'file.json'), CliError); + }); +}); + +describe('countPayloadItems', () => { + it('counts items of every supported shape', () => { + assert.equal(countPayloadItems(null), 0); + assert.equal(countPayloadItems([]), 0); + assert.equal(countPayloadItems([{}, {}]), 2); + assert.equal(countPayloadItems({ AddOrUpdate: [{}, {}, {}] }), 3); + assert.equal(countPayloadItems({ addOrUpdate: [{}] }), 1); + assert.equal(countPayloadItems({ DocumentId: 'a' }), 1); + assert.equal(countPayloadItems({ AddOrUpdate: 'bad' }), 1); + }); +}); + +describe('looksLikePushPayload', () => { + it('accepts arrays and AddOrUpdate objects only', () => { + assert.equal(looksLikePushPayload([]), true); + assert.equal(looksLikePushPayload({ AddOrUpdate: [] }), true); + assert.equal(looksLikePushPayload({ addorupdate: [] }), true); + assert.equal(looksLikePushPayload({ DocumentId: 'a' }), false); + assert.equal(looksLikePushPayload(null), false); + }); +}); + +describe('validatePayload', () => { + const noWarn = { warn: () => {} }; + + it('wraps an array into AddOrUpdate', () => { + const items = [{ DocumentId: 'a', FileExtension: '.txt' }]; + assert.deepEqual(validatePayload(items, noWarn), { AddOrUpdate: items }); + }); + + it('wraps a single document into AddOrUpdate', () => { + const item = { DocumentId: 'a', FileExtension: '.txt' }; + assert.deepEqual(validatePayload(item, noWarn), { AddOrUpdate: [item] }); + }); + + it('keeps a well-formed payload untouched', () => { + const payload = { AddOrUpdate: [{ DocumentId: 'a', FileExtension: '.txt' }], Delete: [] }; + assert.equal(validatePayload(payload, noWarn), payload); + }); + + it('normalizes the case of the AddOrUpdate key and keeps sibling keys', () => { + const items = [{ documentid: 'a', fileextension: '.txt' }]; + assert.deepEqual(validatePayload({ addOrUpdate: items, Delete: [] }, noWarn), { Delete: [], AddOrUpdate: items }); + }); + + it('rejects documents without DocumentId', () => { + assert.throws(() => validatePayload([{ title: 'no id' }], noWarn), { + name: 'CliError', + message: /Missing DocumentId/, + }); + }); + + it('rejects empty or scalar payloads', () => { + assert.throws(() => validatePayload(undefined, noWarn), CliError); + assert.throws(() => validatePayload(42, noWarn), CliError); + assert.throws(() => validatePayload({ AddOrUpdate: 'x' }, noWarn), CliError); + }); + + it('warns once when FileExtension is missing', () => { + const warnings = []; + validatePayload([{ DocumentId: 'a' }, { DocumentId: 'b' }], { warn: (m) => warnings.push(m) }); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /Missing FileExtension/); + }); +});