Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/quiet-hooks-register.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@openfn/runtime': patch
---

Use `module.registerHooks()` for the ESM resolve hook where Node supports it
5 changes: 5 additions & 0 deletions .changeset/worker-node-26-bookworm.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@openfn/ws-worker': minor
---

Run the worker image on Node 26 and Debian bookworm
5 changes: 5 additions & 0 deletions .changeset/worker-sentry-11.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@openfn/ws-worker': patch
---

Upgrade Sentry to v11 to stop a Node deprecation warning at startup, and deduplicate repeated identical errors
77 changes: 68 additions & 9 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,29 +9,47 @@ commands:
name: install CLI globally
command: sudo npm install -g file:dist/openfn-cli.tgz

check_node_version:
parameters:
node_version:
type: string
default: ''
steps:
- run:
name: Check Node versions match .tool-versions
command: scripts/check-node-version.sh << parameters.node_version >>

install_pnpm:
steps:
- run:
name: Install pnpm
command: sudo npm install -g "$(node -p "require('./package.json').packageManager.split('+')[0]")"

# Define a job to be invoked later in a workflow.
# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
jobs:
install:
# Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
# See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
docker:
- image: cimg/node:24.18
- image: cimg/node:26.12.0
resource_class: medium
# Add steps to the job
# See: https://circleci.com/docs/2.0/configuration-reference/#steps
steps:
- checkout
- check_node_version
- install_pnpm
- restore_cache:
# See the configuration reference documentation for more details on using restore_cache and save_cache steps
# https://circleci.com/docs/2.0/configuration-reference/?section=reference#save_cache
keys:
- deps-v1-{{ .Branch }}-{{checksum "pnpm-lock.yaml"}}
- deps-v1-{{ .Branch }}-{{checksum "pnpm-lock.yaml"}}-{{checksum ".tool-versions"}}
- run:
name: Install
command: pnpm install --frozen-lockfile
- save_cache:
key: deps-v1-{{ .Branch }}-{{checksum "pnpm-lock.yaml"}}
key: deps-v1-{{ .Branch }}-{{checksum "pnpm-lock.yaml"}}-{{checksum ".tool-versions"}}
paths:
- node_modules
- persist_to_workspace:
Expand All @@ -41,11 +59,13 @@ jobs:

build:
docker:
- image: cimg/node:24.18
- image: cimg/node:26.12.0
resource_class: medium
steps:
- attach_workspace:
at: ~/project
- check_node_version
- install_pnpm
- run:
name: Build packages
command: pnpm build
Expand All @@ -56,45 +76,53 @@ jobs:

unit_test:
docker:
- image: cimg/node:24.18
- image: cimg/node:26.12.0
resource_class: medium
parallelism: 1
steps:
- attach_workspace:
at: ~/project
- check_node_version
- install_pnpm
- run:
name: Run Tests
command: pnpm test

format:
docker:
- image: cimg/node:24.18
- image: cimg/node:26.12.0
resource_class: medium
steps:
- attach_workspace:
at: ~/project
- check_node_version
- install_pnpm
- run:
name: Check formatting
command: pnpm test:format

type_check:
docker:
- image: cimg/node:24.18
- image: cimg/node:26.12.0
resource_class: medium
steps:
- attach_workspace:
at: ~/project
- check_node_version
- install_pnpm
- run:
name: Type check
command: pnpm test:types

build_openfnx:
docker:
- image: cimg/node:24.18
- image: cimg/node:26.12.0
resource_class: medium
steps:
- attach_workspace:
at: ~/project
- check_node_version
- install_pnpm
- run:
name: Build local tarballs
command: pnpm pack:local dist --no-version
Expand All @@ -114,6 +142,9 @@ jobs:
steps:
- attach_workspace:
at: ~/project
- check_node_version:
node_version: << parameters.node_version >>
- install_pnpm
- install_cli_global
- run:
name: Check install
Expand All @@ -122,6 +153,29 @@ jobs:
name: Run integration test suite
command: pnpm test:integration

worker_image:
docker:
- image: cimg/node:26.12.0
resource_class: medium
steps:
- checkout
- check_node_version
- setup_remote_docker
- run:
name: Build worker image
command: docker build -t ws-worker-ci .
- run:
name: Check Node in the image matches .tool-versions
command: docker run --rm ws-worker-ci /app/scripts/check-node-version.sh
- run:
name: Install and load motherduck as user node
command: |
docker run --rm --user node ws-worker-ci sh -c '
mkdir -p /tmp/repo && cd /tmp/repo &&
npm install --no-audit --no-fund --no-package-lock --ignore-scripts @openfn/language-motherduck@latest &&
node -e "require(\"@openfn/language-motherduck\")"
'

# This ensures that we can run a v1 CLI deploy in node 18,
# which is important for GH sync
legacy_test:
Expand Down Expand Up @@ -165,9 +219,14 @@ workflows:
- integration_test:
matrix:
parameters:
node_version: ['22.20', '24.18.1', '26.5.1']
node_version: ['24.22.0', '26.12.0']
requires:
- build_openfnx
- legacy_test:
requires:
- build_openfnx
- worker_image:
filters:
branches:
ignore:
- main
8 changes: 4 additions & 4 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
{
"name": "Default",
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
"image": "mcr.microsoft.com/devcontainers/base:bullseye",
"image": "mcr.microsoft.com/devcontainers/base:bookworm",

"features": {
"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
Expand All @@ -13,7 +13,7 @@
},
"ghcr.io/devcontainers/features/node:1": {
"nodeGypDependencies": true,
"version": "18"
"version": "none"
}
},

Expand All @@ -25,8 +25,8 @@
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],

// Use 'postCreateCommand' to run commands after the container is created.
"postCreateCommand": "npm install -g pnpm"
// Node comes from .tool-versions, so the node feature only installs nvm.
"onCreateCommand": ".devcontainer/install-node.sh"

// Configure tool-specific properties.
// "customizations": {},
Expand Down
10 changes: 10 additions & 0 deletions .devcontainer/install-node.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail

node_version=$(awk '$1 == "nodejs" { print $2 }' .tool-versions)

. "$NVM_DIR/nvm.sh"
nvm install "$node_version"
nvm alias default "$node_version"

npm install -g "$(node -p "require('./package.json').packageManager.split('+')[0]")"
2 changes: 1 addition & 1 deletion .tool-versions
Original file line number Diff line number Diff line change
@@ -1 +1 @@
nodejs 26.5.1
nodejs 26.12.0
6 changes: 4 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
FROM node:24-alpine AS base
ARG NODE_VERSION=26.12.0

FROM node:${NODE_VERSION}-bookworm-slim AS base
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
COPY . /app
Expand All @@ -18,7 +20,7 @@ RUN npm install -g "$(node -p "require('./package.json').packageManager.split('+

# TODO: remove simple build once prod optimized build is working ---------------
FROM base AS ws-worker
RUN apk add --no-cache git
RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/*
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
RUN pnpm build
WORKDIR /app/packages/ws-worker
Expand Down
9 changes: 9 additions & 0 deletions ava.config.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
const path = require('node:path');
const { pathToFileURL } = require('node:url');
const semver = require("semver")

const loader_arg = semver.lte(process.version, 'v20.5.0') ? '--loader=@swc-node/register/esm' : '--import=@swc-node/register/esm-register'

// Loaded before any test file, so that every module sees the patched fs
const memfs = pathToFileURL(
path.join(__dirname, 'packages/test-helpers/src/memfs.ts')
).href;

module.exports = {
extensions: {
ts: 'module',
Expand All @@ -12,6 +20,7 @@ module.exports = {

nodeArguments: [
loader_arg,
`--import=${memfs}`,
'--no-warnings', // Disable experimental module warnings
'--experimental-vm-modules',
],
Expand Down
2 changes: 0 additions & 2 deletions integration-tests/cli/.dockerignore

This file was deleted.

15 changes: 0 additions & 15 deletions integration-tests/cli/Dockerfile

This file was deleted.

11 changes: 3 additions & 8 deletions integration-tests/cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,14 @@

This package contains a set of integration tests against `openfn` CLI.

The test suite is designed to be run in CircleCI, but can run locally or in a local docker image.
The test suite is designed to be run in CircleCI, but can also run locally.

## Running Tests

During development, you can run the unit tests against a local build (ie, `packages/cli/dist`):

`pnpm test:dev`

You can run the test in isolation with a docker build. This can be useful while developing for CI, or if a test passes locally but fails in CI.

`pnpm run build`
`pnpm run start`

## Architecture

The basic idea here is to test the built (but unreleased) CLI bundle in the global space.
Expand All @@ -29,6 +24,6 @@ How it works:

## Creating tarballs

From this folder, run `build:pack` to create a bunch of tarballs from your built packages. Tarballs will be built to `dist`.
From the repo root, run `pnpm pack:local integration-tests/cli/dist --no-version` to create a bunch of tarballs from your built packages. Tarballs will be built to `dist`.

You can install the tarballs locally with `npm install -g dist openfn-cli.tgz.`
From this folder, install the CLI from the tarballs with `npm install -g ./dist/openfn-cli.tgz`.
3 changes: 0 additions & 3 deletions integration-tests/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@
"type": "module",
"scripts": {
"clean": "rimraf dist repo",
"build:pack": "pnpm clean && cd ../.. && pnpm pack:local integration-tests/cli/dist --no-version",
"build": "pnpm build:pack && docker build --tag cli-integration-tests .",
"start": "docker run cli-integration-tests",
"test": "node prelude.js && npx ava -s --timeout 2m && pnpm clean",
"test:dev": "node prelude.js && pnpm ava -s && pnpm clean"
},
Expand Down
1 change: 0 additions & 1 deletion integration-tests/legacy-deploy/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
"type": "module",
"scripts": {
"clean": "rimraf dist repo",
"start": "docker run cli-integration-tests",
"test": "npx ava -s"
},
"dependencies": {
Expand Down
3 changes: 2 additions & 1 deletion integration-tests/worker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# This dockerfile allows us to run tests against the build in isolation
# (I'm not really sure we need it but it's been a useful dev tool)
FROM node:18.12.0
ARG NODE_VERSION=26.12.0
FROM node:${NODE_VERSION}-bookworm-slim

ENV NODE_ENV=production

Expand Down
3 changes: 1 addition & 2 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,16 @@
"devDependencies": {
"@openfn/language-collections": "^0.9.0",
"@openfn/language-common": "3.2.3",
"@openfn/test-helpers": "workspace:*",
"@types/json-diff": "^1.0.3",
"@types/lodash-es": "~4.17.12",
"@types/mock-fs": "^4.13.4",
"@types/node": "^18.19.130",
"@types/rimraf": "^3.0.2",
"@types/treeify": "^1.0.3",
"@types/ws": "^8.18.1",
"@types/yargs": "^17.0.35",
"ava": "5.3.1",
"lodash-es": "^4.17.23",
"mock-fs": "^5.5.0",
"tslib": "^2.8.1",
"tsup": "^8.5.1",
"typescript": "^5.9.3"
Expand Down
Loading
Loading