diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..108672a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "name": "ChatBotKit Platform", + + "image": "mcr.microsoft.com/devcontainers/javascript-node:24-bookworm", + + "features": { + "ghcr.io/devcontainers/features/git-lfs:1": {}, + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + + "postCreateCommand": "corepack enable" +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6c4b52d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,47 @@ +# Dependencies (installed fresh in the container) +node_modules +**/node_modules + +# Build outputs and caches +**/.next +**/dist +**/.turbo +**/.swc +**/*.tsbuildinfo +**/coverage +**/storybook-static +platform/sites + +# @note content trees stay IN the context deliberately - they are read +# during static generation. Tests do not (the image build skips them via +# SKIP_BUILD_TESTS), and underscore-prefixed files under pages/ are private +# partials and page tests that must not compile into routes - the same set +# the deploy workflow's "Remove private files" step strips, minus the Next.js +# specials at the pages root which are re-included below. +**/*.utest.* +**/*.itest.* +platform/pages/**/_* +!platform/pages/_* + +# Environment files (passed at runtime, never baked in) +**/.env +**/.env.* +!**/.env.example + +# Git metadata and CI definitions - nothing in the build reads them +.git +**/.github + +# Editor and CI +.devcontainer +.vscode +**/.storybook +# @note the storybook stories folder only - platform/content/stories is real +# site content the build imports +platform/stories + +# Logs and scratch +**/*.log + +# pnpm store fallback location (dev container) +**/.pnpm-store diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ede84e0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +# Binary assets - routed through Git LFS, with no text diffs and no eol +# normalization. The tree carries ~75MB of binaries across ~80 files (app +# banners, example imagery, fonts, media, and the generated embeddings +# archives; the largest single file is ~14MB), and that grows every time a +# screenshot or an asset lands. LFS keeps those bytes out of every clone's +# pack and out of PR diffs. +# +# This mirrors the monorepo root .gitattributes, which already LFS-tracks the +# same extensions, so a file's storage does not change depending on which +# repository it is checked out from. +# +# @note SVG is deliberately absent: it is XML, it diffs usefully, and the +# files here are small. +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +*.ico filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.ogg filter=lfs diff=lfs merge=lfs -text +*.ttf filter=lfs diff=lfs merge=lfs -text +*.eot filter=lfs diff=lfs merge=lfs -text +*.woff filter=lfs diff=lfs merge=lfs -text +*.woff2 filter=lfs diff=lfs merge=lfs -text +*.pdf filter=lfs diff=lfs merge=lfs -text +*.doc filter=lfs diff=lfs merge=lfs -text +*.docx filter=lfs diff=lfs merge=lfs -text +*.ppt filter=lfs diff=lfs merge=lfs -text +*.pptx filter=lfs diff=lfs merge=lfs -text +*.xls filter=lfs diff=lfs merge=lfs -text +*.xlsx filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text + +# Generated archives - keep them out of PR diff stats +*.embeddings.json.gz linguist-generated diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..47e9e90 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,52 @@ +name: Bug report +description: Something does not work the way the code says it should +labels: ['bug'] +body: + - type: markdown + attributes: + value: >- + Thanks for taking the time. A reproducible report is the fastest path + to a fix - the closer you get us to the failure, the sooner it dies. + + - type: textarea + id: what-happened + attributes: + label: What happened + description: >- + What you did, what you expected, and what you got instead. Include the + exact error output where there is one. + validations: + required: true + + - type: textarea + id: reproduction + attributes: + label: How to reproduce it + description: >- + The smallest sequence of steps that shows the problem on a fresh + checkout. If it only happens with specific configuration, include the + relevant (redacted) settings. + placeholder: | + 1. pnpm install + 2. ... + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: >- + How you are running the platform and on what. + placeholder: | + - run mode: docker compose / pnpm dev / built image + - node: 24.x, pnpm: 11.x + - os: ... + validations: + required: true + + - type: textarea + id: extra + attributes: + label: Anything else + description: Logs, screenshots, or a theory about the cause. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..c7daa11 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +blank_issues_enabled: false +contact_links: + - name: Security report + url: https://github.com/chatbotkit/platform/blob/main/SECURITY.md + about: >- + Do not report security issues publicly. Email contact@cbk.ai + with "Security report" in the subject - see SECURITY.md. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..e4b9c17 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,28 @@ +name: Feature request +description: Something the platform should do that it does not +labels: ['enhancement'] +body: + - type: textarea + id: problem + attributes: + label: The problem + description: >- + What you are trying to do and what stops you today. Lead with the + problem rather than the solution - it keeps the discussion honest + about alternatives. + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed shape + description: >- + How you imagine it working, if you have a shape in mind. API surface, + configuration, interface - whatever level you have. + + - type: textarea + id: alternatives + attributes: + label: What you do instead today + description: Workarounds you use now, and where they fall short. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..78a53f9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ + + +## What + + + +## Why + + + +## How it was verified + + diff --git a/.github/workflows/_verify.yaml b/.github/workflows/_verify.yaml new file mode 100644 index 0000000..2b46150 --- /dev/null +++ b/.github/workflows/_verify.yaml @@ -0,0 +1,142 @@ +# The repository's quality gate as a reusable workflow: install from the +# committed lockfile and type-check cleanly with the public module defaults +# - the exact environment every fresh checkout gets. +# +# Called from two places: +# - pull-request.yaml, where it gates every pull request (including forks; +# see the security notes there - this workflow uses no secrets and only +# ever needs `contents: read`) +# - publish-ghcr-platform.yaml, where it gates image builds on `next` +# pushes, which land directly without a pull request + +name: Verify + +on: + workflow_call: + +permissions: + contents: read + +env: + # pinned to the devcontainer versions + REQUIRED_PNPM_VERSION: 11.24.0 + REQUIRED_NODE_VERSION: 24.20.0 + +jobs: + verify: + runs-on: ubuntu-latest-8-cores-amd64 + + # @note a healthy sequential run takes low double-digit minutes; anything + # longer is wedged - cut it off and free the hosted runner + timeout-minutes: 45 + + steps: + - uses: actions/checkout@v7 + with: + lfs: true + + - uses: pnpm/setup@v2 + with: + version: ${{ env.REQUIRED_PNPM_VERSION }} + runtime: node@${{ env.REQUIRED_NODE_VERSION }} + cache: true + install: false + + - name: Report runner resources + # @note exit 137 anywhere below is the runner kernel OOM-killing the + # workload: read this step's output for the VM this run actually + # landed on before blaming the workload + run: nproc && grep MemTotal /proc/meminfo && df -h . | tail -1 + + - name: Install from the lockfile + # @note --frozen-lockfile is the point: it fails when pnpm-lock.yaml + # has drifted from the manifests, which is the most common way a + # fresh checkout silently breaks + run: pnpm install --frozen-lockfile + + - name: Audit production dependencies + # @note the published baseline is no known production vulnerabilities, + # so every advisory fails this gate. A temporary exception must name + # the advisory, rationale, owner, compensating controls and expiry in + # the public change that adds the exception before it is ignored here. + run: pnpm audit --prod --audit-level low + + - name: Configure the application from the example environment + # @note the example file is the documented minimum a fresh checkout + # runs on; nothing here is a secret + working-directory: platform + run: cp .env.example .env + + - name: Create the database and generate its client + # @note runs before the package steps: any package that imports the + # database module needs the generated client to build or type-check. + # The client generator reads the typed SQL against a live database, so + # an empty SQLite file is pushed first. The absolute path is + # deliberate - the push runs from the database module's directory, + # the application from its own + working-directory: platform + run: | + mkdir -p .dev + touch .dev/platform.db + export PRISMA_DATABASE_URL="file:$PWD/.dev/platform.db" + pnpm --filter @chatbotkit-dev/db db:push + pnpm db:gen + + - name: Build every package + # @note the application is excluded from this step and the three + # below: its own steps further down provision the environment its + # config asserts at load. Its lint, type-check and tests run below; + # the trusted image-publication workflow builds and smoke-tests it. + run: pnpm turbo run build --continue --filter='!@chatbotkit/platform' + + - name: Lint every package + # @note the lint debt that kept this step out was paid down on + # August 24 2026 (13 packages); the gate is born green + run: pnpm turbo run lint --continue --filter='!@chatbotkit/platform' + + - name: Check every package + run: pnpm turbo run check --continue --filter='!@chatbotkit/platform' + + - name: Test every package + run: pnpm turbo run test --continue --filter='!@chatbotkit/platform' + + - name: Cache TypeScript build info + # @note persists only tsc's non-sensitive incremental project state. + # The SHA suffix lets each successful run advance the baseline, while + # the restore prefix selects the newest state built with the same + # lockfile and application compiler configuration. tsc validates file + # signatures, so a stale restore falls back to checking changed files. + # Fork pull requests can read the base cache but cannot write into its + # scope; never add generated environment or credential files here. + uses: actions/cache@v6 + with: + path: platform/tsconfig.tsbuildinfo + key: ${{ runner.os }}-${{ runner.arch }}-tsbuildinfo-application-${{ hashFiles('pnpm-lock.yaml', 'platform/tsconfig.json') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-tsbuildinfo-application-${{ hashFiles('pnpm-lock.yaml', 'platform/tsconfig.json') }}- + + - name: Cache Jest transform cache + # @note persists only Jest's transform/haste cache - hashed transpiled + # module output, nothing sensitive. Jest validates entries by content + # hash, so a stale restore degrades to re-transforming changed files. + # Same key discipline as the tsbuildinfo cache above: the SHA suffix + # advances the baseline, the restore prefix picks the newest state for + # the same lockfile. + uses: actions/cache@v6 + with: + path: platform/.jest-cache + key: ${{ runner.os }}-${{ runner.arch }}-jest-application-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-jest-application-${{ hashFiles('pnpm-lock.yaml') }}- + + - name: Lint application + working-directory: platform + run: pnpm lint + + - name: Check application types + working-directory: platform + run: pnpm check + + - name: Test application with coverage + working-directory: platform + run: pnpm test diff --git a/.github/workflows/publish-ghcr-platform.yaml b/.github/workflows/publish-ghcr-platform.yaml new file mode 100644 index 0000000..773f249 --- /dev/null +++ b/.github/workflows/publish-ghcr-platform.yaml @@ -0,0 +1,415 @@ +name: Publish GHCR Platform + +on: + workflow_dispatch: + + # @note no `paths` filter here on purpose: the verify job must run on every + # push to next so its check lands on the SHA the promotion pull request + # (next -> main) requires - pull-request.yaml skips its own run for that + # head branch. Path filtering for the image build lives in the changes job + # below instead. + push: + branches: + - main + - next + +permissions: + contents: read + packages: write + +concurrency: + group: publish-ghcr-platform-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + +jobs: + # @note every job is guarded to the canonical repository and its + # `platform-*` siblings: a fork that pushes to main or next gets a clean + # skip rather than a queued build against runners and a registry it does + # not have. + # + # The trigger-level `paths` filter this job replaces could not coexist with + # running verify on every next push, so the build-relevance decision is made + # here from the actual diff of the push. Anything that prevents computing + # that diff (a brand-new branch, a force push, a manual dispatch) falls back + # to building - a spurious build is cheap, a silently skipped one is not. + changes: + if: >- + (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && + github.actor != 'github-actions[bot]' + name: Detect build-relevant changes + runs-on: ubuntu-latest + outputs: + build: ${{ steps.diff.outputs.build }} + steps: + - uses: actions/checkout@v7 + + - name: Diff the push against build-relevant paths + id: diff + env: + BEFORE: ${{ github.event.before }} + run: | + # @note keep this pattern list in sync with the inputs the Docker + # build actually consumes (the former trigger paths filter) + pattern='^(packages/|patches/|platform/|stubs/|docker/|\.dockerignore$|\.pnpmfile\.cjs$|package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$|turbo\.json$|\.github/workflows/(publish-ghcr-platform|_verify)\.yaml$)' + + if [ "$GITHUB_EVENT_NAME" != "push" ] \ + || [ -z "$BEFORE" ] \ + || [ "$BEFORE" = "0000000000000000000000000000000000000000" ] \ + || ! git fetch --quiet --depth=1 origin "$BEFORE" + then + echo "build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if git diff --name-only "$BEFORE" "$GITHUB_SHA" | grep -qE "$pattern"; then + echo "build=true" >> "$GITHUB_OUTPUT" + else + echo "build=false" >> "$GITHUB_OUTPUT" + fi + + # @note `next` receives direct pushes, so nothing has vetted the code yet - + # run the quality gate before spending build minutes on it. It runs on + # every next push, not just build-relevant ones, so the promotion pull + # request always finds a verify check on the head SHA. `main` only moves + # through pull requests that already passed the same gate, so the job is + # skipped there and the build starts immediately. + verify: + if: >- + (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && + github.actor != 'github-actions[bot]' && + github.ref == 'refs/heads/next' + uses: ./.github/workflows/_verify.yaml + + build: + needs: + - changes + - verify + # @note !cancelled() + accepting the skipped verify keeps main builds + # running while a failed gate on next still blocks them + if: >- + !cancelled() && + (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && + needs.changes.outputs.build == 'true' && + (needs.verify.result == 'success' || needs.verify.result == 'skipped') && + github.actor != 'github-actions[bot]' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/next') + name: Build ${{ matrix.flavor.name }} (${{ matrix.architecture.name }}) + runs-on: ${{ matrix.architecture.runner }} + strategy: + fail-fast: false + matrix: + flavor: + # @note package selections are compile-time image flavors. Add a + # flavor here only when its Docker target installs that exact package + # configuration. Runtime services such as Redis are not flavors. + - name: community + application_target: application + initializer_target: initializer + architecture: + - name: amd64 + platform: linux/amd64 + runner: ubuntu-latest-8-cores-amd64 + - name: arm64 + platform: linux/arm64 + runner: ubuntu-latest-8-cores-arm64 + + steps: + - uses: actions/checkout@v7 + with: + lfs: true + + # @note the naming grammar is -[-], with + # the tag carrying only the build axis (channel or sha). The bare + # - name is the Compose distribution artifact - + # the one consumers type - and -app/-init are its digest-pinned + # component images. + - name: Resolve image names + id: image + env: + FLAVOR: ${{ matrix.flavor.name }} + REPOSITORY_NAME: ${{ github.event.repository.name }} + REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + owner="${REPOSITORY_OWNER,,}" + repository="${REPOSITORY_NAME,,}" + stack="${REGISTRY}/${owner}/${repository}-${FLAVOR}" + echo "stack=${stack}" >> "$GITHUB_OUTPUT" + echo "application=${stack}-app" >> "$GITHUB_OUTPUT" + echo "initializer=${stack}-init" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract application metadata + id: application-metadata + uses: docker/metadata-action@v6 + with: + images: ${{ steps.image.outputs.application }} + tags: | + type=raw,value=${{ github.ref_name }} + type=sha,format=short,prefix=sha- + type=raw,value=latest,enable=${{ github.ref_name == 'main' }} + + - name: Build and push application image + id: application + uses: docker/build-push-action@v7 + with: + context: . + file: docker/Dockerfile + target: ${{ matrix.flavor.application_target }} + platforms: ${{ matrix.architecture.platform }} + # @note community images embed full source maps on purpose - the + # source is public and self-hosted debugging needs them + build-args: | + BUILD_SOURCEMAPS=full + labels: ${{ steps.application-metadata.outputs.labels }} + outputs: type=image,name=${{ steps.image.outputs.application }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ github.event.repository.name }}-${{ matrix.flavor.name }}-application-${{ matrix.architecture.name }} + cache-to: type=gha,mode=max,scope=${{ github.event.repository.name }}-${{ matrix.flavor.name }}-application-${{ matrix.architecture.name }} + + - name: Extract initializer metadata + id: initializer-metadata + uses: docker/metadata-action@v6 + with: + images: ${{ steps.image.outputs.initializer }} + tags: | + type=raw,value=${{ github.ref_name }} + type=sha,format=short,prefix=sha- + type=raw,value=latest,enable=${{ github.ref_name == 'main' }} + + - name: Build and push initializer image + id: initializer + uses: docker/build-push-action@v7 + with: + context: . + file: docker/Dockerfile + target: ${{ matrix.flavor.initializer_target }} + platforms: ${{ matrix.architecture.platform }} + build-args: | + BUILD_SOURCEMAPS=full + labels: ${{ steps.initializer-metadata.outputs.labels }} + outputs: type=image,name=${{ steps.image.outputs.initializer }},push-by-digest=true,name-canonical=true,push=true + cache-from: | + type=gha,scope=${{ github.event.repository.name }}-${{ matrix.flavor.name }}-application-${{ matrix.architecture.name }} + type=gha,scope=${{ github.event.repository.name }}-${{ matrix.flavor.name }}-initializer-${{ matrix.architecture.name }} + cache-to: type=gha,mode=max,scope=${{ github.event.repository.name }}-${{ matrix.flavor.name }}-initializer-${{ matrix.architecture.name }} + + - name: Smoke-test published images + env: + APPLICATION_IMAGE: ${{ steps.image.outputs.application }}@${{ steps.application.outputs.digest }} + ARCHITECTURE: ${{ matrix.architecture.name }} + FLAVOR: ${{ matrix.flavor.name }} + INITIALIZER_IMAGE: ${{ steps.image.outputs.initializer }}@${{ steps.initializer.outputs.digest }} + PLATFORM: ${{ matrix.architecture.platform }} + run: | + container="platform-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FLAVOR}-${ARCHITECTURE}" + volume="platform-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FLAVOR}-${ARCHITECTURE}" + + cleanup() { + docker rm --force "$container" >/dev/null 2>&1 || true + docker volume rm --force "$volume" >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + docker volume create "$volume" + docker run --rm \ + --platform "$PLATFORM" \ + --env PRISMA_DATABASE_URL=file:/data/platform.db \ + --volume "$volume:/data" \ + "$INITIALIZER_IMAGE" + + # @note SITE_URL deliberately differs from the image's build-time + # value so the probe below proves the runtime environment actually + # reaches the served pages - a build-time bake cannot pass it + docker run --detach \ + --platform "$PLATFORM" \ + --name "$container" \ + --env NEXTAUTH_SECRET=published-image-smoke-test \ + --env NEXTAUTH_URL=http://smoke-test.invalid \ + --env PRISMA_DATABASE_URL=file:/data/platform.db \ + --env QUEUE_SECRET=published-image-smoke-test \ + --env SITE_URL=http://smoke-test.invalid \ + --volume "$volume:/data" \ + "$APPLICATION_IMAGE" + + # @note /signin renders per request, so its markup must carry the + # runtime site host stamped by the request context + for attempt in $(seq 1 90); do + if docker exec "$container" node -e \ + "fetch('http://127.0.0.1:3000/signin').then(async (response) => { const body = await response.text(); process.exit(response.status < 500 && body.includes('smoke-test.invalid') ? 0 : 1) }).catch(() => process.exit(1))" + then + exit 0 + fi + + sleep 2 + done + + docker logs "$container" + exit 1 + + # @note matrix job outputs cannot be aggregated reliably, so digest + # marker files carry both architecture results into the publish job + - name: Export image digests + env: + APPLICATION_DIGEST: ${{ steps.application.outputs.digest }} + INITIALIZER_DIGEST: ${{ steps.initializer.outputs.digest }} + run: | + mkdir -p "$RUNNER_TEMP/platform-digests/application" + mkdir -p "$RUNNER_TEMP/platform-digests/initializer" + touch "$RUNNER_TEMP/platform-digests/application/${APPLICATION_DIGEST#sha256:}" + touch "$RUNNER_TEMP/platform-digests/initializer/${INITIALIZER_DIGEST#sha256:}" + + - name: Upload image digests + uses: actions/upload-artifact@v7 + with: + name: platform-digests-${{ matrix.flavor.name }}-${{ matrix.architecture.name }} + path: ${{ runner.temp }}/platform-digests + retention-days: 1 + + publish: + # @note !cancelled() is required: the implicit success() looks at the + # whole needs chain, and verify is skipped on main + if: >- + !cancelled() && + (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && + needs.build.result == 'success' && + github.actor != 'github-actions[bot]' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/next') + name: Publish ${{ matrix.flavor.name }} + needs: build + runs-on: ${{ matrix.flavor.runner }} + strategy: + fail-fast: false + matrix: + flavor: + # @note keep this list in sync with the build job's flavor matrix + - name: community + runner: ubuntu-latest-8-cores-amd64 + + steps: + - uses: actions/checkout@v7 + with: + lfs: true + + - name: Resolve image names + id: image + env: + FLAVOR: ${{ matrix.flavor.name }} + REPOSITORY_NAME: ${{ github.event.repository.name }} + REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + owner="${REPOSITORY_OWNER,,}" + repository="${REPOSITORY_NAME,,}" + stack="${REGISTRY}/${owner}/${repository}-${FLAVOR}" + echo "stack=${stack}" >> "$GITHUB_OUTPUT" + echo "application=${stack}-app" >> "$GITHUB_OUTPUT" + echo "initializer=${stack}-init" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + # @note `docker compose publish` requires Compose v2.34 or newer; the + # runner's bundled plugin is not guaranteed to be that new + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Download image digests + uses: actions/download-artifact@v7 + with: + pattern: platform-digests-${{ matrix.flavor.name }}-* + path: ${{ runner.temp }}/platform-digests + merge-multiple: true + + - name: Create multi-platform image manifests + id: manifest + env: + APPLICATION_IMAGE: ${{ steps.image.outputs.application }} + CHANNEL: ${{ github.ref_name }} + DIGESTS_DIR: ${{ runner.temp }}/platform-digests + INITIALIZER_IMAGE: ${{ steps.image.outputs.initializer }} + run: | + application_sources=() + for digest_file in "$DIGESTS_DIR"/application/*; do + [ -f "$digest_file" ] || continue + application_sources+=("${APPLICATION_IMAGE}@sha256:$(basename "$digest_file")") + done + + initializer_sources=() + for digest_file in "$DIGESTS_DIR"/initializer/*; do + [ -f "$digest_file" ] || continue + initializer_sources+=("${INITIALIZER_IMAGE}@sha256:$(basename "$digest_file")") + done + + if [ "${#application_sources[@]}" -ne 2 ] || [ "${#initializer_sources[@]}" -ne 2 ]; then + echo "Expected two architecture digests for each image" >&2 + exit 1 + fi + + short_sha="${GITHUB_SHA:0:7}" + application_tags=(--tag "${APPLICATION_IMAGE}:${CHANNEL}" --tag "${APPLICATION_IMAGE}:sha-${short_sha}") + initializer_tags=(--tag "${INITIALIZER_IMAGE}:${CHANNEL}" --tag "${INITIALIZER_IMAGE}:sha-${short_sha}") + + if [ "$CHANNEL" = "main" ]; then + application_tags+=(--tag "${APPLICATION_IMAGE}:latest") + initializer_tags+=(--tag "${INITIALIZER_IMAGE}:latest") + fi + + docker buildx imagetools create "${application_tags[@]}" "${application_sources[@]}" + docker buildx imagetools create "${initializer_tags[@]}" "${initializer_sources[@]}" + + application_digest=$(docker buildx imagetools inspect "${APPLICATION_IMAGE}:${CHANNEL}" --format '{{.Manifest.Digest}}') + initializer_digest=$(docker buildx imagetools inspect "${INITIALIZER_IMAGE}:${CHANNEL}" --format '{{.Manifest.Digest}}') + echo "application_digest=${application_digest}" >> "$GITHUB_OUTPUT" + echo "initializer_digest=${initializer_digest}" >> "$GITHUB_OUTPUT" + + # @note the one-command distribution route: the flavor's Compose file is + # published as an OCI artifact under -, with every + # image reference resolved to a digest so the artifact tag identifies an + # exact, immutable stack. Consumers run: + # + # docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest up + - name: Publish Compose distribution artifact + env: + APPLICATION_IMAGE: ${{ steps.image.outputs.application }}@${{ steps.manifest.outputs.application_digest }} + CHANNEL: ${{ github.ref_name }} + FLAVOR: ${{ matrix.flavor.name }} + INITIALIZER_IMAGE: ${{ steps.image.outputs.initializer }}@${{ steps.manifest.outputs.initializer_digest }} + STACK_IMAGE: ${{ steps.image.outputs.stack }} + run: | + publish() { + PLATFORM_IMAGE="$APPLICATION_IMAGE" \ + PLATFORM_INIT_IMAGE="$INITIALIZER_IMAGE" \ + docker compose --file "docker/distro/${FLAVOR}/compose.yml" \ + publish -y --resolve-image-digests "$1" + } + + publish "${STACK_IMAGE}:${CHANNEL}" + + if [ "$CHANNEL" = "main" ]; then + publish "${STACK_IMAGE}:latest" + fi + + - name: Smoke-test published distribution artifact + env: + CHANNEL: ${{ github.ref_name }} + STACK_IMAGE: ${{ steps.image.outputs.stack }} + run: | + docker compose --file "oci://${STACK_IMAGE}:${CHANNEL}" config --quiet diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..b3ef4e4 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,52 @@ +# The quality gate for this repository: every pull request must install from +# the committed lockfile and type-check cleanly with the public module defaults +# - the exact environment every fresh checkout gets. The gate itself lives in +# _verify.yaml so pushes to `next` can run the same checks before an image +# build. +# +# SECURITY: this workflow is safe to run on code from strangers, and it is +# designed to stay that way. Fork pull requests run through the plain +# `pull_request` event, which GitHub executes on the fork's merge ref with a +# read-only token and NO access to repository secrets; the job runs on a +# GitHub-hosted runner that is discarded afterwards; the workflow declares +# `contents: read` and uses no secrets of its own. Together those are the +# whole boundary, so there is no fork guard: an approved outside +# contribution gets the same check as a maintainer's branch, which is what +# "required check" has to mean for it to be worth requiring. +# +# What keeps the boundary: +# - never use `pull_request_target` here, and never check out and execute +# pull request code from a job that has secrets or a write token +# - a runner must not survive the job; anything it ran is a foothold. The +# GitHub-hosted runner below is a fresh VM for the job and is discarded +# afterwards. A persistent self-hosted runner must never be used here +# - never add secrets to this workflow or to _verify.yaml; a step that +# needs one belongs in a separate workflow that does not execute +# contributed code +# - keep the repository setting "Require approval for all outside +# collaborators" (Settings > Actions > General) so a stranger's first run +# is still a maintainer's deliberate click, and their compute is not +# spent on drive-by pull requests + +name: Pull Request Checks + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + # @note the promotion pull request (next -> main) is exempt: its head + # commit is the tip of next, and the push-triggered publish workflow has + # already run this exact gate on that SHA - check runs attach to the + # commit, so the pull request inherits the green check without a second + # run. Every other head branch (forks included) still gets its own run. + if: github.event_name != 'pull_request' || github.head_ref != 'next' + uses: ./.github/workflows/_verify.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d0b2c58 --- /dev/null +++ b/.gitignore @@ -0,0 +1,69 @@ +# @note this is the ignore policy for the published repository, where this +# directory is the root. Inside the monorepo the root .gitignore covers most +# of what follows, which is exactly why it has to be repeated here: a +# standalone checkout has no such parent, and without this file a contributor +# running the documented setup sees build output and dependencies reported as +# untracked changes. + +# Dependencies +node_modules/ + +# pnpm store fallback location (dev container) +.pnpm-store/ + +# Local development state (dev-profile database and the like) +.dev/ + +# Environment files - .env.example is the tracked template, real ones never are +.env +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Build output +.next +out +dist +build +*.tsbuildinfo +next-env.d.ts + +# Task and compiler caches +.turbo +.swc +.eslintcache +.cache + +# Test output +coverage +*.lcov +.nyc_output + +# Logs and diagnostics +logs +*.log +.pnpm-debug.log* +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Generated at build time rather than tracked +**/public/integrations/widget/* +**/public/api/*/spec.json +**/public/robots.txt +**/public/sitemap* + +# Storybook build output +storybook-static + +# react-email preview build +.react-email + +# Deployment tooling +.vercel + +# Editor and OS noise +.DS_Store + +# Scratch files +**/*.todo +**/*.bak diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs new file mode 100644 index 0000000..de5abc0 --- /dev/null +++ b/.pnpmfile.cjs @@ -0,0 +1,146 @@ +/** + * @note Install-time dependency fixups for the platform core. When this + * folder is installed standalone (e.g. the public repository) pnpm loads the + * exported `hooks` directly. An enclosing workspace can import + * `createReadPackage` and pass its own path prefix because the `file:`/`link:` + * stub specifiers resolve relative to the install root rather than this file. + * + * @see https://pnpm.io/pnpmfile#hooksreadpackagepkg-context + */ + +/** + * Creates a readPackage hook with stub paths resolved against the given + * install-root-relative prefix ('' when platform is the install root, + * 'platform/' when this folder is embedded in another workspace). + * + * @param {string} pathPrefix + * @returns {(pkg: object) => object} + */ +function createReadPackage(pathPrefix) { + // @note Packages that incorrectly declare peer dependencies as + // devDependencies. pnpm's strict isolation prevents these packages from + // accessing their implicit dependencies, so the missing peerDependencies + // are injected at install time. + + // @note guard against the single most destructive mistake available in this + // folder: running `pnpm install` here while the folder sits inside an + // enclosing workspace. pnpm would faithfully install the standalone + // workspace in place, clobbering the enclosing workspace's module links and + // splitting singletons like react into two instances. The pnpmfile is the + // only hook pnpm loads before touching disk, so the refusal happens before + // any damage. The empty path prefix identifies this folder as the active + // install root; the enclosing workspace imports this hook with its own path + // prefix. Other exemptions are --lockfile-only (writes no modules) and + // PLATFORM_NESTED_INSTALL=1 for deliberate vendored setups. + let installOwnershipValidated = false + + function validateInstallOwnership() { + if (installOwnershipValidated) { + return + } + + installOwnershipValidated = true + + const fs = require('node:fs') + const path = require('node:path') + + const here = __dirname + const cwd = process.cwd() + + const installCommand = ['install', 'i', 'add', 'update', 'up'].some((c) => + process.argv.includes(c) + ) + + const nested = fs.existsSync(path.join(here, '..', 'pnpm-workspace.yaml')) + const inHere = cwd === here || cwd.startsWith(here + path.sep) + const platformOwnsInstall = pathPrefix === '' + + if ( + platformOwnsInstall && + nested && + inHere && + installCommand && + !process.argv.includes('--lockfile-only') && + !process.env.PLATFORM_NESTED_INSTALL + ) { + throw new Error( + 'Refusing to install in place: this folder sits inside an enclosing pnpm workspace whose install owns these directories. Install from the workspace root instead, or use `docker compose up` for an isolated dev environment. Set PLATFORM_NESTED_INSTALL=1 to override for a deliberate vendored setup.' + ) + } + } + + const missingPeerDependenciesMap = { + // @note tailwind-gradient-mask-image requires 'tailwindcss/plugin' but + // declares tailwindcss as a devDependency instead of peerDependency. With + // multiple tailwindcss versions in the workspace, pnpm cannot hoist it + // publicly, causing "Cannot find module 'tailwindcss/plugin'" errors. + // @see https://github.com/juhanakristian/tailwind-gradient-mask-image/issues + + 'tailwind-gradient-mask-image': { + tailwindcss: '*', + }, + + // @note @metascraper/helpers uses re2 but doesn't declare it as a peer + // dependency. We use a local shim at stubs/re2 to avoid native + // compilation issues. + + '@metascraper/helpers': { + re2: `file:${pathPrefix}stubs/re2`, + }, + } + + // @note Packages whose dependencies need to be overridden with stubs or + // alternative implementations. This is used when a package imports a + // dependency that causes build issues (e.g., ESM-only packages in webpack) + // but that functionality is not actually needed. + + const dependencyOverridesMap = { + // @note officeparser imports pdfjs-dist at the top level for PDF parsing, + // but pdfjs-dist is ESM-only and causes webpack bundling issues in + // Next.js. We use officeparser only for docx/pptx/xlsx parsing (not + // PDF), so we stub out pdfjs-dist with an empty implementation at + // stubs/pdfjs-dist. + + officeparser: { + 'pdfjs-dist': `link:${pathPrefix}stubs/pdfjs-dist`, + }, + } + + return function readPackage(pkg) { + validateInstallOwnership() + + // inject missing peer dependencies for packages that incorrectly declare them + { + const missingPeerDependencies = missingPeerDependenciesMap[pkg.name] || {} + + for (const [peerPackage, peerVersion] of Object.entries( + missingPeerDependencies + )) { + pkg.peerDependencies = pkg.peerDependencies || {} + pkg.peerDependencies[peerPackage] = peerVersion + } + } + + // override dependencies with stubs or alternative implementations + { + const dependencyOverrides = dependencyOverridesMap[pkg.name] || {} + + for (const [depPackage, depVersion] of Object.entries( + dependencyOverrides + )) { + if (pkg.dependencies?.[depPackage]) { + pkg.dependencies[depPackage] = depVersion + } + } + } + + return pkg + } +} + +module.exports = { + createReadPackage, + hooks: { + readPackage: createReadPackage(''), + }, +} diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..be7f691 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,10 @@ +{ + "recommendations": [ + "streetsidesoftware.code-spell-checker", + "esbenp.prettier-vscode", + "dbaeumer.vscode-eslint", + "csstools.postcss", + "bradlc.vscode-tailwindcss", + "Prisma.prisma" + ] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..d9e512f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch @chatbotkit/platform", + "runtimeExecutable": "pnpm", + "runtimeArgs": ["-F", "@chatbotkit/platform", "dev"], + "restart": true, + "console": "integratedTerminal" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..879ea08 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib", + "typescript.enablePromptUseWorkspaceTsdk": false, + "typescript.disableAutomaticTypeAcquisition": true, + "typescript.tsserver.maxTsServerMemory": 8192, + "typescript.tsserver.watchOptions": "vscode", + "editor.tabSize": 2, + "editor.rulers": [80], + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + }, + "explorer.sortOrder": "foldersNestsFiles", + "files.autoSave": "off" +} diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..4224695 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,14 @@ +# Initial ownership for the public platform repository. Replace individual +# ownership with maintainer teams as the maintainer group grows. +# +# @note paths here are relative to THIS directory, which becomes the root of +# the published repository. The monorepo keeps its own CODEOWNERS covering the +# wider tree; this file governs reviews on the mirror only. + +* @pdparchitect + +/.github/ @pdparchitect +/packages/ @pdparchitect +/patches/ @pdparchitect +/platform/ @pdparchitect +/stubs/ @pdparchitect diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..a55a436 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,30 @@ +# Code of Conduct + +## Our Standard + +We want participation in the CBK community to be respectful, safe, +and productive. Contributors are expected to: + +- Be considerate and constructive. +- Discuss ideas and code without attacking people. +- Respect different backgrounds, experiences, and levels of expertise. +- Give and receive technical feedback in good faith. +- Protect private, personal, customer, and security-sensitive information. + +Harassment, discrimination, threats, deliberate disruption, and publication of +another person's private information are not acceptable. + +## Enforcement + +Project maintainers may edit or remove contributions, comments, or other +participation that violates these expectations. Repeated or serious violations +may result in temporary or permanent exclusion from project spaces. + +Report conduct concerns privately to `contact@cbk.ai` with +`Code of conduct report` in the subject. Maintainers will review reports fairly +and protect the reporter's privacy as far as practical. + +## Scope + +This code applies in repository discussions, issues, pull requests, community +spaces, and public interactions where someone represents the project. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..07dc87f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,121 @@ +# Contributing + +Thank you for helping improve CBK. + +Contributions are welcome. By submitting a contribution for inclusion in CBK, +you agree to license it under the Apache License 2.0 and confirm that you have +the right to submit it. + +Start with [Architecture](./docs/architecture.md). It explains the two things +every contributor needs before touching the tree: how the swappable module +packages and the application fit together, and the conventions - the two +routers, the `lib/` naming scheme, new TypeScript source with new JavaScript +tests - that are not self-evident from the directory layout. + +## Before starting + +- Search existing issues and pull requests before beginning work. +- Open an issue before a large feature, architectural, or behavioral change - + the issue is where the shape gets agreed, the pull request is where it gets + reviewed. +- Report security vulnerabilities privately, per [SECURITY.md](./SECURITY.md), + never through the issue tracker. +- Keep changes focused. One concern per pull request; unrelated cleanup slows + the review of both. + +## Branches + +Open contributor pull requests against `next`, the development branch. The +`main` branch is the stable release branch and accepts reviewed promotions from +`next`, not direct feature pull requests. + +## Development setup + +Node.js 24.20 and pnpm 11.24 or later, plus Git LFS - binary assets (images, +fonts, sample documents) are LFS pointers, so run `git lfs install` before +cloning or `git lfs pull` afterwards. + +```sh +pnpm install +``` + +To run the application: + +```sh +cd platform +cp .env.example .env +pnpm db:push # provision the local SQLite database +pnpm dev # → http://127.0.0.1:8080 +``` + +A fresh checkout boots after copying `.env.example`, with no vendor or +deployment-specific configuration: the module defaults are a working +vendor-free deployment (email prints to the console, caching is in-process, +queue delivery is immediate and non-durable, and there is no plan or billing +concept). The one default that needs a backing service to do anything is +storage. The public module speaks the S3 protocol, so file flows refuse at the +point of use until the storage block in `.env.example` is uncommented (it +points at the Compose `garage` service: `docker compose up garage +garage-init`). Anything that needs credentials documents them in its own +package README. + +Alternatively, `docker compose up` at this root is the complete default stack - +the dev server with SQLite, Redis, Qdrant and Garage - `docker compose up redis +qdrant garage garage-init` starts the backing services only for host-side +development, and `docker compose --profile distro up --build platform` builds +and serves the full compiled stack. + +## Quality checks + +The CI quality gate (`.github/workflows/_verify.yaml`) runs on every pull +request, and it is exactly what a fresh checkout gets: + +```sh +# every package +pnpm install --frozen-lockfile +pnpm audit --prod --audit-level low +pnpm turbo run build --continue --filter='!@chatbotkit/platform' +pnpm turbo run lint --continue --filter='!@chatbotkit/platform' +pnpm turbo run check --continue --filter='!@chatbotkit/platform' +pnpm turbo run test --continue --filter='!@chatbotkit/platform' + +# the application - from .env.example and a fresh SQLite database +cd platform +pnpm lint # eslint across app and scripts +pnpm check # typescript, no emit +pnpm test # unit suite with coverage +``` + +The package steps exclude the application only because it has its own steps: +the gate copies `.env.example`, pushes an empty SQLite database through the +community database module, generates the client, and then lints, type-checks +and tests the application against that clean-clone module graph. + +**Not in the gate: the application build, a formatting check, and the +self-host smoke test.** The image-publication workflow builds and smoke-tests +the application on pushes to `next`; if your change touches build +configuration, run `pnpm build` locally before you push. + +Two more practical consequences. If you change any `package.json`, refresh +`pnpm-lock.yaml` in the same commit or the frozen install fails. And new +behavior or a bug fix should come with a test - unit tests are `*.utest.js` or +`*.utest.jsx` co-located with their source in the application and `*.test.js` +in packages, written in JavaScript by convention (see `docs/architecture.md` +for why). + +Inside `platform/`, the narrower loops are `pnpm check`, `pnpm lint`, +`pnpm test:unit path/to/file.utest.js`, and `pnpm storybook`. + +## Pull requests + +- Explain the problem and the outcome, not just the diff. +- Link the relevant issue or discussion. +- Describe how the change was verified. +- Call out schema changes, compatibility concerns, security implications, and + follow-up work explicitly. +- Never include credentials, production data, customer information, or + generated build artifacts. + +## Conduct + +Participation is covered by [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSING.md b/LICENSING.md new file mode 100644 index 0000000..58d3e4a --- /dev/null +++ b/LICENSING.md @@ -0,0 +1,19 @@ +# Licensing + +The `platform/` distribution is licensed under the +[Apache License, Version 2.0](./LICENSE), except for the independently licensed +components listed below. The repository attribution is recorded in +[NOTICE](./NOTICE). + +## Scoped licences + +| Path | Licence | Licence file | +| ---------------------------- | ------- | -------------------------------------------------------------------------- | +| `packages/cloak/` | MIT | [`packages/cloak/LICENSE.md`](./packages/cloak/LICENSE.md) | +| `packages/mcp-widgets/` | MIT | [`packages/mcp-widgets/LICENSE`](./packages/mcp-widgets/LICENSE) | +| `packages/react-prompt-kit/` | MIT | [`packages/react-prompt-kit/LICENSE`](./packages/react-prompt-kit/LICENSE) | +| `stubs/` | ISC | [`stubs/LICENSE`](./stubs/LICENSE) | + +The scoped licence for each path takes precedence over the repository licence +for files within that path. Package manifests declare the same SPDX licence +identifier as their effective licence. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..0ca0cfc --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +ChatBotKit Platform +Copyright 2026 CBK.AI LTD + +Licensed under Apache License 2.0. See LICENSE for terms. diff --git a/README.md b/README.md index 02ddf20..e108ad4 100644 --- a/README.md +++ b/README.md @@ -1 +1,99 @@ -# platform \ No newline at end of file +
+ + + + ChatBotKit + + +
+ +

AI platform in a Box

+ +

+ A modern, sovereign AI backend for products
+ and enterprise deployments. +

+ +

+ Node 24+ + pnpm 11 + TypeScript 6 + Next.js 16 + Docker Compose +

+ +

+ Run it · + Try it · + Documentation · + Architecture · + Contributing · +

+ +
+ +

+ AI Platform +

+ +Get the breadth of a managed AI platform with control over the infrastructure, +data and extension points. Use it behind customer products, internal systems, +and regulated deployments without handing the AI control plane to a managed +provider. + +## A complete platform + +- Agent builder and runtime +- Multi-provider model gateway +- Knowledge ingestion and retrieval +- More than 200 typed integrations +- MCP, OpenAPI, GraphQL and code tools +- Sandboxed code and shell execution +- Web widgets, portals and messaging channels +- REST and GraphQL APIs, webhooks and generated client types +- Node.js, Python and Go SDKs and a Terraform provider +- Authentication, users, teams, contacts and multi-tenant identity +- Access control, moderation, PII protection and audit +- Traces, events, ratings, usage and operational logs +- Replaceable database, storage, cache, queue and vector infrastructure + +## Run it + +Run the complete prebuilt stack with one command - no checkout, no build: + +```bash +docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest up +``` + +Open . Sign in with any email address and read the +six-digit code from the platform container log: + +```bash +docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest logs platform +``` + +See [Deployment](./docs/deployment.md) for details. + +## Local development + +Binary assets are stored with Git LFS, so install it (`git lfs install`) +before cloning. From a fresh checkout, run + +```bash +docker compose up +``` + +Open . See +[Getting started](./docs/getting-started.md) for host-side development, +storage configuration and the first model connection. + +## Documentation + +- [Getting started](./docs/getting-started.md) +- [Deployment and production status](./docs/deployment.md) +- [Module defaults](./docs/module-defaults.md) +- [Operator configuration](./docs/configuration.md) +- [Architecture and repository map](./docs/architecture.md) +- [Licensing](./LICENSING.md) +- [Contributing](./CONTRIBUTING.md) +- [Security](./SECURITY.md) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3d8da84 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Reporting a Vulnerability + +Do not report security vulnerabilities in a public issue or discussion. + +Email `contact@cbk.ai` with `Security report` in the subject. Include the +affected component, reproduction steps, potential impact, and any suggested +mitigation. Avoid including sensitive customer data unless it is necessary to +understand the issue. + +We will acknowledge the report as soon as practical, investigate it, and +coordinate disclosure after a fix or mitigation is available. + +## Scope + +This policy covers source maintained in this repository. Product account, +billing, and general support requests should follow [SUPPORT.md](./SUPPORT.md). diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..a5ee489 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,24 @@ +# Support + +## Source Code Questions + +Use a GitHub issue for reproducible defects, documentation problems, and +well-scoped feature proposals related to this repository. Search for an +existing issue before opening a new one and include enough detail to reproduce +the behavior. + +## Product and Account Support + +For the hosted ChatBotKit product, account, and billing support, email +`support@chatbotkit.com`. Do not include API keys, passwords, or other secrets. + +## Security Issues + +Security vulnerabilities must be reported privately according to +[SECURITY.md](./SECURITY.md), not through a public support issue. + +## Private Operations + +The maintainers cannot provide public access to production environments, +customer records, internal logs, or private operational systems when helping +with a source code issue. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..921f6fd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,267 @@ +# ============================================================================= +# ChatBotKit Platform - full local stack in one command +# ============================================================================= +# Three ways to use it - no mode ever builds or starts another mode's +# services: +# +# docker compose up # THE DEFAULT: a ready dev +# # server. The checkout is +# # copied into a container, +# # installed, provisioned +# # (SQLite), and `next dev` +# # runs against redis - hot +# # reload, no image build +# docker compose --profile distro up --build platform +# # the full compiled stack. +# # WARNING: this compiles the +# # entire platform - expect +# # 10-15+ minutes on first +# # build depending on hardware +# docker compose --profile distro up --no-build --pull always platform +# # pull the prebuilt community +# # app and initializer images +# docker compose up redis # backing services only, +# # then `pnpm dev` on the host +# +# The dev server mounts the checkout READ-ONLY and copies it into the +# container, installing there - the host tree is never written, so it runs +# identically from a plain checkout and when nested in another workspace. The +# .devcontainer provides the host-side toolchain and works with all three. +# +# The end-user distribution - the image-only stack published as a Compose OCI +# artifact (`docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest +# up -d`) - lives separately in docker/distro/, one folder per package flavor. +# ============================================================================= + +# @note the storage environment every application service shares. The store +# is Garage, spoken to over the plain S3 protocol - see docker/garage/. +# The credentials are the known development values garage-init provisions. +# +# @note presigned upload/download URLs carry this endpoint, so a browser on +# the host needs to resolve it too: add `127.0.0.1 garage` to /etc/hosts to +# use browser-facing file flows with the containerized modes. (When running +# `pnpm dev` on the host instead, point SERVICE_AWS_ENDPOINT at +# http://localhost:3900 in platform/.env and the question does not arise.) +x-storage-env: &storage-env + SERVICE_AWS_ENDPOINT: http://garage:3900 + SERVICE_AWS_REGION: garage + SERVICE_AWS_ACCESS_KEY_ID: GK31e57eba9df26b2e7e1b0eaa + SERVICE_AWS_SECRET_ACCESS_KEY: 9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde + SERVICE_AWS_FORCE_PATH_STYLE: 'true' + FILE_S3_BUCKET_NAME: file + IMAGE_S3_BUCKET_NAME: image + VIDEO_S3_BUCKET_NAME: video + AUDIO_S3_BUCKET_NAME: audio + CONVERSATION_S3_BUCKET_NAME: conversation + NAMESPACE_S3_BUCKET_NAME: namespace + SESSION_S3_BUCKET_NAME: session + SPACE_S3_BUCKET_NAME: space + TEMP_S3_BUCKET_NAME: temp + OUTPUT_S3_BUCKET_NAME: output + +services: + platform: + profiles: ['distro'] + image: ${PLATFORM_IMAGE:-ghcr.io/chatbotkit/platform-community-app:next} + build: + context: . + dockerfile: docker/Dockerfile + args: + # @note override for a real deployment: the site url is baked into the + # build by Next.js + SITE_URL: ${SITE_URL:-http://localhost:3000} + ports: + - '3000:3000' + environment: + <<: *storage-env + NODE_ENV: production + PORT: 3000 + SITE_URL: ${SITE_URL:-http://localhost:3000} + NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000} + # @note left empty, the image generates these secrets on first boot and + # persists them in the platform-data volume - see docker/entrypoint.sh; + # set explicitly to override + NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-} + QUEUE_SECRET: ${QUEUE_SECRET:-} + JWT_TOKEN_SECRET_KEY: ${JWT_TOKEN_SECRET_KEY:-} + PRISMA_DATABASE_URL: file:/data/chatbotkit.db + # @note optional: encrypts stored credentials in the database; unset, + # they are stored as given. See docs/configuration.md, "Encryption at rest" + PRISMA_FIELD_ENCRYPTION_KEY: ${PRISMA_FIELD_ENCRYPTION_KEY:-} + REDIS_URL: redis://redis:6379 + QDRANT_URL: http://qdrant:6333 + # @note extra configuration (provider keys and the like) goes here; the + # file is optional. Values can also be persisted in the platform-data + # volume with `docker compose run --rm --no-deps platform setup`; the + # environment and this file win over them - see docker/entrypoint.sh + env_file: + - path: .env + required: false + volumes: + - platform-data:/data + depends_on: + db-init: + condition: service_completed_successfully + redis: + condition: service_healthy + qdrant: + condition: service_healthy + garage-init: + condition: service_completed_successfully + restart: unless-stopped + healthcheck: + # Use node for the healthcheck since the alpine image has no wget/curl + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://localhost:3000/').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 90s + + db-init: + profiles: ['distro'] + image: ${PLATFORM_INIT_IMAGE:-ghcr.io/chatbotkit/platform-community-init:next} + build: + context: . + dockerfile: docker/Dockerfile + target: initializer + environment: + PRISMA_DATABASE_URL: file:/data/chatbotkit.db + volumes: + - platform-data:/data + restart: 'no' + + dev: + image: mcr.microsoft.com/devcontainers/javascript-node:24-bookworm + working_dir: /workspace + # @note the checkout is mounted READ-ONLY at /src and copied into the + # container, where the install lives - the host tree is never written. A + # background loop re-syncs edits every 2 seconds for hot reload. + # Provisioning (.env from the example, SQLite client and schema) happens + # here in the compose file; the application itself has no provisioning + # logic. `corepack pnpm` dispatches the exact version pinned in + # package.json's packageManager field, immune to the image's own pnpm. + command: + - bash + - -lc + - >- + command -v rsync >/dev/null || { apt-get update -qq && apt-get install -y -qq rsync >/dev/null; }; + SYNC="rsync -a --delete + --exclude node_modules --exclude .next --exclude .turbo + --exclude .dev --exclude .env --exclude .pnpm-store + --exclude .git --exclude prisma/generated --exclude prisma/zod + /src/ /workspace/"; + $$SYNC && + corepack pnpm install && + { [ -f platform/.env ] || cp platform/.env.example platform/.env; } && + sed -i "s|^PRISMA_DATABASE_URL=.*|PRISMA_DATABASE_URL=file:/workspace/platform/.dev/platform.db|" platform/.env && + { ls node_modules/.pnpm/@pothos+plugin-prisma*/node_modules/@pothos/plugin-prisma/generated.js >/dev/null 2>&1 || corepack pnpm -F @chatbotkit-dev/db db:gen; } && + { [ -f platform/.dev/platform.db ] || { mkdir -p platform/.dev && PRISMA_DATABASE_URL=file:/workspace/platform/.dev/platform.db corepack pnpm -F @chatbotkit-dev/db db:push; }; } && + { while true; do sleep 2; $$SYNC; done & } && + corepack pnpm -F @chatbotkit/platform dev + environment: + <<: *storage-env + REDIS_URL: redis://redis:6379 + QDRANT_URL: http://qdrant:6333 + APP_PORT: 8080 + # @note the store lives in a volume so installs stay warm across + # container recreations + NPM_CONFIG_STORE_DIR: /pnpm-store + volumes: + - .:/src:ro + - dev-workspace:/workspace + - dev-pnpm-store:/pnpm-store + ports: + - '127.0.0.1:8080:8080' + depends_on: + redis: + condition: service_healthy + qdrant: + condition: service_healthy + garage-init: + condition: service_completed_successfully + + redis: + image: redis:7-alpine + command: ['redis-server', '--appendonly', 'yes'] + ports: + # @note published on localhost only, so `pnpm dev` on the host can use + # this same Redis (REDIS_URL=redis://localhost:6379) without building + # anything + - '127.0.0.1:6379:6379' + volumes: + - redis-data:/data + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + qdrant: + image: qdrant/qdrant:v1.15.1 + ports: + # @note published on localhost only, so `pnpm dev` on the host can use + # this same Qdrant (QDRANT_URL=http://localhost:6333) without building + # anything + - '127.0.0.1:6333:6333' + volumes: + - qdrant-data:/qdrant/storage + healthcheck: + # @note the qdrant image ships no curl or wget, so the check is bash + # opening a TCP connection to the HTTP port + test: ['CMD-SHELL', 'bash -c ": > /dev/tcp/127.0.0.1/6333" || exit 1'] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + garage: + image: dxflrs/garage:v2.1.0 + ports: + # @note published on localhost only, so `pnpm dev` on the host can use + # this same store (SERVICE_AWS_ENDPOINT=http://localhost:3900) without + # building anything + - '127.0.0.1:3900:3900' + volumes: + - ./docker/garage/garage.toml:/etc/garage.toml:ro + - garage-data:/var/lib/garage + healthcheck: + # @note the image is a bare binary - no shell - so the check is the + # garage CLI talking to the daemon over RPC + test: ['CMD', '/garage', '-c', '/etc/garage.toml', 'status'] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + garage-init: + # @note one-shot: provisions the single-node layout, the development + # access key and one bucket per storage scope through Garage's admin API. + # Idempotent, so it reruns harmlessly on every `up`. + image: node:24.20.0-alpine + command: ['node', '/init.mjs'] + environment: + GARAGE_ADMIN_URL: http://garage:3903 + GARAGE_ADMIN_TOKEN: dev-admin-token + STORAGE_ACCESS_KEY_ID: GK31e57eba9df26b2e7e1b0eaa + STORAGE_SECRET_ACCESS_KEY: 9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde + volumes: + - ./docker/garage/init.mjs:/init.mjs:ro + depends_on: + garage: + condition: service_healthy + restart: 'no' + +volumes: + platform-data: + redis-data: + qdrant-data: + garage-data: + dev-pnpm-store: + dev-workspace: diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..3177baf --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,206 @@ +# ============================================================================= +# ChatBotKit Platform Image (standalone) +# ============================================================================= +# Builds the platform application from this folder alone, using this folder's +# own workspace manifests and lockfile - it does not reach into any parent +# repository, so it works identically in the public repository and inside the +# monorepo. +# +# In this context every `@chatbotkit-dev/*` module resolves to the open +# default implementation in ./packages: SQLite for the database, the +# in-process or Redis key-value store, and so on. The docker-compose.yml next +# to this file wires the full stack together; to build by hand: +# +# docker build -f Dockerfile -t chatbotkit-platform . # from this folder +# +# This is the only container image. A hosted-deployment image with the +# internal module overrides would be a deliberate variant of this file built +# from the monorepo root, not a separate lineage. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Stage 0: Base +# ----------------------------------------------------------------------------- + +FROM node:24.20.0-alpine AS base + +# Required system dependencies for native modules (better-sqlite3 builds from +# source on alpine) +RUN apk add --no-cache libc6-compat python3 make g++ + +# Enable the pnpm version required by package.json's packageManager field. +RUN corepack enable && corepack prepare pnpm@11.24.0 --activate + +WORKDIR /app + +# ----------------------------------------------------------------------------- +# Stage 1: Workspace dependencies +# ----------------------------------------------------------------------------- + +FROM base AS dependencies + +# @note image builds are non-interactive: pnpm must auto-confirm prompts +# (e.g. recreating the modules directory when hoist settings change) instead +# of aborting for want of a TTY +ENV CI=true + +WORKDIR /app + +# Workspace configuration first, for layer caching +COPY pnpm-workspace.yaml pnpm-lock.yaml package.json .pnpmfile.cjs turbo.json ./ +COPY patches ./patches/ +COPY stubs ./stubs/ + +# @note the one local file: dependency that lives inside the application +# source - fetch scans local dependencies, so it must exist before fetch runs +COPY platform/eslint ./platform/eslint/ + +# @note fetch resolves from the lockfile alone, so editing source does not +# invalidate the dependency layer - the install below links offline from the +# fetched store +RUN pnpm fetch + +# Workspace members +COPY packages ./packages/ +COPY platform ./platform/ + +RUN pnpm install --frozen-lockfile --offline + +# ----------------------------------------------------------------------------- +# Stage 2: Application builder +# ----------------------------------------------------------------------------- + +FROM dependencies AS builder + +ENV NEXT_OUTPUT_MODE=standalone +ENV NODE_ENV=production +ENV NEXT_TELEMETRY_DISABLED=1 + +# @note the Next.js build needs far more heap than node's ~4GB default - the +# hosted CI grants it 80% of a heavy runner's memory. Lower this only if the +# build host must, and expect the build to fail under ~8GB. +ARG NODE_HEAP_MB=16384 + +# @note the emfile guard makes fs.promises retry when the descriptor table +# is full - next's trace collection otherwise dies with EMFILE on hosts with +# modest ulimits, even after the raise below +ENV NODE_OPTIONS="--max-old-space-size=$NODE_HEAP_MB --require /app/platform/scripts/emfile-guard.cjs" + +# @note the build asserts a site url; override for a real deployment. Secrets +# should NOT be passed as build args - use BuildKit secret mounts. +ARG SITE_URL=http://localhost:3000 +ENV SITE_URL=$SITE_URL + +# @note source maps ship without source content by default; pass 'full' +# explicitly to embed the source for debuggable self-hosted images +ARG BUILD_SOURCEMAPS=nosources +ENV BUILD_SOURCEMAPS=$BUILD_SOURCEMAPS + +# @note build-time database: the graphql codegen and static generation +# construct the client, so they need a real (empty) schema to point at. This +# file never leaves the builder stage - the runtime database comes from +# PRISMA_DATABASE_URL at run time (the compose file sets it to the volume). +ENV PRISMA_DATABASE_URL=file:/tmp/build.db +RUN pnpm -F @chatbotkit-dev/db db:push + +# @note unit tests, lint and type validation run as separate CI gates rather +# than during image assembly. Keep the image build focused on producing the +# deployable artifact from an already-validated checkout. +ENV SKIP_BUILD_TESTS=true +ENV SKIP_LINT=1 +ENV SKIP_CHECK=1 + +WORKDIR /app/platform + +# @note the build imports every route module while collecting page data, and +# some modules assert their configuration at import. The example env provides +# documented placeholder values for exactly that; the runtime environment +# comes from the compose file and .env at run time, never from here. +RUN cp .env.example .env + +# @note the cache mount persists Next's incremental compiler cache across +# image rebuilds on the build host, so a source edit costs an incremental +# build rather than a cold one (BuildKit required, which is docker's default) +# @note next's build tracing opens thousands of files concurrently; raise +# the descriptor limit to the hard maximum or the trace dies with EMFILE +RUN --mount=type=cache,target=/app/platform/.next/cache,uid=0,gid=0 ulimit -n $(ulimit -H -n) && pnpm build + +# @note Next copies the builder's .env into the standalone output, where its +# documented placeholder values would silently backfill anything the runtime +# environment leaves unset. Strip it so a missing variable fails loudly at +# boot instead. +RUN rm -f .next/standalone/platform/.env .next/standalone/platform/.env.production + +# ----------------------------------------------------------------------------- +# Stage 3: Database initializer +# ----------------------------------------------------------------------------- + +# Produce a portable package containing the database implementation, schema +# tooling and workspace dependencies, without carrying the application build +# or pnpm store into the published initializer image. +FROM dependencies AS initializer-deployer + +WORKDIR /app + +RUN pnpm --filter @chatbotkit-dev/db deploy --legacy /initializer + +FROM node:24.20.0-alpine AS initializer + +RUN apk add --no-cache libc6-compat + +WORKDIR /app + +COPY --from=initializer-deployer /initializer ./ + +# @note the Garage provisioning script rides along so the distribution stack +# (docker/distro/*) can run it from this image instead of bind-mounting the +# checkout - a published Compose OCI artifact cannot carry bind mounts +COPY docker/garage/init.mjs /garage-init.mjs + +RUN mkdir -p /data + +# @note schema derivation writes inside the installed database package, so the +# one-shot initializer runs as root. The final chown hands SQLite files to the +# uid/gid used by the non-root application image. Network databases ignore the +# otherwise empty /data directory. +CMD ["sh", "-c", "npm run db:push && chown -R 1001:1001 /data"] + +# ----------------------------------------------------------------------------- +# Stage 4: Application +# ----------------------------------------------------------------------------- + +FROM node:24.20.0-alpine AS application + +WORKDIR /app + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +ENV NODE_ENV=production +ENV NEXT_TELEMETRY_DISABLED=1 +ENV PORT=3000 +ENV HOSTNAME="0.0.0.0" + +# Standalone output preserves the workspace structure (platform/...) +COPY --from=builder --chown=nextjs:nodejs /app/platform/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/platform/public ./platform/public +COPY --from=builder --chown=nextjs:nodejs /app/platform/.next/static ./platform/.next/static + +RUN mkdir -p platform/.next +RUN chown nextjs:nodejs platform/.next + +# Writable location for the SQLite database volume +RUN mkdir -p /data +RUN chown nextjs:nodejs /data + +# @note generates missing secrets on first boot - see docker/entrypoint.sh; +# a no-op for deployments that set them +COPY --chmod=755 docker/entrypoint.sh /entrypoint.sh + +USER nextjs + +EXPOSE 3000 + +ENTRYPOINT ["/entrypoint.sh"] + +CMD ["node", "platform/server.js"] diff --git a/docker/distro/AGENTS.md b/docker/distro/AGENTS.md new file mode 100644 index 0000000..a456d3d --- /dev/null +++ b/docker/distro/AGENTS.md @@ -0,0 +1,60 @@ +# Distribution stacks + +One folder per package flavor. Each folder holds a self-contained, image-only +`compose.yml` that CI publishes as a Compose OCI artifact to +`ghcr.io/chatbotkit/platform-`, digest-pinned to the matching +application and initializer images. Consumers run the whole stack with: + +```bash +docker compose -f oci://ghcr.io/chatbotkit/platform-:latest up -d +``` + +## Naming + +- One grammar for every published package: + `platform-[-]:>`. The name says + what it is; the tag says which build it is. +- The bare `platform-` name is the Compose distribution artifact - + the one consumers type. Its component images are `platform--app` + and `platform--init` (e.g. `platform-community-app:next`). +- Tags are single-axis: channels `next`, `main` and `latest` (follows + `main`), plus immutable `sha-` tags on the component images. +- Never mix flavors or revisions between the application and initializer + images inside one stack - schema and client must come from the same build. + +## Hard constraints + +- **No bind mounts, no `build:`, no profiles.** `docker compose publish` + rejects bind mounts, and plain `up -d` must start the stack. Configuration + is inlined via the top-level `configs` element; scripts ship inside + published images (the Garage provisioning script rides in the initializer + image as `/garage-init.mjs`). +- Inline `configs.content` is interpolated: `${...}` must be escaped as + `$${...}`. This is why scripts are baked into images instead of inlined. +- The inline Garage configuration duplicates `docker/garage/garage.toml` - + keep them in sync. +- `env_file` entries are dropped at publish; operator overrides flow through + interpolated variables (`${VAR:-default}`), which Compose presents to the + consumer as a confirmation table on `up`. + +## Adding a flavor + +1. A flavor is a compile-time package selection (e.g. the database package), + not a runtime service swap. Add the Docker build path for it first (see + the note in `docker/Dockerfile` and the matrix comment in the deploy + workflow). +2. Create `docker/distro//compose.yml`, starting from `community/`. + Swap only what the flavor changes (e.g. a `postgres` service replacing the + SQLite volume); keep service names, healthchecks and the variable surface + consistent. +3. Add the flavor to the matrix in + `.github/workflows/publish-ghcr-platform.yaml` with its application and + initializer targets. Everything downstream - images, smoke test, artifact + publish - is parameterized on `matrix.flavor` and needs no other change. +4. Verify before relying on CI: `docker compose -f + docker/distro//compose.yml config --quiet`, then publish to a + scratch registry (ttl.sh works) and boot the artifact from an empty + directory with `docker compose -f oci://... up -d -y`. + +Do not create a flavor folder before the flavor's image build exists - the +convention is the structure; empty placeholders are not. diff --git a/docker/distro/community/compose.yml b/docker/distro/community/compose.yml new file mode 100644 index 0000000..a6744a8 --- /dev/null +++ b/docker/distro/community/compose.yml @@ -0,0 +1,231 @@ +# ============================================================================= +# ChatBotKit Platform - community distribution stack +# ============================================================================= +# The self-contained, image-only Compose application published to +# ghcr.io/chatbotkit/platform-community as an OCI artifact. Consumers run the +# whole platform - application, database initialization, Redis, Qdrant and +# Garage object storage - with a single command and no checkout: +# +# docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest up +# +# The same file runs directly from the repository: +# +# docker compose -f docker/distro/community/compose.yml up -d +# +# Everything a service needs travels inside this file or inside a published +# image: configuration is inlined through the top-level `configs` element and +# the Garage provisioning script ships inside the initializer image, so there +# are no bind mounts - the one restriction `docker compose publish` enforces. +# The developer stack with profiles, source builds and hot reload lives in +# docker-compose.yml at the repository root; this file never builds anything. +# +# One folder per package flavor lives under docker/distro/. Each publishes as +# ghcr.io/chatbotkit/platform-, pinned by CI to the matching +# application and initializer image digests of the same flavor. +# ============================================================================= + +# @note the storage environment every application service shares. The store +# is Garage, spoken to over the plain S3 protocol. Left empty, the access key +# is generated by Garage on first boot and persisted in the platform-data +# volume (garage-init provisions it, the application entrypoint sources it); +# set STORAGE_ACCESS_KEY_ID / STORAGE_SECRET_ACCESS_KEY to use a fixed pair. +# +# @note presigned upload/download URLs carry this endpoint, so a browser on +# the host needs to resolve it too: add `127.0.0.1 garage` to /etc/hosts to +# use browser-facing file flows. +x-storage-env: &storage-env + SERVICE_AWS_ENDPOINT: http://garage:3900 + SERVICE_AWS_REGION: garage + SERVICE_AWS_ACCESS_KEY_ID: ${STORAGE_ACCESS_KEY_ID:-} + SERVICE_AWS_SECRET_ACCESS_KEY: ${STORAGE_SECRET_ACCESS_KEY:-} + SERVICE_AWS_FORCE_PATH_STYLE: 'true' + FILE_S3_BUCKET_NAME: file + IMAGE_S3_BUCKET_NAME: image + VIDEO_S3_BUCKET_NAME: video + AUDIO_S3_BUCKET_NAME: audio + CONVERSATION_S3_BUCKET_NAME: conversation + NAMESPACE_S3_BUCKET_NAME: namespace + SESSION_S3_BUCKET_NAME: session + SPACE_S3_BUCKET_NAME: space + TEMP_S3_BUCKET_NAME: temp + OUTPUT_S3_BUCKET_NAME: output + +services: + platform: + image: ${PLATFORM_IMAGE:-ghcr.io/chatbotkit/platform-community-app:next} + ports: + - '3000:3000' + environment: + <<: *storage-env + NODE_ENV: production + PORT: 3000 + SITE_URL: ${SITE_URL:-http://localhost:3000} + NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000} + # @note left empty, the image generates these secrets on first boot and + # persists them in the platform-data volume; set explicitly to override + NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-} + QUEUE_SECRET: ${QUEUE_SECRET:-} + JWT_TOKEN_SECRET_KEY: ${JWT_TOKEN_SECRET_KEY:-} + PRISMA_DATABASE_URL: file:/data/chatbotkit.db + # @note optional: encrypts stored credentials in the database; unset, + # they are stored as given. See docs/configuration.md, "Encryption at rest" + PRISMA_FIELD_ENCRYPTION_KEY: ${PRISMA_FIELD_ENCRYPTION_KEY:-} + # @note optional platform-wide provider keys; users can also supply + # their own keys through the application itself. Any variable the + # application honours (further providers, the *_CONFIG seams - see + # docs/configuration.md) can be persisted in the platform-data volume + # instead of a .env file, prompted for or set directly: + # docker compose -f oci://... run --rm --no-deps platform setup + # docker compose -f oci://... run --rm --no-deps platform setup OPENROUTER_MODELS_API_KEY=... + # Values given here or in .env win over persisted ones - see + # docker/entrypoint.sh. An override file remains the other route: + # docker compose -f oci://... -f my-override.yml up -d + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENROUTER_MODELS_API_KEY: ${OPENROUTER_MODELS_API_KEY:-} + VERCEL_MODELS_API_KEY: ${VERCEL_MODELS_API_KEY:-} + REDIS_URL: redis://redis:6379 + QDRANT_URL: http://qdrant:6333 + volumes: + - platform-data:/data + depends_on: + db-init: + condition: service_completed_successfully + redis: + condition: service_healthy + qdrant: + condition: service_healthy + garage-init: + condition: service_completed_successfully + restart: unless-stopped + healthcheck: + # Use node for the healthcheck since the alpine image has no wget/curl + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://localhost:3000/').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 90s + + db-init: + image: ${PLATFORM_INIT_IMAGE:-ghcr.io/chatbotkit/platform-community-init:next} + environment: + PRISMA_DATABASE_URL: file:/data/chatbotkit.db + volumes: + - platform-data:/data + restart: 'no' + + redis: + image: redis:7-alpine + command: ['redis-server', '--appendonly', 'yes'] + volumes: + - redis-data:/data + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + qdrant: + image: qdrant/qdrant:v1.15.1 + volumes: + - qdrant-data:/qdrant/storage + healthcheck: + # @note the qdrant image ships no curl or wget, so the check is bash + # opening a TCP connection to the HTTP port + test: ['CMD-SHELL', 'bash -c ": > /dev/tcp/127.0.0.1/6333" || exit 1'] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + garage: + image: dxflrs/garage:v2.1.0 + ports: + # @note published on localhost so presigned URLs (which carry the + # garage:3900 endpoint) work from a host browser with the /etc/hosts + # entry described above + - '127.0.0.1:3900:3900' + configs: + - source: garage-config + target: /etc/garage.toml + volumes: + - garage-data:/var/lib/garage + healthcheck: + # @note the image is a bare binary - no shell - so the check is the + # garage CLI talking to the daemon over RPC + test: ['CMD', '/garage', '-c', '/etc/garage.toml', 'status'] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + garage-init: + # @note one-shot: provisions the single-node layout, the development + # access key and one bucket per storage scope through Garage's admin API. + # Idempotent, so it reruns harmlessly on every `up`. The script ships + # inside the initializer image (the same one db-init runs), so this stack + # needs no bind mount into a checkout. + image: ${PLATFORM_INIT_IMAGE:-ghcr.io/chatbotkit/platform-community-init:next} + command: ['node', '/garage-init.mjs'] + environment: + GARAGE_ADMIN_URL: http://garage:3903 + GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-admin-token} + STORAGE_ACCESS_KEY_ID: ${STORAGE_ACCESS_KEY_ID:-} + STORAGE_SECRET_ACCESS_KEY: ${STORAGE_SECRET_ACCESS_KEY:-} + volumes: + # @note shares the application volume so the generated access key + # persists where the application entrypoint can source it + - platform-data:/data + depends_on: + garage: + condition: service_healthy + restart: 'no' + +configs: + # @note the inline copy of docker/garage/garage.toml, embedded so the + # published OCI artifact is self-contained. Keep the two in sync. + garage-config: + content: | + # Garage (S3-compatible object storage) - single-node configuration. + # + # WARNING: the rpc_secret and admin token default to known development + # values. Neither port is published outside the compose network, but a + # hardened setup overrides them (GARAGE_RPC_SECRET, `openssl rand -hex + # 32`, and GARAGE_ADMIN_TOKEN) - and a real deployment almost certainly + # runs a real store with replication rather than this single-node + # layout. + + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "sqlite" + + replication_factor = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + rpc_secret = "${GARAGE_RPC_SECRET:-1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec}" + + [s3_api] + # @note the region is part of every SigV4 signature: SERVICE_AWS_REGION + # must match it, or every request fails authentication + s3_region = "garage" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage.localhost" + + [admin] + # @note the garage-init service provisions the layout, the access key + # and the buckets through this API + api_bind_addr = "[::]:3903" + admin_token = "${GARAGE_ADMIN_TOKEN:-dev-admin-token}" + +volumes: + platform-data: + redis-data: + qdrant-data: + garage-data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..a1acdcb --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,171 @@ +#!/bin/sh +set -e + +# Configuration precedence, highest first: +# 1. the container environment - Compose `environment:`, `env_file:`, +# `docker run -e`, shell exports Compose interpolates +# 2. $DATA_DIR/config.env - operator values persisted in the data volume, +# written by `setup` below or by hand; one KEY=VALUE per line, no quoting +# 3. $DATA_DIR/.secrets.env - values generated on first boot +# An empty environment value counts as unset, so a Compose default such as +# `${OPENAI_API_KEY:-}` never masks a persisted value. + +DATA_DIR="${PLATFORM_DATA_DIR:-/data}" +CONFIG_FILE="$DATA_DIR/config.env" +SECRETS_FILE="$DATA_DIR/.secrets.env" +STORAGE_FILE="$DATA_DIR/.storage.env" + +DEFAULT_SETUP_KEYS="OPENAI_API_KEY OPENROUTER_MODELS_API_KEY VERCEL_MODELS_API_KEY" + +is_empty() { + eval "[ -z \"\${$1}\" ]" +} + +is_valid_key() { + case "$1" in + '' | [0-9]* | *[!A-Za-z0-9_]*) return 1 ;; + esac +} + +config_get() { + [ -f "$CONFIG_FILE" ] || return 0 + sed -n "s/^$1=//p" "$CONFIG_FILE" | tail -n 1 +} + +# config_set KEY VALUE - an empty VALUE removes the entry +config_set() { + umask 077 + mkdir -p "$DATA_DIR" + { + if [ -f "$CONFIG_FILE" ]; then + grep -v "^$1=" "$CONFIG_FILE" || true + fi + if [ -n "$2" ]; then + printf '%s=%s\n' "$1" "$2" + fi + } > "$CONFIG_FILE.tmp" + mv "$CONFIG_FILE.tmp" "$CONFIG_FILE" +} + +config_load() { + [ -f "$CONFIG_FILE" ] || return 0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in '' | '#'*) continue ;; esac + key="${line%%=*}" + value="${line#*=}" + is_valid_key "$key" || continue + if is_empty "$key"; then + export "$key=$value" + fi + done < "$CONFIG_FILE" +} + +# read_secret KEY - prompts on the terminal without echo; Enter keeps the +# current value, a single "-" clears it +read_secret() { + current="$(config_get "$1")" + if [ -n "$current" ]; then + hint="currently set, ends with ${current#"${current%????}"}" + else + hint="not set" + fi + printf '%s [%s]: ' "$1" "$hint" >&2 + if [ -t 0 ]; then + trap 'stty echo; exit 130' INT TERM + stty -echo + IFS= read -r input + stty echo + trap - INT TERM + echo >&2 + else + IFS= read -r input + fi + case "$input" in + '') ;; + '-') config_set "$1" '' ; echo "$1 cleared" >&2 ;; + *) config_set "$1" "$input" ; echo "$1 saved" >&2 ;; + esac +} + +# setup [KEY | KEY=VALUE ...] - persists operator values in $CONFIG_FILE. +# Bare keys prompt; KEY=VALUE sets without prompting (KEY= removes). With no +# arguments the default provider keys are prompted for. +run_setup() { + [ $# -gt 0 ] || set -- $DEFAULT_SETUP_KEYS + for arg in "$@"; do + key="${arg%%=*}" + if ! is_valid_key "$key"; then + echo "ERROR: invalid variable name: $key" >&2 + exit 1 + fi + case "$arg" in + *=*) config_set "$key" "${arg#*=}" ; echo "$key saved" >&2 ;; + *) + if [ ! -t 0 ]; then + echo "ERROR: no terminal to prompt for $key - pass $key=VALUE or run with -it" >&2 + exit 1 + fi + read_secret "$key" + ;; + esac + done + echo "INFO: values persisted in $CONFIG_FILE - restart the platform service to apply" >&2 +} + +if [ "$1" = "setup" ]; then + shift + run_setup "$@" + exit 0 +fi + +config_load + +# Fills empty NEXTAUTH_SECRET / QUEUE_SECRET / JWT_TOKEN_SECRET_KEY with values +# generated once and persisted in $DATA_DIR, so sessions, queue signatures and +# issued tokens survive restarts as long as it is a volume. +if [ -z "$NEXTAUTH_SECRET" ] || [ -z "$QUEUE_SECRET" ] || [ -z "$JWT_TOKEN_SECRET_KEY" ]; then + generate_secret() { + node -e 'process.stdout.write(require("node:crypto").randomBytes(32).toString("hex"))' + } + + if [ ! -f "$SECRETS_FILE" ]; then + umask 077 + printf 'GENERATED_NEXTAUTH_SECRET=%s\nGENERATED_QUEUE_SECRET=%s\n' "$(generate_secret)" "$(generate_secret)" > "$SECRETS_FILE" + fi + + . "$SECRETS_FILE" + + # @note volumes created before the JWT secret was generated here lack it + if [ -z "$GENERATED_JWT_TOKEN_SECRET_KEY" ]; then + GENERATED_JWT_TOKEN_SECRET_KEY="$(generate_secret)" + printf 'GENERATED_JWT_TOKEN_SECRET_KEY=%s\n' "$GENERATED_JWT_TOKEN_SECRET_KEY" >> "$SECRETS_FILE" + fi + + if [ -z "$NEXTAUTH_SECRET" ]; then + echo "WARNING: NEXTAUTH_SECRET is not set - using a generated value persisted in $SECRETS_FILE" >&2 + export NEXTAUTH_SECRET="$GENERATED_NEXTAUTH_SECRET" + fi + + if [ -z "$QUEUE_SECRET" ]; then + echo "WARNING: QUEUE_SECRET is not set - using a generated value persisted in $SECRETS_FILE" >&2 + export QUEUE_SECRET="$GENERATED_QUEUE_SECRET" + fi + + if [ -z "$JWT_TOKEN_SECRET_KEY" ]; then + echo "WARNING: JWT_TOKEN_SECRET_KEY is not set - using a generated value persisted in $SECRETS_FILE" >&2 + export JWT_TOKEN_SECRET_KEY="$GENERATED_JWT_TOKEN_SECRET_KEY" + fi +fi + +# Storage credentials generated by garage-init land in the shared data +# volume - see docker/garage/init.mjs. +if [ -z "$SERVICE_AWS_ACCESS_KEY_ID" ] && [ -f "$STORAGE_FILE" ]; then + . "$STORAGE_FILE" + + echo "INFO: using generated storage credentials from $STORAGE_FILE" >&2 + + export SERVICE_AWS_ACCESS_KEY_ID="$GENERATED_STORAGE_ACCESS_KEY_ID" + export SERVICE_AWS_SECRET_ACCESS_KEY="$GENERATED_STORAGE_SECRET_ACCESS_KEY" +fi + +exec "$@" diff --git a/docker/garage/garage.toml b/docker/garage/garage.toml new file mode 100644 index 0000000..143961b --- /dev/null +++ b/docker/garage/garage.toml @@ -0,0 +1,36 @@ +# Garage (S3-compatible object storage) - single-node LOCAL configuration, +# used by the docker compose stack. The storage module speaks the S3 protocol, +# so any S3-compatible store works; Garage is the reference store the compose +# file stands up. +# +# WARNING: the rpc_secret and admin token below are known development values +# for the local stack only. A real deployment generates its own +# (`openssl rand -hex 32`) - and almost certainly runs a real store with +# replication rather than this single-node layout. +# +# @note the distribution stack carries an inline copy of this file (a +# published Compose OCI artifact cannot bind-mount it) - keep +# docker/distro/community/compose.yml's garage-config in sync. + +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" + +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +rpc_secret = "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + +[s3_api] +# @note the region is part of every SigV4 signature: SERVICE_AWS_REGION must +# match it, or every request fails authentication +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" + +[admin] +# @note the garage-init compose service provisions the layout, the access key +# and the buckets through this API +api_bind_addr = "[::]:3903" +admin_token = "dev-admin-token" diff --git a/docker/garage/init.mjs b/docker/garage/init.mjs new file mode 100644 index 0000000..39e7261 --- /dev/null +++ b/docker/garage/init.mjs @@ -0,0 +1,160 @@ +// @note one-shot provisioning for the local Garage store, run by the +// garage-init compose service against Garage's admin API: assign the +// single-node layout, provision the access key, create one bucket per +// storage scope and grant the key access to each. Idempotent - every step +// checks before it acts, so the stack restarts cleanly. +// +// The access key comes from STORAGE_ACCESS_KEY_ID / STORAGE_SECRET_ACCESS_KEY +// when set; otherwise one is generated by Garage on first boot and persisted +// in /data (the volume shared with the application, whose entrypoint sources +// it - see docker/entrypoint.sh). +// +// The bucket list mirrors the scopes in @chatbotkit-dev/storage; the compose +// file carries the matching *_S3_BUCKET_NAME variables. +import { chownSync, readFileSync, writeFileSync } from 'node:fs' + +const base = process.env.GARAGE_ADMIN_URL +const token = process.env.GARAGE_ADMIN_TOKEN + +let accessKeyId = process.env.STORAGE_ACCESS_KEY_ID +let secretAccessKey = process.env.STORAGE_SECRET_ACCESS_KEY + +const SECRETS_FILE = '/data/.storage.env' + +const BUCKETS = [ + 'file', + 'image', + 'video', + 'audio', + 'conversation', + 'namespace', + 'session', + 'space', + 'temp', + 'output', +] + +for (const [name, value] of Object.entries({ + GARAGE_ADMIN_URL: base, + GARAGE_ADMIN_TOKEN: token, +})) { + if (!value) { + console.error(`[garage-init] ${name} is not set`) + process.exit(1) + } +} + +if (Boolean(accessKeyId) !== Boolean(secretAccessKey)) { + console.error( + '[garage-init] STORAGE_ACCESS_KEY_ID and STORAGE_SECRET_ACCESS_KEY must be set together' + ) + + process.exit(1) +} + +async function api(path, body) { + const response = await fetch(base + path, { + ...(body && { method: 'POST', body: JSON.stringify(body) }), + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + }) + + const text = await response.text() + + if (!response.ok) { + throw new Error(`${path} -> ${response.status}: ${text.slice(0, 200)}`) + } + + return text ? JSON.parse(text) : null +} + +// @note the compose healthcheck gates on the RPC port; give the admin API a +// short grace period of its own +let status + +for (let attempt = 0; ; attempt++) { + try { + status = await api('/v2/GetClusterStatus') + + break + } catch (error) { + if (attempt >= 30) { + throw error + } + + await new Promise((resolve) => setTimeout(resolve, 1000)) + } +} + +const node = status.nodes[0] + +if (!node.role) { + await api('/v2/UpdateClusterLayout', { + roles: [{ id: node.id, zone: 'dc1', capacity: 1000000000, tags: [] }], + }) + + const layout = await api('/v2/GetClusterLayout') + + await api('/v2/ApplyClusterLayout', { version: layout.version + 1 }) + + console.log('[garage-init] single-node layout applied') +} + +if (!accessKeyId) { + try { + const saved = readFileSync(SECRETS_FILE, 'utf8') + + accessKeyId = saved.match(/^GENERATED_STORAGE_ACCESS_KEY_ID=(.+)$/m)?.[1] + secretAccessKey = saved.match( + /^GENERATED_STORAGE_SECRET_ACCESS_KEY=(.+)$/m + )?.[1] + } catch { + // no persisted key yet - generate one below + } +} + +if (!accessKeyId) { + const key = await api('/v2/CreateKey', { name: 'platform' }) + + accessKeyId = key.accessKeyId + secretAccessKey = key.secretAccessKey + + writeFileSync( + SECRETS_FILE, + `GENERATED_STORAGE_ACCESS_KEY_ID=${accessKeyId}\nGENERATED_STORAGE_SECRET_ACCESS_KEY=${secretAccessKey}\n`, + { mode: 0o600 } + ) + + // @note the application image runs as uid/gid 1001 and must read the file + chownSync(SECRETS_FILE, 1001, 1001) + + console.log('[garage-init] access key generated') +} + +const keys = await api('/v2/ListKeys') + +if (!keys.some((key) => key.id === accessKeyId)) { + await api('/v2/ImportKey', { accessKeyId, secretAccessKey, name: 'platform' }) + + console.log('[garage-init] access key imported') +} + +const buckets = await api('/v2/ListBuckets') + +for (const alias of BUCKETS) { + let bucket = buckets.find((entry) => entry.globalAliases?.includes(alias)) + + if (!bucket) { + bucket = await api('/v2/CreateBucket', { globalAlias: alias }) + } + + await api('/v2/AllowBucketKey', { + bucketId: bucket.id, + accessKeyId, + permissions: { read: true, write: true, owner: true }, + }) +} + +console.log(`[garage-init] buckets ready: ${BUCKETS.join(' ')}`) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..2e23632 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,32 @@ +# Documentation + +The repository landing page explains what the platform is and who it is for. +The documents here cover the operational detail needed to run and evaluate it. + +## Manuals + +- [docs.cbk.ai](https://docs.cbk.ai) - the platform's technical manuals, + generated from this source: API, SDKs, integrations and every configurable + surface +- [chatbotkit.com/docs](https://chatbotkit.com/docs) - the user manuals for the + hosted ChatBotKit product built on the platform + +## Start here + +- [Getting started](./getting-started.md) - run the complete local stack, sign + in and configure a model +- [Deployment](./deployment.md) - Compose profiles, current production status + and operator responsibilities +- [Module defaults](./module-defaults.md) - what each public module does with + nothing set, and what the distribution flavors change + +## Project guides + +- [Architecture](./architecture.md) - application structure, module boundaries + and contribution branches +- [Configuration](./configuration.md) - operator-owned configuration seams and + database credential encryption +- [Contributing](../CONTRIBUTING.md) - development workflow, quality checks and + pull requests +- [Security](../SECURITY.md) - private vulnerability reporting +- [Support](../SUPPORT.md) - community and hosted-product support boundaries diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..be1b0f4 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,343 @@ +# Architecture + +This repository is a self-contained pnpm monorepo with two kinds of members: +the shared and swappable module packages under `packages/`, and the platform +application under `platform/`. This document explains how they fit together +and the conventions that are not self-evident from the tree. All paths below +are relative to the open-code repository root. + +Read this before concluding anything from the directory layout. The two most +common misreadings - that `pages/` and `app/` are a stalled router migration, +and that the flat `lib/` directory is disorganized - are both wrong, and both +are explained below. + +## Repository layout + +| Path | What it is | +| ----------- | ----------------------------------------------------------------- | +| `packages/` | The `@chatbotkit-dev/*` libraries and module contracts (`*-spec`) | +| `platform/` | The platform application, `@chatbotkit/platform` (Next.js) | +| `stubs/` | Local dependency shims applied at install time | +| `patches/` | pnpm dependency patches | + +The `pnpm-workspace.yaml`, lockfile and overrides in the repository root govern +dependency resolution across the application and its packages. + +## Branches + +The `next` branch is the development branch, and contributor pull requests +target it. The `main` branch is the stable release branch and advances through +a reviewed promotion from `next` after the required checks pass. See +`CONTRIBUTING.md` for the current contribution workflow. + +## Swappable modules + +The platform's deployment-specific behavior - configuration catalogues, +storage, email delivery, caching, database engine - lives behind swappable +modules. A swappable module is three packages, not one: + +| Package | Contains | +| ---------------------- | --------------------------------------------------------------------------------------------------------- | +| `packages/-spec` | The contract, plus shared schemas or derivation code where the contract requires them | +| `packages/` | The public default. Must boot with no configuration; a backing service may be required to use the feature | +| (a deployment's own) | An implementation installed over the public name | + +A deployment replaces a module by adding a pnpm override that resolves +`@chatbotkit-dev/` to its own implementation package. Remove the +override and the platform resolves to the public default and still runs - +that is the property every module preserves, and it is why a fresh checkout +of this repository boots with no deployment-specific or vendor configuration. +Booting is the guarantee, not every feature: the public storage module is an +S3-protocol client that needs an endpoint, credentials and buckets before file +flows work (the Compose stack provisions Garage), and the public queue is +immediate and non-durable - delays, retries and ordering are accepted and +ignored. + +[Module defaults](./module-defaults.md) lists the rest. + +The public defaults differ in what "default" means, and the difference is +deliberate. `@chatbotkit-dev/email` logs messages to the console - a working, +if noisy, delivery path. The plan catalogue (`@/config/limits`, read from the +LIMITS_CONFIG environment variable) defaults to empty, which the platform +reads as "this deployment has no plan concept": every entitlement resolves +without limits and no interface renders a plan name. Defaults describe a +working planless, vendor-free deployment, not a crippled one. + +Module conventions, enforced across the set: + +- Configuration is resolved lazily, on first use, never at module load - so + importing the platform never requires a vendor credential. +- Every module exposes an `assertConfigured` readiness check through the + entry point defined by its contract. The application exercises every + installed module in `platform/tests/config/providers.utest.js`, so the + build and CI fail before an incomplete deployment reaches first use. +- A module's README is the authoritative environment-variable reference. The + application's `.env.example` includes only the values useful for the + supported baseline and common local setup. +- Application code depends on the public package name and contract, never on + a deployment implementation. Runtime-specific behavior is generally + concentrated in in-tree adapters such as `lib/queue.ts`, `lib/storage.ts` + and `prisma/client.ts`. + +## The application + +### Two routers, by design + +`platform/pages/` contains the dashboard, product surfaces and the entire +public API under `pages/api/v1/`. It uses the Pages Router. + +`platform/app/` is the apps runtime, and almost nothing else: a root layout and +`app/apps/`. It uses the App Router because its manifest-driven model fits that +router's layout system. + +The two coexist permanently. This is not an unfinished migration: they are +different products sharing one codebase. The platform is a multi-page +application; the apps runtime is a family of focused applications (chat, code, +tasks, usage, and others), served path-based by default or through configured +application hosts. Manifests define the application, while deployment +configuration owns its host topology. + +### The apps runtime + +Every app is a directory under `app/apps/` carrying an `app.manifest` - a +JSON file whose shape is declared in `app/apps/app.manifest.d.ts`. The +minimal complete example: + +```json +{ + "start": "/apps/connect", + "name": "Connect", + "description": "Connect to your favorite apps and services", + "icon": "@lucide/grid-2x2-plus", + "order": 30, + "category": "main", + "config": {} +} +``` + +The parts of the contract that are not obvious: + +- Manifests are discovered at build time by `next.config.d/apps.config.js`, + which walks `app/apps/`, validates every manifest against a schema, and + generates the host routing from the result. The app's slug is its + directory name. +- `start` is the app's entry path. Host mappings do not live in the manifest; + the deployment's app and shell configuration decides whether that path is + served under the main site or a dedicated host. +- `global` is the app's baseline in every context. `config` supplies defaults + on the dashboard and the app's standalone host; a portal uses its own + global, app and user overlays. The complete precedence rules live in + `lib/app.router.app.config.ts`. +- `order` and `category` provide listing defaults. App-shell and portal + configuration can override them for a particular context. +- `category` is one of nine: `main`, `support`, `admin`, `user`, + `developer`, `help`, `other`, `lab`, `service`. + +Two kinds of app directory exist side by side: platform apps with descriptive +names (`chat/`, `code/`, `task/`) and apps under `(adhoc)/` with stable, +opaque 8-hex-digit slugs. "Ad-hoc" describes the route identity, not the app's +importance or maturity. The opaque slug avoids URL churn when a surface is +renamed or repositioned, while the manifest carries its human name and +category. + +A catch-all route (`app/apps/[...path]/route.ts`) claims any `/apps/*` path +no named app claims, and mounts portal static content at the root - which is +what lets a site authored for root deployment resolve its absolute resource +paths when served through a portal. + +### `lib/` - a tree encoded in filenames + +`lib/` is one flat directory, with a co-located test beside most source files. +The organization is in the filename: dot-separated prefixes encode a +two-to-three-level tree, so `action.exec.mcp.ts` reads as _action → execution +→ MCP handler_ and `model.provider.openai.ts` as _model → provider → +OpenAI_. + +The largest families are real architectural units. `model.*` is the model +catalogue and provider layer, with `model.provider.*` holding provider +integrations. `action.*` is the ability execution engine; each +`action.exec.*` file is one action runtime (fetch, shell, image, email, MCP, +and others) following a common internal structure: schemas, operation-name +constants, `do*` implementations, and an `execute*` router whose switch is +exhaustiveness-checked at compile time. +Other families mirror the domain model (`conversation.*`, `bot.*`, +`dataset.*`, `skillset.*`, `user.*`, `usage.*`, `limit.*`, `session.*`) and +the messaging integrations (`slack.*`, `twilio.*`, `telegram.*`, +`whatsapp.*`, `discord.*`, and others) - the flat directory doubles as the +integration registry. + +When adding a file, join an existing family if one fits; a new prefix is a +new subsystem and should be a deliberate choice. + +### `components/` and `hooks/` + +Both are flat like `lib/`, but use word-prefix namespacing instead of dots: +`components/` primarily holds one PascalCase component per file +(`BotBlockStatus.jsx`, `DatasetList.jsx`) with its test and, where useful, a +Storybook story beside it; `hooks/` holds one `useThing` hook per file, with +the same co-location. +The prefix families (`Bot*`, `Dataset*`, `Conversation*`, `Theme*`, +`useConversation*`, `useScroll*`) mirror the same domain nouns as `lib/` and +the database schema. + +UI code prefers plain HTML with the shared class vocabulary from +`styles/globals.css` (`default-button`, `primary-button`, `default-input`, +and friends) over bespoke styled components. + +### `config/` + +`config/` is the application's deployment and product-configuration boundary. +Its TypeScript and JavaScript modules parse operator-owned environment values, +derive origins and host topology, and expose catalogues for apps, models, +limits, navigation and feature defaults. Strict schemas make malformed +operator configuration fail during startup or build instead of silently +changing behavior. + +YAML is used elsewhere for content, prompts and catalogue inputs. The webpack +YAML loader makes those imports available as JavaScript values and can select +entries through `lookupKey` and `lookupValue` resource queries. + +### `next.config.d/` + +`next.config.js` is a loader, not a config: it reads every `*.config.js` in +`next.config.d/`, orders them, and deep-merges the results (with defined +semantics for `webpack`, `headers`, `rewrites`, and `redirects`). Each +module owns one concern. + +They fall into two groups. Portable application configuration covers bundling +and output modes, transpiled packages, security headers and CSP, environment +exposure, image domains, embed script entry points, API discovery headers and +agent content negotiation. Deployment-controlled routing covers app shells, +standalone apps, portals, space sites, partner hosts, request-affine host +mappings and optional multi-zone proxies. With those values unset, the +corresponding rules are inert and the supported single-domain, path-based +topology remains. The module boundaries keep portable behavior and optional +host routing visible file by file. + +### `schemas/` - where authorization lives + +`schemas/` holds one Joi schema per request field, and these are not just +shape validators: identifier schemas resolve the referenced resource and +enforce access on it. `schemas/botId.js`, for example, looks up the bot and +applies use-versus-manipulate access checks, throwing the appropriate +authentication or authorization error. API routes compose their request +validation from these files, so authorization is enforced at the validation +boundary rather than ad hoc inside handlers. `schemas/api/v1/` adds +per-resource response schemas for the public API. + +### The database + +The one hand-edited Prisma schema lives in +`packages/db-spec/prisma/schema.prisma`, kept complete for the richest +supported engine on purpose: engine-specific information only flows +downhill, so deriving is subtractive. Each database implementation derives +its own `schema.prisma` from it - the derived copies are generated, marked +as such, and committed so schema changes show up in review for every engine +they affect. Derivation also runs automatically at the start of every +`db:push` and `db:gen`, so a stale schema cannot reach a database or a +generated client. + +`db-spec` also carries the shared analytics queries in `prisma/sql/`, +written to run unmodified on every supported engine. + +Inside the application, `prisma/` is the data-access layer around the +generated client - the client singleton, custom model methods, field-level +encryption, caching, auditing, and retry - not the schema. + +### Content and layouts + +`content/` contains only the small catalogues still coupled to the product +runtime: FAQs and connection metadata under `other/`. Source-level `@doc` and +`@manual` blocks are publication inputs consumed by the documentation release +tooling. Published documentation is maintained outside the product runtime, so +the application does not depend on generated manuals or documentation content. + +`layouts/` contains the reusable page shells used by the Pages Router, such as +the dashboard, app, exploration and administration layouts. App Router layouts +stay with their routes under `app/`. + +### Everything else, briefly + +`emails/` - React Email components, one per transactional message, +delivered through the swappable email module. `embeds/` - sources for the +embeddable widget and MCP scripts, injected as extra webpack entries. +`graphql/v1/` - the GraphQL schema and resolvers behind +`pages/api/v1/graphql`. `workers/` - browser web workers. `templates/` - +quick-setup wizard definitions. `data/` - ability, secret and other runtime +catalogues expressed as TypeScript DSLs and YAML or OpenAPI inputs. `prompts/` +contains versioned YAML prompt files. `scripts/` contains operational scripts +built mostly on a shared `runScript` harness with CLI and interactive modes. + +## Conventions + +### New source is TypeScript, new tests are JavaScript + +The source tree is being migrated from JavaScript and JSX to TypeScript and +TSX. New source and files converted as part of a change use `.ts` or `.tsx` +and are type-checked by `pnpm check`; existing `.js` and `.jsx` files remain +valid until they are migrated deliberately. + +New tests are JavaScript on purpose: a TypeScript test rejects the +wrong-on-purpose input that a test exists to cover. In the application, unit +tests are `*.utest.js` or `*.utest.jsx` and are normally co-located with their +source; in packages they are `*.test.js`. Existing TypeScript test files +predate this rule - leave them alone unless the surrounding test is already +being rewritten, and do not use them as models. + +One placement exception, about the router: tests under `pages/` stay +co-located but carry an underscore prefix (`pages/api/v1/bot/_create.utest.js`, +`pages/admin/users/[userId]/_index.utest.js`) because the router ignores +`_`-prefixed files, so test files are never exposed as routes. + +Integration tests are `*.itest.js` under `tests/integration/`, run +separately (`pnpm test:integration`). + +### Comments + +`@note` marks gotchas, side effects, and surprising behavior - the things +the next reader would otherwise rediscover the hard way. `@todo` marks +planned work. Both are single sentences, lowercase, no ending period. Plain +comments explain complex logic and use normal punctuation. The codebase +leans heavily on `@note`; when a piece of code depends on something the code +cannot show, that is where it is written down. + +### Custom lint rules + +The application ships its own ESLint rules under `eslint/custom-rules/`, each +encoding a repository invariant. They protect serialization, Prisma deletes, +typed SQL, disposable factory results, the custom router, package +transpilation, directive placement, controlled HTTP egress and centralized +documentation links. If one fires, its error message identifies the invariant; +the rule source and any co-located test show the exact boundary. + +### Build-time machinery worth knowing about + +The webpack layer under `platform/webpack/` carries the YAML loader described +above, a `.json.gz` loader, a markdown frontmatter loader +(`import meta from './file.md?frontmatter'`), and a source-map validation +plugin that fails the build if any emitted map embeds source content. +`app.manifest` files import as JSON via a dedicated rule. + +## Building and verifying + +From the repository root, begin with `pnpm install`. The CI quality gate runs +`build`, `lint`, `check` and `test` across the packages (filtering the +application out with `--filter='!@chatbotkit/platform'`) and then, in a +second job, type-checks the application and runs its unit suite from +`.env.example` and a fresh SQLite database. The application's lint and build +are still local responsibilities. See `CONTRIBUTING.md` for the exact commands +and setup. + +Inside `platform/`: `pnpm dev` starts the development server, `pnpm check` +type-checks, `pnpm lint` lints, `pnpm test:unit` runs the unit suite (or pass a +single test path), and `pnpm storybook` starts the component workbench. A full +`pnpm build` regenerates the database and GraphQL clients, builds templates and +the API specification, runs the unit suite, builds the application, and +generates the sitemap. The `SKIP_*` environment variables it honors exist for +CI stages that cover selected steps separately. + +The `docker-compose.yml` at this root offers two application modes. The +default profile starts a ready development server plus Redis, Qdrant and +Garage; it copies the read-only checkout into a container and preserves hot +reload. The `distro` profile builds and serves the compiled platform. The +backing services can also be started individually for host-side development. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..266014c --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,489 @@ +# Deployment configuration + +Most of what a deployment needs is an ordinary environment variable - a URL, a +credential, a feature toggle - and [`platform/.env.example`](../platform/.env.example) +documents those where they are used. + +This document covers the five variables that are different in kind. Each carries +a JSON document rather than a scalar, and each answers a question the code +cannot answer for you: who administers this deployment, what it sells, what a +plan grants, which hostnames it answers on, who gets exceptions, and which +paths belong to an external application zone. They are **operator-owned data**, +deliberately kept out of the source tree so that a deployment's business model +and routing topology are not code changes. + +Every one of them is optional. A deployment that sets none of them boots and +works: no administrators, no plans, no billing, no exceptions, no additional +host mappings, no external zones, and every surface served path-based on a +single host. That is the supported self-hosted default, not a degraded mode. + +| Variable | Answers | Unset means | +| ---------------------- | ------------------------------------------- | ---------------------- | +| `ADMINS_CONFIG` | Who reaches `/admin` | Nobody | +| `LIMITS_CONFIG` | What each plan grants | No plan concept at all | +| `OVERRIDES_CONFIG` | Per-account exceptions and grants | No exceptions | +| `HOSTS_CONFIG` | Request-affine host mappings | Scalar host defaults | +| `ZONE_CONFIG` | Paths proxied to external application zones | No external zones | + +The four application configuration variables are parsed in `platform/config/`. +`ZONE_CONFIG` is parsed by `platform/next.config.d/zone.config.js` when Next.js +builds its routing table. Those source headers are the authoritative shape +references. This page is the operator's view: what to set, what it costs you to +get it wrong, and what changes safely. + +## Validation rules + +**The four application configuration variables are parsed with strict schemas +when their modules load. An unrecognised key, a missing required key, or a +malformed document fails startup or the build. It does not warn and continue.** + +`ZONE_CONFIG` is different because it generates Next.js rewrites. It is parsed +at build time and validates route shapes as the rewrites are generated. Invalid +JSON or paths fail the build. Changes therefore require a rebuild and redeploy. +The variable must be present in the environment that runs `next build`; setting +it only in the final container's runtime environment is too late. + +This is deliberate. These variables decide entitlements, administrator access +and routing; a table that silently ignored a key it did not understand would +deny a customer, expose an admin route, or 404 a whole surface with nothing in +the logs to say why. Failing at boot is the safe direction. + +The cost is that **adding a required key to one of the four strict schemas is a +breaking change for every operator who sets that variable**. Their deployment +stops booting until they edit it. So: + +- Additive keys should land **optional with a documented default**. +- Unknown keys must keep failing loudly - do not relax `.strict()` to make a + migration easier. +- Any exception travels in the release notes as a configuration migration. + +`HOSTS_CONFIG` is shared by build-time routing and runtime URL selection. The +build flattens every configured API and static target into its routing rules. +At runtime, request-context setup selects a mapping once from the authenticated +frontend host or trusted normalized request host. URL helpers then read only +the resolved context; the raw mapping is not exposed to the browser. The +operator-defined mapping names are only stable configuration identifiers; they +do not select branding, tenancy, or a canonical host. + +## Validating before you deploy + +The four application values are parsed when their modules load, so the cheapest +check is a development boot: + +```bash +# from platform +ADMINS_CONFIG='["ops@example.com"]' pnpm dev +``` + +A malformed document fails immediately with a zod error naming the offending +path. Validating in the shell first is worthwhile for the larger ones: + +```bash +echo "$LIMITS_CONFIG" | jq . > /dev/null && echo "valid JSON" +``` + +Valid JSON is necessary but not sufficient. The application checks the four +strict schemas, while a Next.js build checks `ZONE_CONFIG` and generates its +rewrites. + +Two catalogues are **coupled**. A `plan` grant in `OVERRIDES_CONFIG` is +validated against `LIMITS_CONFIG` when the override module loads, and the +configuration conformity test suite checks the two together. Run the +application quality gate before deploying changes to these values. Set the +limits catalogue first so a granted plan is never left undefined. + +--- + +## `ADMINS_CONFIG` + +Who may reach the administration console at `/admin` and `/api/admin`. + +```json +["ops@example.com", "clxyz0000000000000000000n"] +``` + +An array of identifiers. An entry is **either** a user id **or** an email +address, with no marker saying which - the check compares both fields of the +signed-in user against every entry. + +That distinction matters when you add one. An **email** is a claim about a +person and follows whoever currently holds that address. An **id** is a claim +about an account and survives the person changing their address. Prefer the id +for anything long-lived. + +> **Security.** This is the whole gate. The console ships in every deployment +> and authorizes nobody until this is set, so an empty value is safe and a +> careless value is not. An address you do not control - a former employee's, +> or a domain you have let lapse - is an administrator. + +## `LIMITS_CONFIG` + +The plan catalogue: one **complete** limit table per plan name. + +```json +{ + "pro": { "tokens": 3000000, "conversations": 2500, "...": "every other key" } +} +``` + +The example above is **abbreviated and would not boot** - shown for shape only. +A real table carries every limit key; start from the one documented in +`platform/config/limits.ts`. + +Plan names are yours - `pro`, `team`, `enterprise`, whatever you sell. The +limit _keys_ are platform vocabulary and the table must be complete: the schema +is strict, so a missing or misspelled key fails the boot rather than quietly +resolving to zero. + +Unset is a **working configuration**, not an empty one. With no catalogue the +deployment has no plan concept: every limit lookup resolves to the unlimited +table, enumeration stays empty so no surface renders an invented plan name, and +the entitlement checks short-circuit. It is explicitly _not_ "everyone on the +lowest plan". + +The code reserves three structural names. `free` means no subscription or +grant, and `trial` means a trialing subscription. `unlimited` always resolves to +the unlimited table: it is implicitly available for grants and manual +subscriptions but is never enumerated in plan lists. Do not define +`unlimited` in `LIMITS_CONFIG`. + +## `OVERRIDES_CONFIG` + +Per-account exceptions. The key used depends on the field: plan grants are +looked up by email, while VIP status and enforcement-time limit overrides are +looked up by account id. Some account-summary paths fall back to email for +limits, but enforcement does not, so email-keyed limit entries are not a +dependable deployment contract. + +```json +{ + "ops@example.com": { "plan": "enterprise" }, + "clxyz0000000000000000000n": { + "limits": { "database": { "files": 300 } }, + "plans": { "premium": { "limits": { "tokens": 5000000 } } } + } +} +``` + +- **`plan`** is an email-keyed _grant_: treat this account as if it bought that + plan. No subscription, no billing. This is the comp mechanism. It may name a + catalogue plan other than `free` or `trial`, or the implicit `unlimited` + plan. +- **`vip`** is account-id keyed and skips the hub publishing review queue. +- **`limits`** bends specific values whatever plan the account is on. Key these + entries by account id so the enforcement paths apply them. +- **`plans[name].limits`** bends them only while on that plan, so a + grandfathered exception does not leak into a downgraded one. These keys must + name plans in `LIMITS_CONFIG`; `free` and `trial` are valid here when they are + present in the catalogue. The surrounding entry should also use the account + id. + +A malformed or unknown plan reference fails when the override configuration is +loaded. An id key follows the account; an address-keyed plan grant follows +whoever holds the address. A key matching nothing is silently an override that +never applies, which is why attribution matters. + +> **Operational note.** Record _which customer_ an entry belongs to as a comment +> beside the value in your encrypted environment file. An exception nobody can +> attribute is one nobody can ever remove. + +## Apex hostnames + +Four scalar variables define the apexes beneath which the deployment creates +subdomains. They identify canonical deployment-owned domains and are therefore +separate from the routing table. + +| Variable | Serves | Unset means | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `APP_APEX` | Standalone apps at `.` | Standalone apps are path-based | +| `PORTAL_APEX` | Portals at `.` | Portals use custom domains or the site host | +| `SPACE_APEX` | Space sites at `.` | Deployment-issued space hostnames disabled | +| `PARTNERS_APEX` | Partner experiences at `.` | Partners use custom domains or the site URL | + +Values are hostnames without a protocol or wildcard, for example: + +```dotenv +APP_APEX=example.app +PORTAL_APEX=example.agency +SPACE_APEX=example.site +PARTNERS_APEX=example.partners +``` + +## App shell origins + +Two scalar origins identify the canonical app-shell endpoints. An origin must +include its protocol and must not include a path, query, hash, or trailing +slash. + +| Variable | Serves | Unset means | +| ----------------- | ----------------------- | -------------------------------- | +| `APP_MAIN_ORIGIN` | The main Apps shell | The shell remains path-based | +| `APP_LABS_ORIGIN` | The optional Labs shell | The Labs shell is not registered | + +```dotenv +APP_MAIN_ORIGIN=https://apps.example.com +APP_LABS_ORIGIN=https://labs.example.com +``` + +## `HOSTS_CONFIG` + +Optional request-affine host mappings. Each operator-defined key groups the +site, API, static, and widget hosts that must stay together when the deployment +answers on several domain families. + +```json +{ + "example": { + "match": [ + "example.com", + "api.example.com", + "static.example.com", + "widgets.example.com" + ], + "site": "example.com", + "api": "api.example.com", + "static": "static.example.com", + "widgets": "widgets.example.com" + } +} +``` + +| Field | Purpose | +| --------- | --------------------------------------------------------- | +| `match` | Exact incoming hostnames that select this mapping | +| `site` | Site or application host for request-affine frontend URLs | +| `api` | API host for request-affine API URLs and clean API routes | +| `static` | Static host for public assets and static-host routing | +| `widgets` | Host for private MCP widget bundles | + +Values are exact hostnames without a protocol, wildcard, path, query, or hash. +Every target that can receive a request should also appear in `match`, so a +request arriving on an API or static host selects the same mapping. + +At build time, every `api` and `static` target is enabled unconditionally. At +runtime, context injection selects the mapping once when the authenticated +frontend host or normalized request host appears in `match`. Server URL helpers +read the resolved targets from that context. The HTML document exposes only the +resolved site, API, static, and widget hosts for client hooks. An unknown host +keeps the existing custom-domain behavior. + +`SITE_URL` remains the canonical and requestless default. When no mapping is +selected, `API_URL`, `STATIC_URL`, and `WIDGET_URL` all fall back to +`SITE_URL` - the deployment then serves the API at `/api/v1`, and the static +and widget paths, on its own host, with no host-gated routing derived. The +scalar targets are also routed alongside the mapped ones: an `API_URL` naming +a host other than the site host is routed to the API just as a `STATIC_URL` +host is routed to the static rules, so a deployment with a single dedicated +API subdomain needs only the scalar. There is no implicit `api.` +derivation: advertised API URLs follow `API_URL`, and unset it they stay on +the site host. App shells and apex-based routing are controlled independently +by the scalar variables above. + +The configuration fails validation on malformed hostnames, missing fields, +unknown fields, or a hostname matched by more than one mapping. + +## `ZONE_CONFIG` + +Optional build-time routing for applications deployed separately while this +application remains the public domain gateway. The value is an array of zones; +the legacy single-zone object is also accepted. + +```json +[ + { + "origin": "https://marketing.example.com", + "hosts": ["example.com", "www.example.com"], + "root": true, + "paths": ["/pricing", "/careers"], + "exactPaths": ["/platform"], + "prefixes": ["/media/marketing"], + "exceptions": ["/pricing/internal-tool"], + "aliases": { "/pricing-preview": "/pricing" }, + "assetPrefix": "/marketing-static" + } +] +``` + +| Field | Purpose | +| ------------- | ---------------------------------------------------------------- | +| `origin` | Deployment origin to which the owned routes are proxied | +| `hosts` | Exact incoming hostnames on which this zone applies | +| `root` | Whether the bare `/` belongs to the zone | +| `paths` | Top-level segments, their subtrees, and Pages Router data routes | +| `exactPaths` | Top-level paths owned only at the exact path | +| `prefixes` | Multi-segment subtrees whose parent segment remains shared | +| `exceptions` | Subtrees retained locally beneath a segment listed in `paths` | +| `aliases` | Temporary local paths that proxy to another path in the zone | +| `assetPrefix` | Unique JS and CSS chunk prefix, defaulting to `/zone-static` | + +Only zones with both a non-empty `origin` and at least one host become active. +Hosts not listed in a zone, including white-label and preview hosts, stay with +this application. `paths` and `exactPaths` accept single top-level segments; +`prefixes` and `exceptions` accept multi-segment paths. An exception must sit +beneath an entry in `paths`. + +This variable is deployment topology rather than tenant or branding data. To +return a path to this application, remove it from the table and rebuild. The +experimental `distro` Dockerfile does not currently forward operator `.env` +values into the build, so extending that build definition is required for a +zoned compiled image. + +--- + +## Built-in assistants and public examples + +The assistant and example widgets need two decisions that application code +cannot make for an operator: which model to use, and which account owns an +unauthenticated public conversation. + +| Variable | Purpose | Unset means | +| ------------------------ | ------------------------------------------------------------------ | -------------------------------------- | +| `AUTO_WIDGET_MODEL` | Model for the built-in dashboard, blueprint and website assistants | Built-in assistants are disabled | +| `AUTO_WIDGET_USER_ID` | Service-account owner for the unauthenticated website assistant | The website assistant is not public | +| `EXAMPLE_WIDGET_USER_ID` | Service-account owner for unauthenticated live examples | Live examples require a signed-in user | + +Authenticated dashboard, blueprint and example requests always use the +signed-in account. A service account is only a deliberate owner for an +unauthenticated public surface; it is never substituted for a signed-in user. +If no owner can be resolved, conversation creation fails closed with a 401. + +## Portal rewrite assertions + +Most community deployments do not need internal routing headers. When the +separate portal frontend rewrites a public or custom domain to the platform, +however, the platform still needs the original frontend host for tenant +selection and URL generation, and may need the ingress-provided client address. + +Set the same `INTERNAL_HEADERS_SECRET` value, with at least 16 characters, in +the portal and platform environments. The portal serializes each value as an +independently authenticated assertion under a non-canonical wire name. The +platform verifies allowlisted assertions once and promotes them into request +context. Raw `x-chatbotkit-internal-*` headers, unknown assertions, malformed +values and invalid signatures are ignored. If the secret is unset, the +platform behaves as if no portal assertions were supplied. + +When this value is missing or shorter than 16 characters, the assertion sender +and receiver emit a debug message where the value is used. Incoming portal +assertions are treated as untrusted and internal self-calls emit no assertions. +The portal follows the same fail-closed behavior for outgoing assertions. Use a +randomly generated secret rather than treating the minimum length as an entropy +guarantee. + +This secret does not make ordinary `Host`, `x-forwarded-host` or client-IP +headers trustworthy. A deployment proxy must overwrite those headers and +prevent direct application access according to its own topology. + +## Reverse-proxy headers + +The platform trusts `x-forwarded-host`, `x-forwarded-proto` and the client +address headers (`x-real-ip`, else the last `x-forwarded-for` hop - the one +the proxy itself appended) only when the deployment sets +`TRUST_PROXY_HEADERS=true`. The values are normalized once +into request context and downstream code reads only that context; when trust +is disabled, the forwarded values are ignored and the application uses the +ordinary `Host` header, the request URL protocol where available, and the +directly connected socket address for rate limiting and audit records. + +Behind a reverse proxy this flag also decides whether the sign-in abuse +controls work at all: without it every client shares the proxy's socket +address, so the per-address budgets for code issuance and verification become +one global budget - a handful of failed attempts from anyone locks sign-in for +everyone until the window passes. + +The flag is a deployment-topology assertion, not authentication. Enable it +only when the reverse proxy removes client-supplied forwarded headers, writes +its own values, and prevents clients from reaching the application origin +directly. If those conditions cannot be guaranteed, leave it unset. + +## Platform capacity cap + +`PLATFORM_MAX_TOKENS_PER_MONTH` is an optional deployment-wide safety ceiling, +not a subscription plan. Set it to a positive number to stop non-exempt model +traffic after that many calibrated base tokens in a billing period. Leaving it +unset, or setting it to `Infinity`, gives a community deployment no artificial +hosted quota. Hosted and resource-constrained operators should set a finite +value explicitly. + +## Credential cache + +`PLATFORM_CREDENTIAL_CACHE_TTL` is the number of seconds an API secret key or +OAuth access token lookup may be served from cache when a request +authenticates. It defaults to `0`: every API request reads the credential row, +so revoking a key takes effect on the next request. A deployment whose API +volume makes that read expensive (a metered database, for instance) can set a +small positive value to trade a bounded delay for fewer reads. Be explicit +about what is bought: a revoked key or token keeps working for up to that many +seconds. There is no stale-while-revalidate on top, so the window is exactly +the value set. + +## Encryption at rest + +`PRISMA_FIELD_ENCRYPTION_KEY` encrypts the database columns that hold +credentials - every column carrying a `/// @encrypted` annotation in the +schema: stored secrets and their values, every integration's tokens, app +secrets, API keys, private keys and webhook secrets, the MCP identity +provider's client secret, next-auth's provider tokens and outbound webhook +secrets (`ENCRYPTED_FIELDS` in `platform/prisma/encryption.ts` is the list) - +through the Prisma extension in that module. Not encrypted, by design: the +columns the platform looks up by equality (API keys, its own OAuth server's +tokens and client secrets - `/// @digest` in the schema), because a +ciphertext with a random nonce cannot be searched. It is optional, and **setting it is the +decision to encrypt**: unset, the extension is inert and those columns are +stored as given. It is a separate concern from `CLOAK_ENCRYPTION_KEY`, the +general-purpose key the rest of the application uses (transient OAuth state, +values encrypted out of band with `pnpm script:encrypt`); the two are never +substituted for each other. + +**What you get.** Writes are encrypted on the way in, reads decrypted on the +way out, including `include`-d relations; a test fails if the schema +annotations and the extension's field map ever disagree. AES-256-GCM with a +fresh random nonce per value, and each ciphertext bound to its column +(`.` as authenticated data), so a value copied by raw SQL from +one column or model to another is rejected on read rather than granting the +target row a credential it never had. Audit rows only ever see ciphertext. +A copy of the database - a backup, a replica, an injection, a script gone +wrong - is useless without the key. It does not protect against someone who +holds the runtime, and it is one key for the whole deployment rather than +per tenant. + +**Generating a key.** + +```bash +# from platform +pnpm script:generate-encryption-key +``` + +prints a `k1.aesgcm256.<43 base64url characters>=` value - 32 random bytes. +Back it up somewhere that is not the database: losing every key loses every +encrypted value, and there is no recovery. A malformed value is not +silently ignored; the first write to an encrypted column fails. + +**Rotation.** The variable is a comma-separated keychain: the first key +encrypts, every key decrypts. Rotation therefore needs no downtime and no +window during which reads fail: + +1. Generate a new key and **prepend** it: `PRISMA_FIELD_ENCRYPTION_KEY=,`. + Deploy. New writes use the new key; existing rows still read. +2. Run `pnpm script:backfill-database-encryption`. It walks every encrypted + column and rewrites each value that is not already under the first key + with its column binding, in batches, and reports counts. Without + `--execute` it is a dry run and only counts. It is safe to re-run, and it + stops - rather than skipping - on a value that no configured key accepts. +3. Remove the old key. Do not do this before step 2 completes: values still + under it become unreadable, and the script cannot recover them. + +**Turning it on later.** The same script is the one-time migration for a +deployment that ran without a key: set the key, deploy, run the script once. +Reads pass plaintext rows through, so nothing breaks in the meantime, but a +production estate should not rely on "re-save to encrypt". Removing the key +later does not decrypt anything - rows encrypted while it was set come back +as ciphertext until it is restored. + +--- + +## Related + +- [`platform/.env.example`](../platform/.env.example) - every other variable +- [Architecture](./architecture.md) - the configuration boundary and module + architecture +- `packages/*/README.md` - each swappable module's own variables diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..a5c28f2 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,258 @@ +# Deployment + +The repository provides a complete development stack, a locally compiled image +profile and prebuilt community images produced by trusted pushes. These are +application distribution artifacts, not infrastructure provisioning recipes. +Production operators remain responsible for the surrounding deployment and +its operational guarantees. + +## Compose profiles + +Run the complete development stack: + +```bash +docker compose up +``` + +Run only backing services for host-side development: + +```bash +docker compose up redis qdrant garage garage-init +``` + +Build and run the experimental compiled image: + +```bash +docker compose --profile distro up --build platform +``` + +The first compiled build can take more than 15 minutes depending on available +CPU, memory and network cache state. + +## Pull the community image + +Trusted pushes to `main` and `next` publish matching application and database +initializer images. Names follow `platform--`; tags carry +only the build: the moving `main` and `next` tags are channels (`latest` +follows `main`), and `sha-` tags identify an immutable source +revision. + +The publication workflow derives the registry owner and image name from the +GitHub repository. In `chatbotkit/platform` this resolves to the official image +names below without repository-specific workflow configuration. + +The public ChatBotKit images use `ghcr.io/chatbotkit/platform-community-app` +and `ghcr.io/chatbotkit/platform-community-init`. Start the Compose profile +without allowing a source build. Each tag is a multi-platform image for +`linux/amd64` and `linux/arm64`, so Docker selects the host architecture +automatically: + +```bash +docker compose --profile distro up --no-build --pull always platform +``` + +By default this pulls: + +```text +ghcr.io/chatbotkit/platform-community-app:next +ghcr.io/chatbotkit/platform-community-init:next +``` + +Select another matching channel or immutable revision by setting both image +references: + +```bash +PLATFORM_IMAGE=ghcr.io/chatbotkit/platform-community-app:main \ +PLATFORM_INIT_IMAGE=ghcr.io/chatbotkit/platform-community-init:main \ + docker compose --profile distro up --no-build --pull always platform +``` + +Never mix application and initializer revisions. The database schema and +generated application client must come from the same source and package flavor. + +## One-command distribution stack + +Each trusted push also publishes the flavor's complete Compose application as +an OCI artifact under `ghcr.io/chatbotkit/platform-`. The artifact is +self-contained - the application, database initializer, Redis, Qdrant and +Garage with its configuration and provisioning - and every image reference in +it is resolved to a digest, so an artifact tag identifies an exact, immutable +stack. No checkout, no bind mounts: + +```bash +docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest up +``` + +The `latest` tag follows `main`; `next` follows the `next` branch. Compose +v2.34 or newer is required. On `up`, Compose shows the stack's variables - +site URL, secrets, optional provider keys - and their defaults before +proceeding; set them in the shell, in a `.env` file in the directory the +command runs from (picked up automatically), or via an explicit `--env-file`, +and pass `-y` to skip the confirmation in scripts. Shell values win over +`.env`, and only variables the stack declares are consumed - the published +artifact carries no `env_file` mounts, so arbitrary extra entries do nothing. + +### Persisted configuration + +Values can also live in the platform data volume, where they survive +restarts and upgrades and never touch a file on the host. The application +entrypoint reads `/data/config.env` (one `KEY=VALUE` per line, no quoting) +and exports every entry the container environment does not already set. Any +variable the application honours is accepted, not only the ones the stack +declares. `setup` writes the file: + +```bash +# prompts for the provider keys, input hidden; Enter keeps, "-" clears +docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest run --rm --no-deps platform setup + +# prompts for named variables, or sets them without a terminal +docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest run --rm --no-deps platform setup OPENROUTER_MODELS_API_KEY +docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest run --rm --no-deps platform setup OPENROUTER_MODELS_API_KEY=sk-or-... OPENAI_API_KEY= +``` + +Restart the `platform` service afterwards. Precedence, highest first: the +container environment (shell, `.env`, `--env-file`, `-e`), then +`config.env`, then the secrets generated on first boot. An empty environment +value counts as unset, so the stack's `${OPENAI_API_KEY:-}` defaults never +mask a persisted value; to override one for a single run, set it in the +shell. The file is owned by the application user with mode `0600`; back it +up with the volume, and prefer `PRISMA_FIELD_ENCRYPTION_KEY` in the +environment rather than next to the database it protects. + +### Several instances on one host + +The stack declares no project name, so Compose derives one. Two instances +started from the same artifact would share that project - and with it the +`platform-data` volume - and both would try to publish port 3000. Give each +instance its own project with `-p` and move its published ports with an +override file; the override applies after the artifact, so pass both, in that +order, on every command for that instance: + +```yaml +# staging.yml +services: + platform: + ports: !override + - '3001:3000' + garage: + ports: !override + - '127.0.0.1:3901:3900' +``` + +```bash +docker compose -p cbk-staging \ + -f oci://ghcr.io/chatbotkit/platform-community:latest -f staging.yml up -d +docker compose -p cbk-staging \ + -f oci://ghcr.io/chatbotkit/platform-community:latest -f staging.yml logs platform +``` + +Set `SITE_URL` and `NEXTAUTH_URL` to the instance's published address +(`http://localhost:3001` here) - in the shell or through `--env-file`, since a +single `.env` in the working directory cannot describe both instances. Volumes, +networks and container names are all prefixed with the project name, so each +instance keeps its own database, generated secrets, object store and vector +index, and `-p` is also how `logs`, `ps` and `down` find the right one. +Presigned storage URLs carry `garage:3900`, so the `/etc/hosts` entry serves the +instance that keeps port 3900; browser-facing file flows on the others need +their own store host name and endpoint. + +The artifact is published from +[docker/distro/community/compose.yml](../docker/distro/community/compose.yml), +which also runs directly from a checkout. One folder per package flavor lives +under `docker/distro/`; a future PostgreSQL flavor publishes as +`ghcr.io/chatbotkit/platform-postgresql` from its own folder, built from the +matching image flavor. + +Browser-facing file upload and download flows presign URLs against the +in-stack store; add `127.0.0.1 garage` to `/etc/hosts` on the host to use +them, as with the development stack. + +### Distribution flavors + +A flavor is the baseline of [module defaults](./module-defaults.md) plus the +backing services its stack provisions. Everything not listed keeps the +default - in the community flavor the database is SQLite in the platform data +volume, the queue is immediate and non-durable, sign-in codes are read from +the container log, and the sandbox refuses under `NODE_ENV=production`. + +| Flavor | Database | Cache | Vector | Storage | +| ----------- | ----------------------- | ----- | ------ | ------- | +| `community` | SQLite (default module) | Redis | Qdrant | Garage | + +A PostgreSQL flavor would swap the database column only; the other services +travel unchanged. + +## Production boundary + +The `distro` profile demonstrates that the application can be compiled and run +from the published tree. It deliberately does not stand up an operator's +production infrastructure. A production deployment still needs: + +- TLS termination and a trusted reverse proxy that overwrites forwarded headers +- protection and backup of the runtime secrets generated into the persistent + platform data volume, or explicit operator-provided values +- durable database, object-storage and backup policies +- a durable queue when delayed delivery, retries, callbacks or ordering matter +- a production-safe isolated sandbox implementation if agent code execution is + enabled +- monitoring, restore testing and an upgrade and rollback procedure + +The repository does not yet publish versioned releases, SBOMs or signed +provenance, so branch and commit images remain pre-release artifacts. This +status concerns release provenance and compatibility, not whether Compose +should provision the operator-owned infrastructure listed above. + +The experimental Dockerfile builds with `.env.example`; Compose attaches the +optional operator `.env` file only to the running container. Configuration +consumed by Next.js while it builds, including `ZONE_CONFIG` and the build-time +parts of host and subscription configuration, is therefore not baked into the +`distro` image. A production pipeline must supply those values during the build +and keep secrets out of image layers. + +The current community image deliberately bakes the neutral single-host +topology: `SITE_URL=http://localhost:3000`, with no app-shell origins, apexes or +external zones. Runtime service variables such as the database, Redis, Qdrant +and S3-compatible storage endpoints remain configurable. Deployment identity +that Next currently exposes through `next.config.js` is still frozen at build +time; do not present the same digest as portable across arbitrary public domains +until that migration is complete. + +## API endpoint + +Every deployment serves the API at `/api/v1` on its own host - nothing to +configure. To advertise and serve it on a dedicated origin instead, set +`API_URL` (e.g. `https://api.example.com`), point that DNS name at the +deployment, and rebuild: the host is then routed to the API (answering under +the clean `/v1` path) and every externally advertised URL - webhook +registrations, embeds, the OpenAPI spec - follows it. Unset, advertised URLs +stay on the site host under `/api`. Multi-domain deployments name their API +hosts in `HOSTS_CONFIG` instead; see +[Configuration](./configuration.md#hosts_config). Both are read at build time, +so changing them requires a rebuild, not just a restart. + +## Reverse proxy trust + +Set `TRUST_PROXY_HEADERS=true` only when the reverse proxy overwrites forwarded +host, protocol and client-address headers and the application origin cannot be +reached directly. See [Configuration](./configuration.md) for the complete +trust-boundary requirements. + +## Persistent state + +The development stack stores state in Compose volumes. A production design must +make the retention, backup and restore behavior explicit for: + +- the application database +- object-storage buckets +- Redis when it carries shared rate-limit or cache state +- Qdrant or the selected vector implementation +- encryption keys and other runtime secrets + +Backing up ciphertext without its encryption key is not a recoverable backup. + +## Module limits + +The public defaults prioritize a vendor-free boot and an honest development +experience. Some defaults are intentionally not production implementations. +Read [Module defaults](./module-defaults.md) before selecting production +backends. diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..1465ac8 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,168 @@ +# Getting started + +The complete supported local baseline runs with Docker Compose. A host-side +development setup is available when you want a faster edit loop. + +To run the prebuilt platform without a checkout, use the one-command +distribution stack described in [Deployment](./deployment.md) instead. + +## Run the complete local stack + +From the repository root (with Git LFS installed before cloning, so the binary +assets arrive as files rather than pointers): + +```bash +docker compose up +``` + +Open . Sign in with any email address and read the +six-digit sign-in code from the `dev` service log (`docker compose logs -f +dev`). + +This command starts: + +- the platform development server on SQLite +- Redis for shared caching +- Qdrant for vector storage +- Garage as an S3-protocol object store +- `garage-init`, which provisions the development key and storage buckets + +The checkout is mounted read-only and synchronized into the development +container. Editing the host working tree still triggers hot reload. + +Browser-facing file flows use presigned URLs containing the Compose service +hostname. Add `127.0.0.1 garage` to the host machine's hosts file before testing +uploads or downloads in a containerized mode. Host-side development configured +with `SERVICE_AWS_ENDPOINT=http://localhost:3900` does not need that entry. + +No hosted account, billing configuration or vendor credential is required to +boot. Model-backed agent responses require at least one model provider key. + +## Configure the development container + +The Compose service keeps its working copy in a writable volume and +deliberately excludes the host `.env` file from source synchronization. In a +second terminal, create or edit the host file and copy it into the running +container: + +```bash +test -f platform/.env || cp platform/.env.example platform/.env +# Edit platform/.env, then: +docker compose cp platform/.env dev:/workspace/platform/.env +docker compose restart dev +``` + +Repeat the copy and restart after changing environment variables. A host-side +development server reads `platform/.env` directly and does not need this step. + +## Add a model provider + +The platform only advertises provider models when the matching credential is +configured. For example, add this to `platform/.env`, then update the running +container as described above when using Compose: + +```bash +OPENAI_API_KEY=sk-... +``` + +The same key also powers dataset embeddings in the default vector setup. Other +provider variables are documented in `platform/.env.example`. + +## Run the application on the host + +Requirements: + +- Node.js 24.20 or newer +- pnpm 11.24.0 or newer +- Git LFS - binary assets are LFS pointers; run `git lfs install` before + cloning, or `git lfs pull` in an existing checkout + +From the repository root: + +```bash +pnpm install +cd platform +cp .env.example .env +pnpm db:push +pnpm dev +``` + +Open . + +Run `pnpm db:push` from the application directory rather than invoking the +database module's CLI directly. The wrapper resolves the relative SQLite URL +against the application directory before the module runs. + +The host-side baseline uses: + +- SQLite on disk +- an in-process cache when `REDIS_URL` is unset +- local JSON vector files when `QDRANT_URL` is unset +- console email delivery +- no plan, billing or entitlement catalogue + +## Configure object storage on the host + +The public storage module is an S3-protocol client, not a local-filesystem +store. The application boots without storage, but file uploads, generated +images and speech, space assets and sandbox storage mounts refuse at the point +of use until a store is configured. + +The quickest local option is the Compose Garage service: + +```bash +docker compose up garage garage-init +``` + +It publishes a provisioned store on `127.0.0.1:3900`. Uncomment the matching +block in `.env.example`, including: + +- `SERVICE_AWS_ENDPOINT` +- `SERVICE_AWS_REGION` +- `SERVICE_AWS_ACCESS_KEY_ID` +- `SERVICE_AWS_SECRET_ACCESS_KEY` +- `SERVICE_AWS_FORCE_PATH_STYLE` +- the `*_S3_BUCKET_NAME` variables + +AWS S3, Cloudflare R2, SeaweedFS and other S3-compatible stores can be used with +their own values. Sandbox storage mounts additionally require +`SERVICE_AWS_STORAGE_ROLE_ARN` and an STS-capable store. + +## Configure shared cache and vector storage + +These services are optional for a host-side development server: + +```bash +docker compose up redis qdrant +``` + +Then configure: + +```bash +REDIS_URL=redis://localhost:6379 +QDRANT_URL=http://localhost:6333 +``` + +## Protect stored credentials + +Before storing real credentials, configure `PRISMA_FIELD_ENCRYPTION_KEY`. An +unset value means scalar credential columns are stored as provided. + +Generate a key from the application directory: + +```bash +pnpm script:generate-encryption-key +``` + +Keep the key safe. Losing it makes encrypted credentials unrecoverable. See +[Configuration](./configuration.md#encryption-at-rest) for backfill and key +rotation. + +## Next steps + +- Review the [module defaults](./module-defaults.md) before evaluating + queueing, storage, sandboxing or multi-process behavior. +- Read [deployment](./deployment.md) before exposing the application outside a + local development environment. +- Read [configuration](./configuration.md) before setting any `*_CONFIG` + variable. Leaving them unset is the supported default. diff --git a/docs/module-defaults.md b/docs/module-defaults.md new file mode 100644 index 0000000..aba4a4d --- /dev/null +++ b/docs/module-defaults.md @@ -0,0 +1,80 @@ +# Module defaults + +Every swappable module ships a public default that boots with nothing set. +This page records what each of those defaults actually does, so a deployment +can tell a working baseline from a feature that needs an operator +implementation. Booting is the guarantee, not production semantics. + +How modules are swapped, and the conventions every module keeps, are in +[Architecture](./architecture.md#swappable-modules). The distribution flavors +bundle backing services over these defaults; +[Deployment](./deployment.md#distribution-flavors) lists what each flavor +changes. Each module's package README owns its environment-variable reference. + +## Default behavior + +### Database + +The public database module uses SQLite in a file. It is suitable for the local +single-process baseline. Production operators own database durability, +concurrency, backup and restore behavior. + +### Storage + +The public storage module is an S3-protocol client with no built-in store. Bare +`docker compose up` provisions Garage. A host-side checkout must configure an +S3-compatible endpoint and buckets before storage-backed features work. + +### Queue + +The public queue delivers immediately and is non-durable. Publishing sends a +request back to the local route. Nothing outlives that request. + +It accepts `delayInSeconds`, retries, flow ordering and callbacks but does not +act on them. A delayed message fires immediately, a failed delivery is not +retried and `parallel: 1` does not serialize work. Install a durable queue for +features that depend on those semantics. It does suppress duplicate deliveries +within one process for 30 minutes, but that memory is neither shared nor +durable. + +### Cache + +The public cache is a bounded in-process LRU unless `REDIS_URL` is set. State is +per process and is lost on restart. In a multi-process deployment, rate-limit +and cache state does not coordinate without a shared implementation. + +### Vector storage + +The public vector module stores records in local JSON files unless `QDRANT_URL` +is set. Embedding still requires a configured model provider, such as +`OPENAI_API_KEY`. + +### Email + +The public email module writes delivery information to the console. This makes +local email-code sign-in usable without SMTP but does not deliver external +mail. + +### Sandbox + +The public sandbox runs code in the application process for development and +refuses under `NODE_ENV=production`. Production code execution requires an +isolated implementation with explicit CPU, memory, disk, network, lifetime and +tenant boundaries. + +### Unavailable service defaults + +The public batch runner, realtime relay, screenshot capture and response +delivery modules keep the application importable but refuse their service +operations. Their `assertConfigured` checks fail so deployment readiness tests +cannot mistake an unavailable capability for a production backend. Features +that need scheduled batch work, live relay channels, captured pages, or +outbound response delivery require an operator implementation. + +### Optional and no-op defaults + +The default search engine finds nothing and the PII module passes content +through without detecting or redacting anything. Platform-secret and partner +catalogues are empty. Observability writes exceptions and messages to the +console; tags and spans are debug-only, and the framework adapters are no-ops. +Callers handle these states without requiring a vendor. diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..8395090 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,22 @@ +import { FlatCompat } from '@eslint/eslintrc' + +const compat = new FlatCompat({ + baseDirectory: import.meta.dirname, + resolvePluginsRelativeTo: import.meta.dirname, +}) + +const config = [ + { + ignores: [ + '**/node_modules/**', + '**/dist/**', + '**/types/**', + '**/.next/**', + '**/coverage/**', + '**/cdn/**', + ], + }, + ...compat.extends('@chatbotkit-dev/eslint-config'), +] + +export default config diff --git a/package.json b/package.json new file mode 100644 index 0000000..c57b743 --- /dev/null +++ b/package.json @@ -0,0 +1,39 @@ +{ + "name": "platform", + "version": "0.0.0", + "private": true, + "license": "Apache-2.0", + "packageManager": "pnpm@11.24.0", + "engines": { + "node": ">=24.20.0", + "pnpm": ">=11.24.0 <12.0.0" + }, + "scripts": { + "build": "turbo run build", + "check": "turbo run check", + "clean": "turbo run clean", + "format": "run-p format:*", + "format:00-all": "turbo run format", + "format:01-package": "format-package -w", + "format:02-prettier": "prettier -w .", + "lint": "turbo run lint", + "test": "turbo run test" + }, + "dependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@eslint/eslintrc": "^3.3.6", + "@trivago/prettier-plugin-sort-imports": "^4.3.0", + "@tsconfig/recommended": "^1.0.2", + "@types/node": "^24.0.0", + "dotenv": "^16.0.3", + "eslint": "^9.0.0", + "format-package": "^7.0.0", + "npm-run-all2": "^9.0.3", + "prettier": "^2.8.7", + "rimraf": "^5.0.5", + "ts-node": "^10.9.1", + "tsconfig-paths": "^4.2.0", + "turbo": "^2.10.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/AGENTS.md b/packages/AGENTS.md new file mode 100644 index 0000000..83cf122 --- /dev/null +++ b/packages/AGENTS.md @@ -0,0 +1,261 @@ +# Packages + +Shared code that the platform consumes as real dependencies. The complete tree +is published with the platform, so every package must work from a standalone +checkout. A package may also be published independently when its own manifest +and release configuration say so. + +Some packages here are the public default half of a swappable module and can be +replaced at install time by a deployment-specific implementation. Those have +the extra requirements below. + +## Nothing here may depend on an unpublished counterpart + +Deployment operators may replace a public default with their own implementation. +That implementation is not part of this repository and must not be required to +understand, install or run the public package. + +Three things must never appear in a public package - not in source, not in a +comment, not in a runtime message, not in a README: + +| Never | Instead | +| ------------------------------------------------------------------ | ------------------------------- | +| a deployment-specific package's concrete name | the contract it satisfies | +| an operator's service or directory name | what that class of backend does | +| the infrastructure behind one (a named VMM, orchestrator, or host) | the property that matters | + +The rule is about _identity_, not about technical detail. Explaining that a +backend which embeds server-side cannot be handed vectors is exactly the kind of +reasoning these files should carry. Naming the service that does it is what +turns a design note into a disclosure. + +A public third-party library is not covered by this. `@chatbotkit-dev/memcache` +documents at length that its serialization is a deliberate port of a published +Redis client's, and it should: a reader can go and check that claim. + +### The two places this leaks without anyone noticing + +**Runtime messages.** `assertConfigured` and unsupported-operation errors are +written for whoever is deploying, so the temptation is to name the package they +should install. That string ends up in logs, in support tickets and in the +terminal of someone who has never heard of the private tree. Name the override +point and the contract: + +```ts +'no batch backend is installed, so container jobs cannot run - override +@chatbotkit-dev/batch with a package whose default export satisfies +BatchProvider from @chatbotkit-dev/batch-spec' +``` + +**README override examples.** Use a placeholder rather than an implementation +from one deployment, and say what qualifies: + +```yaml +overrides: + '@chatbotkit-dev/': npm:your--implementation@* +``` + +### Checking + +Search for deployment-specific package scopes, service prefixes, repository +names and infrastructure nouns before publishing. This is especially important +for a new spec: a contract is usually written by someone who has just finished +reading an implementation, and that implementation's vocabulary comes along +with it. + +## Source is TypeScript, tests are JavaScript + +This split is deliberate, not an accident of history. + +**Source files are TypeScript.** Every `.ts` file is type checked by +`pnpm check`, which is what makes a shared package safe to depend on: a change +to an exported signature fails at the call site rather than at runtime in +whichever site imported it. A package whose source is `.js` is not checked at +all — `checkJs` is off — so its exports are effectively untyped no matter how +much JSDoc it carries. + +Typing means real types on the exported surface, not a `.ts` extension. JSDoc +annotations are **ignored** in a `.ts` file, so moving a documented `.js` file +across without writing its signatures silently turns every optional parameter +into a required one and breaks arity at every call site. + +The same applies to JSDoc _cast expressions_, which are easier to miss because +the code still reads as though it asserts something: + +```ts +// inert in a .ts file - the cast does nothing and the type is whatever was +// inferred, which here loses the inner wrapper's options entirely +export const fetchPlusPlus = + /** @type {FetchFn} */ ( + withRetry(withTimeout(fetch)) + ) + +// what it has to become +export const fetchPlusPlus = withRetry(withTimeout(fetch)) as FetchFn< + withTimeoutOptions & withRetryOptions +> +``` + +Grep a converted file for `/** @type` before calling it done. Nothing warns you: +the file compiles, and the error surfaces at a call site in another package. + +**Test files are JavaScript** — `*.test.js`, and `*.utest.js` in the sites. +Tests exercise the package the way a consumer does, including the shapes a +consumer can actually pass. Writing them in TypeScript makes the compiler reject +the wrong-on-purpose input a test exists to cover, so the test either gets +weakened until it type checks or acquires casts that assert away the very thing +under test. Tests are run, not compiled — see the `checkJs` note under Jest +configuration for what actually enforces that, and exclude `**/*.test.js` from +the package `tsconfig.json` so `pnpm check` does not pick them up either. + +``` +src/index.ts <- TypeScript, type checked by `pnpm check` +src/index.test.js <- JavaScript, run by `pnpm test` +``` + +**Two exceptions, and both are load-bearing.** `partners` has JavaScript source +because `platform/next.config.d/partner.config.js` reads the partner catalogue +and Node loads `next.config.js` directly, with no bundler and no transpile. A +`.ts` entry point there fails to import at build time, so renaming it breaks +`next build` rather than `pnpm check`. The contract is still enforced: the +package sets `checkJs`, its catalogue's JSDoc annotation is checked against +`partners-spec`, and the exported surface is declared in a hand-written +`src/index.d.ts`. Anything the catalogue imports at module scope inherits the +same constraint, which is why the mail transport in `packages/partners` defers +its `@chatbotkit-dev/email` import to send time. + +The `observability` package's `next/config` entry point is JavaScript for the +same reason. `next.config.js` loads it directly before webpack exists, including +from the materialized `node_modules` tree produced by `pnpm deploy`. Its runtime +client and server entry points remain TypeScript because Next transpiles those. + +Some older packages still have `.test.ts` files. They predate this rule; leave +them alone unless you are already rewriting the test, and do not add new ones. + +## Jest configuration + +A package's `jest.config.js` is three or four lines. Anything longer is usually +a workaround for something that already works. + +```js +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} +``` + +with `"test": "NODE_OPTIONS=--experimental-vm-modules jest"`. + +**Never add a `moduleNameMapper` for a workspace package.** pnpm already +symlinks `@chatbotkit-dev/x` into the package's `node_modules`, and that +package's `exports` already points at `./src/index.ts`, so jest resolves it with +no help. A hand-written map is redundant on the day it is written and wrong +soon after: it has to be extended for every new dependency, and nothing fails +when an entry is stale, so entries rot in place. One such block shipped a +mapping for `@chatbotkit-dev/http` against a package actually named +`http-codes` — dead the moment it was written, silently. + +**Use the CommonJS preset only when the tests need it.** Under the ESM preset +there is no `jest` global and no `jest.mock` hoisting; the ESM way is +`import { jest } from '@jest/globals'` plus `jest.unstable_mockModule`. Tests +that rely on hoisted `jest.mock` need the CommonJS preset instead, and the +config should say why: + +```js +// @note CommonJS transform: these tests use `jest.mock` hoisting and the `jest` +// global, neither of which is available under the ESM preset. + +export default { + preset: 'ts-jest', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', + + transform: { + '^.+\\.[jt]sx?$': [ + 'ts-jest', + { + useESM: false, + // @note transpile only. Type checking is the `check` script's job. + isolatedModules: true, + tsconfig: { module: 'commonjs', esModuleInterop: true, allowJs: true }, + }, + ], + }, +} +``` + +**`checkJs` defeats the tests-are-not-type-checked rule.** ts-jest compiles a +test with the package's own `tsconfig.json`, so `"checkJs": true` makes the ESM +preset type check `.test.js` files and reject exactly the loose fixtures a test +exists to hold. Since every source in a package is TypeScript, `checkJs` has +nothing legitimate left to check — leave it `false`. + +## Everything in a package is TypeScript + +Not just the entry point. A package with `index.ts` next to an unchecked +`helpers.js` gets no error when `helpers.js` stops matching how `index.ts` calls +it, which is the failure the extension was supposed to prevent. + +There is a second reason, and it only bites in a materialized deployment. +**TypeScript does not read +`.js` files inside `node_modules`** - `maxNodeModuleJsDepth` defaults to `0`. +See the section below for why the local workspace does not expose the problem. + +So a JavaScript source in a package is not a slightly-weaker package. It can +pass through workspace symlinks and fail after deployment packaging, with an +error that points at the importing application rather than the cause. If a file +is hard to type, that is a reason to type it carefully, not a reason to leave +it. + +## Packages resolve differently after deployment packaging + +This is the single most expensive thing to know about this repository, and it +has produced four separate CI failures that were all invisible on the branch. + +Locally, pnpm **symlinks** a workspace package into `node_modules`. Its real +path is `packages//src/index.ts` — outside `node_modules` — so every tool +treats it as ordinary source. + +The production packaging path runs `pnpm deploy --legacy`, which +**materialises** the same package at +`node_modules/.pnpm/@scope+name@file+packages+name_/…`. That +path contains `/node_modules/`, and every tool in the chain has a rule about not +processing `node_modules`: + +| Tool | The rule | What breaks | +| ---------------- | ------------------------- | --------------------------------------------------------------------------------- | +| `tsc6` | `maxNodeModuleJsDepth: 0` | a `.js` source in a package loses all its named exports | +| `jest` | `transformIgnorePatterns` | a `.ts` source is fed to node raw: "Cannot use import statement outside a module" | +| `next` / webpack | `transpilePackages` | "Module parse failed: Unexpected token" on the first `import type` | + +Two further traps in the same family: + +- **Undeclared `@types/*` resolve by accident.** A package that uses typings it + does not declare finds nothing locally, so the import degrades to `any` and is + never checked. In the deploy layout, resolution walks into pnpm's hoist + directory, finds the typings some _other_ package declared, and type checks the + file for the first time — surfacing bugs that were always there. Declare every + `@types/*` your source uses, in `dependencies` rather than `devDependencies`, + because `pnpm deploy` prunes devDependencies of transitive workspace packages. +- **The `.pnpm` directory name is not a semver.** A workspace package appears as + `@scope+name@file+packages+name_`, so allowlist patterns written as + `\.pnpm/@\d+\.\d+\.\d+` never match it. + +### Verifying + +`pnpm check`, `pnpm test` and `pnpm lint` all run in the local layout, so none +of them can see this class of failure. Reproduce the packaged layout directly: + +```bash +pnpm deploy --legacy -F @chatbotkit/platform platform/build-artifacts +cd platform/build-artifacts +# then whichever of tsc6 / jest / next build is in question +``` + +Do this after moving code into a package, after adding a package to the +platform's dependencies, and after changing anything that names packages — +`transpilePackages`, `transformIgnorePatterns`, `.pnpmfile.cjs`. `pnpm deploy` +takes a few minutes; each of the four failures above cost considerably more. + +`platform/build-artifacts` is not gitignored. Remove it when you are done. diff --git a/packages/README.md b/packages/README.md new file mode 100644 index 0000000..74dcfce --- /dev/null +++ b/packages/README.md @@ -0,0 +1,11 @@ +# Packages + +This directory contains the public libraries and module contracts used by the +platform application. Some pairs define a swappable module: a `*-spec` package +owns the contract, while its sibling package supplies the default implementation +that a standalone checkout runs. + +Every package must build, test and document its configuration without relying on +code outside this repository. Package-specific behavior and environment +variables are documented in each package's README. The shared authoring and +verification rules live in [AGENTS.md](./AGENTS.md). diff --git a/packages/auxiliary-google-calendar/README.md b/packages/auxiliary-google-calendar/README.md new file mode 100644 index 0000000..02bbe05 --- /dev/null +++ b/packages/auxiliary-google-calendar/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-google-calendar diff --git a/packages/auxiliary-google-calendar/jest.config.js b/packages/auxiliary-google-calendar/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-google-calendar/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-google-calendar/package.json b/packages/auxiliary-google-calendar/package.json new file mode 100644 index 0000000..0f8795d --- /dev/null +++ b/packages/auxiliary-google-calendar/package.json @@ -0,0 +1,31 @@ +{ + "name": "@chatbotkit-dev/auxiliary-google-calendar", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-google-calendar/src/index.test.ts b/packages/auxiliary-google-calendar/src/index.test.ts new file mode 100644 index 0000000..1df72bb --- /dev/null +++ b/packages/auxiliary-google-calendar/src/index.test.ts @@ -0,0 +1,5 @@ +describe('true', () => { + it('should be true', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/auxiliary-google-calendar/src/index.ts b/packages/auxiliary-google-calendar/src/index.ts new file mode 100644 index 0000000..d68947d --- /dev/null +++ b/packages/auxiliary-google-calendar/src/index.ts @@ -0,0 +1 @@ +export const PLACEHOLDER = 'placeholder' diff --git a/packages/auxiliary-google-calendar/tsconfig.json b/packages/auxiliary-google-calendar/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-google-calendar/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-google-docs/README.md b/packages/auxiliary-google-docs/README.md new file mode 100644 index 0000000..c8dda16 --- /dev/null +++ b/packages/auxiliary-google-docs/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-google-docs diff --git a/packages/auxiliary-google-docs/jest.config.js b/packages/auxiliary-google-docs/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-google-docs/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-google-docs/package.json b/packages/auxiliary-google-docs/package.json new file mode 100644 index 0000000..da555b8 --- /dev/null +++ b/packages/auxiliary-google-docs/package.json @@ -0,0 +1,31 @@ +{ + "name": "@chatbotkit-dev/auxiliary-google-docs", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-google-docs/src/index.test.ts b/packages/auxiliary-google-docs/src/index.test.ts new file mode 100644 index 0000000..1df72bb --- /dev/null +++ b/packages/auxiliary-google-docs/src/index.test.ts @@ -0,0 +1,5 @@ +describe('true', () => { + it('should be true', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/auxiliary-google-docs/src/index.ts b/packages/auxiliary-google-docs/src/index.ts new file mode 100644 index 0000000..d68947d --- /dev/null +++ b/packages/auxiliary-google-docs/src/index.ts @@ -0,0 +1 @@ +export const PLACEHOLDER = 'placeholder' diff --git a/packages/auxiliary-google-docs/tsconfig.json b/packages/auxiliary-google-docs/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-google-docs/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-google-mail/README.md b/packages/auxiliary-google-mail/README.md new file mode 100644 index 0000000..ea4c072 --- /dev/null +++ b/packages/auxiliary-google-mail/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-google-mail diff --git a/packages/auxiliary-google-mail/jest.config.js b/packages/auxiliary-google-mail/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-google-mail/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-google-mail/package.json b/packages/auxiliary-google-mail/package.json new file mode 100644 index 0000000..47bfecb --- /dev/null +++ b/packages/auxiliary-google-mail/package.json @@ -0,0 +1,31 @@ +{ + "name": "@chatbotkit-dev/auxiliary-google-mail", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-google-mail/src/index.test.ts b/packages/auxiliary-google-mail/src/index.test.ts new file mode 100644 index 0000000..1df72bb --- /dev/null +++ b/packages/auxiliary-google-mail/src/index.test.ts @@ -0,0 +1,5 @@ +describe('true', () => { + it('should be true', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/auxiliary-google-mail/src/index.ts b/packages/auxiliary-google-mail/src/index.ts new file mode 100644 index 0000000..d68947d --- /dev/null +++ b/packages/auxiliary-google-mail/src/index.ts @@ -0,0 +1 @@ +export const PLACEHOLDER = 'placeholder' diff --git a/packages/auxiliary-google-mail/tsconfig.json b/packages/auxiliary-google-mail/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-google-mail/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-google-meet/README.md b/packages/auxiliary-google-meet/README.md new file mode 100644 index 0000000..f9fa624 --- /dev/null +++ b/packages/auxiliary-google-meet/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-google-meet diff --git a/packages/auxiliary-google-meet/jest.config.js b/packages/auxiliary-google-meet/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-google-meet/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-google-meet/package.json b/packages/auxiliary-google-meet/package.json new file mode 100644 index 0000000..e9caafc --- /dev/null +++ b/packages/auxiliary-google-meet/package.json @@ -0,0 +1,31 @@ +{ + "name": "@chatbotkit-dev/auxiliary-google-meet", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-google-meet/src/index.test.ts b/packages/auxiliary-google-meet/src/index.test.ts new file mode 100644 index 0000000..1df72bb --- /dev/null +++ b/packages/auxiliary-google-meet/src/index.test.ts @@ -0,0 +1,5 @@ +describe('true', () => { + it('should be true', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/auxiliary-google-meet/src/index.ts b/packages/auxiliary-google-meet/src/index.ts new file mode 100644 index 0000000..d68947d --- /dev/null +++ b/packages/auxiliary-google-meet/src/index.ts @@ -0,0 +1 @@ +export const PLACEHOLDER = 'placeholder' diff --git a/packages/auxiliary-google-meet/tsconfig.json b/packages/auxiliary-google-meet/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-google-meet/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-graphql/README.md b/packages/auxiliary-graphql/README.md new file mode 100644 index 0000000..de941a1 --- /dev/null +++ b/packages/auxiliary-graphql/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-graphql diff --git a/packages/auxiliary-graphql/jest.config.js b/packages/auxiliary-graphql/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-graphql/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-graphql/package.json b/packages/auxiliary-graphql/package.json new file mode 100644 index 0000000..b365b6e --- /dev/null +++ b/packages/auxiliary-graphql/package.json @@ -0,0 +1,45 @@ +{ + "name": "@chatbotkit-dev/auxiliary-graphql", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./notion": { + "import": "./src/notion.ts" + }, + "./slack": { + "import": "./src/slack.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@pothos/core": "^4.8.1", + "graphql": "^16.11.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-graphql/src/builder.ts b/packages/auxiliary-graphql/src/builder.ts new file mode 100644 index 0000000..af7b342 --- /dev/null +++ b/packages/auxiliary-graphql/src/builder.ts @@ -0,0 +1,24 @@ +import SchemaBuilder from '@pothos/core' + +/** + * Context interface for the auxiliary GraphQL schema. + * Contains authentication session and caller information. + */ +export interface Context { + session?: { + user?: { + id?: string | null + email?: string | null + name?: string | null + } | null + } | null + caller?: string | null +} + +/** + * Shared schema builder instance for all auxiliary service schemas. + * This builder is used across notion.ts, slack.ts, and other service modules. + */ +export const builder = new SchemaBuilder<{ + Context: Context +}>({}) diff --git a/packages/auxiliary-graphql/src/http-field.ts b/packages/auxiliary-graphql/src/http-field.ts new file mode 100644 index 0000000..eafebd2 --- /dev/null +++ b/packages/auxiliary-graphql/src/http-field.ts @@ -0,0 +1,107 @@ +/** + * HTTP Field Helper for Pothos GraphQL + * + * This module provides utilities for creating GraphQL fields that automatically + * make HTTP requests and handle responses. + */ + +/** + * HTTP request configuration for automatic resolver generation + */ +export interface HttpFieldConfig> { + // @note http method to use + method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' + + // @note url template (can use {arg} placeholders) or function + url: string | ((args: TArgs) => string) + + // @note optional headers + headers?: Record | ((args: TArgs) => Record) + + // @note optional body transformer (for POST/PUT/PATCH) + body?: (args: TArgs) => unknown + + // @note optional response transformer + transform?: (response: unknown) => unknown + + // @note optional error handler + onError?: (error: Error) => unknown +} + +/** + * Creates an HTTP field resolver that automatically handles requests + * + * @example + * ```typescript + * search: t.field({ + * type: SearchResult, + * args: { query: t.arg.string({ required: true }) }, + * ...createHttpField({ + * method: 'GET', + * url: (args) => `https://api.example.com/search?q=${args.query}`, + * headers: { 'Authorization': 'Bearer token' }, + * transform: (data) => ({ items: data.results, total: data.count }) + * }) + * }) + * ``` + */ +export function createHttpField>( + config: HttpFieldConfig +) { + return { + resolve: async (_parent: unknown, args: TArgs) => { + try { + // @note build url from template or function + const url = + typeof config.url === 'function' + ? config.url(args) + : config.url.replace(/\{(\w+)\}/g, (_, key) => + encodeURIComponent( + String((args as Record)[key] || '') + ) + ) + + // @note build headers + const headers = + typeof config.headers === 'function' + ? config.headers(args) + : config.headers || {} + + // @note build request body if applicable + const body = + config.body && ['POST', 'PUT', 'PATCH'].includes(config.method) + ? JSON.stringify(config.body(args)) + : undefined + + // @note make http request using native fetch (this is a runtime-agnostic library package) + // eslint-disable-next-line no-restricted-globals + const response = await fetch(url, { + method: config.method, + headers: { + 'Content-Type': 'application/json', + ...headers, + }, + body, + }) + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`) + } + + const data = await response.json() + + // @note transform response if transformer provided + return config.transform ? config.transform(data) : data + } catch (error) { + // @note handle error if handler provided + if (config.onError) { + return config.onError( + error instanceof Error ? error : new Error(String(error)) + ) + } + + throw error + } + }, + } +} diff --git a/packages/auxiliary-graphql/src/index.test.ts b/packages/auxiliary-graphql/src/index.test.ts new file mode 100644 index 0000000..6511909 --- /dev/null +++ b/packages/auxiliary-graphql/src/index.test.ts @@ -0,0 +1,41 @@ +import { schema } from './index' + +describe('Auxiliary GraphQL Schema', () => { + it('should build a valid schema', () => { + expect(schema).toBeDefined() + expect(schema.getQueryType()).toBeDefined() + expect(schema.getMutationType()).toBeDefined() + }) + + it('should have notion namespace in query', () => { + const queryType = schema.getQueryType() + const fields = queryType?.getFields() + + expect(fields?.notion).toBeDefined() + expect(fields?.notion.type.toString()).toContain('NotionNamespace') + }) + + it('should have slack namespace in query', () => { + const queryType = schema.getQueryType() + const fields = queryType?.getFields() + + expect(fields?.slack).toBeDefined() + expect(fields?.slack.type.toString()).toContain('SlackNamespace') + }) + + it('should have notion namespace in mutation', () => { + const mutationType = schema.getMutationType() + const fields = mutationType?.getFields() + + expect(fields?.notion).toBeDefined() + expect(fields?.notion.type.toString()).toContain('NotionNamespace') + }) + + it('should have slack namespace in mutation', () => { + const mutationType = schema.getMutationType() + const fields = mutationType?.getFields() + + expect(fields?.slack).toBeDefined() + expect(fields?.slack.type.toString()).toContain('SlackNamespace') + }) +}) diff --git a/packages/auxiliary-graphql/src/index.ts b/packages/auxiliary-graphql/src/index.ts new file mode 100644 index 0000000..5d2f25c --- /dev/null +++ b/packages/auxiliary-graphql/src/index.ts @@ -0,0 +1,12 @@ +// @note export the builder and context type for external use +export { builder, type Context } from './builder' + +// @note export HTTP field helper +export { createHttpField, type HttpFieldConfig } from './http-field' + +// @note export service namespaces +export { NotionNamespace } from './notion' +export { SlackNamespace } from './slack' + +// @note export the combined schema +export { schema } from './schema' diff --git a/packages/auxiliary-graphql/src/notion.ts b/packages/auxiliary-graphql/src/notion.ts new file mode 100644 index 0000000..9ecca73 --- /dev/null +++ b/packages/auxiliary-graphql/src/notion.ts @@ -0,0 +1,97 @@ +import { builder } from './builder' + +/** + * Notion auxiliary schema. + * This module provides namespaced GraphQL types and operations for Notion integration. + */ + +// @note notion search result item type +const NotionSearchResultItem = builder + .objectRef<{ + id: string + title: string + type: string + url?: string + }>('NotionSearchResultItem') + .implement({ + fields: (t) => ({ + id: t.exposeString('id', { + description: 'Unique identifier of the Notion item', + }), + title: t.exposeString('title', { + description: 'Title of the Notion page or database', + }), + type: t.exposeString('type', { + description: 'Type of the item (page, database, etc.)', + }), + url: t.exposeString('url', { + nullable: true, + description: 'URL to the Notion item', + }), + }), + }) + +// @note notion search result type +const NotionSearchResult = builder + .objectRef<{ + items: Array<{ + id: string + title: string + type: string + url?: string + }> + total: number + }>('NotionSearchResult') + .implement({ + fields: (t) => ({ + items: t.field({ + type: [NotionSearchResultItem], + resolve: (parent) => parent.items, + description: 'Array of search result items', + }), + total: t.exposeInt('total', { + description: 'Total number of results found', + }), + }), + }) + +// @note notion namespace type for organizing all notion-related queries and mutations +export const NotionNamespace = builder + .objectRef('NotionNamespace') + .implement({ + fields: (t) => ({ + // @todo add notion-specific query fields here + version: t.string({ + resolve: () => '1.0.0', + }), + + // @note search notion content + search: t.field({ + type: NotionSearchResult, + args: { + query: t.arg.string({ + required: true, + description: 'Search query string', + }), + }, + resolve: (_parent, args) => { + // @todo implement actual notion search logic + return { + items: [ + { + id: 'notion-1', + title: `Notion result for: ${args.query}`, + type: 'page', + url: 'https://notion.so/example', + }, + ], + total: 1, + } + }, + }), + }), + }) + +// @todo add notion-specific object types +// @todo add notion-specific queries +// @todo add notion-specific mutations diff --git a/packages/auxiliary-graphql/src/schema.ts b/packages/auxiliary-graphql/src/schema.ts new file mode 100644 index 0000000..eab4c9e --- /dev/null +++ b/packages/auxiliary-graphql/src/schema.ts @@ -0,0 +1,48 @@ +import { builder } from './builder' +// @note import all service schemas to register their types +import { NotionNamespace } from './notion' +import { SlackNamespace } from './slack' + +// @note create the root query type +builder.queryType({ + fields: (t) => ({ + // @note notion namespace for all notion-related queries + notion: t.field({ + type: NotionNamespace, + resolve: () => ({}), + }), + + // @note slack namespace for all slack-related queries + slack: t.field({ + type: SlackNamespace, + resolve: () => ({}), + }), + + // @todo add additional auxiliary service namespaces here + }), +}) + +// @note create the root mutation type +builder.mutationType({ + fields: (t) => ({ + // @note notion namespace for all notion-related mutations + notion: t.field({ + type: NotionNamespace, + resolve: () => ({}), + }), + + // @note slack namespace for all slack-related mutations + slack: t.field({ + type: SlackNamespace, + resolve: () => ({}), + }), + + // @todo add additional auxiliary service namespaces here + }), +}) + +/** + * Build and export the final GraphQL schema. + * This schema combines all auxiliary service schemas into a single executable schema. + */ +export const schema = builder.toSchema() diff --git a/packages/auxiliary-graphql/src/slack.ts b/packages/auxiliary-graphql/src/slack.ts new file mode 100644 index 0000000..082c510 --- /dev/null +++ b/packages/auxiliary-graphql/src/slack.ts @@ -0,0 +1,183 @@ +import { builder } from './builder' + +/** + * Slack auxiliary schema. + * This module provides namespaced GraphQL types and operations for Slack integration. + */ + +// @note slack search result item type +const SlackSearchResultItem = builder + .objectRef<{ + id: string + text: string + channel: string + user?: string + timestamp: string + }>('SlackSearchResultItem') + .implement({ + fields: (t) => ({ + id: t.exposeString('id', { + description: 'Unique identifier of the Slack message', + }), + text: t.exposeString('text', { + description: 'Message text content', + }), + channel: t.exposeString('channel', { + description: 'Channel ID or name where the message was posted', + }), + user: t.exposeString('user', { + nullable: true, + description: 'User ID who posted the message', + }), + timestamp: t.exposeString('timestamp', { + description: 'Message timestamp', + }), + }), + }) + +// @note slack search result type +const SlackSearchResult = builder + .objectRef<{ + items: Array<{ + id: string + text: string + channel: string + user?: string + timestamp: string + }> + total: number + }>('SlackSearchResult') + .implement({ + fields: (t) => ({ + items: t.field({ + type: [SlackSearchResultItem], + resolve: (parent) => parent.items, + description: 'Array of search result items', + }), + total: t.exposeInt('total', { + description: 'Total number of results found', + }), + }), + }) + +// @note slack send message result type +const SlackSendMessageResult = builder + .objectRef<{ + success: boolean + messageId?: string + timestamp?: string + error?: string + }>('SlackSendMessageResult') + .implement({ + fields: (t) => ({ + success: t.exposeBoolean('success', { + description: 'Whether the message was sent successfully', + }), + messageId: t.exposeString('messageId', { + nullable: true, + description: 'ID of the sent message', + }), + timestamp: t.exposeString('timestamp', { + nullable: true, + description: 'Timestamp of the sent message', + }), + error: t.exposeString('error', { + nullable: true, + description: 'Error message if the operation failed', + }), + }), + }) + +// @note slack namespace type for organizing all slack-related queries and mutations +export const SlackNamespace = builder + .objectRef('SlackNamespace') + .implement({ + fields: (t) => ({ + // @todo add slack-specific query fields here + version: t.string({ + resolve: () => '1.0.0', + }), + + // @note search slack messages and channels + search: t.field({ + type: SlackSearchResult, + args: { + query: t.arg.string({ + required: true, + description: 'Search query string', + }), + }, + resolve: (_parent, args) => { + // @todo implement actual slack search logic + return { + items: [ + { + id: 'slack-1', + text: `Slack message containing: ${args.query}`, + channel: 'general', + user: 'U12345', + timestamp: new Date().toISOString(), + }, + ], + total: 1, + } + }, + }), + + // @note send a message to a slack channel + sendMessage: t.field({ + type: SlackSendMessageResult, + args: { + channel: t.arg.string({ + required: true, + description: 'Channel ID or name to send the message to', + }), + text: t.arg.string({ + required: true, + description: 'Message text content', + }), + }, + resolve: (_parent, _args) => { + // @todo implement actual slack send message logic + return { + success: true, + messageId: `msg-${Date.now()}`, + timestamp: new Date().toISOString(), + } + }, + }), + }), + }) + +// @todo add slack-specific object types +// @todo add slack-specific queries +// @todo add slack-specific mutations + +/** + * Example usage of createHttpField helper: + * + * import { createHttpField } from './http-field' + * + * getChannels: t.field({ + * type: ChannelListResult, + * args: { + * limit: t.arg.int({ required: false }), + * }, + * ...createHttpField({ + * method: 'GET', + * url: (args) => `https://slack.com/api/conversations.list?limit=${args.limit || 100}`, + * headers: (args) => ({ + * 'Authorization': `Bearer ${process.env.SLACK_TOKEN}`, + * }), + * transform: (data) => ({ + * channels: data.channels, + * total: data.channels.length, + * }), + * onError: (error) => ({ + * channels: [], + * total: 0, + * error: error.message, + * }), + * }), + * }), + */ diff --git a/packages/auxiliary-graphql/tsconfig.json b/packages/auxiliary-graphql/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-graphql/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-microsoft-drive/README.md b/packages/auxiliary-microsoft-drive/README.md new file mode 100644 index 0000000..7487051 --- /dev/null +++ b/packages/auxiliary-microsoft-drive/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-microsoft-drive diff --git a/packages/auxiliary-microsoft-drive/jest.config.js b/packages/auxiliary-microsoft-drive/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-microsoft-drive/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-microsoft-drive/package.json b/packages/auxiliary-microsoft-drive/package.json new file mode 100644 index 0000000..2c76b67 --- /dev/null +++ b/packages/auxiliary-microsoft-drive/package.json @@ -0,0 +1,35 @@ +{ + "name": "@chatbotkit-dev/auxiliary-microsoft-drive", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-microsoft-drive/src/index.test.ts b/packages/auxiliary-microsoft-drive/src/index.test.ts new file mode 100644 index 0000000..1df72bb --- /dev/null +++ b/packages/auxiliary-microsoft-drive/src/index.test.ts @@ -0,0 +1,5 @@ +describe('true', () => { + it('should be true', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/auxiliary-microsoft-drive/src/index.ts b/packages/auxiliary-microsoft-drive/src/index.ts new file mode 100644 index 0000000..d68947d --- /dev/null +++ b/packages/auxiliary-microsoft-drive/src/index.ts @@ -0,0 +1 @@ +export const PLACEHOLDER = 'placeholder' diff --git a/packages/auxiliary-microsoft-drive/tsconfig.json b/packages/auxiliary-microsoft-drive/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-microsoft-drive/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-microsoft-sharepoint/README.md b/packages/auxiliary-microsoft-sharepoint/README.md new file mode 100644 index 0000000..df45888 --- /dev/null +++ b/packages/auxiliary-microsoft-sharepoint/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-microsoft-sharepoint diff --git a/packages/auxiliary-microsoft-sharepoint/jest.config.js b/packages/auxiliary-microsoft-sharepoint/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-microsoft-sharepoint/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-microsoft-sharepoint/package.json b/packages/auxiliary-microsoft-sharepoint/package.json new file mode 100644 index 0000000..71fe408 --- /dev/null +++ b/packages/auxiliary-microsoft-sharepoint/package.json @@ -0,0 +1,35 @@ +{ + "name": "@chatbotkit-dev/auxiliary-microsoft-sharepoint", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-microsoft-sharepoint/src/index.test.ts b/packages/auxiliary-microsoft-sharepoint/src/index.test.ts new file mode 100644 index 0000000..1df72bb --- /dev/null +++ b/packages/auxiliary-microsoft-sharepoint/src/index.test.ts @@ -0,0 +1,5 @@ +describe('true', () => { + it('should be true', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/auxiliary-microsoft-sharepoint/src/index.ts b/packages/auxiliary-microsoft-sharepoint/src/index.ts new file mode 100644 index 0000000..d68947d --- /dev/null +++ b/packages/auxiliary-microsoft-sharepoint/src/index.ts @@ -0,0 +1 @@ +export const PLACEHOLDER = 'placeholder' diff --git a/packages/auxiliary-microsoft-sharepoint/tsconfig.json b/packages/auxiliary-microsoft-sharepoint/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/auxiliary-microsoft-sharepoint/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/auxiliary-notion/README.md b/packages/auxiliary-notion/README.md new file mode 100644 index 0000000..9f04f51 --- /dev/null +++ b/packages/auxiliary-notion/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/auxiliary-notion diff --git a/packages/auxiliary-notion/jest.config.js b/packages/auxiliary-notion/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/auxiliary-notion/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/auxiliary-notion/package.json b/packages/auxiliary-notion/package.json new file mode 100644 index 0000000..2fad6c3 --- /dev/null +++ b/packages/auxiliary-notion/package.json @@ -0,0 +1,55 @@ +{ + "name": "@chatbotkit-dev/auxiliary-notion", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./properties": { + "import": "./src/properties.ts" + }, + "./client": { + "import": "./src/client.ts" + }, + "./contents": { + "import": "./src/contents.ts" + }, + "./database": { + "import": "./src/database.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/fetch": "workspace:*", + "@chatbotkit-dev/sql": "workspace:*", + "@notionhq/client": "^2.2.4", + "notion-to-md": "2.5.5" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@chatbotkit-dev/typescript-utils": "workspace:*", + "@jest/globals": "^29.7.0", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/auxiliary-notion/src/client.test.ts b/packages/auxiliary-notion/src/client.test.ts new file mode 100644 index 0000000..ff03d3f --- /dev/null +++ b/packages/auxiliary-notion/src/client.test.ts @@ -0,0 +1,19 @@ +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils' + +import { getClient } from './client' + +describe('getClient', () => { + it('should create a Notion client with the provided auth token', () => { + expect(getClient('test_token' as TrimmedNonEmptyString)).toBeDefined() + }) + + it('should create a Notion client with the provided bearer token', () => { + expect( + getClient('Bearer test_token' as TrimmedNonEmptyString) + ).toBeDefined() + }) + + it('should throw if no auth token is provided', () => { + expect(() => getClient('' as TrimmedNonEmptyString)).toThrow() + }) +}) diff --git a/packages/auxiliary-notion/src/client.ts b/packages/auxiliary-notion/src/client.ts new file mode 100644 index 0000000..43d8668 --- /dev/null +++ b/packages/auxiliary-notion/src/client.ts @@ -0,0 +1,28 @@ +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils' + +import { fetch as cbkFetch, withRetry, withTimeout } from '@chatbotkit-dev/fetch' +import { Client } from '@notionhq/client' + +export function getClient( + auth: TrimmedNonEmptyString, + options?: { fetch?: typeof cbkFetch } +): Client { + const clientAuth = auth.replace(/^Bearer\s+/i, '').trim() + + if (!clientAuth) { + throw new Error(`Authentication token not provided`) + } + + return new Client({ + auth: clientAuth as TrimmedNonEmptyString, + + fetch: withRetry( + withTimeout(options?.fetch || cbkFetch, { timeout: 10000 }), + { + retries: 5, + retryDelay: 250, + retryTimeout: true, + } + ), + }) +} diff --git a/packages/auxiliary-notion/src/contents.ts b/packages/auxiliary-notion/src/contents.ts new file mode 100644 index 0000000..2150cf3 --- /dev/null +++ b/packages/auxiliary-notion/src/contents.ts @@ -0,0 +1,26 @@ +import type { fetch as chatbotkitFetch } from '@chatbotkit-dev/fetch' + +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils' + +import { getClient } from './client' + +import { NotionToMarkdown } from 'notion-to-md' + +export async function getContents({ + auth, + pageId, + fetch, +}: { + auth: TrimmedNonEmptyString + pageId: TrimmedNonEmptyString + fetch?: typeof chatbotkitFetch +}): Promise { + const notionClient = getClient(auth, { fetch }) + + const n2m = new NotionToMarkdown({ notionClient }) + + const mdblocks = await n2m.pageToMarkdown(pageId) + const contents = n2m.toMarkdownString(mdblocks) + + return contents +} diff --git a/packages/auxiliary-notion/src/database.test.ts b/packages/auxiliary-notion/src/database.test.ts new file mode 100644 index 0000000..379b3b2 --- /dev/null +++ b/packages/auxiliary-notion/src/database.test.ts @@ -0,0 +1,577 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { + DatabaseProperties} from './database'; +import { + convertDatabaseCreatePropertiesFromKnownProperties, + getMultiSelectUpsertValues, + getSimplifiedDatabaseProperties, +} from './database' + +describe('getSimplifiedDatabaseProperties', () => { + test('should handle title property', () => { + const properties: DatabaseProperties = { + Name: { + id: 'title', + type: 'title', + name: 'Test Title', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Name: 'Test Title' }) + }) + + test('should handle title property when null', () => { + const properties: DatabaseProperties = { + Name: { + id: 'title', + type: 'title', + name: null, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Name: null }) + }) + + test('should handle status property', () => { + const properties: DatabaseProperties = { + Status: { + id: 'status', + type: 'status', + name: 'In Progress', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Status: 'In Progress' }) + }) + + test('should handle rich_text property', () => { + const properties: DatabaseProperties = { + Description: { + id: 'desc', + type: 'rich_text', + name: 'Text Description', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Description: 'Text Description' }) + }) + + test('should handle number property', () => { + const properties: DatabaseProperties = { + Count: { + id: 'count', + type: 'number', + number: 42, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Count: 42 }) + }) + + test('should handle checkbox property', () => { + const properties: DatabaseProperties = { + Completed: { + id: 'completed', + type: 'checkbox', + checkbox: true, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Completed: true }) + }) + + test('should handle select property', () => { + const properties: DatabaseProperties = { + Category: { + id: 'category', + type: 'select', + select: { + options: [ + { name: 'Work', id: '1', color: 'blue' }, + { name: 'Personal', id: '2', color: 'green' }, + ], + }, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Category: ['Work', 'Personal'] }) + }) + + test('should handle multi_select property', () => { + const properties: DatabaseProperties = { + Tags: { + id: 'tags', + type: 'multi_select', + multi_select: { + options: [ + { name: 'urgent', id: '1', color: 'red' }, + { name: 'bug', id: '2', color: 'orange' }, + ], + }, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Tags: ['urgent', 'bug'] }) + }) + + test('should handle date property', () => { + const properties: DatabaseProperties = { + DueDate: { + id: 'due', + type: 'date', + date: { start: '2023-01-01' }, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ DueDate: '2023-01-01' }) + }) + + test('should handle date property when null', () => { + const properties: DatabaseProperties = { + DueDate: { + id: 'due', + type: 'date', + date: null, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ DueDate: null }) + }) + + test('should handle url property', () => { + const properties: DatabaseProperties = { + Website: { + id: 'website', + type: 'url', + url: 'https://example.com', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Website: 'https://example.com' }) + }) + + test('should handle url property when null', () => { + const properties: DatabaseProperties = { + Website: { + id: 'website', + type: 'url', + url: null, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Website: null }) + }) + + test('should handle email property', () => { + const properties: DatabaseProperties = { + Email: { + id: 'email', + type: 'email', + email: 'test@example.com', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Email: 'test@example.com' }) + }) + + test('should handle phone_number property', () => { + const properties: DatabaseProperties = { + Phone: { + id: 'phone', + type: 'phone_number', + phone_number: '123-456-7890', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Phone: '123-456-7890' }) + }) + + test('should handle created_time property', () => { + const timestamp = '2023-01-01T12:00:00Z' + const properties: DatabaseProperties = { + Created: { + id: 'created', + type: 'created_time', + created_time: timestamp, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Created: timestamp }) + }) + + test('should handle last_edited_time property', () => { + const timestamp = '2023-01-01T12:00:00Z' + const properties: DatabaseProperties = { + LastEdited: { + id: 'edited', + type: 'last_edited_time', + last_edited_time: timestamp, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ LastEdited: timestamp }) + }) + + test('should skip ignored property types', () => { + const properties: DatabaseProperties = { + UniqueID: { + id: 'uid', + type: 'unique_id', + unique_id: { number: 123 }, + }, + Formula: { + id: 'formula', + type: 'formula', + formula: { string: 'result' }, + }, + Rollup: { + id: 'rollup', + type: 'rollup', + rollup: { number: 42 }, + }, + Relation: { + id: 'relation', + type: 'relation', + relation: {}, + }, + People: { + id: 'people', + type: 'people', + people: {}, + }, + Files: { + id: 'files', + type: 'files', + files: [], + }, + CreatedBy: { + id: 'created_by', + type: 'created_by', + created_by: {}, + }, + LastEditedBy: { + id: 'last_edited_by', + type: 'last_edited_by', + last_edited_by: {}, + }, + Name: { + id: 'title', + type: 'title', + name: 'Test', + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ Name: 'Test' }) + }) + + test('should handle multiple properties', () => { + const properties: DatabaseProperties = { + Name: { + id: 'title', + type: 'title', + name: 'Test Database', + }, + Done: { + id: 'done', + type: 'checkbox', + checkbox: false, + }, + Priority: { + id: 'priority', + type: 'select', + select: { + options: [ + { name: 'High', id: '1', color: 'red' }, + { name: 'Medium', id: '2', color: 'yellow' }, + { name: 'Low', id: '3', color: 'green' }, + ], + }, + }, + } as any + + const result = getSimplifiedDatabaseProperties(properties) + + expect(result).toEqual({ + Name: 'Test Database', + Done: false, + Priority: ['High', 'Medium', 'Low'], + }) + }) +}) + +describe('getMultiSelectUpsertValues', () => { + test('should return array of trimmed strings from array input', () => { + const input = ['foo', 'bar', 'baz'] + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar', 'baz']) + }) + + test('should split comma-separated strings in array input', () => { + const input = ['foo, bar', 'baz'] + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar', 'baz']) + }) + + test('should trim whitespace from each value in array input', () => { + const input = [' foo ', ' bar ', 'baz'] + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar', 'baz']) + }) + + test('should filter out empty strings in array input', () => { + const input = ['foo', '', ' ', 'bar'] + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar']) + }) + + test('should ignore non-string values in array input', () => { + const input = ['foo', 123 as any, null as any, 'bar'] + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar']) + }) + + test('should return array of trimmed strings from comma-separated string input', () => { + const input = 'foo, bar, baz' + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar', 'baz']) + }) + + test('should handle single string input', () => { + const input = 'foo' + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo']) + }) + + test('should trim whitespace from string input', () => { + const input = ' foo , bar ,baz ' + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar', 'baz']) + }) + + test('should filter out empty strings in string input', () => { + const input = 'foo, , ,bar' + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar']) + }) + + test('should return empty array for non-string, non-array input', () => { + expect(getMultiSelectUpsertValues(123)).toEqual([]) + expect(getMultiSelectUpsertValues(null)).toEqual([]) + expect(getMultiSelectUpsertValues(undefined)).toEqual([]) + expect(getMultiSelectUpsertValues({})).toEqual([]) + }) + + test('should handle String object input', () => { + const input = new String('foo,bar') as any + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar']) + }) + + test('should handle array with String objects', () => { + const input = [new String('foo, bar'), 'baz'] as any + const result = getMultiSelectUpsertValues(input) + + expect(result).toEqual(['foo', 'bar', 'baz']) + }) +}) + +describe('convertDatabaseCreatePropertiesFromKnownProperties', () => { + test('should truncate rich_text content exceeding 2000 characters', () => { + const knownDatabaseProperties = { + Description: { + format: 'rich_text' as const, + type: 'string' as const, + }, + } + + const longContent = 'a'.repeat(2060) + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties: { Description: longContent }, + }) + + const richText = result.properties.Description as { + rich_text: { text: { content: string } }[] + } + + expect(richText.rich_text[0].text.content.length).toBeLessThanOrEqual(2000) + expect(richText.rich_text[0].text.content).toBe('a'.repeat(2000)) + }) + + test('should truncate title content exceeding 2000 characters', () => { + const knownDatabaseProperties = { + Name: { + format: 'title' as const, + type: 'string' as const, + }, + } + + const longContent = 'b'.repeat(2050) + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties: { Name: longContent }, + }) + + const title = result.properties.Name as { + title: { text: { content: string } }[] + } + + expect(title.title[0].text.content.length).toBeLessThanOrEqual(2000) + expect(title.title[0].text.content).toBe('b'.repeat(2000)) + }) + + test('should not truncate rich_text content within 2000 characters', () => { + const knownDatabaseProperties = { + Description: { + format: 'rich_text' as const, + type: 'string' as const, + }, + } + + const normalContent = 'Hello world' + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties: { Description: normalContent }, + }) + + const richText = result.properties.Description as { + rich_text: { text: { content: string } }[] + } + + expect(richText.rich_text[0].text.content).toBe('Hello world') + }) + + test('should handle email property with empty string', () => { + const knownDatabaseProperties = { + Email: { + format: 'email' as const, + type: 'string' as const, + description: 'Valid email address', + }, + } + + const properties = { + Email: '', + } + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, + }) + + // @note Notion API requires email to be null, not empty string + expect(result.properties.Email).toEqual({ email: null }) + }) + + test('should handle email property with valid email', () => { + const knownDatabaseProperties = { + Email: { + format: 'email' as const, + type: 'string' as const, + description: 'Valid email address', + }, + } + + const properties = { + Email: 'test@example.com', + } + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, + }) + + expect(result.properties.Email).toEqual({ email: 'test@example.com' }) + }) + + test('should handle url property with empty string', () => { + const knownDatabaseProperties = { + Website: { + format: 'url' as const, + type: 'string' as const, + description: 'Valid URL', + }, + } + + const properties = { + Website: '', + } + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, + }) + + // @note Notion API requires url to be null, not empty string + expect(result.properties.Website).toEqual({ url: null }) + }) + + test('should handle phone_number property with empty string', () => { + const knownDatabaseProperties = { + Phone: { + format: 'phone_number' as const, + type: 'string' as const, + description: 'Valid phone number', + }, + } + + const properties = { + Phone: '', + } + + const result = convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, + }) + + // @note Notion API requires phone_number to be null, not empty string + expect(result.properties.Phone).toEqual({ phone_number: null }) + }) +}) diff --git a/packages/auxiliary-notion/src/database.ts b/packages/auxiliary-notion/src/database.ts new file mode 100644 index 0000000..e1ba80d --- /dev/null +++ b/packages/auxiliary-notion/src/database.ts @@ -0,0 +1,766 @@ +import type { fetch as chatbotkitFetch } from '@chatbotkit-dev/fetch' + +import type { Immutable } from '@chatbotkit-dev/typescript-utils/object' +import type { ToReadonlyRecord } from '@chatbotkit-dev/typescript-utils/record' +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils/string' + +import { getClient } from './client' + +import type { + CreatePageParameters, + DatabaseObjectResponse, + QueryDatabaseParameters, + UpdatePageParameters, +} from '@notionhq/client/build/src/api-endpoints' + +export type DatabaseProperties = DatabaseObjectResponse['properties'] + +export type SimplifiedDatabaseProperties = Record + +export function getSimplifiedDatabaseProperties( + properties: Immutable +): SimplifiedDatabaseProperties { + const simplified: SimplifiedDatabaseProperties = {} + + for (const [key, value] of Object.entries(properties)) { + if ( + value.type === 'unique_id' || + value.type === 'formula' || + value.type === 'rollup' || + value.type === 'relation' || + value.type === 'people' || + value.type === 'files' || + value.type === 'created_by' || + value.type === 'last_edited_by' + ) { + continue + } + + switch (value.type) { + case 'title': { + simplified[key] = value.name || null + + break + } + + case 'status': { + simplified[key] = value.name || null + + break + } + + case 'rich_text': { + simplified[key] = value.name || null + + break + } + + case 'number': { + simplified[key] = value.number + + break + } + + case 'checkbox': { + simplified[key] = value.checkbox + + break + } + + case 'select': { + simplified[key] = value.select.options.map( + (item: { name: string }) => item.name + ) + + break + } + + case 'multi_select': { + simplified[key] = value.multi_select.options.map( + (item: { name: string }) => item.name + ) + + break + } + + case 'date': { + if (value.date) { + simplified[key] = value.date.start + } else { + simplified[key] = null + } + + break + } + + case 'url': { + simplified[key] = value.url || null + + break + } + + case 'email': { + simplified[key] = value.email || null + + break + } + + case 'phone_number': { + simplified[key] = value.phone_number || null + + break + } + + case 'created_time': { + simplified[key] = value.created_time + + break + } + + case 'last_edited_time': { + simplified[key] = value.last_edited_time + + break + } + + default: { + const x: never = value + + x + } + } + } + + return simplified +} + +export interface IntrospectedDatabaseProperty { + format: Exclude< + DatabaseProperties[string]['type'], + | 'unique_id' + | 'formula' + | 'rollup' + | 'relation' + | 'people' + | 'files' + | 'created_by' + | 'last_edited_by' + > + type: 'string' | 'number' | 'boolean' + enum?: string[] + description?: string +} + +export async function introspectDatabaseProperties({ + auth, + databaseId, + fetch, +}: { + auth: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + fetch?: typeof chatbotkitFetch +}): Promise> { + const client = getClient(auth, { fetch }) + + const data = await client.databases.retrieve({ + database_id: databaseId, + }) + + const properties: Record = {} + + for (const [name, value] of Object.entries(data.properties)) { + if ( + value.type === 'unique_id' || + value.type === 'formula' || + value.type === 'rollup' || + value.type === 'relation' || + value.type === 'people' || + value.type === 'files' || + value.type === 'created_by' || + value.type === 'last_edited_by' + ) { + continue + } + + switch (value.type) { + case 'title': { + properties[name] = { + format: 'title', + type: 'string', + } + + break + } + + case 'status': { + properties[name] = { + format: 'status', + type: 'string', + enum: value.status.options.map((option) => option.name), + } + + break + } + + case 'rich_text': { + properties[name] = { + format: 'rich_text', + type: 'string', + } + + break + } + + case 'number': { + properties[name] = { + format: 'number', + type: 'number', + } + + break + } + + case 'checkbox': { + properties[name] = { + format: 'checkbox', + type: 'boolean', + } + + break + } + + case 'select': { + properties[name] = { + format: 'select', + type: 'string', + enum: value.select.options.map((option) => option.name), + } + + break + } + + case 'multi_select': { + properties[name] = { + format: 'multi_select', + type: 'string', + description: 'Comma-separated values', + } + + break + } + + case 'date': { + properties[name] = { + format: 'date', + type: 'string', + description: 'Date in ISO 8601 format', + } + + break + } + + case 'url': { + properties[name] = { + format: 'url', + type: 'string', + description: 'Valid URL', + } + + break + } + + case 'email': { + properties[name] = { + format: 'email', + type: 'string', + description: 'Valid email address', + } + + break + } + + case 'phone_number': { + properties[name] = { + format: 'phone_number', + type: 'string', + description: 'Valid phone number', + } + + break + } + + case 'created_time': { + properties[name] = { + format: 'created_time', + type: 'string', + description: 'Creation time in ISO 8601 format', + } + + break + } + + case 'last_edited_time': { + properties[name] = { + format: 'last_edited_time', + type: 'string', + description: 'Last edited time in ISO 8601 format', + } + + break + } + + default: { + const x: never = value + + x + } + } + } + + return properties +} + +export function getMultiSelectUpsertValues(input: unknown): string[] { + if (Array.isArray(input)) { + return input + .filter((item) => typeof item === 'string' || item instanceof String) + .flatMap((item) => item.split(',')) + .map((item) => item.trim()) + .filter(Boolean) + } + + if (typeof input === 'string' || input instanceof String) { + return input + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + } + + return [] +} + +export type DatabaseCreateItemProperties = CreatePageParameters['properties'] + +export function convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, +}: { + knownDatabaseProperties: ToReadonlyRecord< + Record + > + properties: ToReadonlyRecord> +}): { + unsupported: string[] + properties: DatabaseCreateItemProperties +} { + const databaseProperties: DatabaseCreateItemProperties = {} + + const unsupportedProperties: string[] = Object.keys(properties).filter( + (key) => !(key in knownDatabaseProperties) + ) + + for (const [key, value] of Object.entries(knownDatabaseProperties)) { + if (!(key in properties)) { + continue + } + + if ( + value.format === 'created_time' || + value.format === 'last_edited_time' + ) { + continue + } + + switch (value.format) { + case 'title': { + // @note Notion API limits rich_text content to 2000 characters per block + databaseProperties[key] = { + title: [ + { + type: 'text', + text: { + content: String(properties[key]).slice(0, 2000), + }, + }, + ], + } + + break + } + + case 'status': { + databaseProperties[key] = { + status: { + name: String(properties[key]), + }, + } + + break + } + + case 'rich_text': { + // @note Notion API limits rich_text content to 2000 characters per block + databaseProperties[key] = { + rich_text: [ + { + type: 'text', + text: { + content: String(properties[key]).slice(0, 2000), + }, + }, + ], + } + + break + } + + case 'number': { + databaseProperties[key] = { + number: parseFloat(String(properties[key])), + } + + break + } + + case 'checkbox': { + databaseProperties[key] = { + checkbox: Boolean(properties[key]), + } + + break + } + + case 'select': { + databaseProperties[key] = { + select: { + name: String(properties[key]), + }, + } + + break + } + + case 'multi_select': { + const values = getMultiSelectUpsertValues(properties[key]) + + if (values.length > 0) { + databaseProperties[key] = { + multi_select: values.map((item) => ({ + name: item, + })), + } + } + + break + } + + case 'date': { + databaseProperties[key] = { + date: { + start: String(properties[key]), + }, + } + + break + } + + case 'url': { + const urlValue = String(properties[key]) + + databaseProperties[key] = { + url: urlValue.trim() === '' ? null : urlValue, + } + + break + } + + case 'email': { + const emailValue = String(properties[key]) + + databaseProperties[key] = { + email: emailValue.trim() === '' ? null : emailValue, + } + + break + } + + case 'phone_number': { + const phoneValue = String(properties[key]) + + databaseProperties[key] = { + phone_number: phoneValue.trim() === '' ? null : phoneValue, + } + + break + } + + default: { + const x: never = value.format + + x + } + } + } + + return { + unsupported: unsupportedProperties, + properties: databaseProperties, + } +} + +export async function convertDatabaseCreateProperties({ + auth, + databaseId, + properties, +}: { + auth: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + properties: ToReadonlyRecord> +}): Promise<{ + unsupported: string[] + properties: DatabaseCreateItemProperties +}> { + const knownDatabaseProperties = await introspectDatabaseProperties({ + auth, + databaseId, + }) + + return convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, + }) +} + +export type DatabaseUpdateItemProperties = UpdatePageParameters['properties'] + +export function convertDatabaseUpdatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, +}: { + knownDatabaseProperties: ToReadonlyRecord< + Record + > + properties: ToReadonlyRecord> +}): { + unsupported: string[] + properties: DatabaseUpdateItemProperties +} { + return convertDatabaseCreatePropertiesFromKnownProperties({ + knownDatabaseProperties, + properties, + }) +} + +export async function convertDatabaseUpdateProperties({ + auth, + databaseId, + properties, +}: { + auth: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + properties: ToReadonlyRecord> +}): Promise<{ + unsupported: string[] + properties: DatabaseUpdateItemProperties +}> { + const { + unsupported: unsupportedProperties, + properties: convertedProperties, + } = await convertDatabaseCreateProperties({ + auth, + databaseId, + properties, + }) + + return { + unsupported: unsupportedProperties, + properties: convertedProperties, + } +} + +export type DatabaseItemFilter = QueryDatabaseParameters['filter'] + +export function convertDatabaseItemFilterFromKnownProperties({ + knownDatabaseProperties, + query, +}: { + knownDatabaseProperties: ToReadonlyRecord< + Record + > + query: ToReadonlyRecord> +}): { unsupported: string[]; filter: DatabaseItemFilter | undefined } { + const filter: DatabaseItemFilter = { + or: [], + } + + const unsupported: string[] = Object.keys(query).filter( + (key) => !(key in knownDatabaseProperties) + ) + + for (const [key, value] of Object.entries(knownDatabaseProperties)) { + if (!(key in query)) { + continue + } + + if ( + value.format === 'created_time' || + value.format === 'last_edited_time' + ) { + continue + } + + switch (value.format) { + case 'title': { + filter.or.push({ + property: key, + title: { + contains: String(query[key]), + }, + }) + + break + } + + case 'status': { + filter.or.push({ + property: key, + status: { + equals: String(query[key]), + }, + }) + + break + } + + case 'rich_text': { + filter.or.push({ + property: key, + rich_text: { + contains: String(query[key]), + }, + }) + + break + } + + case 'number': { + filter.or.push({ + property: key, + number: { + equals: parseFloat(String(query[key])), + }, + }) + + break + } + + case 'checkbox': { + filter.or.push({ + property: key, + checkbox: { + equals: Boolean(query[key]), + }, + }) + + break + } + + case 'select': { + filter.or.push({ + property: key, + select: { + equals: String(query[key]), + }, + }) + + break + } + + case 'multi_select': { + filter.or.push({ + property: key, + multi_select: { + contains: String(query[key]), + }, + }) + + break + } + + case 'date': { + filter.or.push({ + property: key, + date: { + equals: String(query[key]), + }, + }) + + break + } + + case 'url': { + filter.or.push({ + property: key, + url: { + equals: String(query[key]), + }, + }) + + break + } + + case 'email': { + filter.or.push({ + property: key, + email: { + equals: String(query[key]), + }, + }) + + break + } + + case 'phone_number': { + filter.or.push({ + property: key, + phone_number: { + equals: String(query[key]), + }, + }) + + break + } + + default: { + const x: never = value.format + + x + } + } + } + + return { + unsupported: unsupported, + filter: filter.or.length > 0 ? filter : undefined, + } +} + +export async function convertDatabaseItemFilter({ + auth, + databaseId, + query, +}: { + auth: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + query: ToReadonlyRecord> +}): Promise<{ unsupported: string[]; filter: DatabaseItemFilter | undefined }> { + const knownDatabaseProperties = await introspectDatabaseProperties({ + auth, + databaseId, + }) + + return convertDatabaseItemFilterFromKnownProperties({ + knownDatabaseProperties, + query, + }) +} diff --git a/packages/auxiliary-notion/src/errors.test.ts b/packages/auxiliary-notion/src/errors.test.ts new file mode 100644 index 0000000..76894ce --- /dev/null +++ b/packages/auxiliary-notion/src/errors.test.ts @@ -0,0 +1,23 @@ +import { UnsupportedPropertiesError } from './errors' + +describe('UnsupportedPropertiesError', () => { + it('should format the message from the property names', () => { + const error = new UnsupportedPropertiesError(['Long Summary', 'Priority']) + + expect(error.message).toBe('Unsupported properties: Long Summary, Priority') + }) + + it('should expose the property names', () => { + const error = new UnsupportedPropertiesError(['Long Summary']) + + expect(error.properties).toEqual(['Long Summary']) + }) + + it('should be an instance of Error and carry its own name', () => { + const error = new UnsupportedPropertiesError(['x']) + + expect(error).toBeInstanceOf(Error) + expect(error).toBeInstanceOf(UnsupportedPropertiesError) + expect(error.name).toBe('UnsupportedPropertiesError') + }) +}) diff --git a/packages/auxiliary-notion/src/errors.ts b/packages/auxiliary-notion/src/errors.ts new file mode 100644 index 0000000..7101456 --- /dev/null +++ b/packages/auxiliary-notion/src/errors.ts @@ -0,0 +1,16 @@ +/** + * Thrown when the caller supplies property (or filter) names that do not exist + * in the target Notion database schema. This is a caller-side validation + * problem, not a server or upstream fault, so consumers should surface it as a + * bad request rather than capturing it as an unexpected exception. + */ +export class UnsupportedPropertiesError extends Error { + readonly properties: string[] + + constructor(properties: string[]) { + super(`Unsupported properties: ${properties.join(', ')}`) + + this.name = 'UnsupportedPropertiesError' + this.properties = properties + } +} diff --git a/packages/auxiliary-notion/src/handler.ts b/packages/auxiliary-notion/src/handler.ts new file mode 100644 index 0000000..b296510 --- /dev/null +++ b/packages/auxiliary-notion/src/handler.ts @@ -0,0 +1,847 @@ +import type { fetch as chatbotkitFetch } from '@chatbotkit-dev/fetch' + +import type { PositiveNumber } from '@chatbotkit-dev/typescript-utils/number' +import type { ToReadonlyRecord } from '@chatbotkit-dev/typescript-utils/record' +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils/string' + +import { getClient } from './client' +import { getContents } from './contents' +import type { + DatabaseProperties, + IntrospectedDatabaseProperty, + SimplifiedDatabaseProperties, +} from './database' +import { + convertDatabaseCreateProperties, + convertDatabaseItemFilter, + convertDatabaseUpdateProperties, + getSimplifiedDatabaseProperties, + introspectDatabaseProperties, +} from './database' +import { UnsupportedPropertiesError } from './errors' +import type { PageProperties, SimplifiedPageProperties } from './page' +import { getSimplifiedPageProperties } from './page' + +export type PageEnumerationItem = { + id: string + object: 'page' + parent?: object + created_time?: string + last_edited_time?: string + properties: + | ReturnType + | Record + url?: string +} + +export type DatabaseEnumerationItem = { + id: string + object: 'database' + parent?: object + title?: string + created_time?: string + last_edited_time?: string + properties: + | ReturnType + | Record + url?: string +} + +export type EnumerationItem = PageEnumerationItem | DatabaseEnumerationItem + +export async function searchHandler({ + token, + query, + startCursor: _startCursor, + pageSize: _pageSize, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + query?: TrimmedNonEmptyString + startCursor?: TrimmedNonEmptyString + pageSize?: PositiveNumber + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + items: EnumerationItem[] + cursor: string | undefined +}> { + const client = getClient(token, { fetch }) + + let startCursor = _startCursor ? _startCursor.trim() : undefined + + startCursor = startCursor && startCursor.length > 0 ? startCursor : undefined + + let pageSize = _pageSize ? _pageSize : 20 + + pageSize = Math.min(pageSize, 100) + + const data = await client.search({ + query: query, + start_cursor: startCursor, + page_size: pageSize, + }) + + const { results } = data + + const items: EnumerationItem[] = [] + + for (const item of results) { + switch (item.object) { + case 'page': { + const id = item.id + + const object = item.object + + const parent = 'parent' in item ? item.parent : undefined + + const created_time = + 'created_time' in item ? item.created_time : undefined + const last_edited_time = + 'last_edited_time' in item ? item.last_edited_time : undefined + + const properties = 'properties' in item ? item.properties : {} + + const url = 'url' in item ? item.url : undefined + + items.push({ + id, + + object, + + parent, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedPageProperties(properties) + : properties, + + url, + }) + + break + } + + case 'database': { + const id = item.id + + const object = item.object + + const parent = 'parent' in item ? item.parent : undefined + + const title = + 'title' in item + ? item.title.map(({ plain_text }) => plain_text).join(' ') + : undefined + + const created_time = + 'created_time' in item ? item.created_time : undefined + const last_edited_time = + 'last_edited_time' in item ? item.last_edited_time : undefined + + const properties = 'properties' in item ? item.properties : {} + + const url = 'url' in item ? item.url : undefined + + items.push({ + id, + + object, + + parent, + + title, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedDatabaseProperties(properties) + : properties, + + url, + }) + + break + } + + default: { + const x: never = item + + x + } + } + } + + return { + items, + cursor: data.next_cursor ? data.next_cursor : undefined, + } +} + +export async function listHandler({ + token, + startCursor, + pageSize, + simplifiedProperties, +}: { + token: TrimmedNonEmptyString + startCursor?: TrimmedNonEmptyString + pageSize?: PositiveNumber + simplifiedProperties?: boolean +}): Promise<{ + items: EnumerationItem[] + cursor: string | undefined +}> { + return await searchHandler({ + token, + startCursor, + pageSize, + simplifiedProperties, + }) +} + +export async function listPagesHandler({ + token, + startCursor: _startCursor, + pageSize: _pageSize, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + startCursor?: TrimmedNonEmptyString + pageSize?: PositiveNumber + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + pages: PageEnumerationItem[] + cursor: string | undefined +}> { + const client = getClient(token, { fetch }) + + let startCursor = _startCursor ? _startCursor.trim() : undefined + + startCursor = startCursor && startCursor.length > 0 ? startCursor : undefined + + let pageSize = _pageSize ? _pageSize : 20 + + pageSize = Math.min(pageSize, 100) + + const data = await client.search({ + filter: { + property: 'object', + value: 'page', + }, + start_cursor: startCursor, + page_size: pageSize, + }) + + const { results } = data + + const pages: PageEnumerationItem[] = [] + + for (const item of results) { + switch (item.object) { + case 'page': { + const id = item.id + + const object = item.object + + const parent = 'parent' in item ? item.parent : undefined + + const created_time = + 'created_time' in item ? item.created_time : undefined + const last_edited_time = + 'last_edited_time' in item ? item.last_edited_time : undefined + + const properties = 'properties' in item ? item.properties : {} + + const url = 'url' in item ? item.url : undefined + + pages.push({ + id, + + object, + + parent, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedPageProperties(properties) + : properties, + + url, + }) + + break + } + + case 'database': { + break + } + + default: { + const x: never = item + + x + } + } + } + + return { + pages, + cursor: data.next_cursor ? data.next_cursor : undefined, + } +} + +export async function listDatabasesHandler({ + token, + startCursor: _startCursor, + pageSize: _pageSize, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + startCursor?: TrimmedNonEmptyString + pageSize?: PositiveNumber + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + databases: DatabaseEnumerationItem[] + cursor: string | undefined +}> { + const client = getClient(token, { fetch }) + + let startCursor = _startCursor ? _startCursor.trim() : undefined + + startCursor = startCursor && startCursor.length > 0 ? startCursor : undefined + + let pageSize = _pageSize ? _pageSize : 20 + + pageSize = Math.min(pageSize, 100) + + const data = await client.search({ + filter: { + property: 'object', + value: 'database', + }, + start_cursor: startCursor, + page_size: pageSize, + }) + + const { results } = data + + const databases: DatabaseEnumerationItem[] = [] + + for (const item of results) { + switch (item.object) { + case 'page': { + break + } + + case 'database': { + const id = item.id + + const object = item.object + + const parent = 'parent' in item ? item.parent : undefined + + const title = + 'title' in item + ? item.title.map(({ plain_text }) => plain_text).join(' ') + : undefined + + const created_time = + 'created_time' in item ? item.created_time : undefined + const last_edited_time = + 'last_edited_time' in item ? item.last_edited_time : undefined + + const properties = 'properties' in item ? item.properties : {} + + const url = 'url' in item ? item.url : undefined + + databases.push({ + id, + + object, + + parent, + + title, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedDatabaseProperties(properties) + : properties, + + url, + }) + + break + } + + default: { + const x: never = item + + x + } + } + } + + return { + databases, + cursor: data.next_cursor ? data.next_cursor : undefined, + } +} + +export async function fetchPageHandler({ + token, + pageId, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + pageId: TrimmedNonEmptyString + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + page: { + id: string + + object: 'page' + + created_time?: string + last_edited_time?: string + + properties: SimplifiedPageProperties | PageProperties + + url?: string + } + contents: string +}> { + const client = getClient(token, { fetch }) + + const page = await client.pages.retrieve({ + page_id: pageId, + }) + + const id = page.id + + const object = page.object + + const created_time = 'created_time' in page ? page.created_time : undefined + const last_edited_time = + 'last_edited_time' in page ? page.last_edited_time : undefined + + const properties = 'properties' in page ? page.properties : {} + + const url = 'url' in page ? page.url : undefined + + const contents = await getContents({ auth: token, pageId }) + + return { + page: { + id, + + object, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedPageProperties(properties) + : properties, + + url, + }, + contents, + } +} + +export async function introspectDatabaseHandler({ + token, + databaseId, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + fetch?: typeof chatbotkitFetch +}): Promise> { + return await introspectDatabaseProperties({ auth: token, databaseId, fetch }) +} + +export async function searchDatabaseHandler({ + token, + databaseId, + query, + startCursor: _startCursor, + pageSize: _pageSize, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + query?: ToReadonlyRecord> + startCursor?: TrimmedNonEmptyString + pageSize?: PositiveNumber + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + items: EnumerationItem[] + cursor: string | undefined +}> { + const client = getClient(token, { fetch }) + + let filter + + { + if (query) { + const { unsupported, filter: _filter } = await convertDatabaseItemFilter({ + auth: token, + databaseId, + query, + }) + + if (unsupported.length > 0) { + throw new UnsupportedPropertiesError(unsupported) + } + + filter = _filter + } else { + filter = undefined + } + } + + let startCursor = _startCursor ? _startCursor.trim() : undefined + + startCursor = startCursor && startCursor.length > 0 ? startCursor : undefined + + let pageSize = _pageSize ? _pageSize : 20 + + pageSize = Math.min(pageSize, 100) + + const data = await client.databases.query({ + database_id: databaseId, + filter: filter, + start_cursor: startCursor, + page_size: pageSize, + }) + + const { results } = data + + const items: EnumerationItem[] = [] + + for (const item of results) { + switch (item.object) { + case 'page': { + const id = item.id + + const object = item.object + + const parent = 'parent' in item ? item.parent : undefined + + const created_time = + 'created_time' in item ? item.created_time : undefined + const last_edited_time = + 'last_edited_time' in item ? item.last_edited_time : undefined + + const properties = 'properties' in item ? item.properties : {} + + const url = 'url' in item ? item.url : undefined + + items.push({ + id, + + object, + + parent, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedPageProperties(properties) + : properties, + + url, + }) + + break + } + + case 'database': { + const id = item.id + + const object = item.object + + const parent = 'parent' in item ? item.parent : undefined + + const title = + 'title' in item + ? item.title.map(({ plain_text }) => plain_text).join(' ') + : undefined + + const created_time = + 'created_time' in item ? item.created_time : undefined + const last_edited_time = + 'last_edited_time' in item ? item.last_edited_time : undefined + + const properties = 'properties' in item ? item.properties : {} + + const url = 'url' in item ? item.url : undefined + + items.push({ + id, + + object, + + parent, + + title, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedDatabaseProperties(properties) + : properties, + + url, + }) + + break + } + + default: { + const x: never = item + + x + } + } + } + + return { + items, + cursor: data.next_cursor ? data.next_cursor : undefined, + } +} + +export async function listDatabaseItemsHandler({ + token, + databaseId, + startCursor, + pageSize, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + startCursor?: TrimmedNonEmptyString + pageSize?: PositiveNumber + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + items: EnumerationItem[] + cursor: string | undefined +}> { + return await searchDatabaseHandler({ + token, + databaseId, + startCursor, + pageSize, + simplifiedProperties, + fetch, + }) +} + +export async function fetchDatabaseItemHandler({ + token, + databaseId, + itemId, + simplifiedProperties, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + itemId: TrimmedNonEmptyString + simplifiedProperties?: boolean + fetch?: typeof chatbotkitFetch +}): Promise<{ + item: { + id: string + + object: 'page' | 'database' + + created_time?: string + last_edited_time?: string + + properties: SimplifiedDatabaseProperties | DatabaseProperties + + url?: string + } + contents: string +}> { + databaseId // @todo validate the item belongs to the database + + const client = getClient(token, { fetch }) + + const data = await client.pages.retrieve({ + page_id: itemId, + }) + + const id = data.id + + const created_time = 'created_time' in data ? data.created_time : undefined + const last_edited_time = + 'last_edited_time' in data ? data.last_edited_time : undefined + + const properties = 'properties' in data ? data.properties : {} + + const url = 'url' in data ? data.url : undefined + + return { + item: { + id, + + object: data.object, + + created_time, + last_edited_time, + + properties: simplifiedProperties + ? getSimplifiedPageProperties(properties) + : properties, + + url, + }, + contents: await getContents({ auth: token, pageId: itemId }), + } +} + +export async function createDatabaseItemHandler({ + token, + databaseId, + properties, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + properties: ToReadonlyRecord> + fetch?: typeof chatbotkitFetch +}): Promise<{ + id: string + object: 'page' + url?: string +}> { + const client = getClient(token, { fetch }) + + const { unsupported: unsupportedProperties, properties: databaseProperties } = + await convertDatabaseCreateProperties({ + auth: token, + databaseId, + properties, + }) + + if (unsupportedProperties.length > 0) { + throw new UnsupportedPropertiesError(unsupportedProperties) + } + + const data = await client.pages.create({ + parent: { + database_id: databaseId, + }, + properties: databaseProperties, + }) + + return { + id: data.id, + + object: data.object, + + url: 'url' in data ? data.url : undefined, + } +} + +export async function updateDatabaseItemHandler({ + token, + databaseId, + itemId, + properties, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + itemId: TrimmedNonEmptyString + properties: ToReadonlyRecord> + fetch?: typeof chatbotkitFetch +}): Promise<{ + id: string + object: 'page' + url?: string +}> { + const client = getClient(token, { fetch }) + + const { unsupported: unsupportedProperties, properties: databaseProperties } = + await convertDatabaseUpdateProperties({ + auth: token, + databaseId, + properties, + }) + + if (unsupportedProperties.length > 0) { + throw new UnsupportedPropertiesError(unsupportedProperties) + } + + const data = await client.pages.update({ + page_id: itemId, + properties: databaseProperties, + }) + + return { + id: data.id, + + object: data.object, + + url: 'url' in data ? data.url : undefined, + } +} + +export async function deleteDatabaseItemHandler({ + token, + databaseId, + itemId, + fetch, +}: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + itemId: TrimmedNonEmptyString + fetch?: typeof chatbotkitFetch +}): Promise<{ + id: string + object: 'page' + url?: string +}> { + databaseId // @todo validate the item belongs to the database + + const client = getClient(token, { fetch }) + + const data = await client.pages.update({ + page_id: itemId, + archived: true, + }) + + return { + id: data.id, + + object: data.object, + + url: 'url' in data ? data.url : undefined, + } +} diff --git a/packages/auxiliary-notion/src/index.ts b/packages/auxiliary-notion/src/index.ts new file mode 100644 index 0000000..e0ce61f --- /dev/null +++ b/packages/auxiliary-notion/src/index.ts @@ -0,0 +1,6 @@ +export * from './client' +export * from './contents' +export * from './errors' +export * from './page' +export * from './database' +export * from './handler' diff --git a/packages/auxiliary-notion/src/page.test.ts b/packages/auxiliary-notion/src/page.test.ts new file mode 100644 index 0000000..07c94d9 --- /dev/null +++ b/packages/auxiliary-notion/src/page.test.ts @@ -0,0 +1,319 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { PageProperties} from './page'; +import { getSimplifiedPageProperties } from './page' + +describe('getSimplifiedPageProperties', () => { + test('should handle title property', () => { + const properties: PageProperties = { + Title: { + id: 'title', + type: 'title', + title: [ + { plain_text: 'Hello', type: 'text', annotations: {}, href: null }, + ], + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Title: 'Hello' }) + }) + + test('should handle status property', () => { + const properties: PageProperties = { + Status: { + id: 'status', + type: 'status', + status: { name: 'Done', color: 'green', id: '1' }, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Status: 'Done' }) + }) + + test('should handle status property when null', () => { + const properties: PageProperties = { + Status: { + id: 'status', + type: 'status', + status: null, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Status: null }) + }) + + test('should handle rich_text property', () => { + const properties: PageProperties = { + Description: { + id: 'desc', + type: 'rich_text', + rich_text: [ + { plain_text: 'Hello', type: 'text', annotations: {}, href: null }, + { plain_text: 'World', type: 'text', annotations: {}, href: null }, + ], + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Description: 'Hello World' }) + }) + + test('should handle number property', () => { + const properties: PageProperties = { + Count: { + id: 'count', + type: 'number', + number: 42, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Count: 42 }) + }) + + test('should handle checkbox property', () => { + const properties: PageProperties = { + Completed: { + id: 'completed', + type: 'checkbox', + checkbox: true, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Completed: true }) + }) + + test('should handle select property', () => { + const properties: PageProperties = { + Category: { + id: 'category', + type: 'select', + select: { name: 'Work', color: 'blue', id: '1' }, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Category: 'Work' }) + }) + + test('should handle select property when null', () => { + const properties: PageProperties = { + Category: { + id: 'category', + type: 'select', + select: null, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Category: null }) + }) + + test('should handle multi_select property', () => { + const properties: PageProperties = { + Tags: { + id: 'tags', + type: 'multi_select', + multi_select: [ + { name: 'urgent', color: 'red', id: '1' }, + { name: 'bug', color: 'orange', id: '2' }, + ], + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Tags: ['urgent', 'bug'] }) + }) + + test('should handle date property', () => { + const properties: PageProperties = { + DueDate: { + id: 'due', + type: 'date', + date: { start: '2023-01-01', end: null, time_zone: null }, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ DueDate: '2023-01-01' }) + }) + + test('should handle date property when null', () => { + const properties: PageProperties = { + DueDate: { + id: 'due', + type: 'date', + date: null, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ DueDate: null }) + }) + + test('should handle url property', () => { + const properties: PageProperties = { + Website: { + id: 'website', + type: 'url', + url: 'https://example.com', + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Website: 'https://example.com' }) + }) + + test('should handle url property when null', () => { + const properties: PageProperties = { + Website: { + id: 'website', + type: 'url', + url: null, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Website: null }) + }) + + test('should handle email property', () => { + const properties: PageProperties = { + Email: { + id: 'email', + type: 'email', + email: 'test@example.com', + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Email: 'test@example.com' }) + }) + + test('should handle phone_number property', () => { + const properties: PageProperties = { + Phone: { + id: 'phone', + type: 'phone_number', + phone_number: '123-456-7890', + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Phone: '123-456-7890' }) + }) + + test('should handle created_time property', () => { + const timestamp = '2023-01-01T12:00:00Z' + const properties: PageProperties = { + Created: { + id: 'created', + type: 'created_time', + created_time: timestamp, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Created: timestamp }) + }) + + test('should handle last_edited_time property', () => { + const timestamp = '2023-01-01T12:00:00Z' + const properties: PageProperties = { + LastEdited: { + id: 'edited', + type: 'last_edited_time', + last_edited_time: timestamp, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ LastEdited: timestamp }) + }) + + test('should skip ignored property types', () => { + const properties: PageProperties = { + UniqueID: { + id: 'uid', + type: 'unique_id', + unique_id: { number: 123, prefix: null }, + }, + Formula: { + id: 'formula', + type: 'formula', + formula: { string: 'result' }, + }, + Rollup: { + id: 'rollup', + type: 'rollup', + rollup: { number: 42 }, + }, + Name: { + id: 'title', + type: 'title', + title: [ + { plain_text: 'Test', type: 'text', annotations: {}, href: null }, + ], + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ Name: 'Test' }) + }) + + test('should handle multiple properties', () => { + const properties: PageProperties = { + Name: { + id: 'title', + type: 'title', + title: [ + { + plain_text: 'Test Task', + type: 'text', + annotations: {}, + href: null, + }, + ], + }, + Done: { + id: 'done', + type: 'checkbox', + checkbox: false, + }, + Priority: { + id: 'priority', + type: 'select', + select: { name: 'High', color: 'red', id: '1' }, + }, + } as any + + const result = getSimplifiedPageProperties(properties) + + expect(result).toEqual({ + Name: 'Test Task', + Done: false, + Priority: 'High', + }) + }) +}) diff --git a/packages/auxiliary-notion/src/page.ts b/packages/auxiliary-notion/src/page.ts new file mode 100644 index 0000000..4e3d4a9 --- /dev/null +++ b/packages/auxiliary-notion/src/page.ts @@ -0,0 +1,128 @@ +import type { Immutable } from '@chatbotkit-dev/typescript-utils/object' + +import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +export type PageProperties = PageObjectResponse['properties'] + +export type SimplifiedPageProperties = Record + +export function getSimplifiedPageProperties( + properties: Immutable +): SimplifiedPageProperties { + const simplified: SimplifiedPageProperties = {} + + for (const [key, value] of Object.entries(properties)) { + if ( + value.type === 'unique_id' || + value.type === 'formula' || + value.type === 'rollup' || + value.type === 'relation' || + value.type === 'people' || + value.type === 'files' || + value.type === 'button' || + value.type === 'created_by' || + value.type === 'last_edited_by' || + value.type === 'verification' + ) { + continue + } + + switch (value.type) { + case 'title': { + simplified[key] = value.title + .map((item: { plain_text: string }) => item.plain_text) + .join(' ') + + break + } + + case 'status': { + simplified[key] = value.status?.name || null + + break + } + + case 'rich_text': { + simplified[key] = value.rich_text + .map((item: { plain_text: string }) => item.plain_text) + .join(' ') + + break + } + + case 'number': { + simplified[key] = value.number + + break + } + + case 'checkbox': { + simplified[key] = value.checkbox + + break + } + + case 'select': { + simplified[key] = value.select?.name || null + + break + } + + case 'multi_select': { + simplified[key] = value.multi_select.map( + (item: { name: string }) => item.name + ) + + break + } + + case 'date': { + if (value.date) { + simplified[key] = value.date.start + } else { + simplified[key] = null + } + + break + } + + case 'url': { + simplified[key] = value.url || null + + break + } + + case 'email': { + simplified[key] = value.email || null + + break + } + + case 'phone_number': { + simplified[key] = value.phone_number || null + + break + } + + case 'created_time': { + simplified[key] = value.created_time + + break + } + + case 'last_edited_time': { + simplified[key] = value.last_edited_time + + break + } + + default: { + const x: never = value + + x + } + } + } + + return simplified +} diff --git a/packages/auxiliary-notion/src/sql.test.ts b/packages/auxiliary-notion/src/sql.test.ts new file mode 100644 index 0000000..401dc35 --- /dev/null +++ b/packages/auxiliary-notion/src/sql.test.ts @@ -0,0 +1,397 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils' + +import { jest } from '@jest/globals' + +jest.unstable_mockModule('./client', () => ({ + getClient: jest.fn(), +})) + +jest.unstable_mockModule('./database', () => ({ + introspectDatabaseProperties: jest.fn(), + convertDatabaseItemFilter: jest.fn(), + convertDatabaseCreateProperties: jest.fn(), + convertDatabaseUpdateProperties: jest.fn(), + getSimplifiedDatabaseProperties: jest.fn(), +})) + +jest.unstable_mockModule('./page', () => ({ + getSimplifiedPageProperties: jest.fn(), +})) + +const { getClient } = await import('./client') +const { + introspectDatabaseProperties, + convertDatabaseItemFilter, + convertDatabaseCreateProperties, + convertDatabaseUpdateProperties, +} = await import('./database') +const { getSimplifiedPageProperties } = await import('./page') +const { UnsupportedPropertiesError } = await import('./errors') +const { DatabaseDriver } = await import('./sql') + +const TOKEN = 'test_token' as TrimmedNonEmptyString +const DATABASE_ID = 'test_database_id' as TrimmedNonEmptyString + +const mockedGetClient = getClient as jest.MockedFunction +const mockedIntrospect = introspectDatabaseProperties as jest.MockedFunction< + typeof introspectDatabaseProperties +> +const mockedConvertFilter = convertDatabaseItemFilter as jest.MockedFunction< + typeof convertDatabaseItemFilter +> +const mockedConvertCreate = + convertDatabaseCreateProperties as jest.MockedFunction< + typeof convertDatabaseCreateProperties + > +const mockedConvertUpdate = + convertDatabaseUpdateProperties as jest.MockedFunction< + typeof convertDatabaseUpdateProperties + > +const mockedGetSimplifiedPage = + getSimplifiedPageProperties as jest.MockedFunction< + typeof getSimplifiedPageProperties + > + +describe('DatabaseDriver', () => { + let mockClient: any + + beforeEach(() => { + jest.clearAllMocks() + + mockClient = { + pages: { + retrieve: jest.fn(), + create: jest.fn(), + update: jest.fn(), + }, + databases: { + query: jest.fn(), + }, + } + + mockedGetClient.mockReturnValue(mockClient) + }) + + describe('describeColumns', () => { + it('should return id column plus introspected database columns', async () => { + mockedIntrospect.mockResolvedValue({ + Name: { type: 'string', format: 'title', enum: undefined }, + Status: { + type: 'string', + format: 'select', + enum: ['Active', 'Inactive'], + }, + }) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + const columns = await driver.describeColumns() + + expect(columns).toEqual([ + { name: 'id', type: 'string' }, + { name: 'Name', type: 'string', options: undefined }, + { name: 'Status', type: 'string', options: ['Active', 'Inactive'] }, + ]) + + expect(introspectDatabaseProperties).toHaveBeenCalledWith({ + auth: TOKEN, + databaseId: DATABASE_ID, + }) + }) + }) + + describe('doSelect', () => { + it('should query by page id when id is in where clause', async () => { + const mockPage = { + id: 'page_123', + object: 'page', + properties: { + Name: { type: 'title', title: [{ plain_text: 'Test' }] }, + }, + url: 'https://notion.so/test', + } + + mockClient.pages.retrieve.mockResolvedValue(mockPage) + mockedGetSimplifiedPage.mockReturnValue({ Name: 'Test' } as any) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + const results = await driver.doSelect([], { + or: [ + { + and: [ + { + column: 'id', + operator: 'EQ', + criteria: { type: 'string', value: 'page_123' }, + }, + ], + }, + ], + }) + + expect(mockClient.pages.retrieve).toHaveBeenCalledWith({ + page_id: 'page_123', + }) + + expect(results).toEqual([ + { + row: { id: 'page_123', Name: 'Test' }, + url: 'https://notion.so/test', + }, + ]) + }) + + it('should query database using constructor databaseId, not where properties', async () => { + mockedConvertFilter.mockResolvedValue({ + unsupported: [], + filter: { property: 'Name', rich_text: { equals: 'John' } }, + }) + + mockClient.databases.query.mockResolvedValue({ + results: [ + { + object: 'page', + properties: {}, + url: 'https://notion.so/test', + }, + ], + }) + + mockedGetSimplifiedPage.mockReturnValue({ Name: 'John' } as any) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await driver.doSelect([], { + or: [ + { + and: [ + { + column: 'Name', + operator: 'EQ', + criteria: { type: 'string', value: 'John' }, + }, + ], + }, + ], + }) + + expect(mockClient.databases.query).toHaveBeenCalledWith( + expect.objectContaining({ + database_id: DATABASE_ID, + }) + ) + }) + + it('should not use database_id from where properties', async () => { + mockedConvertFilter.mockResolvedValue({ + unsupported: [], + filter: undefined, + }) + + mockClient.databases.query.mockResolvedValue({ results: [] }) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await driver.doSelect([], { + or: [ + { + and: [ + { + column: 'database_id', + operator: 'EQ', + criteria: { type: 'string', value: 'some_other_database' }, + }, + ], + }, + ], + }) + + const callArg = mockClient.databases.query.mock.calls[0][0] + + expect(callArg.database_id).toBe(DATABASE_ID) + expect(callArg.database_id).not.toBe('some_other_database') + }) + + it('should throw when unsupported properties are present in filter', async () => { + mockedConvertFilter.mockResolvedValue({ + unsupported: ['unknownField'], + filter: undefined, + }) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await expect( + driver.doSelect([], { + or: [ + { + and: [ + { + column: 'unknownField', + operator: 'EQ', + criteria: { type: 'string', value: 'x' }, + }, + ], + }, + ], + }) + ).rejects.toThrow('Unsupported properties: unknownField') + }) + + it('should return empty array when database query returns no results', async () => { + mockedConvertFilter.mockResolvedValue({ + unsupported: [], + filter: undefined, + }) + + mockClient.databases.query.mockResolvedValue({ results: [] }) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + const results = await driver.doSelect([]) + + expect(results).toEqual([]) + }) + }) + + describe('doInsert', () => { + it('should create a page in the correct database', async () => { + mockedConvertCreate.mockResolvedValue({ + unsupported: [], + properties: { Name: { title: [{ text: { content: 'New Item' } }] } }, + }) + + mockClient.pages.create.mockResolvedValue({}) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await driver.doInsert({ Name: 'New Item' }) + + expect(mockClient.pages.create).toHaveBeenCalledWith({ + parent: { database_id: DATABASE_ID }, + properties: { Name: { title: [{ text: { content: 'New Item' } }] } }, + }) + }) + + it('should throw when insert has unsupported properties', async () => { + mockedConvertCreate.mockResolvedValue({ + unsupported: ['unknownProp'], + properties: {}, + }) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await expect(driver.doInsert({ unknownProp: 'value' })).rejects.toThrow( + 'Unsupported properties: unknownProp' + ) + + await expect( + driver.doInsert({ unknownProp: 'value' }) + ).rejects.toBeInstanceOf(UnsupportedPropertiesError) + }) + }) + + describe('doUpdate', () => { + it('should update a page using the row id', async () => { + mockedConvertUpdate.mockResolvedValue({ + unsupported: [], + properties: { Status: { select: { name: 'Done' } } }, + }) + + mockClient.pages.update.mockResolvedValue({}) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await driver.doUpdate({ row: { id: 'page_456' } }, { Status: 'Done' }) + + expect(mockClient.pages.update).toHaveBeenCalledWith({ + page_id: 'page_456', + properties: { Status: { select: { name: 'Done' } } }, + }) + }) + + it('should throw when row id is missing', async () => { + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await expect( + driver.doUpdate({ row: {} }, { Status: 'Done' }) + ).rejects.toThrow('Cannot update row: missing id') + }) + + it('should throw when update has unsupported properties', async () => { + mockedConvertUpdate.mockResolvedValue({ + unsupported: ['unknownProp'], + properties: {}, + }) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await expect( + driver.doUpdate({ row: { id: 'page_456' } }, { unknownProp: 'value' }) + ).rejects.toThrow('Unsupported properties: unknownProp') + }) + }) + + describe('doDelete', () => { + it('should archive a page using the row id', async () => { + mockClient.pages.update.mockResolvedValue({}) + + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await driver.doDelete({ row: { id: 'page_789' } }) + + expect(mockClient.pages.update).toHaveBeenCalledWith({ + page_id: 'page_789', + archived: true, + }) + }) + + it('should throw when row id is missing', async () => { + const driver = new DatabaseDriver({ + token: TOKEN, + databaseId: DATABASE_ID, + }) + + await expect(driver.doDelete({ row: {} })).rejects.toThrow( + 'Cannot delete row: missing id' + ) + }) + }) +}) diff --git a/packages/auxiliary-notion/src/sql.ts b/packages/auxiliary-notion/src/sql.ts new file mode 100644 index 0000000..cfd2d4b --- /dev/null +++ b/packages/auxiliary-notion/src/sql.ts @@ -0,0 +1,221 @@ +import type { fetch as chatbotkitFetch } from '@chatbotkit-dev/fetch' + +import type { + Column, + WhereStatement} from '@chatbotkit-dev/sql'; +import { + GenericDriver, + getWhereProperties, +} from '@chatbotkit-dev/sql' +import type { TrimmedNonEmptyString } from '@chatbotkit-dev/typescript-utils' + +import { getClient } from './client' +import { + convertDatabaseCreateProperties, + convertDatabaseItemFilter, + convertDatabaseUpdateProperties, + getSimplifiedDatabaseProperties, + introspectDatabaseProperties, +} from './database' +import { UnsupportedPropertiesError } from './errors' +import { getSimplifiedPageProperties } from './page' + +type DatabaseRow = Record + +export class DatabaseDriver extends GenericDriver { + #token: TrimmedNonEmptyString + #databaseId: TrimmedNonEmptyString + fetch?: typeof chatbotkitFetch + + constructor({ + token, + databaseId, + fetch, + }: { + token: TrimmedNonEmptyString + databaseId: TrimmedNonEmptyString + fetch?: typeof chatbotkitFetch + }) { + super() + + this.#token = token + this.#databaseId = databaseId + this.fetch = fetch + } + + async describeColumns(): Promise { + const properties = await introspectDatabaseProperties({ + auth: this.#token, + databaseId: this.#databaseId, + }) + + return [ + { + name: 'id', + type: 'string', + }, + + ...Object.entries(properties).map( + ([name, { type, ['enum']: options }]) => { + return { + name, + type, + options, + } + } + ), + ] + } + + async doSelect(columns: string[], where?: WhereStatement) { + const properties = where ? getWhereProperties(where) : {} + + const client = getClient(this.#token, { fetch: this.fetch }) + + if ('id' in properties) { + const data = await client.pages.retrieve({ + page_id: properties.id as string, + }) + + // @todo validate against this.#databaseId + + return [ + { + row: { + id: data.id, + ...('properties' in data + ? getSimplifiedPageProperties(data.properties) + : {}), + }, + + url: 'url' in data ? data.url : undefined, + }, + ] + } else { + const { unsupported, filter } = await convertDatabaseItemFilter({ + auth: this.#token, + databaseId: this.#databaseId, + query: properties, + }) + + if (unsupported.length > 0) { + throw new UnsupportedPropertiesError(unsupported) + } + + const data = await client.databases.query({ + database_id: this.#databaseId, + filter: filter, + }) + + const { results } = data + + return results.map((result) => { + switch (result.object) { + case 'page': { + return { + row: { + id: result.id, + ...('properties' in result + ? getSimplifiedPageProperties(result.properties) + : {}), + }, + + url: 'url' in result ? result.url : undefined, + } + } + + case 'database': { + return { + row: { + id: result.id, + ...('properties' in result + ? getSimplifiedDatabaseProperties(result.properties) + : {}), + }, + + url: 'url' in result ? result.url : undefined, + } + } + + default: { + const x: never = result + + x + + throw new Error(`Unsupported property type`) + } + } + }) + } + } + + async doInsert(parameters: Record) { + const client = getClient(this.#token, { fetch: this.fetch }) + + const { + unsupported: unsupportedProperties, + properties: databaseProperties, + } = await convertDatabaseCreateProperties({ + auth: this.#token, + databaseId: this.#databaseId, + properties: parameters, + }) + + if (unsupportedProperties.length > 0) { + throw new UnsupportedPropertiesError(unsupportedProperties) + } + + await client.pages.create({ + parent: { + database_id: this.#databaseId, + }, + properties: databaseProperties, + }) + } + + async doUpdate( + { row }: { row: DatabaseRow }, + parameters: Record + ) { + const pageId = row.id as string + + if (!pageId) { + throw new Error('Cannot update row: missing id') + } + + const client = getClient(this.#token, { fetch: this.fetch }) + + const { + unsupported: unsupportedProperties, + properties: databaseProperties, + } = await convertDatabaseUpdateProperties({ + auth: this.#token, + databaseId: this.#databaseId, + properties: parameters, + }) + + if (unsupportedProperties.length > 0) { + throw new UnsupportedPropertiesError(unsupportedProperties) + } + + await client.pages.update({ + page_id: pageId, + properties: databaseProperties, + }) + } + + async doDelete({ row }: { row: DatabaseRow }) { + const pageId = row.id as string + + if (!pageId) { + throw new Error('Cannot delete row: missing id') + } + + const client = getClient(this.#token, { fetch: this.fetch }) + + await client.pages.update({ + page_id: pageId, + archived: true, + }) + } +} diff --git a/packages/auxiliary-notion/tsconfig.json b/packages/auxiliary-notion/tsconfig.json new file mode 100644 index 0000000..2000c74 --- /dev/null +++ b/packages/auxiliary-notion/tsconfig.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": [ + "jest" + ], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2019" + ], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": false, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strictNullChecks": true, + "skipLibCheck": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/packages/batch-spec/package.json b/packages/batch-spec/package.json new file mode 100644 index 0000000..f7a80f2 --- /dev/null +++ b/packages/batch-spec/package.json @@ -0,0 +1,34 @@ +{ + "name": "@chatbotkit-dev/batch-spec", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "true" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "eslint": "^9.0.0", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/batch-spec/src/index.ts b/packages/batch-spec/src/index.ts new file mode 100644 index 0000000..604d3be --- /dev/null +++ b/packages/batch-spec/src/index.ts @@ -0,0 +1,209 @@ +// @note the batch execution contract. +// +// The platform occasionally needs a container image run somewhere that is not +// the request that asked for it: a crawl of someone's site, an import that +// takes twenty minutes, work that wants eight gigabytes of disk and must not +// hold a Node process open while it uses them. Where that runs is a +// deployment's choice. +// +// The altitude is the decision worth explaining, for the same reason it was on +// the sandbox contract. A deployment that already has a typed client for its own +// job-running service will be tempted to wrap that, because the client exists +// and is already typed. It is the wrong shape. Such a surface carries warm pool +// status, job cloning, server log tailing, registry credentials, a manifest TTL +// on the create body and a vCPU count nobody sets, of which the platform calls +// exactly one method. A contract shaped like one vendor's REST API is a contract +// only that vendor can implement. +// +// So there are two operations here, and the vocabulary is the platform's. There +// is no VM, no queue, and no pool. `id` is whatever the implementation issues +// and the platform only ever hands it back; making it resolve to the same job +// later is the implementation's problem. That is what lets one deployment map +// it to a microVM and another to a Kubernetes Job, a container on the host, or +// a row in a table, without the platform holding a lifecycle only one of them +// has. +// +// What is deliberately absent is as much of the design as what is here. See +// `BatchProvider` at the bottom of the file. + +/** + * Where a job is in its life. + * + * @note five states because the platform can distinguish five outcomes and no + * more. An implementation with a richer machine - pulling, scheduling, + * uploading - collapses into these on the way out, the way the error codes do. + */ +export type BatchJobStatus = + | 'queued' + | 'running' + | 'completed' + | 'failed' + | 'cancelled' + +/** + * How much machine to give a job, when the implementation has a say. + * + * @note advisory, like the sandbox contract's resources. A backend with nothing + * to allocate ignores this rather than failing, and no caller checks whether it + * was honoured. It is a preference, not a requirement. + */ +export interface BatchResources { + memoryMb?: number + diskMb?: number +} + +export interface BatchRunOptions { + /** + * The container image to run, as a registry reference. + * + * @note the one field with no plausible alternative. A backend that cannot + * pull OCI images cannot implement this contract at all, which is why the + * community default refuses rather than approximating. + */ + image: string + + /** + * Environment for the container. + * + * @note this is how the platform passes a job its input - there is no + * argument list here, because the platform has never set one and every image + * it runs reads its own `BATCH_INPUT`. Adding `command` would be adding a way + * to run something other than what an image was built to do. + */ + env?: Record + + /** + * Wall clock limit, in **seconds**. + * + * @note seconds rather than the milliseconds the sandbox contract uses, and + * the inconsistency is deliberate: these jobs are budgeted in minutes by the + * plan limits that produce this number, and a millisecond field would invite + * the conversion to be done twice + */ + timeout?: number + + resources?: BatchResources + + /** + * How long a registry may serve a cached manifest for `image`, in seconds. + * + * @note advisory, and the only field here that leaks an implementation detail + * - it exists because a `:latest` tag that is re-pushed hourly is otherwise + * pinned by whatever proxy sits in front of the registry. A backend with no + * proxy ignores it. It is kept rather than dropped because the caller that + * sets it is expressing something real: this image moves, do not cache it + */ + manifestTtl?: number +} + +export interface BatchRunResult { + /** The implementation's identifier for the job. */ + id: string +} + +/** + * A job as the platform can observe it. + * + * @note there are no timestamps on this, and that is not an oversight. The + * platform starts jobs and finds out how they ended; nothing reads when one was + * queued, and a `startedAt` that some backends can supply and others cannot is + * a field every caller has to treat as absent anyway. + */ +export interface BatchJob { + id: string + + status: BatchJobStatus + + /** The container's exit code. Present once the job has finished. */ + exitCode?: number + + /** Why it failed, when it did. For logs; never shown to a user unedited. */ + error?: string +} + +// --- errors --- + +/** + * @note coarser than the codes a VM-backed service returns, and the collapsing + * is not lossy in any way the platform can observe. One such service separates + * a VM it could not acquire from one it could not reach from one it could not + * release; the platform cannot respond to those differently, and codes naming + * VMs would be unimplementable by a backend that has none. + * + * `UNSUPPORTED_OPERATION` is the one code with no vendor ancestor, and it is + * what makes a partial implementation honest - see `run` on the provider. + */ +export type BatchErrorCode = + | 'JOB_NOT_FOUND' + | 'BATCH_UNAVAILABLE' + | 'NOT_AUTHORIZED' + | 'VALIDATION_FAILED' + | 'UNSUPPORTED_OPERATION' + | 'UNKNOWN' + +/** + * The shape an implementation's errors have to carry so the platform can decide + * what to tell the caller and whether to report the failure. + * + * @note an interface rather than a base class, and detected structurally rather + * than with `instanceof`, for the same two reasons the sandbox contract gives: + * the spec packages in this repository hold no behaviour, and `instanceof` + * across a package boundary is a bet on module identity that a bundler is free + * to lose. A structural brand cannot fail that way. + */ +export interface BatchErrorLike extends Error { + /** The brand. Always `true`, present so the check is not a guess. */ + readonly batch: true + + readonly code: BatchErrorCode + + /** The underlying failure, for logs. */ + readonly detail?: string +} + +// --- provider --- + +export interface BatchProvider { + /** + * Starts a job and returns as soon as it has been accepted. + * + * @note asynchronous by construction, and there is no synchronous sibling. + * Every job the platform runs reports its own results back through the API + * when it is done, so the caller has nothing to wait for and the request that + * started it is long gone by the time the job finishes. + * + * An implementation that cannot run containers must throw + * `UNSUPPORTED_OPERATION` naming the override that can, rather than + * approximating. A batch runner that silently does nothing looks exactly like + * one whose jobs are slow, and the deployment finds out weeks later from a + * dataset that never filled. + */ + run(options: BatchRunOptions): Promise + + /** + * Reports a job's current state. + * + * @note there is no `waitForCompletion` here, and the omission is the reason + * this method exists at all. Waiting is a caller's timing policy - how often + * to look, how long to give up after - and putting it in the contract makes + * every backend reimplement the same poll loop slightly differently. The + * platform runs one loop over this. + * + * @throws `JOB_NOT_FOUND` for an id the implementation has forgotten, which + * it is entitled to do: jobs are retained for a while after they finish and + * then they are not. + */ + get(id: string): Promise + + // @note there is no `cancel`, no `logs` and no `list`, and their absence was + // checked rather than assumed. The service client offers all three and + // nothing in the repository calls any of them; three operations no caller + // wants are still three operations every future backend has to implement. + // Whoever needs one should add it here first, which is the point of the seam. + + /** + * @note the convention every swappable module follows. See + * packages/AGENTS.md. + */ + assertConfigured(): Promise +} diff --git a/packages/batch-spec/tsconfig.json b/packages/batch-spec/tsconfig.json new file mode 100644 index 0000000..6e97651 --- /dev/null +++ b/packages/batch-spec/tsconfig.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/batch/jest.config.js b/packages/batch/jest.config.js new file mode 100644 index 0000000..19ff3e6 --- /dev/null +++ b/packages/batch/jest.config.js @@ -0,0 +1,11 @@ +// @note the node environment, not the shared jsdom one. `just-bash` picks its +// bundle from the resolver's export conditions, and under jsdom it resolves the +// browser build - whose `python3` is a stub that reports "not available in +// browser environments". The platform runs these commands in node, so testing +// against the browser bundle would be testing a build nothing uses. + +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: 'node', +} diff --git a/packages/batch/package.json b/packages/batch/package.json new file mode 100644 index 0000000..2d23d08 --- /dev/null +++ b/packages/batch/package.json @@ -0,0 +1,41 @@ +{ + "name": "@chatbotkit-dev/batch", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "@chatbotkit-dev/batch-spec": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/batch/src/index.test.js b/packages/batch/src/index.test.js new file mode 100644 index 0000000..5584a12 --- /dev/null +++ b/packages/batch/src/index.test.js @@ -0,0 +1,58 @@ +import { assertConfigured, get, run } from './index' + +// @note this default refuses, so what is worth testing is that it refuses +// *legibly*. A backend that cannot run containers is a normal deployment state; +// one that fails without naming the override is a support ticket. + +describe('run', () => { + it('refuses with UNSUPPORTED_OPERATION', async () => { + await expect( + run({ image: 'ghcr.io/chatbotkit/runner-sitemap:latest' }) + ).rejects.toMatchObject({ + batch: true, + code: 'UNSUPPORTED_OPERATION', + }) + }) + + // @note the message has to name the public override point and contract + it('names the override point and the contract in the message', async () => { + await expect(run({ image: 'example:latest' })).rejects.toThrow( + /@chatbotkit-dev\/batch.*BatchProvider.*@chatbotkit-dev\/batch-spec/ + ) + }) + + it('names the image it was asked to run in the detail', async () => { + await expect(run({ image: 'example:latest' })).rejects.toMatchObject({ + detail: expect.stringContaining('example:latest'), + }) + }) + + // @note the brand is what the platform detects errors with - structurally, + // never `instanceof` - so a missing one is silently a different failure path + it('brands the error so the platform recognises it', async () => { + const error = await run({ image: 'example:latest' }).catch((e) => e) + + expect(error).toBeInstanceOf(Error) + expect(error.batch).toBe(true) + expect(typeof error.code).toBe('string') + }) +}) + +describe('get', () => { + it('reports JOB_NOT_FOUND rather than refusing', async () => { + await expect(get('job-123')).rejects.toMatchObject({ + batch: true, + code: 'JOB_NOT_FOUND', + }) + }) +}) + +describe('assertConfigured', () => { + // @note unlike most public defaults this one throws, because nothing can be + // served from it - see the note in index.ts + it('fails the deployment readiness check', async () => { + await expect(assertConfigured()).rejects.toThrow( + /@chatbotkit-dev\/batch-spec/ + ) + }) +}) diff --git a/packages/batch/src/index.ts b/packages/batch/src/index.ts new file mode 100644 index 0000000..886f1e6 --- /dev/null +++ b/packages/batch/src/index.ts @@ -0,0 +1,126 @@ +// @note the community default for batch execution. +// +// This one refuses, and the choice is worth stating rather than discovering. +// The other public defaults in this repository degrade: `@chatbotkit-dev/email` +// logs to the console, `@chatbotkit-dev/searchengine` finds nothing, and +// `@chatbotkit-dev/sandbox` genuinely interprets bash in-process. Each of those +// has a cheap, honest approximation of the real thing. This module does not. +// Running an arbitrary OCI image with its own kernel, disk and network is not +// something a Node process can approximate, and every approximation on offer is +// worse than refusing: +// +// shell out to `docker` - needs a daemon, which is configuration, and turns +// a missing socket into a failure that reads like a +// broken job rather than an absent backend. +// +// run the entrypoint - runs the platform's own process as the job, with +// in-process the platform's credentials. Not a sandbox at all. +// +// accept and drop - the worst one, and the reason this file exists. +// A batch runner that silently succeeds and does +// nothing is indistinguishable from one whose jobs +// are slow, so the deployment finds out weeks later +// from a dataset that never filled. +// +// So `run` throws `UNSUPPORTED_OPERATION` naming the override, and +// `assertConfigured` throws too, rather than following `@chatbotkit-dev/email` +// and resolving anyway: a module that cannot serve any request +// should fail the deployment's readiness check instead of waiting to fail the +// first user. See packages/AGENTS.md. +// +// The platform still imports and boots on this. What it loses is the features +// that launch jobs - today, sitemap and crawl imports - which fail at the point +// of use with a message naming what to install. + +import type { + BatchErrorCode, + BatchErrorLike, + BatchJob, + BatchProvider, + BatchRunOptions, + BatchRunResult, +} from '@chatbotkit-dev/batch-spec' + +export type * from '@chatbotkit-dev/batch-spec' + +// @note the parameters below are named for the contract rather than for what +// this implementation does with them, which is nothing +/* eslint-disable unused-imports/no-unused-vars */ + +export class BatchError extends Error implements BatchErrorLike { + readonly batch = true as const + + readonly code: BatchErrorCode + + readonly detail?: string + + constructor( + code: BatchErrorCode, + message: string, + options?: { detail?: string; cause?: unknown } + ) { + super(message) + + this.name = 'BatchError' + this.code = code + this.detail = options?.detail + + // @note assigned rather than passed to `super`, because the two-argument + // `Error` constructor is ES2022 and these packages compile against ES2021 + + if (options?.cause !== undefined) { + ;(this as { cause?: unknown }).cause = options.cause + } + } +} + +const UNSUPPORTED = + 'no batch backend is installed, so container jobs cannot run - override @chatbotkit-dev/batch with a package whose default export satisfies BatchProvider from @chatbotkit-dev/batch-spec' + +/** + * Refuses, because nothing here can run a container image. + * + * @throws always, with `UNSUPPORTED_OPERATION` + */ +export async function run(options: BatchRunOptions): Promise { + throw new BatchError('UNSUPPORTED_OPERATION', UNSUPPORTED, { + detail: `cannot run ${options.image}`, + }) +} + +/** + * Reports that the job is not here, because no job ever started here. + * + * @note `JOB_NOT_FOUND` rather than `UNSUPPORTED_OPERATION`, and the difference + * matters to the caller. An id can only reach this function if something else + * issued it, which means the caller is polling a job started by a backend that + * has since been swapped out - and "that job is gone" is both true and the + * thing it needs to stop polling. + */ +export async function get(id: string): Promise { + throw new BatchError('JOB_NOT_FOUND', `no batch job ${id}`, { + detail: UNSUPPORTED, + }) +} + +/** + * @note throws, unlike most of the public defaults. Nothing can be served from + * this module - every job would fail at the point of use - so it fails the + * deployment's readiness check instead, the way an empty model catalogue does. + * A deployment that genuinely wants no batch backend removes the case from + * `platform/tests/config/providers.utest.js`, which is a decision someone + * makes on purpose rather than one that happens quietly. + */ +export async function assertConfigured(): Promise { + throw new Error( + `@chatbotkit-dev/batch is the community default and ${UNSUPPORTED}` + ) +} + +const provider: BatchProvider = { + run, + get, + assertConfigured, +} + +export default provider diff --git a/packages/batch/tsconfig.json b/packages/batch/tsconfig.json new file mode 100644 index 0000000..eb04375 --- /dev/null +++ b/packages/batch/tsconfig.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/billing-spec/package.json b/packages/billing-spec/package.json new file mode 100644 index 0000000..a180b64 --- /dev/null +++ b/packages/billing-spec/package.json @@ -0,0 +1,34 @@ +{ + "name": "@chatbotkit-dev/billing-spec", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "true" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "eslint": "^9.0.0", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/billing-spec/src/index.ts b/packages/billing-spec/src/index.ts new file mode 100644 index 0000000..fd56bc7 --- /dev/null +++ b/packages/billing-spec/src/index.ts @@ -0,0 +1,148 @@ +// @note the billing module contract: the surface a billing module exposes, +// as function and value shapes an implementation conforms to. How an +// implementation configures itself - environment variables, configuration +// formats, payment provider - is its own business and deliberately absent +// here. The default implementation lives in @chatbotkit-dev/billing. + +/** + * The classic surface most call sites read: trial policy plus a plan-to-price + * table. A null/absent price presents as Infinity, matching how the unbounded + * tier has always rendered. + */ +export interface Subscriptions { + trialDays: number + + trialPlans?: string[] + + pricing: Record +} + +/** + * The platform facts the subscription model closes over: its plan + * vocabulary, its installed catalogue and its grant mechanism. Nothing else + * crosses the boundary. + */ +export interface SubscriptionFacts { + /** + * The structural plan names: `free` is the plan of an account with no + * subscription and no grant, `trial` the plan of a trialing subscription, + * and `unlimited` the implicit full-access plan every deployment has. + */ + structuralPlans: { + free: string + trial: string + unlimited: string + } + + /** The plan names of the installed limits catalogue. */ + planKeys: readonly string[] + + /** The plan granted to an email address by a platform override, if any. */ + grantedPlan: (email: string) => string | undefined +} + +/** + * The account shape the subscription checks read. + * + * @note the property names are the account schema's billing columns. They + * are the billing module's own recording (null when the account has none) + * and an implementation detail everywhere else: the platform transports them + * on account rows but never interprets them. + */ +export interface SubscriptionHolder { + email: string + + billingSubscriptionId?: string | null + + billingSubscriptionStatus?: string | null +} + +/** + * The subscription model surface: the sellable catalogue, the id/name + * mappings and the subscription checks, closed over one deployment's facts. + */ +export interface SubscriptionModel { + /** + * The display name of the plan an account is recorded on - undefined for + * an account with no subscription or one recorded on an id the deployment + * no longer knows. Tolerant on purpose: it feeds listings and exports. + */ + recordedPlanName( + account: Pick + ): string | undefined + + /** Whether the account holds a live subscription or a grant. */ + hasSubscription(user: SubscriptionHolder): boolean + + /** + * Whether the account has ever consumed its trial - one per account, + * regardless of the plan it ran on or how the trial ended. + */ + hasTrialed(account: { + billingSubscriptionTrialedAt?: Date | string | null + }): boolean + + /** The plan the account is on, resolved from its own subscription facts. */ + userToPlan(user: SubscriptionHolder): string +} + +/** Builds the subscription model for one deployment's facts. */ +export type CreateSubscriptionModel = ( + facts: SubscriptionFacts +) => SubscriptionModel + +/** + * The platform facts the billing gates close over. + */ +export interface BillingGatesFacts { + /** Whether the platform has an installed plan catalogue. */ + hasPlans: boolean + + /** Whether an account is structurally owned by another account. */ + isChildUser: (user: unknown) => boolean +} + +/** + * The deployment gates: whether this deployment sells at all, and whether a + * given account may use billing. + */ +export interface BillingGates { + /** + * Whether this deployment sells plans at all. False means no pricing + * surface, no upgrade affordance and no trials. + */ + isSellable: boolean + + /** Whether a payment provider is configured. Read lazily. */ + isBillingConfigured(): boolean + + /** Whether this account may reach the billing surface at all. */ + canDoBilling(user: unknown): boolean +} + +/** Builds the billing gates for one deployment's facts. */ +export type CreateBillingGates = (facts: BillingGatesFacts) => BillingGates + +/** + * The trial policy, derived from a subscriptions configuration. + */ +export interface TrialPolicy { + /** + * The plans on which a free trial can be started - the single switch for + * trials, an empty list disabling them everywhere. + */ + trialPlans: readonly string[] + + /** + * The plan a trial is presented as by default, undefined when trials are + * disabled. + */ + primaryTrialPlan: string | undefined + + canTrialPlan(plan: string): boolean +} + +/** Derives the trial policy from a subscriptions configuration. */ +export type CreateTrialPolicy = ( + subscriptions: Pick +) => TrialPolicy diff --git a/packages/billing-spec/tsconfig.json b/packages/billing-spec/tsconfig.json new file mode 100644 index 0000000..6e97651 --- /dev/null +++ b/packages/billing-spec/tsconfig.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/billing/package.json b/packages/billing/package.json new file mode 100644 index 0000000..4add1ac --- /dev/null +++ b/packages/billing/package.json @@ -0,0 +1,45 @@ +{ + "name": "@chatbotkit-dev/billing", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + }, + "./assert": { + "import": "./src/assert.ts", + "default": "./src/assert.ts" + }, + "./provider": { + "import": "./src/provider.ts", + "default": "./src/provider.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "true" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "@chatbotkit-dev/billing-spec": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "eslint": "^9.0.0", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/billing/src/assert.ts b/packages/billing/src/assert.ts new file mode 100644 index 0000000..5b2cd1b --- /dev/null +++ b/packages/billing/src/assert.ts @@ -0,0 +1,8 @@ +// @note boot/CI-time validation, exposed at `@chatbotkit-dev/billing/assert` +// by convention. See packages/AGENTS.md. + +/** + * Throws when this module is not usable with the current environment. This + * module sells nothing and has nothing to misconfigure, so it resolves. + */ +export async function assertConfigured(_input?: unknown): Promise {} diff --git a/packages/billing/src/config.ts b/packages/billing/src/config.ts new file mode 100644 index 0000000..0a6c17e --- /dev/null +++ b/packages/billing/src/config.ts @@ -0,0 +1,15 @@ +import type { Subscriptions } from '@chatbotkit-dev/billing-spec' + +// @note nothing is sold and there is nothing to configure - no pricing +// surface, no upgrade affordance, no trials. + +export const isConfigured = false + +export const subscriptionsConfig: Subscriptions = Object.freeze({ + trialDays: 0, + + trialPlans: [], + + pricing: {}, +}) + diff --git a/packages/billing/src/gates.ts b/packages/billing/src/gates.ts new file mode 100644 index 0000000..7afd9dd --- /dev/null +++ b/packages/billing/src/gates.ts @@ -0,0 +1,22 @@ +import type { + BillingGates, + BillingGatesFacts, +} from '@chatbotkit-dev/billing-spec' + +/** + * Builds the billing gates for one deployment's facts. With nothing sold and + * no payment provider, every gate is closed. + */ +export function createBillingGates(_facts: BillingGatesFacts): BillingGates { + return Object.freeze({ + isSellable: false, + + isBillingConfigured(): boolean { + return false + }, + + canDoBilling(_user: unknown): boolean { + return false + }, + }) +} diff --git a/packages/billing/src/index.ts b/packages/billing/src/index.ts new file mode 100644 index 0000000..65614fd --- /dev/null +++ b/packages/billing/src/index.ts @@ -0,0 +1,10 @@ +// @note the public default billing module: a deployment that sells nothing. +// Plans remain pure entitlement tiers assigned through grants and overrides - +// a working deployment shape, not a broken one. A deployment that sells +// overrides this package with an implementation satisfying +// @chatbotkit-dev/billing-spec. + +export * from './config' +export * from './model' +export * from './trial' +export * from './gates' diff --git a/packages/billing/src/model.ts b/packages/billing/src/model.ts new file mode 100644 index 0000000..893a9be --- /dev/null +++ b/packages/billing/src/model.ts @@ -0,0 +1,49 @@ +import type { + SubscriptionFacts, + SubscriptionModel, +} from '@chatbotkit-dev/billing-spec' + +export type { SubscriptionHolder } from '@chatbotkit-dev/billing-spec' + +/** + * Builds the subscription model for one deployment's facts. With nothing + * sold, an account's plan comes from its grant alone. + */ +export function createSubscriptionModel( + facts: SubscriptionFacts +): SubscriptionModel { + const { structuralPlans, planKeys, grantedPlan } = facts + + // @note `free` and `trial` are not tiers - they are the no-subscription + // and trialing states - so they are never grantable. The structural + // unlimited plan is implicitly part of every deployment. + const planNames: ReadonlySet = new Set( + [...planKeys, structuralPlans.unlimited].filter( + (plan) => plan !== structuralPlans.free && plan !== structuralPlans.trial + ) + ) + + return Object.freeze({ + recordedPlanName() { + return undefined + }, + + hasSubscription(user) { + return grantedPlan(user.email) !== undefined + }, + + hasTrialed() { + return false + }, + + userToPlan(user) { + const granted = grantedPlan(user.email) + + if (granted && planNames.has(granted)) { + return granted + } + + return structuralPlans.free + }, + } satisfies SubscriptionModel) +} diff --git a/packages/billing/src/provider.ts b/packages/billing/src/provider.ts new file mode 100644 index 0000000..4d95f46 --- /dev/null +++ b/packages/billing/src/provider.ts @@ -0,0 +1,135 @@ +// @note the payment-provider operations, exposed at +// `@chatbotkit-dev/billing/provider`. This default module has no payment +// provider, so every operation refuses. +// +// @note this subpath is not part of @chatbotkit-dev/billing-spec: the +// platform's billing routes import these operations and their result shapes +// directly, which couples them to whichever implementation is installed. The +// shapes below mirror that de-facto contract so the routes compile - a +// deployment that sells must override @chatbotkit-dev/billing with an +// implementation providing the same operations. + +/** The account shape the customer operations read. */ +export interface BillingAccount { + billingCustomerId?: string | null + + billingSubscriptionId?: string | null +} + +export interface CustomerAccount extends BillingAccount { + id: string + + email?: string | null + + name?: string | null +} + +function unsupported(): never { + throw new Error( + 'no payment provider is installed, so billing operations cannot run - ' + + 'override @chatbotkit-dev/billing with a package that satisfies ' + + '@chatbotkit-dev/billing-spec and provides these operations' + ) +} + +/** + * Retrieves one price from the provider - the probe the boot-time assertions + * check the selling configuration against. + */ +export async function retrievePrice(_priceId: string): Promise { + return unsupported() +} + +export type OpenBillingPortalResult = + | { outcome: 'redirect'; url: string } + | { outcome: 'failed' } + +export async function openBillingPortal( + _db: unknown, + _account: CustomerAccount, + _options: { returnUrl: string } +): Promise { + return unsupported() +} + +export type StartCheckoutResult = + | { outcome: 'redirect'; url: string } + | { outcome: 'unknown_plan' } + | { outcome: 'trial_unavailable' } + | { outcome: 'possibly_fraudulent' } + | { outcome: 'already_subscribed' } + | { outcome: 'customer_gone' } + | { outcome: 'delinquent' } + | { outcome: 'failed' } + +export interface CheckoutIntent { + plan: string + + trial: boolean + + coupon?: string | null + + referral?: string | null + + returnUrl: string +} + +export async function startCheckout( + _db: unknown, + _account: CustomerAccount, + _intent: CheckoutIntent +): Promise { + return unsupported() +} + +export type SkipTrialResult = + | { outcome: 'skipped'; subscriptionId: string } + | { outcome: 'not_trialing' } + | { outcome: 'no_subscription' } + +export async function skipTrial( + _db: unknown, + _account: BillingAccount & { + id: string + billingSubscriptionStatus?: string | null + } +): Promise { + return unsupported() +} + +export async function deleteCustomer( + _account: Pick +): Promise { + return unsupported() +} + +export type WebhookFollowUp = + | { action: 'notify_trial_start' } + | { action: 'notify_trial_duplicate_card' } + | { action: 'reset_account_limits' } + | { action: 'notify_subscription_deleted' } + | { action: 'notify_invoice_payment_succeeded' } + | { action: 'notify_invoice_payment_failed' } + | { action: 'credit_booster_tokens'; userId: string } + | { action: 'delete_account' } + +export type HandleWebhookEventResult = + | { outcome: 'missing_signature' } + | { outcome: 'unconfigured' } + | { outcome: 'invalid'; message: string } + | { outcome: 'unknown_account'; customerId: string } + | { + outcome: 'handled' + type: string + account?: { id?: string } | null + followUps: WebhookFollowUp[] + messages: string[] + } + +export async function handleWebhookEvent( + _db: unknown, + _slidingWindow: unknown, + _request: { payload: string; headers: Record } +): Promise { + return unsupported() +} diff --git a/packages/billing/src/trial.ts b/packages/billing/src/trial.ts new file mode 100644 index 0000000..ee0de1f --- /dev/null +++ b/packages/billing/src/trial.ts @@ -0,0 +1,27 @@ +import type { Subscriptions, TrialPolicy } from '@chatbotkit-dev/billing-spec' + +import { subscriptionsConfig } from './config' + +/** + * Derives the trial policy from a subscriptions configuration. + */ +export function createTrialPolicy( + subscriptions: Pick +): TrialPolicy { + const trialPlans: readonly string[] = Object.freeze([ + ...(subscriptions.trialPlans ?? []), + ]) + + return Object.freeze({ + trialPlans, + + primaryTrialPlan: trialPlans[0], + + canTrialPlan(plan: string): boolean { + return trialPlans.includes(plan) + }, + }) +} + +export const { trialPlans, primaryTrialPlan, canTrialPlan } = + createTrialPolicy(subscriptionsConfig) diff --git a/packages/billing/tsconfig.json b/packages/billing/tsconfig.json new file mode 100644 index 0000000..6e97651 --- /dev/null +++ b/packages/billing/tsconfig.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/blacklists/jest.config.js b/packages/blacklists/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/blacklists/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/blacklists/package.json b/packages/blacklists/package.json new file mode 100644 index 0000000..87dd965 --- /dev/null +++ b/packages/blacklists/package.json @@ -0,0 +1,45 @@ +{ + "name": "@chatbotkit-dev/blacklists", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + }, + "./domains": { + "import": "./src/domains.ts", + "default": "./src/domains.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "disposable-email-domains": "^1.0.62" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/blacklists/src/domains.ts b/packages/blacklists/src/domains.ts new file mode 100644 index 0000000..0e1423a --- /dev/null +++ b/packages/blacklists/src/domains.ts @@ -0,0 +1,87 @@ +// @note the signup domain blacklist - a built-in anti-abuse control against +// disposable and throwaway addresses, maintained here for every deployment. +// One shared list is better protection than a swappable one that most +// deployments would leave empty. +// +// An entry matches the domain itself and every subdomain of it (`example.com` +// bans `example.com` and `mail.example.com`). A bare TLD entry (`cfd`) +// therefore bans every domain on that TLD - use those only for TLDs that +// produce nothing but abuse. + +import wildcard from 'disposable-email-domains/wildcard.json' + +/** Email domains refused at signup. */ +export const domains: string[] = [ + // The maintained disposable-email wildcard list. + ...wildcard, + + // Hand-collected abuse intelligence gathered from signup traffic. + 'myhome-server.de', + 'lebetrust.org', + 'throwawaymail.com', + 'lanxiu233.com', + 'nimail.cn', + 'indoxs.bond', + 'sportcornwall.org', + 'dmcelements.org', + 'havertz.tk', + '0cpub2.tech', + 'dynv6.net', + 'yorushika.one', + 'siempre.gratis', + 'aaconservation.org', + 'linlin.cloud', + 'linux-do.me', + 'indevs.in', + 'hulisang.edu.kg', + 'freemails.pp.ua', + 'abrdns.com', + 'plugintonature.org.uk', + 'ycglobalmovement.com', + 'nyc.mn', + 'novaprime.vip', + 'tokenized.name', + 'thinktank.edu.kg', + 'ahavaexperience.com', + 'chanceforfuture.org', + 'mwalshabs.dev', + 'positivevoiceteesvalley.co.uk', + 'us.ci', + 'alarafoundation.com', + 'mailtao.me', + 'acg-news.com', + 'churchinsouthampton.org.uk', + 'wearvault.biz.id', + 'sylu.net', + 'thameschamberorchestra.co.uk', + 'baileybridge.org', + 'jobsma.pp.ua', + 'freeddns.org', + 'abilityaccesslegal.org', + 'xiwinnie.icu', + 'cuvrs.info', + 'lordfortescue.org.uk', + 'qzz.io', + 'fightingzebras.org', + 'vvvv.ee', + 'slimirin.com', + 'de5.net', + 'zutomayo.best', + 'idu4aa4.info', + 'markableytrust.org.uk', + 'x80la.shop', + 'bcmail.pro', + 'caowo.online', + 'dpdns.org', + 'ggff.net', + 'supergrok.site', + 'eu.org', + 'cc.cd', + + // Bare TLD bans. + 'cfd', + 'cc', + 'top', + 'xyz', + 'sbs', +] diff --git a/packages/blacklists/src/index.test.ts b/packages/blacklists/src/index.test.ts new file mode 100644 index 0000000..29cf4d3 --- /dev/null +++ b/packages/blacklists/src/index.test.ts @@ -0,0 +1,29 @@ +import blacklist, { domains } from './index' + +describe('blacklists', () => { + it('exports the domain list both ways', () => { + expect(blacklist.domains).toBe(domains) + }) + + it('carries the maintained disposable-email list plus the curated entries', () => { + // @note the wildcard list alone is hundreds of entries; a shrunken list + // means the dependency stopped resolving and signups lost their cover + expect(domains.length).toBeGreaterThan(300) + + expect(domains).toContain('throwawaymail.com') + expect(domains).toContain('qzz.io') + }) + + it('carries the bare TLD bans', () => { + for (const tld of ['cfd', 'cc', 'top', 'xyz', 'sbs']) { + expect(domains).toContain(tld) + } + }) + + it('contains only plausible domain entries', () => { + for (const domain of domains) { + expect(domain).toMatch(/^[a-z0-9*.-]+$/i) + expect(domain).not.toMatch(/\s|@|\/$/) + } + }) +}) diff --git a/packages/blacklists/src/index.ts b/packages/blacklists/src/index.ts new file mode 100644 index 0000000..855c5d4 --- /dev/null +++ b/packages/blacklists/src/index.ts @@ -0,0 +1,18 @@ +// @note built-in blacklists the platform enforces for every deployment. Not a +// swappable configuration module - the lists ship with the platform and are +// maintained for everyone. See src/domains.ts for the signup domain list. + +import { domains } from './domains' + +export { domains } + +export interface Blacklist { + /** Email domains refused at signup. */ + domains: string[] +} + +const blacklist: Blacklist = { + domains, +} + +export default blacklist diff --git a/packages/blacklists/tsconfig.json b/packages/blacklists/tsconfig.json new file mode 100644 index 0000000..72b364c --- /dev/null +++ b/packages/blacklists/tsconfig.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node", + "jest" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/buffer/README.md b/packages/buffer/README.md new file mode 100644 index 0000000..b8b0fec --- /dev/null +++ b/packages/buffer/README.md @@ -0,0 +1,12 @@ +``` + .d8888b. 888888b. 888 d8P + d88P Y88b 888 "88b 888 d8P + 888 888 888 .88P 888 d8P + 888 8888888K. 888d88K + 888 888 "Y88b 8888888b + 888 888 888 888 888 Y88b + Y88b d88P 888 d88P 888 Y88b + "Y8888P" 8888888P" 888 Y88b +``` + +For external use where it is needed. diff --git a/packages/buffer/jest.config.js b/packages/buffer/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/buffer/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/buffer/package.json b/packages/buffer/package.json new file mode 100644 index 0000000..15bdfda --- /dev/null +++ b/packages/buffer/package.json @@ -0,0 +1,40 @@ +{ + "name": "@chatbotkit-dev/buffer", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "js-base64": "^3.7.7" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/buffer/src/index.test.ts b/packages/buffer/src/index.test.ts new file mode 100644 index 0000000..b255dc3 --- /dev/null +++ b/packages/buffer/src/index.test.ts @@ -0,0 +1,135 @@ +import { + b64d2buf, + buf2b64d, + buf2hex, + buf2str, + buf2stream, + concatBufs, + hex2buf, + str2buf, + stream2buf, +} from './index' + +describe('buf2str', () => { + test('converts an ArrayBuffer to a string', () => { + const testArrayBuffer = str2buf('Hello, World!') + + expect(buf2str(testArrayBuffer)).toBe('Hello, World!') + }) +}) + +describe('str2buf', () => { + test('converts a string to an ArrayBuffer', () => { + const testString = 'Hello, World!' + + expect(new TextDecoder().decode(str2buf(testString))).toBe(testString) + }) +}) + +describe('hex2buf', () => { + test('converts a hex string to a Uint8Array', () => { + const testHex = '48656c6c6f2c20576f726c6421' + const expectedArray = Uint8Array.from( + new TextEncoder().encode('Hello, World!') + ) + + expect(hex2buf(testHex)).toEqual(expectedArray) + }) + + test('should match Buffer from Node.js', () => { + const testHexBuf = hex2buf('48656c6c6f2c20576f726c6421') + const expectedHexBuf = new Uint8Array( + Buffer.from('48656c6c6f2c20576f726c6421', 'hex') + ) + + expect(testHexBuf).toEqual(expectedHexBuf) + }) + + test('should throw on odd-length hex string', () => { + expect(() => hex2buf('abc')).toThrow('odd length') + expect(() => hex2buf('a')).toThrow('odd length') + }) +}) + +describe('buf2hex', () => { + test('converts a Uint8Array to a hex string', () => { + const testBuffer = new TextEncoder().encode('Hello, World!') + const testHex = '48656c6c6f2c20576f726c6421' + + expect(buf2hex(testBuffer)).toBe(testHex) + }) +}) + +describe('b64d2buf', () => { + test('converts a base64 encoded string to a Uint8Array', () => { + const testBase64 = 'SGVsbG8sIFdvcmxkIQ==' + const testBuffer = new TextEncoder().encode('Hello, World!') + + expect(b64d2buf(testBase64)).toEqual(testBuffer) + }) +}) + +describe('buf2b64d', () => { + test('converts a Uint8Array to a base64 string', () => { + const testBuffer = new TextEncoder().encode('Hello, World!') + const testBase64 = 'SGVsbG8sIFdvcmxkIQ==' + + expect(buf2b64d(testBuffer)).toBe(testBase64) + }) +}) + +describe('concatBufs', () => { + test('concatenates multiple ArrayBuffers into a single ArrayBuffer', () => { + const buffer1 = str2buf('Hello') + const buffer2 = str2buf(' ') + const buffer3 = str2buf('World') + const concatenatedBuffer = concatBufs(buffer1, buffer2, buffer3) + + expect(new TextDecoder().decode(concatenatedBuffer)).toBe('Hello World') + }) + + test('matches Buffer.concat from Node.js', () => { + const testBuf = new Uint8Array( + concatBufs(str2buf('Hello'), str2buf(' '), str2buf('World')) + ) + const expectedBuf = new Uint8Array( + Buffer.concat([ + Buffer.from('Hello'), + Buffer.from(' '), + Buffer.from('World'), + ]) + ) + + expect(testBuf).toEqual(expectedBuf) + }) +}) + +describe('stream2buf', () => { + test('converts a ReadableStream to an ArrayBuffer', async () => { + const testArrayBuffer = str2buf('Hello, World!') + const stream = await buf2stream(testArrayBuffer) + const outputBuffer = await stream2buf(stream) + + expect(new Uint8Array(outputBuffer)).toEqual( + new Uint8Array(testArrayBuffer) + ) + }) +}) + +describe('buf2stream', () => { + test('converts an ArrayBuffer to a ReadableStream', async () => { + const testArrayBuffer = str2buf('Hello, World!') + const stream = await buf2stream(testArrayBuffer) + const reader = stream.getReader() + const result = await reader.read() + + expect(result.done).toBeFalsy() + expect(new Uint8Array(result.value!)).toEqual( + new Uint8Array(testArrayBuffer) + ) + + const end = await reader.read() + + expect(end.done).toBeTruthy() + }) +}) diff --git a/packages/buffer/src/index.ts b/packages/buffer/src/index.ts new file mode 100644 index 0000000..9465fda --- /dev/null +++ b/packages/buffer/src/index.ts @@ -0,0 +1,111 @@ +import { Base64 } from 'js-base64' + +// we need to polyfill the ReadableStream for chrome and Safari +{ + if ( + typeof globalThis.ReadableStream === 'function' && + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Symbol is not defined + typeof globalThis.ReadableStream.prototype[Symbol.asyncIterator] !== + 'function' + ) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore Symbol is not defined + globalThis.ReadableStream.prototype[Symbol.asyncIterator] = function () { + const reader = this.getReader() + + return { + next: () => reader.read(), + return: () => { + reader.releaseLock() + + return Promise.resolve({ done: true }) + }, + } + } + } +} + +export function buf2str( + buf: ArrayBuffer | ArrayBufferView, + encoding: string | undefined = 'utf-8' +): string { + return new TextDecoder(encoding).decode(buf) +} + +export function str2buf(str: string): Uint8Array { + return new TextEncoder().encode(str) +} + +export function hex2buf(hex: string): Uint8Array { + if (hex.length % 2 !== 0) { + throw new Error(`Invalid hex string: odd length (${hex.length})`) + } + + const byteArray = new Uint8Array(hex.length / 2) + + for (let i = 0; i < byteArray.length; i++) { + const byteCode = hex.substring(i * 2, i * 2 + 2) + + byteArray[i] = parseInt(byteCode, 16) + } + + return byteArray +} + +export function buf2hex(buf: Uint8Array): string { + return Array.from(buf) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') +} + +export function b64d2buf(b64d: string): Uint8Array { + return Base64.toUint8Array(b64d) +} + +export function buf2b64d(buf: Uint8Array): string { + return Base64.fromUint8Array(buf) +} + +export function concatBufs(...bufs: (Uint8Array | ArrayBuffer)[]): ArrayBuffer { + const totalLength = bufs.reduce((acc, buf) => acc + buf.byteLength, 0) + + const result = new Uint8Array(totalLength) + + let offset = 0 + + for (const buf of bufs) { + const view = new Uint8Array(buf) + + result.set(view, offset) + + offset += buf.byteLength + } + + return result.buffer +} + +export async function stream2buf( + stream: ReadableStream +): Promise { + const chunks: (ArrayBuffer | Uint8Array)[] = [] + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore this may not longer be necessary as streams are now iterable + for await (const chunk of stream) { + chunks.push(chunk) + } + + return concatBufs(...chunks) +} + +export async function buf2stream( + buf: ArrayBuffer | Uint8Array +): Promise> { + return new ReadableStream({ + start(controller) { + controller.enqueue(buf) + controller.close() + }, + }) +} diff --git a/packages/buffer/tsconfig.json b/packages/buffer/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/buffer/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/cloak/LICENSE.md b/packages/cloak/LICENSE.md new file mode 100644 index 0000000..e81cad1 --- /dev/null +++ b/packages/cloak/LICENSE.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2019 François Best +Copyright (c) 2026 CBK.AI LTD (modifications) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/cloak/README.md b/packages/cloak/README.md new file mode 100644 index 0000000..4f5c749 --- /dev/null +++ b/packages/cloak/README.md @@ -0,0 +1,12 @@ +# @chatbotkit-dev/cloak + +AES-GCM encryption, decryption, and key management for the Web Crypto API. + +This package is based on +[`@47ng/cloak` v1.2.0](https://github.com/47ng/cloak/tree/v1.2.0) +(`fc619b746ce25b862d7510ad51231264516db798`). It preserves the upstream data +formats while adapting the implementation for this platform and adding features +such as additional authenticated data support. + +The original work and CBK.AI's modifications are distributed under the MIT +License. See [LICENSE.md](./LICENSE.md). diff --git a/packages/cloak/jest.config.js b/packages/cloak/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/cloak/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/cloak/package.json b/packages/cloak/package.json new file mode 100644 index 0000000..dd6082b --- /dev/null +++ b/packages/cloak/package.json @@ -0,0 +1,39 @@ +{ + "name": "@chatbotkit-dev/cloak", + "version": "0.0.0", + "license": "MIT", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@47ng/codec": "^1.1.0" + }, + "devDependencies": { + "@47ng/cloak": "^1.2.0", + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/cloak/src/ciphers/aes-gcm.ts b/packages/cloak/src/ciphers/aes-gcm.ts new file mode 100644 index 0000000..c028be0 --- /dev/null +++ b/packages/cloak/src/ciphers/aes-gcm.ts @@ -0,0 +1,63 @@ +import { utf8 } from '@47ng/codec' + +export interface AesCipher { + iv: Uint8Array + text: Uint8Array +} + +export interface AesGcmOptions { + /** + * Additional authenticated data. Not encrypted and not carried in the + * ciphertext, but bound to it: decryption fails unless the exact same + * bytes are supplied again. Callers use it to tie a ciphertext to its + * context (a column, a record) so it cannot be moved somewhere else. + */ + additionalData?: Uint8Array +} + +export async function encryptAesGcm( + key: CryptoKey, + message: string, + options: AesGcmOptions = {} +): Promise { + const buf = utf8.encode(message) + + const iv = crypto.getRandomValues(new Uint8Array(12)) + + const cipherText = await crypto.subtle.encrypt( + { + name: 'AES-GCM', + iv, + ...(options.additionalData + ? { additionalData: new Uint8Array(options.additionalData) } + : {}), + }, + key, + new Uint8Array(buf) + ) + + return { + text: new Uint8Array(cipherText), + iv, + } +} + +export async function decryptAesGcm( + key: CryptoKey, + cipher: AesCipher, + options: AesGcmOptions = {} +): Promise { + const buf = await crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: new Uint8Array(cipher.iv), + ...(options.additionalData + ? { additionalData: new Uint8Array(options.additionalData) } + : {}), + }, + key, + new Uint8Array(cipher.text) + ) + + return utf8.decode(new Uint8Array(buf)) +} diff --git a/packages/cloak/src/index.test.ts b/packages/cloak/src/index.test.ts new file mode 100644 index 0000000..2e95285 --- /dev/null +++ b/packages/cloak/src/index.test.ts @@ -0,0 +1,197 @@ +import { + decryptString, + encryptString, + findKeyForMessage, + generateKey, + makeKeychain, + parseCloakedString, + parseKey, +} from './index' + +import { + decryptString as decryptString47ng, + encryptString as encryptString47ng, +} from '@47ng/cloak' + +test('Key generation', () => { + const key = generateKey() + + expect(key.startsWith('k1.aesgcm256.')).toBeTruthy() + expect(key.length).toEqual(57) +}) + +describe('v1 format', () => { + test('Encrypt / decrypt', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const expected = 'Hello, World !' + const cipher = await encryptString(expected, key) + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Encrypt / decrypt compatibility 001', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const expected = 'Hello, World !' + const cipher = await encryptString(expected, key) + const received = await decryptString47ng(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Encrypt / decrypt compatibility 002', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const expected = 'Hello, World !' + const cipher = await encryptString47ng(expected, key) + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Encrypt / decrypt 4 MiB string', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const expected = 'a'.repeat(4_194_304) // 2 ** 22 = 4 MiB + const cipher = await encryptString(expected, key) + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Encrypt empty string', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const expected = '' + const cipher = await encryptString(expected, key) + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Decrypt known message (empty string)', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const cipher = + 'v1.aesgcm256.710bb0e2.9tZkprVBt4L7ZW_U.GDrlM3U_P0UnHf38HvOCgQ==' + const expected = '' + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Decrypt known message', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const cipher = + 'v1.aesgcm256.710bb0e2.F5wkSytfdVv4xvtN.8uNajc7ufhVmMFpDdzWgKMKhOY4ZR2OSv1DFjvnm' + const expected = 'Hello, World !' + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Decrypt known message from browser', async () => { + const key = 'k1.aesgcm256.CO6hoJ8l1nAmXpuCcuNg-l5g3Nn63X36lBwhsNepUEY=' + const cipher = + 'v1.aesgcm256.4eb11c57.UAuPXcQZV_e40NP6.OvVOoWCXhMB_G-giNtAbDYZI0sfJomHUAW0vpxKV' + const expected = 'Hello, World !' + const received = await decryptString(cipher, key) + + expect(received).toEqual(expected) + }) + + test('Ciphertext & IV are rotated', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const cipher1 = await encryptString('Hello, World !', key) + const cipher2 = await encryptString('Hello, World !', key) + + expect(cipher1).not.toEqual(cipher2) + }) + + test('Fingerprinting & keychain', async () => { + const keyA = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const keyB = 'k1.aesgcm256.caNwte-JDsVUATl3qCQgu9ZPuHAiJhWSOn0pcgGhwyE=' + const cipherA = await encryptString('Hello', keyA) + const cipherB = await encryptString('Hello', keyB) + const keychain = await makeKeychain([keyA, keyB]) + const keyForA = findKeyForMessage(cipherA, keychain) + const keyForB = findKeyForMessage(cipherB, keychain) + + expect(keyForA).toEqual(await parseKey(keyA)) + + expect(keyForB).toEqual(await parseKey(keyB)) + }) + + test('Parse key', async () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + const parsedKey = await parseKey(key) + const expected = 'Hello, World !' + const cipher = await encryptString(expected, parsedKey) + const received = await decryptString(cipher, parsedKey) + + expect(received).toEqual(expected) + }) + + test('Rejects ciphertext with invalid characters', () => { + // @note ensures parseCloakedString returns false for malformed ciphertexts + // instead of passing through to b64.decode which would throw an unexpected error + const malformed = + 'v1.aesgcm256.710bb0e2.9tZkprVBt4L7ZW_U.!!!!!!!!!!!!!!!!!!!!!!!!!=' + + expect(parseCloakedString(malformed)).toBe(false) + }) +}) + +describe('additional authenticated data', () => { + const key = 'k1.aesgcm256.2itF7YmMYIP4b9NNtKMhIx2axGi6aI50RcwGBiFq-VA=' + + test('round trips with matching AAD', async () => { + const cipher = await encryptString('Hello', key, { + additionalData: 'Secret.value', + }) + const received = await decryptString(cipher, key, { + additionalData: 'Secret.value', + }) + + expect(received).toEqual('Hello') + }) + + test('AAD is not carried in the message', async () => { + const cipher = await encryptString('Hello', key, { + additionalData: 'Secret.value', + }) + + expect(parseCloakedString(cipher)).toBeTruthy() + expect(cipher).not.toContain('Secret') + }) + + test('rejects a different AAD', async () => { + const cipher = await encryptString('Hello', key, { + additionalData: 'Secret.value', + }) + + await expect( + decryptString(cipher, key, { additionalData: 'SecretValue.value' }) + ).rejects.toThrow() + }) + + test('rejects a missing AAD', async () => { + const cipher = await encryptString('Hello', key, { + additionalData: 'Secret.value', + }) + + await expect(decryptString(cipher, key)).rejects.toThrow() + }) + + test('rejects an unexpected AAD on a message encrypted without one', async () => { + const cipher = await encryptString('Hello', key) + + await expect( + decryptString(cipher, key, { additionalData: 'Secret.value' }) + ).rejects.toThrow() + }) + + test('accepts raw bytes', async () => { + const aad = new Uint8Array([1, 2, 3]) + const cipher = await encryptString('Hello', key, { additionalData: aad }) + const received = await decryptString(cipher, key, { additionalData: aad }) + + expect(received).toEqual('Hello') + }) +}) diff --git a/packages/cloak/src/index.ts b/packages/cloak/src/index.ts new file mode 100644 index 0000000..27724e1 --- /dev/null +++ b/packages/cloak/src/index.ts @@ -0,0 +1,10 @@ +export { + cloakKeyRegex, + exportCryptoKey, + generateKey, + parseKey, + serializeKey, +} from './key' +export type { TextCloakKey, ParsedCloakKey } from './key' +export * from './keychain' +export * from './message' diff --git a/packages/cloak/src/key.test.ts b/packages/cloak/src/key.test.ts new file mode 100644 index 0000000..9dbba82 --- /dev/null +++ b/packages/cloak/src/key.test.ts @@ -0,0 +1,32 @@ +import { + cloakKeyRegex, + exportKey, + formatKey, + generateKey, + parseKey, +} from './key' + +describe('key', () => { + test('formatKey', () => { + const bytes = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]) + const received = formatKey(bytes) + const expected = 'k1.aesgcm256.AQIDBAUGBwg=' + + expect(received).toEqual(expected) + }) + + test('parseKey + formatKey', async () => { + const key = 'k1.aesgcm256.Q46Y_L1Vx3KBVQ1POmtuGo2IdWalnbWQzxigxC-vEqo=' + const parsed = await parseKey(key) + const received = formatKey(await exportKey(parsed.key)) + + expect(parsed.fingerprint).toEqual('1fb314a0') + expect(received).toEqual(key) + }) + + test('generateKey', () => { + const key = generateKey() + + expect(key).toMatch(cloakKeyRegex) + }) +}) diff --git a/packages/cloak/src/key.ts b/packages/cloak/src/key.ts new file mode 100644 index 0000000..2e4c33a --- /dev/null +++ b/packages/cloak/src/key.ts @@ -0,0 +1,84 @@ +import { b64, hex, utf8 } from '@47ng/codec' + +export const FINGERPRINT_LENGTH = 8 + +export const cloakKeyRegex = /^k1\.aesgcm256\.(?[a-zA-Z0-9-_]{43}=?)$/ + +export type TextCloakKey = string + +export interface ParsedCloakKey { + key: CryptoKey + fingerprint: string +} + +export function formatKey(raw: Uint8Array) { + return ['k1', 'aesgcm256', b64.encode(raw)].join('.') +} + +export async function parseKey( + key: TextCloakKey, + usage?: 'encrypt' | 'decrypt' +): Promise { + return { + key: await importKey(key, usage), + fingerprint: await getKeyFingerprint(key), + } +} + +export async function serializeKey(key: ParsedCloakKey): Promise { + return formatKey(await exportKey(key.key)) +} + +export function generateKey(): TextCloakKey { + const keyLength = 32 + + const key = crypto.getRandomValues(new Uint8Array(keyLength)) + + return formatKey(key) +} + +export async function exportCryptoKey(key: CryptoKey): Promise { + const algo = key.algorithm as AesKeyAlgorithm + + if (algo.name !== 'AES-GCM' || algo.length !== 256) { + throw new Error('Unsupported key type') + } + + return formatKey(await exportKey(key)) +} + +export async function importKey( + key: TextCloakKey, + usage?: 'encrypt' | 'decrypt' +): Promise { + const match = key.match(cloakKeyRegex) + + if (!match) { + throw new Error('Unknown key format') + } + + const raw = b64.decode(match.groups!.key) + + return await crypto.subtle.importKey( + 'raw', + new Uint8Array(raw), + { + name: 'AES-GCM', + length: 256, + }, + true, + usage ? [usage] : ['encrypt', 'decrypt'] + ) +} + +export async function exportKey(key: CryptoKey): Promise { + return new Uint8Array(await crypto.subtle.exportKey('raw', key)) +} + +export async function getKeyFingerprint(key: TextCloakKey): Promise { + const data = utf8.encode(key) + + const hash = await crypto.subtle.digest('SHA-256', new Uint8Array(data)) + + return hex.encode(new Uint8Array(hash)).slice(0, FINGERPRINT_LENGTH) +} diff --git a/packages/cloak/src/keychain.ts b/packages/cloak/src/keychain.ts new file mode 100644 index 0000000..3b2a8f9 --- /dev/null +++ b/packages/cloak/src/keychain.ts @@ -0,0 +1,104 @@ +import type { ParsedCloakKey, TextCloakKey } from './key' +import { parseKey, serializeKey } from './key' +import type { CloakedString } from './message' +import { + decryptString, + encryptString, + getMessageKeyFingerprint, +} from './message' + +export interface KeychainEntry { + key: ParsedCloakKey + createdAt: number // timestamp + label?: string +} + +interface SerializedKeychainEntry { + key: TextCloakKey + createdAt: number // timestamp + label?: string +} + +export type CloakKeychain = { + [fingerprint: string]: KeychainEntry +} + +export async function makeKeychain( + keys: TextCloakKey[] +): Promise { + const keychain: CloakKeychain = {} + + for (const key of keys) { + const parsedKey = await parseKey(key) + + keychain[parsedKey.fingerprint] = { + key: parsedKey, + createdAt: Date.now(), + } + } + + return keychain +} + +export async function importKeychain( + encryptedKeychain: CloakedString, + masterKey: TextCloakKey +): Promise { + const json = await decryptString(encryptedKeychain, masterKey) + const keys: SerializedKeychainEntry[] = JSON.parse(json) + const keychain: CloakKeychain = {} + + for (const { key, ...rest } of keys) { + const parsedKey = await parseKey(key) + + keychain[parsedKey.fingerprint] = { + key: parsedKey, + ...rest, + } + } + + return keychain +} + +export async function exportKeychain( + keychain: CloakKeychain, + masterKey: TextCloakKey | ParsedCloakKey +): Promise { + const rawEntries: KeychainEntry[] = Object.values(keychain) + const entries: SerializedKeychainEntry[] = [] + + for (const entry of rawEntries) { + entries.push({ + key: await serializeKey(entry.key), + createdAt: entry.createdAt, + label: entry.label, + }) + } + + return await encryptString(JSON.stringify(entries), masterKey) +} + +export function findKeyForMessage( + message: CloakedString, + keychain: CloakKeychain +): ParsedCloakKey { + const fingerprint = getMessageKeyFingerprint(message) + + if (!(fingerprint in keychain)) { + throw new Error('Key is not available') + } + + return keychain[fingerprint].key +} + +export function getKeyAge( + fingerprint: string, + keychain: CloakKeychain, + now: number = Date.now() +) { + if (!(fingerprint in keychain)) { + throw new Error('Key is not available') + } + + return now - keychain[fingerprint].createdAt +} diff --git a/packages/cloak/src/message.ts b/packages/cloak/src/message.ts new file mode 100644 index 0000000..73ffce0 --- /dev/null +++ b/packages/cloak/src/message.ts @@ -0,0 +1,145 @@ +import { decryptAesGcm, encryptAesGcm } from './ciphers/aes-gcm' +import type { ParsedCloakKey, TextCloakKey } from './key' +import { importKey, parseKey } from './key' + +import { b64, utf8 } from '@47ng/codec' + +export type CloakedString = string + +export interface CloakOptions { + /** + * Additional authenticated data (AEAD). A string is UTF-8 encoded. The + * value is not stored in the message; the same value must be supplied to + * `decryptString` or decryption fails with an authentication error. + */ + additionalData?: string | Uint8Array +} + +function toAdditionalData( + value: string | Uint8Array | undefined +): Uint8Array | undefined { + if (value === undefined) { + return undefined + } + + return typeof value === 'string' ? new Uint8Array(utf8.encode(value)) : value +} + +export function encodeEncryptedString( + fingerprint: string, + iv: Uint8Array, + ciphertext: Uint8Array +) { + return [ + 'v1', + 'aesgcm256', + fingerprint, + b64.encode(iv), + b64.encode(ciphertext), + ].join('.') +} + +export async function encryptString( + input: string, + key: TextCloakKey | ParsedCloakKey, + options: CloakOptions = {} +): Promise { + if (typeof key === 'string') { + key = await parseKey(key, 'encrypt') + } + + const { text: ciphertext, iv } = await encryptAesGcm(key.key, input, { + additionalData: toAdditionalData(options.additionalData), + }) + + return encodeEncryptedString(key.fingerprint, iv, ciphertext) +} + +export const cloakedStringRegex = + /^v1\.aesgcm256\.(?[0-9a-fA-F]{8})\.(?[a-zA-Z0-9-_]{16})\.(?[a-zA-Z0-9-_]{22,})={0,2}$/ + +function isBase64(str: string) { + // @note validate all characters are valid base64url characters before checking padding + if (!/^[a-zA-Z0-9-_]*={0,2}$/.test(str)) { + return false + } + + const len = str.length + const firstPaddingChar = str.indexOf('=') + + return ( + firstPaddingChar === -1 || + firstPaddingChar === len - 1 || + (firstPaddingChar === len - 2 && str[len - 1] === '=') + ) +} + +export function parseCloakedString(input: CloakedString) { + const [version, algorithm, fingerprint, iv, ciphertext, nothing] = + input.split('.') + + const isCloakedString = + version === 'v1' && + algorithm === 'aesgcm256' && + /^[0-9a-f]{8}$/i.test(fingerprint) && + /^[a-zA-Z0-9-_]{16}$/.test(iv) && + isBase64(ciphertext) && + ciphertext.length >= 24 && + nothing === undefined + + if (isCloakedString === false) { + return false + } else { + return { + groups: { + fingerprint, + iv, + ciphertext, + }, + } + } +} + +export async function decryptString( + input: CloakedString, + key: TextCloakKey | ParsedCloakKey, + options: CloakOptions = {} +): Promise { + const match = parseCloakedString(input) + + if (!match) { + throw new Error(`Unknown message format`) + } + + const iv = match.groups.iv + const ciphertext = match.groups.ciphertext + + let aesKey: CryptoKey + + if (typeof key === 'string') { + aesKey = await importKey(key, 'decrypt') + } else { + aesKey = key.key + } + + return await decryptAesGcm( + aesKey, + { + iv: b64.decode(iv), + text: b64.decode(ciphertext), + }, + { + additionalData: toAdditionalData(options.additionalData), + } + ) +} + +export function getMessageKeyFingerprint(message: CloakedString) { + const match = parseCloakedString(message) + + if (!match) { + throw new Error('Unknown message format') + } + + return match.groups.fingerprint +} diff --git a/packages/cloak/tsconfig.json b/packages/cloak/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/cloak/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/db-spec/README.md b/packages/db-spec/README.md new file mode 100644 index 0000000..2d082db --- /dev/null +++ b/packages/db-spec/README.md @@ -0,0 +1,35 @@ +# @chatbotkit-dev/db-spec + +The database contract, laid out like an implementation: `prisma/schema.prisma` +is THE schema - the one hand-edited source - with the shared analytics SQL and +the zod generator config beside it. + +Every implementation derives its own `prisma/` from this one by importing the +renderers from `@chatbotkit-dev/db-spec/derive` in a small script of its own. +The direction is deliberate: this package does not know who implements it. +Implementations depend on the spec, never the reverse. + +## Changing the schema + +1. Edit `prisma/schema.prisma` +2. `pnpm -r derive` - every implementation re-derives itself; outputs are + committed, so the change shows up in review on every engine it affects +3. Push and generate through the platform as usual (`pnpm db` in `platform`) - + each implementation also re-derives automatically at the start of its own + `db:push` and `db:gen`, so a stale schema cannot reach a database or a + generated client + +The blueprint stays MySQL-complete on purpose - the engine-specific information +only flows downhill, so deriving is subtractive. The 48 queries in `prisma/sql` +are shared by every engine: `?` placeholders and `DATE()` work on MySQL and +SQLite alike. Keep them free of engine-specific expressions - dates are computed +by the caller and passed as parameters. + +## Why `zod` is pinned exactly + +The Json column shapes this package exports are zod schema *instances*, and the +platform passes them into code that checks `instanceof` against its own zod. Two +resolved zod versions mean two instances and a failure that reads as "not a Zod +schema" far from the cause. The exact pin keeps this package on the platform's +resolved version - if the platform upgrades zod, bump this pin in the same +change. diff --git a/packages/db-spec/jest.config.js b/packages/db-spec/jest.config.js new file mode 100644 index 0000000..add9213 --- /dev/null +++ b/packages/db-spec/jest.config.js @@ -0,0 +1,16 @@ +// @note diagnostics are off. This package sets `checkJs`, because its source is +// JavaScript and the JSDoc annotation is what enforces the spec - see +// src/index.js - and ts-jest would then type check every .js file it transforms, +// including the test environment itself, which was never written against this +// tsconfig. Tests are run, not compiled; `pnpm check` is what type checks this +// package. See packages/AGENTS.md. + +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', + + transform: { + '^.+\\.[tj]sx?$': ['ts-jest', { useESM: true, diagnostics: false }], + }, +} diff --git a/packages/db-spec/package.json b/packages/db-spec/package.json new file mode 100644 index 0000000..828a925 --- /dev/null +++ b/packages/db-spec/package.json @@ -0,0 +1,40 @@ +{ + "name": "@chatbotkit-dev/db-spec", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "rimraf node_modules *.tsbuildinfo", + "format": "prisma format --schema prisma/schema.prisma", + "lint": "eslint src scripts --ext .ts,.js", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "eslint": "^9.0.0", + "jest": "^29", + "prisma": "^7.3.0", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + }, + "exports": { + "./derive": { + "import": "./src/derive.js", + "default": "./src/derive.js" + }, + "./types": { + "import": "./src/types.ts", + "default": "./src/types.ts" + }, + "./scripts/post-generate-zod.js": "./scripts/post-generate-zod.js" + }, + "dependencies": { + "zod": "3.25.67" + } +} diff --git a/packages/db-spec/prisma/schema.prisma b/packages/db-spec/prisma/schema.prisma new file mode 100644 index 0000000..5d38f67 --- /dev/null +++ b/packages/db-spec/prisma/schema.prisma @@ -0,0 +1,3500 @@ +// @note THE schema - the one hand-edited source, here in +// @chatbotkit-dev/db-spec. Every implementation derives its own +// prisma/schema.prisma from this file; none of those is edited directly. It stays +// MySQL-complete on purpose: the engine-specific information (native types, the +// expression defaults TEXT columns force) only flows downhill, so deriving is +// subtractive. See src/derive.js for exactly what each engine changes. + +generator client { + provider = "prisma-client" + output = "./generated/prisma" + previewFeatures = ["typedSql"] +} + +generator json { + provider = "prisma-json-types-generator" + namespace = "PrismaJson" + allowAny = false +} + +generator zod { + provider = "prisma-zod-generator" + output = "./zod/schemas" + config = "./zod-generator.config.json" +} + +generator pothos { + provider = "prisma-pothos-types" + output = "./generated/pothos.ts" + generateDatamodel = true + documentation = false + clientOutput = "./prisma" +} + +// --- +// --- +// --- + +datasource db { + provider = "mysql" + relationMode = "prisma" +} + +// --- +// --- +// --- + +// @see https://next-auth.js.org/adapters/prisma + +model Account { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + type String + + provider String + providerAccountId String + + refresh_token String? @db.Text /// @encrypted + access_token String? @db.Text /// @encrypted + id_token String? @db.Text /// @encrypted + token_type String? + + expires_at Int? + + scope String? + + session_state String? + + refresh_token_expires_in Int? + + meta Json? /// [Meta] + + // timestamps + // @note disabled because new + // createdAt DateTime @default(now()) + // updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([provider, providerAccountId]) + // indexes: other + @@index(fields: [userId]) +} + +model Session { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String? // @note deliberately optional + description String? // @note deliberately optional + + sessionToken String @unique + + expires DateTime + + audience String? + + payload Json? /// [JsonRecord] + + options Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + // @note disabled because new + // createdAt DateTime @default(now()) + // updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) +} + +model VerificationToken { + // fields + identifier String + + token String @unique + + expires DateTime + + meta Json? /// [Meta] + + // timestamps + // @note disabled because new + // createdAt DateTime @default(now()) + // updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([identifier, token]) +} + +// --- +// --- +// --- + +model User { + id String @id @default(cuid()) + + // relationships + parentId String? + parent User? @relation("ParentChild", fields: [parentId], references: [id], onDelete: NoAction, onUpdate: NoAction) // @note prevents deletion - must use deleteUser function + + // ref + alias String? // @note only used for child Users (parentId must be set) + + // fields + name String? // @note deliberately optional + description String? @db.Text // @note deliberately optional + + email String @unique + emailVerified DateTime? + + parentContextName String? + + parentContextEmail String? + parentContextEmailVerified DateTime? + + image String? + + billingCustomerId String? @unique + billingSubscriptionId String? + billingSubscriptionStatus String? + billingSubscriptionStartedAt DateTime? + billingSubscriptionTrialedAt DateTime? + + channel String? + organization String? + industry String? + role String? + goal String? @db.Text + + limits Json? /// [Limits] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: account + accounts Account[] + sessions Session[] + + // connections: children + children User[] @relation("ParentChild") + + // connections: teams + teams Team[] + + // connections: developer + tokens Token[] + webhooks Webhook[] + + // connections: objects + contacts Contact[] + conversations Conversation[] + tasks Task[] + ratings Rating[] + memories Memory[] + spaces Space[] + + // connections: executions + taskExecutions TaskExecution[] + + // connections: resources + blueprints Blueprint[] + bots Bot[] + datasets Dataset[] + skillsets Skillset[] + abilities Ability[] + secrets Secret[] + files File[] + portals Portal[] + policies Policy[] + + // connections: publishing + spaceSites SpaceSite[] + + // connections: special + contexts Context[] + + // connections: integrations + triggerIntegrations TriggerIntegration[] + widgetIntegrations WidgetIntegration[] + slackIntegrations SlackIntegration[] + discordIntegrations DiscordIntegration[] + microsoftteamsIntegrations MicrosoftteamsIntegration[] + googlechatIntegrations GooglechatIntegration[] + whatsappIntegrations WhatsappIntegration[] + messengerIntegrations MessengerIntegration[] + instagramIntegrations InstagramIntegration[] + telegramIntegrations TelegramIntegration[] + twilioIntegrations TwilioIntegration[] + anamIntegrations AnamIntegration[] + avatarIntegrations AvatarIntegration[] + recallIntegrations RecallIntegration[] + githubIntegrations GithubIntegration[] + emailIntegrations EmailIntegration[] + sitemapIntegrations SitemapIntegration[] + notionIntegrations NotionIntegration[] + supportIntegrations SupportIntegration[] + extractIntegrations ExtractIntegration[] + mcpserverIntegrations McpserverIntegration[] + skillserverIntegrations SkillserverIntegration[] + + // connections: oauth + oAuthConnections OAuthConnection[] + oAuthApplications OAuthApplication[] + oAuthApplicationTokens OAuthApplicationToken[] + + // connections: hub + hubBotPages HubBotPage[] + hubDatasetPages HubDatasetPage[] + hubSkillsetPages HubSkillsetPage[] + hubBlueprintPages HubBlueprintPage[] + hubWidgetPages HubWidgetPage[] + + // connections: secret values + secretValues SecretValue[] + + // connections: lock + locks Lock[] + + // connections: usage + usages Usage[] @relation("UserUsage") + childUsages Usage[] @relation("ParentUserUsage") + + // connections: observability + eventLogs EventLog[] + eventMetrics EventMetric[] + auditLogs AuditLog[] + + // indexes: unique + @@unique([parentId, alias]) // for alias lookup by parent + @@unique([parentId, parentContextEmail]) + // indexes: other + @@index(fields: [parentId]) + @@index(fields: [id, parentId]) +} + +// --- +// --- +// --- + +enum ResourceState { + // @note the order is important because the default value is the first one + + enabled + disabled +} + +// --- +// --- +// --- + +model Team { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: team + memberships TeamMembership[] + + // indexes: other + @@index(fields: [userId]) +} + +model TeamMembership { + id String @id @default(cuid()) + + // relationships + teamId String + team Team @relation(fields: [teamId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + email String + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([teamId, email]) + // indexes: other + @@index(fields: [email]) +} + +// --- +// --- +// --- + +model OAuthConnection { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + issuer String? + clientId String? + clientSecret String? @db.Text /// @encrypted + + scopes String @default("openid email profile") + + allowedDomains String? @db.Text + requiredClaims Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: integrations + mcpserverIntegrations McpserverIntegration[] + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) +} + +model OAuthApplication { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + clientId String @unique + clientSecret String @unique /// @digest + + redirectUris Json @default("[]") /// [OAuthRedirectUris] + + scopes Json @default("[]") /// [OAuthScopes] + + grants Json @default("[]") /// [OAuthGrants] + + accessTokenLifetime Int? + refreshTokenLifetime Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: oauth + oAuthApplicationTokens OAuthApplicationToken[] + + // indexes: other + @@index(fields: [userId]) +} + +model OAuthApplicationToken { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + applicationId String + application OAuthApplication @relation(fields: [applicationId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + accessToken String @unique /// @digest + accessTokenExpiresAt DateTime? + + refreshToken String? @unique /// @digest + refreshTokenExpiresAt DateTime? + + scopes Json @default("[]") /// [OAuthScopes] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [applicationId]) +} + +// --- +// --- +// --- + +enum Visibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +// --- +// --- +// --- + +model Lock { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + blueprints Blueprint[] + bots Bot[] + datasets Dataset[] + skillsets Skillset[] + files File[] + secrets Secret[] + policies Policy[] + portals Portal[] + + // connections: integrations + triggerIntegrations TriggerIntegration[] + widgetIntegrations WidgetIntegration[] + slackIntegrations SlackIntegration[] + discordIntegrations DiscordIntegration[] + microsoftteamsIntegrations MicrosoftteamsIntegration[] + googlechatIntegrations GooglechatIntegration[] + whatsappIntegrations WhatsappIntegration[] + messengerIntegrations MessengerIntegration[] + instagramIntegrations InstagramIntegration[] + telegramIntegrations TelegramIntegration[] + twilioIntegrations TwilioIntegration[] + githubIntegrations GithubIntegration[] + emailIntegrations EmailIntegration[] + sitemapIntegrations SitemapIntegration[] + notionIntegrations NotionIntegration[] + supportIntegrations SupportIntegration[] + extractIntegrations ExtractIntegration[] + mcpserverIntegrations McpserverIntegration[] + skillserverIntegrations SkillserverIntegration[] +} + +// --- +// --- +// --- + +model Context { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + payload Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) + @@index(fields: [datasetId]) + @@index(fields: [skillsetId]) + @@index(fields: [contactId]) +} + +// --- +// --- +// --- + +enum BlueprintVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Blueprint { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + visibility BlueprintVisibility @default(private) + + config Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: hub + hubBlueprintPage HubBlueprintPage? + + // connections: resources + bots Bot[] /// @resource + datasets Dataset[] /// @resource + skillsets Skillset[] /// @resource + abilities Ability[] /// @resource + secrets Secret[] /// @resource + files File[] /// @resource + spaces Space[] /// @resource + portals Portal[] /// @resource + policies Policy[] /// @resource + tasks Task[] /// @resource + + // connections: special + contexts Context[] + + // connections: oauth + oAuthConnections OAuthConnection[] /// @resource + + // connections: integrations + triggerIntegrations TriggerIntegration[] /// @resource + widgetIntegrations WidgetIntegration[] /// @resource + slackIntegrations SlackIntegration[] /// @resource + discordIntegrations DiscordIntegration[] /// @resource + microsoftteamsIntegrations MicrosoftteamsIntegration[] /// @resource + googlechatIntegrations GooglechatIntegration[] /// @resource + whatsappIntegrations WhatsappIntegration[] /// @resource + messengerIntegrations MessengerIntegration[] /// @resource + instagramIntegrations InstagramIntegration[] /// @resource + telegramIntegrations TelegramIntegration[] /// @resource + twilioIntegrations TwilioIntegration[] /// @resource + avatarIntegrations AvatarIntegration[] /// @resource + anamIntegrations AnamIntegration[] /// @resource + recallIntegrations RecallIntegration[] /// @resource + githubIntegrations GithubIntegration[] /// @resource + emailIntegrations EmailIntegration[] /// @resource + supportIntegrations SupportIntegration[] /// @resource + extractIntegrations ExtractIntegration[] /// @resource + sitemapIntegrations SitemapIntegration[] /// @resource + notionIntegrations NotionIntegration[] /// @resource + mcpserverIntegrations McpserverIntegration[] /// @resource + skillserverIntegrations SkillserverIntegration[] /// @resource + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +enum BotVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Bot { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + backstory String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.MediumText + + model String @default("") @db.VarChar(512) + + privacy Boolean @default(false) + moderation Boolean @default(false) + + visibility BotVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: hub + hubBotPage HubBotPage? + + // connections: objects + conversations Conversation[] + tasks Task[] + ratings Rating[] + memories Memory[] + + // connections: resources + abilities Ability[] + policies Policy[] + + // connections: special + contexts Context[] + + // connections: integrations + triggerIntegrations TriggerIntegration[] + widgetIntegrations WidgetIntegration[] + slackIntegrations SlackIntegration[] + discordIntegrations DiscordIntegration[] + microsoftteamsIntegrations MicrosoftteamsIntegration[] + googlechatIntegrations GooglechatIntegration[] + whatsappIntegrations WhatsappIntegration[] + messengerIntegrations MessengerIntegration[] + instagramIntegrations InstagramIntegration[] + telegramIntegrations TelegramIntegration[] + twilioIntegrations TwilioIntegration[] + anamIntegrations AnamIntegration[] + avatarIntegrations AvatarIntegration[] + recallIntegrations RecallIntegration[] + githubIntegrations GithubIntegration[] + emailIntegrations EmailIntegration[] + supportIntegrations SupportIntegration[] + extractIntegrations ExtractIntegration[] + + // connections: connections + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [datasetId]) + @@index(fields: [skillsetId]) + @@index(fields: [id, datasetId]) +} + +// --- +// --- +// --- + +// @idea A/B Testing / Experiments +// +// This is a proposed design for A/B testing bots. The idea is that an +// Experiment acts as a "router" that distributes traffic across multiple +// bot variants based on configurable weights. +// +// How it works: +// 1. User creates multiple bots (each bot is effectively a "version") +// 2. User creates an Experiment with ExperimentVariants pointing to those bots +// 3. Integration (Widget, Slack, etc.) references the experimentId +// 4. Session creation selects a variant based on weights and routes to that bot +// 5. Conversation records experimentId + experimentVariantId for analytics +// +// Precedence logic for integrations: +// - If experimentId is set and experiment is active → use experiment routing +// - Otherwise fall back to botId +// +// Example: +// Experiment "New Prompt Test" +// ├─ Variant "Control" (50%) → Bot A +// └─ Variant "Friendly Tone" (50%) → Bot B +// +// Analytics can then compare ratings, engagement, etc. per variant. +// +// Schema: +// +// enum ExperimentStatus { +// draft // not yet running +// active // accepting traffic +// paused // temporarily stopped +// completed // finished, winner declared +// } +// +// model Experiment { +// id String @id @default(cuid()) +// +// // owner +// userId String +// user User @relation(fields: [userId], references: [id], onDelete: Cascade) +// +// // relationships +// blueprintId String? +// blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) +// +// // fields +// name String @default("") +// description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text +// +// status ExperimentStatus @default(draft) +// +// startedAt DateTime? +// completedAt DateTime? +// +// // optional: reference to winning variant after experiment concludes +// winnerVariantId String? +// +// meta Json? /// [Meta] +// +// // timestamps +// createdAt DateTime @default(now()) +// updatedAt DateTime @updatedAt +// +// // connections +// variants ExperimentVariant[] +// conversations Conversation[] +// +// // indexes +// @@index(fields: [userId, status, createdAt(sort: Desc)]) +// @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) +// } +// +// model ExperimentVariant { +// id String @id @default(cuid()) +// +// // relationships +// experimentId String +// experiment Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade) +// botId String +// bot Bot @relation(fields: [botId], references: [id], onDelete: Cascade) +// +// // fields +// name String @default("") // e.g., "Control", "Variant A", "Friendly Tone" +// weight Int @default(1) // relative weight (not percentage) - traffic is distributed proportionally +// isControl Boolean @default(false) +// +// meta Json? /// [Meta] +// +// // timestamps +// createdAt DateTime @default(now()) +// updatedAt DateTime @updatedAt +// +// // connections +// conversations Conversation[] +// +// // indexes +// @@index(fields: [experimentId]) +// @@index(fields: [botId]) +// } +// +// Additional changes needed: +// +// 1. Add to integrations (WidgetIntegration, SlackIntegration, etc.): +// experimentId String? +// experiment Experiment? @relation(fields: [experimentId], references: [id], onDelete: SetNull) +// +// 2. Add to Conversation: +// experimentId String? +// experiment Experiment? @relation(fields: [experimentId], references: [id], onDelete: SetNull) +// experimentVariantId String? +// experimentVariant ExperimentVariant? @relation(fields: [experimentVariantId], references: [id], onDelete: SetNull) +// +// 3. Add to Bot: +// experimentVariants ExperimentVariant[] + +// --- +// --- +// --- + +enum DatasetVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Dataset { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + reranker String? + + recordMaxTokens Int? + + searchMinScore Float? + searchMaxRecords Int? + searchMaxTokens Int? + + separators String? + + matchInstruction String? @db.Text // do not tempt to set to empty string + mismatchInstruction String? @db.Text // do not tempt to set to empty string + + visibility DatasetVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: hub + hubDatasetPage HubDatasetPage? + + // connections: objects + conversations Conversation[] + + // connections: resources + bots Bot[] + files DatasetFileAttachment[] + + // connections: special + contexts Context[] + + // connections: integrations + sitemapIntegrations SitemapIntegration[] + notionIntegrations NotionIntegration[] + + // connections: many-to-many + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) +} + +// --- +// --- +// --- + +enum SkillsetVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Skillset { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + visibility SkillsetVisibility @default(private) + + // lifecycle state - toggle the whole skillset on/off without deleting it + state ResourceState @default(enabled) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: conversation + conversations Conversation[] + + // connections: resources + bots Bot[] + abilities Ability[] + + // connections: special + contexts Context[] + + // connections: integrations + mcpserverIntegrations McpserverIntegration[] + skillserverIntegrations SkillserverIntegration[] + + // connections: hub + hubSkillsetPage HubSkillsetPage? + + // connections: many-to-many + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) +} + +model Ability { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + // @note the resource an ability is linked to (what it acts on), as opposed + // to the owner/container relations above + linkedSecretId String? + linkedSecret Secret? @relation(fields: [linkedSecretId], references: [id], onDelete: SetNull) + linkedFileId String? + linkedFile File? @relation(fields: [linkedFileId], references: [id], onDelete: SetNull) + linkedBotId String? + linkedBot Bot? @relation(fields: [linkedBotId], references: [id], onDelete: SetNull) + linkedSpaceId String? + linkedSpace Space? @relation(fields: [linkedSpaceId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + instruction String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + // lifecycle state - toggle the ability on/off without deleting it + state ResourceState @default(enabled) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [blueprintId]) + @@index(fields: [skillsetId]) +} + +// --- +// --- +// --- + +enum FileVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model File { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + visibility FileVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + abilities Ability[] + datasets DatasetFileAttachment[] + widgetIntegrations WidgetIntegrationFileAttachment[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +enum SecretKind { + shared + personal +} + +enum SecretType { + plain + basic + bearer + jwt + oauth + template + reference +} + +enum SecretVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Secret { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + kind SecretKind @default(shared) + + type SecretType @default(plain) + + value String? @db.Text /// @encrypted + + config Json? /// [SecretConfig] @encrypted @see prisma/post-generate-zod.js + + visibility SecretVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + abilities Ability[] + + // connections: secret values + secretValues SecretValue[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) +} + +model SecretValue { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + secretId String? + secret Secret? @relation(fields: [secretId], references: [id], onDelete: Cascade) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + value String @db.Text /// @encrypted + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, secretId, contactId]) + // indexes: other + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +model Portal { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + slug String @unique + + config Json? /// [PortalConfig] @see prisma/post-generate-zod.js + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +enum Trigger { + never + + automatic +} + +// --- + +enum Schedule { + never + + quarterhourly + halfhourly + hourly + + twicedaily + daily + + twiceweekly + weekly + + twicemonthly + monthly +} + +// --- + +enum SyncStatus { + pending + synced + error +} + +// --- +// --- +// --- + +model TriggerIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + secret String @db.Text /// @encrypted + + authenticate Boolean @default(true) + + sessionDuration Float? + + schedule String? + timezone String? + + lastTriggerAt DateTime? + nextTriggerAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [schedule, lastTriggerAt]) + @@index(fields: [nextTriggerAt]) + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model WidgetIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + theme String? @db.Text + + layout String? @db.Text + + title String? + + intro String? @db.Text + + initial String? @db.Text + + placeholder String? + + origin String? @db.Text + + sessionDuration Float? + + language String? + + plugins String? @db.Text + + stream Boolean @default(true) + + verbose Boolean @default(true) + + tools Boolean @default(false) + + unfurl Boolean @default(true) + + math Boolean @default(false) + + carousel Boolean @default(false) + + form Boolean @default(false) + + attachments Boolean @default(false) + + autoScroll Boolean @default(true) + + startFirst Boolean @default(false) + + contactCollection Boolean @default(false) + + exportConversation Boolean @default(true) + restartConversation Boolean @default(true) + + maximize Boolean @default(true) + + messagePeek Boolean @default(true) + + voiceIn Boolean @default(false) + voiceOut Boolean @default(false) + + poweredBy Boolean @default(true) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + files WidgetIntegrationFileAttachment[] + + // connections: hub + hubWidgetPage HubWidgetPage? + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model SlackIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + signingSecret String? @db.Text /// @encrypted + + botToken String? @db.Text /// @encrypted + userToken String? @db.Text /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + references Boolean @default(false) + + ratings Boolean @default(false) + + visibleMessages Int? + + autoRespond String? @db.Text + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model DiscordIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + appId String? + botToken String? @db.Text /// @encrypted + publicKey String? + + handle String? + + ephemeral Boolean? + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model MicrosoftteamsIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + botFrameworkAppId String? + botFrameworkAppSecret String? @db.Text /// @encrypted + tenantId String? + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model GooglechatIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + serviceAccountKey String? @db.Text /// @encrypted + + projectNumber String? // @note Google Cloud project number used to verify the JWT audience on incoming events + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + autoRespond String? @db.Text + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model WhatsappIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + verifyToken String @db.Text /// @encrypted + + appSecret String? @db.Text /// @encrypted + + phoneNumberId String? + + accessToken String? @db.Text /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model MessengerIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + verifyToken String @db.Text /// @encrypted + + accessToken String? @db.Text /// @encrypted + + // @note the customer's own Meta APP secret - the key X-Hub-Signature-256 + // callbacks are signed with. Optional: without it callbacks are accepted + // unverified, with a logged bypass. + appSecret String? @db.Text /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model InstagramIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + verifyToken String @db.Text /// @encrypted + + accessToken String? @db.Text /// @encrypted + + // @note the customer's own Meta APP secret - the key X-Hub-Signature-256 + // callbacks are signed with. Optional: without it callbacks are accepted + // unverified, with a logged bypass. + appSecret String? @db.Text /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model TelegramIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + botToken String? @db.Text /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model TwilioIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + accountSid String? @db.Text + authToken String? @db.Text /// @encrypted + + voice String? @db.Text + + contactCollection Boolean @default(false) + + sessionDuration Float? + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model AnamIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + apiKey String? @db.Text /// @encrypted + + personaId String? + + visibility Visibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model AvatarIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + visibility Visibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model RecallIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + apiKey String? @db.Text /// @encrypted + + // @note the signing secret of the Recall webhook endpoint (Svix, `whsec_...`). + // Optional: without it status callbacks are accepted unverified, with a logged + // bypass. + webhookSecret String? @db.Text /// @encrypted + + region String? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model GithubIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + // this integration's GitHub App identity + credentials. Each integration is + // its own GitHub App. The installation id is NOT stored: it rides in every + // event payload and is combined with the App key to mint a token to reply. + appId String? // the GitHub App id (public-ish; signs the App JWT as `iss`) + privateKey String? @db.Text /// @encrypted - the App's RSA private key (PEM) + webhookSecret String? @db.Text /// @encrypted - validates x-hub-signature-256 HMAC-SHA256 + + contactCollection Boolean @default(false) + + sessionDuration Float? + + // @note unlike the other integrations this defaults to `@collaborators` and + // not `*`: an installed App hears from everyone who can comment, which on a + // public repository is every GitHub account. Rows predating this field were + // backfilled to `*` to preserve their behaviour, so the column default only + // governs new integrations. See lib/github.validation.ts + allowFrom String? @default(dbgenerated("(_utf8mb4\\'@collaborators\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model EmailIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default(dbgenerated("(_utf8mb4\\'*\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model SitemapIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + url String? + + glob String? + + selectors String? + + javascript Boolean? + + expiresIn Float? + + syncStatus SyncStatus @default(pending) + syncSchedule Schedule @default(never) + + lastSyncedAt DateTime @default(now()) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [syncSchedule, lastSyncedAt]) // used to find sitemap integrations to sync +} + +// --- +// --- +// --- + +model NotionIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + token String @db.Text /// @encrypted + + expiresIn Float? + + syncStatus SyncStatus @default(pending) + syncSchedule Schedule @default(never) + + lastSyncedAt DateTime @default(now()) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [syncSchedule, lastSyncedAt]) // used to find notion integrations to sync +} + +// --- +// --- +// --- + +model SupportIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + email String? + + trigger Trigger? @default(automatic) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId]) + @@index(fields: [userId, botId]) +} + +// --- +// --- +// --- + +model ExtractIntegrationItem { + id String @id @default(cuid()) + + // relationships + extractIntegrationId String + extractIntegration ExtractIntegration @relation(fields: [extractIntegrationId], references: [id], onDelete: Cascade) + conversationId String + conversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade) + + // fields + data Json? /// [JsonRecord] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([extractIntegrationId, conversationId]) + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [extractIntegrationId, createdAt(sort: Desc)]) // used for listing +} + +model ExtractIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + schema Json? /// [JsonRecord] + + request String? @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + model String? @db.VarChar(512) + + trigger Trigger? @default(automatic) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: items + items ExtractIntegrationItem[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model McpserverIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + oAuthConnectionId String? + oAuthConnection OAuthConnection? @relation(fields: [oAuthConnectionId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + accessToken String @db.Text /// @encrypted + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model SkillserverIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + accessToken String @db.Text /// @encrypted + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +enum PolicyType { + // @note the order is important because of the default value is the first one + + retention + usage +} + +model Policy { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: Cascade) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + type PolicyType @default(retention) + + // lifecycle state - toggle the policy on/off without deleting it + state ResourceState @default(enabled) + + config Json? /// [PolicyConfig] @see prisma/post-generate-zod.js + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) + @@index(fields: [userId, type]) + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId, type]) +} + +// --- +// --- +// --- + +model HubBotPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + botId String @unique + bot Bot @relation(fields: [botId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubDatasetPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + datasetId String @unique + dataset Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubSkillsetPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + skillsetId String @unique + skillset Skillset @relation(fields: [skillsetId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubBlueprintPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String @unique + blueprint Blueprint @relation(fields: [blueprintId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + slug String? @unique + icon String? + rank Int? + + shareLog Boolean @default(false) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubWidgetPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + widgetId String @unique + widget WidgetIntegration @relation(fields: [widgetId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +// @note UseType was migrated from an enum to a plain String to avoid requiring +// database migrations every time a new model is added. Valid use type values +// are now derived from the model config at runtime. See lib/usage.types.js. + +model Usage { + id String @id @default(cuid()) + + // owner + // @note usage records must survive user deletion with userId intact + userId String + user User @relation("UserUsage", fields: [userId], references: [id], onDelete: NoAction) + + // parent + // @note parent usage records must survive user deletion with parentUserId intact + parentUserId String? + parentUser User? @relation("ParentUserUsage", fields: [parentUserId], references: [id], onDelete: NoAction) + + // relationships: shallow + conversationId String? + messageId String? + taskId String? + contactId String? + blueprintId String? + botId String? + datasetId String? + skillsetId String? + abilityId String? + + // fields + type String @db.VarChar(191) + + count Int + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [type, createdAt(sort: Desc)]) // used for listing + @@index(fields: [userId, type, createdAt(sort: Desc)]) + @@index(fields: [userId, botId, createdAt(sort: Desc)]) // used by bot usage stats query + @@index(fields: [parentUserId, type, createdAt(sort: Desc)]) // used for parent account usage aggregation +} + +// --- +// --- +// --- + +model EventLog { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships: shallow + conversationId String? + taskId String? + contactId String? + spaceId String? + blueprintId String? + botId String? + datasetId String? + recordId String? + skillsetId String? + abilityId String? + fileId String? + secretId String? + portalId String? + policyId String? + widgetIntegrationId String? + slackIntegrationId String? + discordIntegrationId String? + microsoftteamsIntegrationId String? @map("teamsIntegrationId") + googlechatIntegrationId String? + whatsappIntegrationId String? + messengerIntegrationId String? + instagramIntegrationId String? + telegramIntegrationId String? + twilioIntegrationId String? + githubIntegrationId String? + emailIntegrationId String? + sitemapIntegrationId String? + notionIntegrationId String? + triggerIntegrationId String? + supportIntegrationId String? + extractIntegrationId String? + mcpserverIntegrationId String? + skillserverIntegrationId String? + // @note the event log tracks conversational integrations only; anam, + // avatar and recall are excluded by design (they emit no events). If that + // ever changes, add the three `*IntegrationId` columns here, the matching + // entries in the event log/metric list+export whitelists and the + // EventLogItem type, then regenerate the client. + webhookId String? + taskExecutionId String? + triggerExecutionId String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + type String + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, type, createdAt(sort: Desc)]) +} + +model EventMetric { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships: shallow + conversationId String? + taskId String? + contactId String? + spaceId String? + blueprintId String? + botId String? + datasetId String? + recordId String? + skillsetId String? + abilityId String? + fileId String? + secretId String? + portalId String? + policyId String? + widgetIntegrationId String? + slackIntegrationId String? + discordIntegrationId String? + microsoftteamsIntegrationId String? @map("teamsIntegrationId") + googlechatIntegrationId String? + whatsappIntegrationId String? + messengerIntegrationId String? + instagramIntegrationId String? + telegramIntegrationId String? + twilioIntegrationId String? + githubIntegrationId String? + emailIntegrationId String? + sitemapIntegrationId String? + notionIntegrationId String? + triggerIntegrationId String? + supportIntegrationId String? + extractIntegrationId String? + mcpserverIntegrationId String? + skillserverIntegrationId String? + // @note the event log tracks conversational integrations only; anam, + // avatar and recall are excluded by design (they emit no events). If that + // ever changes, add the three `*IntegrationId` columns here, the matching + // entries in the event log/metric list+export whitelists and the + // EventLogItem type, then regenerate the client. + webhookId String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + type String + + value Float? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, type, createdAt(sort: Desc)]) +} + +// --- + +model AuditLog { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + conversationId String? + taskId String? + contactId String? + spaceId String? + blueprintId String? + botId String? + datasetId String? + recordId String? + skillsetId String? + abilityId String? + fileId String? + secretId String? + portalId String? + policyId String? + webhookId String? + sessionId String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + action String + + oldValues Json? /// [JsonRecord] + newValues Json? /// [JsonRecord] + + ipAddress String? + userAgent String? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, action, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model Contact { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + preferences String? @db.Text + + fingerprint String @default(cuid()) + + email String? + phone String? + nick String? + + verifiedAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: objects + conversations Conversation[] + tasks Task[] + ratings Rating[] + memories Memory[] + spaces Space[] + contexts Context[] + + // connections: secret values + secretValues SecretValue[] + + // indexes: unique + @@unique([userId, fingerprint]) + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, email]) +} + +// --- + +model Conversation { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: SetNull) + spaceId String? + space Space? @relation(fields: [spaceId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + taskId String? + task Task? @relation(fields: [taskId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + backstory String? @default(dbgenerated("(_utf8mb4\\'\\')")) @db.MediumText + + model String? @default("") @db.VarChar(512) + + privacy Boolean? @default(false) + moderation Boolean? @default(false) + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: objects + messages Message[] + ratings Rating[] + + // connections: tasks + taskExecutions TaskExecution[] + + // connections: extractIntegrationItems + extractIntegrationItems ExtractIntegrationItem[] + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) // used for listing + @@index(fields: [userId, contactId, createdAt(sort: Desc)]) // used for listing + @@index(fields: [id, botId]) + @@index(fields: [id, datasetId]) + @@index(fields: [id, skillsetId]) + @@index(fields: [expiresAt]) // used for cleaning up expired conversations + @@index(fields: [createdAt]) // used for cleaning up empty conversations + @@index(fields: [datasetId]) + @@index(fields: [skillsetId]) + @@index(fields: [botId]) + @@index(fields: [contactId, createdAt(sort: Desc)]) // used by conversation fetch in actions + @@index(fields: [spaceId, createdAt(sort: Desc)]) // used by conversation fetch in actions + @@index(fields: [taskId, createdAt(sort: Desc)]) // used by task fetch in actions + @@index(fields: [contactId, taskId, createdAt(sort: Desc)]) // used by task fetch in actions +} + +// --- + +enum MessageType { + // stable types + + user // a message from the user + bot // a message from the bot + reasoning // a message that carries reasoning such as a thought process + context // a message that carries some context such as additional information + instruction // a message that carries some instruction such as a command + backstory // a message that describes the backstory of the bot + activity // a message that describes an activity + checkpoint // a message that describes a compact summary of the conversation + // notification // a message that describes a notification +} + +model Message { + id String @id @default(cuid()) + + // relationships + conversationId String + conversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + type MessageType + + text String @db.Text + + nps Int? + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: objects + ratings Rating[] + + // connections: tasks + taskExecutionsAsStart TaskExecution[] @relation("TaskExecutionStart") + taskExecutionsAsEnd TaskExecution[] @relation("TaskExecutionEnd") + + // indexes: other + @@index(fields: [conversationId, type, createdAt(sort: Desc), id(sort: Desc)]) // supports reverse-chronological engine reads within one conversation and message type; id breaks timestamp ties + @@index(fields: [conversationId, type, createdAt(sort: Desc)]) // specifically used in /api/v1/conversation/[conversationId]/send to get messages fast + @@index(fields: [conversationId, createdAt(sort: Desc)]) // @note all messages are almost always pulled in reverse order +} + +// --- + +model Rating { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: SetNull) + conversationId String? + conversation Conversation? @relation(fields: [conversationId], references: [id], onDelete: SetNull) + messageId String? + message Message? @relation(fields: [messageId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + value Int + + reason String? @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [botId]) + @@index(fields: [conversationId]) + @@index(fields: [messageId]) +} + +// --- + +enum TaskStatus { + idle // not currently running + running // actively executing + canceled // execution was canceled before completion +} + +enum TaskOutcome { + pending // never run yet + success // last run completed successfully + failure // last run failed or was incomplete +} + +model TaskExecution { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + taskId String + task Task @relation(fields: [taskId], references: [id], onDelete: Cascade) + conversationId String? + conversation Conversation? @relation(fields: [conversationId], references: [id], onDelete: SetNull) + startMessageId String? + startMessage Message? @relation("TaskExecutionStart", fields: [startMessageId], references: [id], onDelete: SetNull) + endMessageId String? + endMessage Message? @relation("TaskExecutionEnd", fields: [endMessageId], references: [id], onDelete: SetNull) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + status TaskStatus @default(idle) + outcome TaskOutcome @default(pending) + + completedAt DateTime? + + resumeAt DateTime? + + keepAliveUntil DateTime? + + summary String? @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [taskId, createdAt(sort: Desc)]) + // indexes: reaper (stalled sweep over running executions) + @@index(fields: [status, keepAliveUntil]) + // indexes: foreign keys + @@index(fields: [userId]) + @@index(fields: [taskId]) + @@index(fields: [conversationId]) + @@index(fields: [startMessageId]) + @@index(fields: [endMessageId]) +} + +// --- + +model Task { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + status TaskStatus @default(idle) + outcome TaskOutcome @default(pending) + + sessionDuration Float? + + maxIterations Int? + maxTime Float? + maxCalls Int? + + schedule String? + timezone String? + + nextRunAt DateTime? + lastRunAt DateTime? + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: tasks + taskExecutions TaskExecution[] + + // connections: objects + conversations Conversation[] + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [schedule, lastRunAt]) + @@index(fields: [nextRunAt]) +} + +// --- + +model Memory { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + text String @db.Text + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [botId]) +} + +// --- + +model Space { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: SetNull) + + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: conversation + conversations Conversation[] + + // connections: resources + abilities Ability[] + + // connections: publishing + sites SpaceSite[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) +} + +// --- + +model SpaceSite { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // attachment (many sites per space) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + + // ref + alias String? + + // basic information + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + // host binding + slug String @unique + + // serving config + prefix String? + index String @default("index.html") + notFound String @default("404.html") + + // meta and others + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) + @@index(fields: [spaceId]) +} + +// --- +// --- +// --- + +// @note `Token` is the API token (auth credential); LLM token counting is +// `usage` and `limits` vocabulary. Both keep the word by decision - the two +// never share a type or a table, and prose distinguishes "API token" from +// "usage tokens". + +model Token { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + token String @unique /// @digest + + config Json? /// [TokenConfig] @see prisma/post-generate-zod.js + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [createdAt(sort: Desc)]) // used for listing + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +model Webhook { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + request String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + events String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text + + secret String @unique /// @encrypted + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +enum DatasetFileAttachmentType { + source +} + +model DatasetFileAttachment { + // relationships + datasetId String + dataset Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade) + fileId String + file File @relation(fields: [fileId], references: [id], onDelete: Cascade) + + // fields + type DatasetFileAttachmentType + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@id([datasetId, fileId]) + @@index(fields: [fileId]) // because of @relation + @@index(fields: [datasetId, createdAt(sort: Desc)]) // used for listing + @@index(fields: [datasetId, type]) +} + +// --- + +enum WidgetIntegrationFileAttachmentType { + bar + user + bot + button +} + +model WidgetIntegrationFileAttachment { + // relationships + widgetIntegrationId String + widgetIntegration WidgetIntegration @relation(fields: [widgetIntegrationId], references: [id], onDelete: Cascade) + fileId String + file File @relation(fields: [fileId], references: [id], onDelete: Cascade) + + // fields + type WidgetIntegrationFileAttachmentType + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@id([widgetIntegrationId, type]) + @@index(fields: [fileId]) // because of @relation +} diff --git a/packages/db-spec/prisma/sql/breakdownTotalContactsWithConversationsOverPeriod.sql b/packages/db-spec/prisma/sql/breakdownTotalContactsWithConversationsOverPeriod.sql new file mode 100644 index 0000000..cba8623 --- /dev/null +++ b/packages/db-spec/prisma/sql/breakdownTotalContactsWithConversationsOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(c.createdAt) AS date, + COUNT(DISTINCT c.contactId) AS total +FROM Conversation c +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY DATE(c.createdAt) +ORDER BY date ASC diff --git a/packages/db-spec/prisma/sql/breakdownTotalConversationsOverPeriod.sql b/packages/db-spec/prisma/sql/breakdownTotalConversationsOverPeriod.sql new file mode 100644 index 0000000..16f2b42 --- /dev/null +++ b/packages/db-spec/prisma/sql/breakdownTotalConversationsOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(c.createdAt) AS date, + COUNT(c.id) AS total +FROM Conversation c +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY DATE(c.createdAt) +ORDER BY date ASC diff --git a/packages/db-spec/prisma/sql/breakdownTotalMessagesOfTypeOverPeriod.sql b/packages/db-spec/prisma/sql/breakdownTotalMessagesOfTypeOverPeriod.sql new file mode 100644 index 0000000..45a966a --- /dev/null +++ b/packages/db-spec/prisma/sql/breakdownTotalMessagesOfTypeOverPeriod.sql @@ -0,0 +1,16 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The message type ('user', 'bot', 'activity') +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT + DATE(m.createdAt) AS date, + COUNT(m.id) AS total +FROM Message m +JOIN Conversation c ON m.conversationId = c.id +WHERE c.userId = ? + AND m.type = ? + AND c.contactId IS NOT NULL + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY DATE(m.createdAt) +ORDER BY date ASC diff --git a/packages/db-spec/prisma/sql/breakdownTotalMessagesOverPeriod.sql b/packages/db-spec/prisma/sql/breakdownTotalMessagesOverPeriod.sql new file mode 100644 index 0000000..c67aae1 --- /dev/null +++ b/packages/db-spec/prisma/sql/breakdownTotalMessagesOverPeriod.sql @@ -0,0 +1,14 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(m.createdAt) AS date, + COUNT(m.id) AS total +FROM Message m +JOIN Conversation c ON m.conversationId = c.id +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY DATE(m.createdAt) +ORDER BY date ASC diff --git a/packages/db-spec/prisma/sql/breakdownTotalRatingsOverPeriod.sql b/packages/db-spec/prisma/sql/breakdownTotalRatingsOverPeriod.sql new file mode 100644 index 0000000..b923cd8 --- /dev/null +++ b/packages/db-spec/prisma/sql/breakdownTotalRatingsOverPeriod.sql @@ -0,0 +1,14 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT + DATE(createdAt) as date, + COUNT(*) as total, + COUNT(CASE WHEN value > 0 THEN 1 END) as thumbsUp, + COUNT(CASE WHEN value < 0 THEN 1 END) as thumbsDown +FROM Rating +WHERE userId = ? + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/breakdownTotalUsageTokensOverPeriod.sql b/packages/db-spec/prisma/sql/breakdownTotalUsageTokensOverPeriod.sql new file mode 100644 index 0000000..c312f22 --- /dev/null +++ b/packages/db-spec/prisma/sql/breakdownTotalUsageTokensOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(u.createdAt) AS date, + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +WHERE u.userId = ? + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ? +GROUP BY DATE(u.createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/getAverageMessagesOfTypeOverPeriod.sql b/packages/db-spec/prisma/sql/getAverageMessagesOfTypeOverPeriod.sql new file mode 100644 index 0000000..18c984f --- /dev/null +++ b/packages/db-spec/prisma/sql/getAverageMessagesOfTypeOverPeriod.sql @@ -0,0 +1,15 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:messageType The type of the message (e.g., 'bot') +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT COALESCE(AVG(msg_count), 0) AS average +FROM ( + SELECT m.conversationId, COUNT(*) AS msg_count + FROM Message m + JOIN Conversation c ON c.id = m.conversationId + WHERE c.userId = ? + AND m.type = ? + AND m.createdAt >= ? + AND m.createdAt <= ? + GROUP BY m.conversationId +) AS messages_per_conversation_in_period; diff --git a/packages/db-spec/prisma/sql/getBotConversationCountByDay.sql b/packages/db-spec/prisma/sql/getBotConversationCountByDay.sql new file mode 100644 index 0000000..0d876f3 --- /dev/null +++ b/packages/db-spec/prisma/sql/getBotConversationCountByDay.sql @@ -0,0 +1,12 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:botId The ID of the bot +-- @param {DateTime} $3:fromDate Inclusive start of the period +-- @param {DateTime} $4:toDate Inclusive end of the period +SELECT DATE(createdAt) AS date, COUNT(*) AS total +FROM Conversation +WHERE userId = ? + AND botId = ? + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/getBotConversationsWithUserMessageCount.sql b/packages/db-spec/prisma/sql/getBotConversationsWithUserMessageCount.sql new file mode 100644 index 0000000..bc9feb2 --- /dev/null +++ b/packages/db-spec/prisma/sql/getBotConversationsWithUserMessageCount.sql @@ -0,0 +1,16 @@ +-- @param {String} $1:userMessageType The message type that marks a user turn +-- @param {String} $2:userId The ID of the user +-- @param {String} $3:botId The ID of the bot +-- @param {DateTime} $4:fromDate Inclusive start of the period +-- @param {DateTime} $5:toDate Inclusive end of the period +SELECT COUNT(*) AS total +FROM ( + SELECT c.id + FROM Conversation c + JOIN Message m ON m.conversationId = c.id AND m.type = ? + WHERE c.userId = ? + AND c.botId = ? + AND c.createdAt >= ? + AND c.createdAt <= ? + GROUP BY c.id +) AS conversationsWithUserMessages; diff --git a/packages/db-spec/prisma/sql/getBotMessageCountByDay.sql b/packages/db-spec/prisma/sql/getBotMessageCountByDay.sql new file mode 100644 index 0000000..4243a0e --- /dev/null +++ b/packages/db-spec/prisma/sql/getBotMessageCountByDay.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:botId The ID of the bot +-- @param {DateTime} $3:fromDate Inclusive start of the period +-- @param {DateTime} $4:toDate Inclusive end of the period +SELECT DATE(m.createdAt) AS date, COUNT(*) AS total +FROM Message m +JOIN Conversation c ON c.id = m.conversationId +WHERE c.userId = ? + AND c.botId = ? + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY DATE(m.createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/getBotSingleTurnConversationCount.sql b/packages/db-spec/prisma/sql/getBotSingleTurnConversationCount.sql new file mode 100644 index 0000000..74f9adf --- /dev/null +++ b/packages/db-spec/prisma/sql/getBotSingleTurnConversationCount.sql @@ -0,0 +1,17 @@ +-- @param {String} $1:userMessageType The message type that marks a user turn +-- @param {String} $2:userId The ID of the user +-- @param {String} $3:botId The ID of the bot +-- @param {DateTime} $4:fromDate Inclusive start of the period +-- @param {DateTime} $5:toDate Inclusive end of the period +SELECT COUNT(*) AS total +FROM ( + SELECT c.id + FROM Conversation c + LEFT JOIN Message m ON m.conversationId = c.id AND m.type = ? + WHERE c.userId = ? + AND c.botId = ? + AND c.createdAt >= ? + AND c.createdAt <= ? + GROUP BY c.id + HAVING COUNT(m.id) = 1 +) AS singleTurnConvs; diff --git a/packages/db-spec/prisma/sql/getBotUsageStats.sql b/packages/db-spec/prisma/sql/getBotUsageStats.sql new file mode 100644 index 0000000..226b040 --- /dev/null +++ b/packages/db-spec/prisma/sql/getBotUsageStats.sql @@ -0,0 +1,22 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:botId The ID of the bot +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_BASE_TOKEN' THEN count + ELSE 0 + END), 0) AS totalTokens, + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_CONVERSATION' THEN count + ELSE 0 + END), 0) AS totalConversations, + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_MESSAGE' THEN count + ELSE 0 + END), 0) AS totalMessages +FROM Usage +WHERE userId = ? + AND botId = ? + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getConversationUsageStats.sql b/packages/db-spec/prisma/sql/getConversationUsageStats.sql new file mode 100644 index 0000000..0afa09b --- /dev/null +++ b/packages/db-spec/prisma/sql/getConversationUsageStats.sql @@ -0,0 +1,18 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:conversationId The ID of the conversation +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_BASE_TOKEN' THEN count + ELSE 0 + END), 0) AS totalTokens, + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_MESSAGE' THEN count + ELSE 0 + END), 0) AS totalMessages +FROM Usage +WHERE userId = ? + AND conversationId = ? + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getDailyNegativeRatingCount.sql b/packages/db-spec/prisma/sql/getDailyNegativeRatingCount.sql new file mode 100644 index 0000000..a374304 --- /dev/null +++ b/packages/db-spec/prisma/sql/getDailyNegativeRatingCount.sql @@ -0,0 +1,11 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Inclusive start of the period +-- @param {DateTime} $3:toDate Inclusive end of the period +SELECT DATE(createdAt) AS date, COUNT(*) AS total +FROM Rating +WHERE userId = ? + AND value < 0 + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/getEventMetricSeriesOverPeriod.sql b/packages/db-spec/prisma/sql/getEventMetricSeriesOverPeriod.sql new file mode 100644 index 0000000..a920e11 --- /dev/null +++ b/packages/db-spec/prisma/sql/getEventMetricSeriesOverPeriod.sql @@ -0,0 +1,12 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The event metric type +-- @param {DateTime} $3:fromDate Inclusive start of the period (DateTime) +SELECT + DATE(createdAt) AS date, + COALESCE(SUM(value), 0) AS total +FROM EventMetric +WHERE userId = ? + AND type = ? + AND createdAt >= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/getTotalAbilitiesForUser.sql b/packages/db-spec/prisma/sql/getTotalAbilitiesForUser.sql new file mode 100644 index 0000000..a478fd6 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalAbilitiesForUser.sql @@ -0,0 +1,7 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Ability a +JOIN Skillset s ON a.skillsetId = s.id +JOIN User u ON s.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalBotsForUser.sql b/packages/db-spec/prisma/sql/getTotalBotsForUser.sql new file mode 100644 index 0000000..2cf95ec --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalBotsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Bot b +JOIN User u ON b.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalContacts.sql b/packages/db-spec/prisma/sql/getTotalContacts.sql new file mode 100644 index 0000000..3f89622 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalContacts.sql @@ -0,0 +1,4 @@ +-- @param {String} $1:userId The ID of the user +SELECT count(*) as total +FROM Contact +WHERE userId = ?; diff --git a/packages/db-spec/prisma/sql/getTotalContactsWithConversationsOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalContactsWithConversationsOverPeriod.sql new file mode 100644 index 0000000..ca20908 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalContactsWithConversationsOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM ( + SELECT c.contactId + FROM Conversation c + WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? + GROUP BY c.contactId +) AS distinct_contacts_with_conversations_in_period; diff --git a/packages/db-spec/prisma/sql/getTotalConversationsOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalConversationsOverPeriod.sql new file mode 100644 index 0000000..91e1662 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalConversationsOverPeriod.sql @@ -0,0 +1,8 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM Conversation c +WHERE c.userId = ? + AND c.createdAt >= ? + AND c.createdAt <= ? diff --git a/packages/db-spec/prisma/sql/getTotalConversationsWithNumberedMessagesOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalConversationsWithNumberedMessagesOverPeriod.sql new file mode 100644 index 0000000..a1c74ff --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalConversationsWithNumberedMessagesOverPeriod.sql @@ -0,0 +1,15 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:minMessages Minimum number of messages per conversation in the period +SELECT COUNT(*) AS total +FROM ( + SELECT m.conversationId + FROM Message m + JOIN Conversation c ON c.id = m.conversationId + WHERE c.userId = ? + AND m.createdAt >= ? + AND m.createdAt <= ? + GROUP BY m.conversationId + HAVING COUNT(*) >= ? +) AS conversations_with_min_messages_in_period; diff --git a/packages/db-spec/prisma/sql/getTotalDatasetsForUser.sql b/packages/db-spec/prisma/sql/getTotalDatasetsForUser.sql new file mode 100644 index 0000000..4b37715 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalDatasetsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Dataset d +JOIN User u ON d.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalFilesForUser.sql b/packages/db-spec/prisma/sql/getTotalFilesForUser.sql new file mode 100644 index 0000000..7f90e5c --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalFilesForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM File f +JOIN User u ON f.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalMessagesOfTypeOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalMessagesOfTypeOverPeriod.sql new file mode 100644 index 0000000..62da574 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalMessagesOfTypeOverPeriod.sql @@ -0,0 +1,11 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:messageType The type of the message +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM Message m +JOIN Conversation c ON c.id = m.conversationId +WHERE c.userId = ? + AND m.type = ? + AND m.createdAt >= ? + AND m.createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getTotalMessagesOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalMessagesOverPeriod.sql new file mode 100644 index 0000000..b67c687 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalMessagesOverPeriod.sql @@ -0,0 +1,9 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM Message m +JOIN Conversation c ON c.id = m.conversationId +WHERE c.userId = ? + AND m.createdAt >= ? + AND m.createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getTotalPoliciesForUser.sql b/packages/db-spec/prisma/sql/getTotalPoliciesForUser.sql new file mode 100644 index 0000000..4bcf107 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalPoliciesForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Policy p +JOIN User u ON p.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalPortalsForUser.sql b/packages/db-spec/prisma/sql/getTotalPortalsForUser.sql new file mode 100644 index 0000000..63b09e3 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalPortalsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Portal p +JOIN User u ON p.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalRatingsOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalRatingsOverPeriod.sql new file mode 100644 index 0000000..159ba8d --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalRatingsOverPeriod.sql @@ -0,0 +1,8 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT count(*) as total +FROM Rating +WHERE userId = ? + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getTotalSkillsetsForUser.sql b/packages/db-spec/prisma/sql/getTotalSkillsetsForUser.sql new file mode 100644 index 0000000..cf28f88 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalSkillsetsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Skillset s +JOIN User u ON s.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalTeamMembersForUser.sql b/packages/db-spec/prisma/sql/getTotalTeamMembersForUser.sql new file mode 100644 index 0000000..8295fc1 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalTeamMembersForUser.sql @@ -0,0 +1,7 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM TeamMembership tm +JOIN Team t ON tm.teamId = t.id +JOIN User u ON t.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalTeamsForUser.sql b/packages/db-spec/prisma/sql/getTotalTeamsForUser.sql new file mode 100644 index 0000000..564fd21 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalTeamsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Team t +JOIN User u ON t.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db-spec/prisma/sql/getTotalThumbsDownOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalThumbsDownOverPeriod.sql new file mode 100644 index 0000000..3bae679 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalThumbsDownOverPeriod.sql @@ -0,0 +1,9 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT count(*) as total +FROM Rating +WHERE userId = ? + AND value <= 0 + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getTotalThumbsUpOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalThumbsUpOverPeriod.sql new file mode 100644 index 0000000..75769a7 --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalThumbsUpOverPeriod.sql @@ -0,0 +1,9 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT count(*) as total +FROM Rating +WHERE userId = ? + AND value > 0 + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getTotalUsageTokensOverPeriod.sql b/packages/db-spec/prisma/sql/getTotalUsageTokensOverPeriod.sql new file mode 100644 index 0000000..4e6b89c --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalUsageTokensOverPeriod.sql @@ -0,0 +1,10 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +WHERE u.userId = ? + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ?; diff --git a/packages/db-spec/prisma/sql/getTotalUsersForUser.sql b/packages/db-spec/prisma/sql/getTotalUsersForUser.sql new file mode 100644 index 0000000..5b87daf --- /dev/null +++ b/packages/db-spec/prisma/sql/getTotalUsersForUser.sql @@ -0,0 +1,4 @@ +-- @param {String} $1:userId The ID of the user +SELECT COUNT(*) as count +FROM User u +WHERE u.parentId = ?; diff --git a/packages/db-spec/prisma/sql/getUsageCountByTypeOverPeriod.sql b/packages/db-spec/prisma/sql/getUsageCountByTypeOverPeriod.sql new file mode 100644 index 0000000..6472805 --- /dev/null +++ b/packages/db-spec/prisma/sql/getUsageCountByTypeOverPeriod.sql @@ -0,0 +1,14 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The usage type +-- @param {DateTime} $3:fromDate Inclusive start of the period +-- @param {DateTime} $4:toDate Inclusive end of the period +SELECT + DATE(createdAt) AS date, + COALESCE(SUM(count), 0) AS total +FROM Usage +WHERE userId = ? + AND type = ? + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/getUsageCountByTypeSince.sql b/packages/db-spec/prisma/sql/getUsageCountByTypeSince.sql new file mode 100644 index 0000000..1bdf8b3 --- /dev/null +++ b/packages/db-spec/prisma/sql/getUsageCountByTypeSince.sql @@ -0,0 +1,12 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The usage type +-- @param {DateTime} $3:since Inclusive start of the period +SELECT + DATE(createdAt) AS date, + COALESCE(SUM(count), 0) AS total +FROM Usage +WHERE userId = ? + AND type = ? + AND createdAt >= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db-spec/prisma/sql/listContacts.sql b/packages/db-spec/prisma/sql/listContacts.sql new file mode 100644 index 0000000..2836206 --- /dev/null +++ b/packages/db-spec/prisma/sql/listContacts.sql @@ -0,0 +1,7 @@ +-- @param {String} $1:userId The ID of the user +-- @param {Int} $2:limit The maximum number of contacts to return (optional) +SELECT id, name, description, email, nick, meta, createdAt +FROM Contact +WHERE userId = ? +ORDER BY createdAt DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listContactsWithConversationsOverPeriod.sql b/packages/db-spec/prisma/sql/listContactsWithConversationsOverPeriod.sql new file mode 100644 index 0000000..6315e7b --- /dev/null +++ b/packages/db-spec/prisma/sql/listContactsWithConversationsOverPeriod.sql @@ -0,0 +1,23 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of contacts to return (optional) +SELECT + co.id, + co.name, + co.description, + co.email, + co.nick, + co.meta, + co.createdAt, + COUNT(c.id) AS _countValue, + 'conversation' AS _countType +FROM Conversation c +JOIN Contact co ON c.contactId = co.id +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY co.id, co.name, co.description, co.email, co.nick +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listContactsWithMessagesOverPeriod.sql b/packages/db-spec/prisma/sql/listContactsWithMessagesOverPeriod.sql new file mode 100644 index 0000000..fd3e718 --- /dev/null +++ b/packages/db-spec/prisma/sql/listContactsWithMessagesOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of contacts to return (optional) +SELECT + co.id, + co.name, + co.description, + co.email, + co.nick, + co.meta, + co.createdAt, + COUNT(m.id) AS _countValue, + 'message' AS _countType +FROM Message m +JOIN Conversation c ON m.conversationId = c.id +JOIN Contact co ON c.contactId = co.id +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY co.id, co.name, co.description, co.email, co.nick +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listContactsWithRatingsOverPeriod.sql b/packages/db-spec/prisma/sql/listContactsWithRatingsOverPeriod.sql new file mode 100644 index 0000000..1347c6a --- /dev/null +++ b/packages/db-spec/prisma/sql/listContactsWithRatingsOverPeriod.sql @@ -0,0 +1,26 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +-- @param {Int} $4:limit The maximum number of contacts to return +SELECT + c.id, + c.name, + c.description, + c.email, + c.nick, + c.meta, + c.createdAt, + COUNT(CASE WHEN r.value > 0 THEN 1 END) as _upvoteCount, + COUNT(CASE WHEN r.value < 0 THEN 1 END) as _downvoteCount, + COUNT(r.id) as _countValue, + 'rating' as _countType +FROM Contact c +LEFT JOIN Rating r ON c.id = r.contactId + AND r.userId = ? + AND r.createdAt >= ? + AND r.createdAt <= ? +WHERE c.userId = r.userId +GROUP BY c.id, c.name, c.description, c.email, c.nick, c.meta, c.createdAt +HAVING _countValue > 0 +ORDER BY _countValue DESC, _upvoteCount DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listConversationsOverPeriod.sql b/packages/db-spec/prisma/sql/listConversationsOverPeriod.sql new file mode 100644 index 0000000..c85e953 --- /dev/null +++ b/packages/db-spec/prisma/sql/listConversationsOverPeriod.sql @@ -0,0 +1,25 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of conversations to return (optional) +SELECT + c.id, + co.id AS contactId, + co.name, + co.email, + co.nick, + co.description, + co.meta, + c.createdAt, + COUNT(m.id) AS _countValue, + 'message' AS _countType +FROM Conversation c +JOIN Contact co ON c.contactId = co.id +LEFT JOIN Message m ON c.id = m.conversationId +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY c.id, c.createdAt, co.id, co.name, co.email, co.nick, co.description +ORDER BY _countValue DESC, c.createdAt DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listConversationsWithNumberedMessagesOverPeriod.sql b/packages/db-spec/prisma/sql/listConversationsWithNumberedMessagesOverPeriod.sql new file mode 100644 index 0000000..8949a8e --- /dev/null +++ b/packages/db-spec/prisma/sql/listConversationsWithNumberedMessagesOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:messageCount Minimum number of messages for follow-ups +-- @param {Int} $5:limit The maximum number of conversations to return (optional) +SELECT + c.id, + c.createdAt, + co.name, + co.email, + co.nick, + co.description, + COUNT(m.id) AS messageCount +FROM Conversation c +JOIN Contact co ON c.contactId = co.id +LEFT JOIN Message m ON c.id = m.conversationId +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY c.id, c.createdAt, co.name, co.email, co.nick, co.description +HAVING COUNT(m.id) >= ? +ORDER BY messageCount DESC, c.createdAt DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listEventLogsOfTypeActionsGroupedByTypeOverPeriod.sql b/packages/db-spec/prisma/sql/listEventLogsOfTypeActionsGroupedByTypeOverPeriod.sql new file mode 100644 index 0000000..b46093b --- /dev/null +++ b/packages/db-spec/prisma/sql/listEventLogsOfTypeActionsGroupedByTypeOverPeriod.sql @@ -0,0 +1,18 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of action types to return (optional) +SELECT + el.type, + el.name, + el.description, + COUNT(el.id) AS _countValue, + 'action' AS _countType +FROM EventLog el +WHERE el.userId = ? + AND el.type LIKE 'action.%' + AND el.createdAt >= ? + AND el.createdAt <= ? +GROUP BY el.type, el.name, el.description +ORDER BY _countValue DESC, el.type ASC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listTopBotsByTokenUsageOverPeriod.sql b/packages/db-spec/prisma/sql/listTopBotsByTokenUsageOverPeriod.sql new file mode 100644 index 0000000..3e2cfe8 --- /dev/null +++ b/packages/db-spec/prisma/sql/listTopBotsByTokenUsageOverPeriod.sql @@ -0,0 +1,19 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of results to return +SELECT + u.botId AS id, + b.name AS name, + b.description AS description, + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +LEFT JOIN Bot b ON u.botId = b.id +WHERE u.userId = ? + AND u.botId IS NOT NULL + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ? +GROUP BY u.botId, b.name, b.description +ORDER BY total DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listTopContactsByTokenUsageOverPeriod.sql b/packages/db-spec/prisma/sql/listTopContactsByTokenUsageOverPeriod.sql new file mode 100644 index 0000000..4852c9c --- /dev/null +++ b/packages/db-spec/prisma/sql/listTopContactsByTokenUsageOverPeriod.sql @@ -0,0 +1,19 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of results to return +SELECT + u.contactId AS id, + c.name AS name, + c.description AS description, + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +LEFT JOIN Contact c ON u.contactId = c.id +WHERE u.userId = ? + AND u.contactId IS NOT NULL + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ? +GROUP BY u.contactId, c.name, c.description +ORDER BY total DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listTopDownvotersOverPeriod.sql b/packages/db-spec/prisma/sql/listTopDownvotersOverPeriod.sql new file mode 100644 index 0000000..f37a07c --- /dev/null +++ b/packages/db-spec/prisma/sql/listTopDownvotersOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +-- @param {Int} $4:limit The maximum number of contacts to return +SELECT + c.id, + c.name, + c.description, + c.email, + c.nick, + c.meta, + c.createdAt, + COUNT(CASE WHEN r.value < 0 THEN 1 END) as _countValue, + 'downvote' as _countType +FROM Contact c +LEFT JOIN Rating r ON c.id = r.contactId + AND r.userId = ? + AND r.createdAt >= ? + AND r.createdAt <= ? +WHERE c.userId = r.userId +GROUP BY c.id, c.name, c.description, c.email, c.nick, c.meta, c.createdAt +HAVING _countValue > 0 +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/sql/listTopUpvotersOverPeriod.sql b/packages/db-spec/prisma/sql/listTopUpvotersOverPeriod.sql new file mode 100644 index 0000000..4c8ac91 --- /dev/null +++ b/packages/db-spec/prisma/sql/listTopUpvotersOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +-- @param {Int} $4:limit The maximum number of contacts to return +SELECT + c.id, + c.name, + c.description, + c.email, + c.nick, + c.meta, + c.createdAt, + COUNT(CASE WHEN r.value > 0 THEN 1 END) as _countValue, + 'upvote' as _countType +FROM Contact c +LEFT JOIN Rating r ON c.id = r.contactId + AND r.userId = ? + AND r.createdAt >= ? + AND r.createdAt <= ? +WHERE c.userId = r.userId +GROUP BY c.id, c.name, c.description, c.email, c.nick, c.meta, c.createdAt +HAVING _countValue > 0 +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db-spec/prisma/zod-generator.config.json b/packages/db-spec/prisma/zod-generator.config.json new file mode 100644 index 0000000..eec0afa --- /dev/null +++ b/packages/db-spec/prisma/zod-generator.config.json @@ -0,0 +1,24 @@ +{ + "mode": "custom", + "pureModels": true, + "pureModelsLean": true, + "pureModelsIncludeRelations": false, + "dateTimeStrategy": "date", + "emit": { + "enums": true, + "objects": false, + "crud": false, + "results": false, + "pureModels": true, + "variants": false + }, + "naming": { + "pureModel": { + "filePattern": "{model}.ts", + "schemaSuffix": "Model", + "typeSuffix": "", + "exportNamePattern": "{Model}Model", + "legacyAliases": false + } + } +} diff --git a/packages/db-spec/scripts/post-generate-zod.js b/packages/db-spec/scripts/post-generate-zod.js new file mode 100644 index 0000000..307541f --- /dev/null +++ b/packages/db-spec/scripts/post-generate-zod.js @@ -0,0 +1,145 @@ +/* eslint-disable no-console */ + +/** + * @file post-generate-zod.js + * + * Post-generation script for prisma-zod-generator compatibility. + * + * Creates re-export files at the old zod-prisma output paths so that + * existing consumer imports (e.g. `from 'prisma/zod/schemas/bot'`) + * continue to work without changes. + * + * Also patches the 4 models that use custom Zod types for their + * `config` JSON field (Secret, Portal, Policy, Token). + * + * Run after `prisma generate`: + * node prisma/post-generate-zod.js + * + * @todo Simplify this script by migrating all consumer imports to use the new + * prisma-zod-generator paths directly (e.g. `from 'prisma/zod/schemas/models/Bot'`). + * Once imports are updated, remove the lowercase re-export logic. Consider using + * a codemod or find-replace to update ~100+ import statements across the codebase. + * The custom config type overrides (Secret, Portal, Policy, Token) will still need + * special handling - explore if prisma-zod-generator supports custom type injection + * via schema annotations or generator options. + */ +import { readFileSync, readdirSync, writeFileSync } from 'fs' +import { basename, join } from 'path' + +// @note the prisma directory to post-process comes from the caller - this +// script is shared by every db module, each of which generates its own zod +// output next to its own schema +const PRISMA_DIR = process.argv[2] + +if (!PRISMA_DIR) { + console.error('usage: node post-generate-zod.js ') + + process.exit(1) +} + +const SCHEMAS_DIR = new URL( + 'zod/schemas', + `file://${PRISMA_DIR.endsWith('/') ? PRISMA_DIR : PRISMA_DIR + '/'}` +).pathname +const MODELS_DIR = join(SCHEMAS_DIR, 'models') + +/** + * Models that need custom type overrides for their `config` field. + * Maps model file name (in models/) to the custom type import. + */ +const CUSTOM_CONFIG_MODELS = { + 'secret.ts': { type: 'SecretConfig', nullable: true }, + 'portal.ts': { type: 'PortalConfig', nullable: true }, + 'policy.ts': { type: 'PolicyConfig', nullable: true }, + 'token.ts': { type: 'TokenConfig', nullable: true }, +} + +// Read all generated model files +const modelFiles = readdirSync(MODELS_DIR).filter( + (f) => f.endsWith('.ts') && f !== 'index.ts' +) + +// @note Patch models whose exported type name shadows a TypeScript built-in. +// The zod generator creates `export type Record = z.infer<...>` which shadows +// the global `Record` utility type, breaking any `Record` +// usage inside the same file's refinement functions. + +const TS_BUILTIN_SHADOWS = ['record.ts'] + +for (const shadowFile of TS_BUILTIN_SHADOWS) { + const filePath = join(MODELS_DIR, shadowFile) + + try { + let content = readFileSync(filePath, 'utf-8') + + // Replace `Record<` with `globalThis.Record<` inside refinement code + // but NOT in the `export type Record = ...` line + content = content.replace(/(\bas\s+)Record { + // @note comments are rewritten like everything else - a commented-out + // field must stay valid for this engine when uncommented, and the + // patterns are token-anchored so prose never matches them + return line + .replace(/\s*@db\.\w+(\([^)]*\))?/g, '') + .replace( + /@default\(dbgenerated\("\(_utf8mb4\\\\'(.*?)\\\\'\)"\)\)/g, + '@default("$1")' + ) + .replace( + 'onDelete: NoAction, onUpdate: NoAction', + 'onDelete: Restrict, onUpdate: Restrict' + ) + .replace('onDelete: NoAction', 'onDelete: Restrict') + .replace(/provider\s*=\s*"mysql"/, 'provider = "sqlite"') + }) + .join('\n') +} + +/** + * Derives an implementation's prisma directory from the blueprint. + * + * Writes the rendered schema and copies the shared SQL and the zod generator + * config alongside it, which is everything TypedSQL and the generators resolve + * relative to the schema. + * + * @param {object} options + * @param {string} options.prismaDir - the implementation's prisma directory + * @param {(source: string) => string} options.render - the engine's renderer + * @returns {Promise} + */ +export async function derive({ prismaDir, render }) { + const source = await fs.readFile( + path.join(SPEC_PRISMA, 'schema.prisma'), + 'utf8' + ) + + // @note the source file opens with a note describing itself as the one + // hand-edited schema - true there, false in anything derived from it. Strip + // the leading comment block; the derived header names the source instead. + const lines = source.split('\n') + + let body = 0 + + while ( + body < lines.length && + (lines[body].trim() === '' || lines[body].trimStart().startsWith('//')) + ) { + body += 1 + } + + const stripped = lines.slice(body).join('\n') + + await fs.mkdir(path.join(prismaDir, 'sql'), { recursive: true }) + + await fs.writeFile( + path.join(prismaDir, 'schema.prisma'), + HEADER + render(stripped) + ) + + const sqlDir = path.join(SPEC_PRISMA, 'sql') + + for (const file of await fs.readdir(sqlDir)) { + await fs.copyFile( + path.join(sqlDir, file), + path.join(prismaDir, 'sql', file) + ) + } + + await fs.copyFile( + path.join(SPEC_PRISMA, 'zod-generator.config.json'), + path.join(prismaDir, 'zod-generator.config.json') + ) + + console.log(`derived ${path.relative(process.cwd(), prismaDir)}`) +} diff --git a/packages/db-spec/src/types.test.js b/packages/db-spec/src/types.test.js new file mode 100644 index 0000000..fbff386 --- /dev/null +++ b/packages/db-spec/src/types.test.js @@ -0,0 +1,296 @@ +import { PortalConfig, SecretConfig } from './types' + +import { ZodError } from 'zod' + +describe('PortalConfig', () => { + describe('valid configurations', () => { + it('should accept empty object', () => { + const validConfig = {} + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual({}) + }) + + it('should accept valid apps configuration', () => { + const validConfig = { + apps: { + app1: {}, + app2: {}, + }, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + + it('should accept valid apps with random configuration properties', () => { + const validConfig = { + apps: { + app1: { customProp: 'value', anotherProp: 123 }, + app2: { enabled: true, version: '1.0.0' }, + }, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + + it('should accept valid users configuration', () => { + const validConfig = { + users: { + user1: {}, + user2: {}, + }, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + + it('should accept valid auth configuration', () => { + const validConfig = { + auth: {}, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + + it('should accept valid signin configuration', () => { + const validConfig = { + signin: { + title: 'Welcome', + headline: 'Sign in to your account', + }, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + + it('should accept valid layout configuration', () => { + const validConfig = { + layout: { + madeWith: true, + }, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + + it('should accept complete valid configuration', () => { + const validConfig = { + apps: { app1: {} }, + users: { user1: {} }, + auth: {}, + signin: { + title: 'Portal', + headline: 'Welcome back', + }, + layout: { + madeWith: false, + }, + } + + const result = PortalConfig.parse(validConfig) + + expect(result).toEqual(validConfig) + }) + }) + + describe('non-standard properties handling', () => { + // @note for now we allow this to happen + it.skip('should strip non-standard properties at root level (strict behavior)', () => { + const configWithExtra = { + apps: { app1: {} }, + extraProperty: 'this will be stripped', + } + + const result = PortalConfig.parse(configWithExtra) + + expect(result).toEqual({ + apps: { app1: {} }, + }) + expect(result.extraProperty).toBeUndefined() + }) + + // @note for now we allow this to happen + it.skip('should strip non-standard properties in nested objects', () => { + const configWithExtra = { + signin: { + title: 'Welcome', + headline: 'Sign in', + customProperty: 'will be stripped', + }, + layout: { + madeWith: true, + customLayoutProp: 'will be stripped', + }, + } + + const result = PortalConfig.parse(configWithExtra) + + expect(result).toEqual({ + signin: { + title: 'Welcome', + headline: 'Sign in', + }, + layout: { + madeWith: true, + }, + }) + expect(result.signin.customProperty).toBeUndefined() + expect(result.layout.customLayoutProp).toBeUndefined() + }) + + // @note for now we allow this to happen + it.skip('should strip all extra properties while preserving valid ones', () => { + const configWithMultipleExtras = { + apps: { app1: {} }, + users: { user1: {} }, + customProp1: 'stripped', + customProp2: 42, + customProp3: { nested: 'object' }, + signin: { + title: 'Valid title', + invalidProp: 'stripped', + }, + } + + const result = PortalConfig.parse(configWithMultipleExtras) + + expect(result).toEqual({ + apps: { app1: {} }, + users: { user1: {} }, + signin: { + title: 'Valid title', + }, + }) + + expect(result.customProp1).toBeUndefined() + expect(result.customProp2).toBeUndefined() + expect(result.customProp3).toBeUndefined() + expect(result.signin.invalidProp).toBeUndefined() + }) + + // @note for now we allow this to happen + it.skip('should demonstrate that PortalConfig does not accept non-standard properties', () => { + const input = { + invalidProperty: 'will be removed', + } + + const result = PortalConfig.parse(input) + + expect(Object.keys(result)).toEqual([]) + expect(result.invalidProperty).toBeUndefined() + }) + }) + + describe('type validation', () => { + it('should reject invalid types for signin properties', () => { + const invalidConfig = { + signin: { + title: 123, // should be string + headline: true, // should be string + }, + } + + expect(() => PortalConfig.parse(invalidConfig)).toThrow(ZodError) + }) + + it('should reject invalid type for layout.footer.madeWith', () => { + const invalidConfig = { + layout: { + footer: { + madeWith: 'yes', // should be boolean + }, + }, + } + + expect(() => PortalConfig.parse(invalidConfig)).toThrow(ZodError) + }) + + it('should reject invalid type for apps', () => { + const invalidConfig = { + apps: 'not an object', // should be record/object + } + + expect(() => PortalConfig.parse(invalidConfig)).toThrow(ZodError) + }) + + it('should reject invalid type for users', () => { + const invalidConfig = { + users: [], // should be record/object + } + + expect(() => PortalConfig.parse(invalidConfig)).toThrow(ZodError) + }) + }) +}) + +describe('SecretConfig', () => { + describe('valid configurations', () => { + it('should accept empty object', () => { + const validConfig = {} + + const result = SecretConfig.parse(validConfig) + + expect(result).toEqual({}) + }) + }) + + describe('non-standard properties handling', () => { + // @note for now we allow this to happen + it.skip('should strip any additional properties (strict empty object schema)', () => { + const configWithExtra = { + someProperty: 'will be stripped', + } + + const result = SecretConfig.parse(configWithExtra) + + expect(result).toEqual({}) + expect(result.someProperty).toBeUndefined() + }) + + // @note for now we allow this to happen + it.skip('should strip all additional properties regardless of type', () => { + const configWithMultipleProps = { + prop1: 'string value', + prop2: 123, + prop3: true, + prop4: { nested: 'object' }, + } + + const result = SecretConfig.parse(configWithMultipleProps) + + expect(result).toEqual({}) + expect(Object.keys(result)).toHaveLength(0) + expect(result.prop1).toBeUndefined() + expect(result.prop2).toBeUndefined() + expect(result.prop3).toBeUndefined() + expect(result.prop4).toBeUndefined() + }) + + // @note for now we allow this to happen + it.skip('should demonstrate that SecretConfig does not accept any non-standard properties', () => { + const input = { + secret: 'password', + apiKey: 'key123', + token: 'abc123', + } + + const result = SecretConfig.parse(input) + + expect(result).toEqual({}) + expect(Object.keys(result)).toHaveLength(0) + }) + }) +}) diff --git a/packages/db-spec/src/types.ts b/packages/db-spec/src/types.ts new file mode 100644 index 0000000..f343501 --- /dev/null +++ b/packages/db-spec/src/types.ts @@ -0,0 +1,271 @@ +import { z } from 'zod' + +/** + * SECRET CONFIG + */ +export const SecretConfig = z.union([ + // null + z.null(), + + // oauth + z + .object({ + clientId: z.string().optional(), + clientSecret: z.string().optional(), + authorizationUrl: z.string().url().optional(), + tokenUrl: z.string().url().optional(), + revokeUrl: z.string().url().optional(), + scope: z.string().optional(), + grantType: z.string().optional(), + }) + .passthrough(), + + // basic + z + .object({ + username: z.string().optional(), + password: z.string().optional(), + user: z.string().optional(), + pass: z.string().optional(), + }) + .passthrough(), +]) + +export type SecretConfigType = z.infer + +/** + * PORTAL CONFIG + */ +export const PortalConfig = z + .object({ + apps: z + .record( + z + .object({ + // @todo add app specific configs here + }) + .passthrough() + ) + .optional(), + + groups: z + .record( + z + .object({ + users: z.record( + z + .object({ + // @todo add user specific configs here + }) + .passthrough() + ), + + apps: z.record( + z + .object({ + // @todo add app specific configs here + }) + .passthrough() + ), + }) + .passthrough() + ) + .optional(), + + users: z + .record( + z + .object({ + // @todo add user specific configs here + }) + .passthrough() + ) + .optional(), + + auth: z + .object({ + // @todo add auth specific configs here + }) + .passthrough() + .optional(), + + signin: z + .object({ + title: z.string().optional(), + headline: z.string().optional(), + }) + .passthrough() + .optional(), + + layout: z + .object({ + header: z + .union([ + z.boolean(), + z + .object({ + // @todo add header specific configs here + }) + .passthrough(), + ]) + .optional(), + + footer: z + .union([ + z.boolean(), + z + .object({ + madeWith: z.boolean().optional(), + }) + .passthrough(), + ]) + .optional(), + + sidebar: z.union([ + z.boolean(), + z + .object({ + title: z.string().optional(), + logo: z.string().optional(), + icon: z.string().optional(), + link: z.string().url().optional(), + }) + .passthrough() + .optional(), + ]), + }) + .passthrough() + .optional(), + }) + .passthrough() + +export type PortalConfigType = z.infer + +/** + * TOKEN CONFIG + */ +export const TokenConfig = z + .object({ + allowedRoutes: z.array(z.string()).optional(), + + // @note when set, the token is bound to this contact: the session loader + // propagates it into the session payload so handlers such as + // conversation/complete attribute interactions to the contact as a hard + // override. @see @/lib/session.get and @/schemas/contactId + contactId: z.string().optional(), + }) + .passthrough() + +export type TokenConfigType = z.infer + +/** + * POLICY CONFIG + * + * Each policy type has its own config shape. The authoritative discriminator is + * the Policy row's `type` column (so policies can be selected by a real column + * rather than by querying inside the JSON), which is why `type` is intentionally + * not duplicated inside the config. The right shape is selected by `type` via + * `parsePolicyConfig` in `@/lib/policy.config`. + */ + +// retention: how long (in days) to keep a conversation before it expires. +export const RetentionPolicyConfig = z.object({ + expiresInDays: z.number().int().positive().optional(), +}) + +export type RetentionPolicyConfigType = z.infer + +const UsagePolicyEmailRecipient = z.string().email() + +const UsagePolicyEmailRecipients = z.array(UsagePolicyEmailRecipient).min(1) + +const UsagePolicyEmailAction = z.union([ + z.string().email(), + UsagePolicyEmailRecipients, + z.object({ + to: z + .union([UsagePolicyEmailRecipient, UsagePolicyEmailRecipients]) + .optional(), + }), +]) + +// usage: trip one or more actions when a bot's usage of `metric` exceeds +// `threshold` within a rolling `windowInSeconds` window. +export const UsagePolicyConfig = z.object({ + metric: z.enum(['tokens', 'messages', 'conversations']), + + threshold: z.number().int().positive(), + + windowInSeconds: z.number().int().positive(), + + actions: z + .object({ + // temporarily block the bot for `durationInSeconds`. + block: z + .object({ + durationInSeconds: z.number().int().positive(), + }) + .optional(), + + // send an email notification. A string or array sends to explicit + // recipients. An object without `to` notifies the policy owner. + email: UsagePolicyEmailAction.optional(), + }) + .refine((actions) => !!actions.block || !!actions.email, { + message: + 'a usage policy must define at least one action (block or email)', + }), +}) + +export type UsagePolicyConfigType = z.infer + +// @note a plain (non-discriminated) union: with `type` removed from the config +// this only describes the JSON column / provides a soft client-side hint. The +// authoritative validation is `parsePolicyConfig`, which selects by row `type`. +// UsagePolicyConfig must be tried first: zod unions return the first matching +// branch, and RetentionPolicyConfig (all-optional keys, strip mode) matches any +// object and would strip a usage config down to `{}`. +export const PolicyConfig = z.union([UsagePolicyConfig, RetentionPolicyConfig]) + +export type PolicyConfigType = z.infer + +/** + * USER LIMITS + */ +export const UserLimits = z + .object({ + tokens: z.number().int().min(0).optional(), + conversations: z.number().int().min(0).optional(), + messages: z.number().int().min(0).optional(), + database: z + .object({ + datasets: z.number().int().min(0).optional(), + records: z.number().int().min(0).optional(), + skillsets: z.number().int().min(0).optional(), + abilities: z.number().int().min(0).optional(), + files: z.number().int().min(0).optional(), + }) + .passthrough() + .optional(), + file: z + .object({ + maxFileSize: z.number().int().min(0).optional(), + }) + .passthrough() + .optional(), + attachment: z + .object({ + maxFileSize: z.number().int().min(0).optional(), + }) + .passthrough() + .optional(), + }) + .passthrough() + .nullable() + +export type UserLimitsType = z.infer + +/** + * CONVENIENCE EXPORT + */ +export default z diff --git a/packages/db-spec/tsconfig.json b/packages/db-spec/tsconfig.json new file mode 100644 index 0000000..b2a7c56 --- /dev/null +++ b/packages/db-spec/tsconfig.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node" + ], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js", + "./scripts/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/db/.gitignore b/packages/db/.gitignore new file mode 100644 index 0000000..731cdef --- /dev/null +++ b/packages/db/.gitignore @@ -0,0 +1,3 @@ +/prisma/generated +/prisma/zod +/prisma/.dev.db diff --git a/packages/db/README.md b/packages/db/README.md new file mode 100644 index 0000000..2c3a6cd --- /dev/null +++ b/packages/db/README.md @@ -0,0 +1,21 @@ +# @chatbotkit-dev/db + +The community database: SQLite in a file. The schema is derived from the +blueprint in `@chatbotkit-dev/db-spec`; the client is generated against a file +database this package creates itself, so generation needs nothing provisioned +and nothing reachable. + +## Environment + +| Variable | Purpose | +| -------- | ------- | +| `DATABASE_URL` | A `file:` url, e.g. `file:./data/cbk.db` | + +## Known differences from the MySQL module + +- `DATE(...)` expression columns in the analytics queries come back as + `'YYYY-MM-DD'` strings (typed loosely), where MySQL returns `Date` objects. + Plain columns are identical. Affects the date-bucketed usage/report series + only. +- Writes are as concurrent as SQLite is - fine for a single-process deployment, + which is what this module is for. diff --git a/packages/db/package.json b/packages/db/package.json new file mode 100644 index 0000000..0944e0e --- /dev/null +++ b/packages/db/package.json @@ -0,0 +1,68 @@ +{ + "name": "@chatbotkit-dev/db", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + }, + "./constraints": { + "import": "./src/constraints.ts", + "default": "./src/constraints.ts" + }, + "./client": { + "import": "./prisma/generated/prisma/client.ts", + "default": "./prisma/generated/prisma/client.ts" + }, + "./browser": { + "import": "./prisma/generated/prisma/browser.ts", + "default": "./prisma/generated/prisma/browser.ts" + }, + "./sql": { + "import": "./prisma/generated/prisma/sql.ts", + "default": "./prisma/generated/prisma/sql.ts" + }, + "./zod-models": { + "import": "./prisma/zod/schemas/models/index.ts", + "default": "./prisma/zod/schemas/models/index.ts" + }, + "./pothos": { + "import": "./prisma/generated/pothos.ts", + "default": "./prisma/generated/pothos.ts" + }, + "./package.json": "./package.json", + "./schema": "./prisma/schema.prisma" + }, + "main": "./src/index.ts", + "scripts": { + "build": "true", + "check": "if [ -d prisma/generated ]; then tsc6 --noEmit --incremental; else echo 'skipping check: prisma/generated missing (run db:gen via pnpm run with-env)'; fi", + "clean": "rimraf node_modules prisma/generated prisma/.dev.db *.tsbuildinfo", + "format": "true", + "lint": "eslint src scripts --ext .ts,.js", + "test": "true", + "db:gen": "node scripts/derive.js && node scripts/generate.js", + "derive": "node scripts/derive.js", + "db:push": "node scripts/derive.js && prisma db push" + }, + "access": "restricted", + "dependencies": { + "@prisma/adapter-better-sqlite3": "^7.3.0", + "@prisma/client": "^7.3.0", + "@pothos/plugin-prisma": "^4.10.0", + "prisma": "^7.3.0", + "prisma-json-types-generator": "^3.1.1", + "prisma-zod-generator": "^2.1.2", + "zod": "3.25.67" + }, + "devDependencies": { + "@chatbotkit-dev/db-spec": "workspace:*", + "@chatbotkit-dev/eslint-config": "workspace:*", + "eslint": "^9.0.0", + "rimraf": "^5.0.5", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/db/prisma.config.ts b/packages/db/prisma.config.ts new file mode 100644 index 0000000..d034018 --- /dev/null +++ b/packages/db/prisma.config.ts @@ -0,0 +1,19 @@ +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'prisma/config' + +const ROOT = path.dirname(fileURLToPath(import.meta.url)) + +// @note one variable, shared with the runtime. The generate step overrides it +// with a throwaway file so generation never touches a real database - TypedSQL +// needs a live database to typecheck the queries in prisma/sql against, and a +// file this package creates itself is the whole point of the default. +export default defineConfig({ + schema: 'prisma/schema.prisma', + + datasource: { + url: + process.env.PRISMA_DATABASE_URL || + `file:${path.join(ROOT, 'prisma', '.dev.db')}`, + }, +}) diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma new file mode 100644 index 0000000..7f1452a --- /dev/null +++ b/packages/db/prisma/schema.prisma @@ -0,0 +1,3500 @@ +// GENERATED - do not edit. +// +// The source is prisma/schema.prisma in @chatbotkit-dev/db-spec - that is the +// one hand-edited schema, and this file is derived from it by this package's +// `pnpm derive` (see @chatbotkit-dev/db-spec/derive for exactly what this +// engine changes). Edit the source, then run `pnpm -r derive`. + +generator client { + provider = "prisma-client" + output = "./generated/prisma" + previewFeatures = ["typedSql"] +} + +generator json { + provider = "prisma-json-types-generator" + namespace = "PrismaJson" + allowAny = false +} + +generator zod { + provider = "prisma-zod-generator" + output = "./zod/schemas" + config = "./zod-generator.config.json" +} + +generator pothos { + provider = "prisma-pothos-types" + output = "./generated/pothos.ts" + generateDatamodel = true + documentation = false + clientOutput = "./prisma" +} + +// --- +// --- +// --- + +datasource db { + provider = "sqlite" + relationMode = "prisma" +} + +// --- +// --- +// --- + +// @see https://next-auth.js.org/adapters/prisma + +model Account { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + type String + + provider String + providerAccountId String + + refresh_token String? /// @encrypted + access_token String? /// @encrypted + id_token String? /// @encrypted + token_type String? + + expires_at Int? + + scope String? + + session_state String? + + refresh_token_expires_in Int? + + meta Json? /// [Meta] + + // timestamps + // @note disabled because new + // createdAt DateTime @default(now()) + // updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([provider, providerAccountId]) + // indexes: other + @@index(fields: [userId]) +} + +model Session { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String? // @note deliberately optional + description String? // @note deliberately optional + + sessionToken String @unique + + expires DateTime + + audience String? + + payload Json? /// [JsonRecord] + + options Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + // @note disabled because new + // createdAt DateTime @default(now()) + // updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) +} + +model VerificationToken { + // fields + identifier String + + token String @unique + + expires DateTime + + meta Json? /// [Meta] + + // timestamps + // @note disabled because new + // createdAt DateTime @default(now()) + // updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([identifier, token]) +} + +// --- +// --- +// --- + +model User { + id String @id @default(cuid()) + + // relationships + parentId String? + parent User? @relation("ParentChild", fields: [parentId], references: [id], onDelete: Restrict, onUpdate: Restrict) // @note prevents deletion - must use deleteUser function + + // ref + alias String? // @note only used for child Users (parentId must be set) + + // fields + name String? // @note deliberately optional + description String? // @note deliberately optional + + email String @unique + emailVerified DateTime? + + parentContextName String? + + parentContextEmail String? + parentContextEmailVerified DateTime? + + image String? + + billingCustomerId String? @unique + billingSubscriptionId String? + billingSubscriptionStatus String? + billingSubscriptionStartedAt DateTime? + billingSubscriptionTrialedAt DateTime? + + channel String? + organization String? + industry String? + role String? + goal String? + + limits Json? /// [Limits] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: account + accounts Account[] + sessions Session[] + + // connections: children + children User[] @relation("ParentChild") + + // connections: teams + teams Team[] + + // connections: developer + tokens Token[] + webhooks Webhook[] + + // connections: objects + contacts Contact[] + conversations Conversation[] + tasks Task[] + ratings Rating[] + memories Memory[] + spaces Space[] + + // connections: executions + taskExecutions TaskExecution[] + + // connections: resources + blueprints Blueprint[] + bots Bot[] + datasets Dataset[] + skillsets Skillset[] + abilities Ability[] + secrets Secret[] + files File[] + portals Portal[] + policies Policy[] + + // connections: publishing + spaceSites SpaceSite[] + + // connections: special + contexts Context[] + + // connections: integrations + triggerIntegrations TriggerIntegration[] + widgetIntegrations WidgetIntegration[] + slackIntegrations SlackIntegration[] + discordIntegrations DiscordIntegration[] + microsoftteamsIntegrations MicrosoftteamsIntegration[] + googlechatIntegrations GooglechatIntegration[] + whatsappIntegrations WhatsappIntegration[] + messengerIntegrations MessengerIntegration[] + instagramIntegrations InstagramIntegration[] + telegramIntegrations TelegramIntegration[] + twilioIntegrations TwilioIntegration[] + anamIntegrations AnamIntegration[] + avatarIntegrations AvatarIntegration[] + recallIntegrations RecallIntegration[] + githubIntegrations GithubIntegration[] + emailIntegrations EmailIntegration[] + sitemapIntegrations SitemapIntegration[] + notionIntegrations NotionIntegration[] + supportIntegrations SupportIntegration[] + extractIntegrations ExtractIntegration[] + mcpserverIntegrations McpserverIntegration[] + skillserverIntegrations SkillserverIntegration[] + + // connections: oauth + oAuthConnections OAuthConnection[] + oAuthApplications OAuthApplication[] + oAuthApplicationTokens OAuthApplicationToken[] + + // connections: hub + hubBotPages HubBotPage[] + hubDatasetPages HubDatasetPage[] + hubSkillsetPages HubSkillsetPage[] + hubBlueprintPages HubBlueprintPage[] + hubWidgetPages HubWidgetPage[] + + // connections: secret values + secretValues SecretValue[] + + // connections: lock + locks Lock[] + + // connections: usage + usages Usage[] @relation("UserUsage") + childUsages Usage[] @relation("ParentUserUsage") + + // connections: observability + eventLogs EventLog[] + eventMetrics EventMetric[] + auditLogs AuditLog[] + + // indexes: unique + @@unique([parentId, alias]) // for alias lookup by parent + @@unique([parentId, parentContextEmail]) + // indexes: other + @@index(fields: [parentId]) + @@index(fields: [id, parentId]) +} + +// --- +// --- +// --- + +enum ResourceState { + // @note the order is important because the default value is the first one + + enabled + disabled +} + +// --- +// --- +// --- + +model Team { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: team + memberships TeamMembership[] + + // indexes: other + @@index(fields: [userId]) +} + +model TeamMembership { + id String @id @default(cuid()) + + // relationships + teamId String + team Team @relation(fields: [teamId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + email String + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([teamId, email]) + // indexes: other + @@index(fields: [email]) +} + +// --- +// --- +// --- + +model OAuthConnection { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + + // fields + name String @default("") + description String @default("") + + issuer String? + clientId String? + clientSecret String? /// @encrypted + + scopes String @default("openid email profile") + + allowedDomains String? + requiredClaims Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: integrations + mcpserverIntegrations McpserverIntegration[] + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) +} + +model OAuthApplication { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + clientId String @unique + clientSecret String @unique /// @digest + + redirectUris Json @default("[]") /// [OAuthRedirectUris] + + scopes Json @default("[]") /// [OAuthScopes] + + grants Json @default("[]") /// [OAuthGrants] + + accessTokenLifetime Int? + refreshTokenLifetime Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: oauth + oAuthApplicationTokens OAuthApplicationToken[] + + // indexes: other + @@index(fields: [userId]) +} + +model OAuthApplicationToken { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + applicationId String + application OAuthApplication @relation(fields: [applicationId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + accessToken String @unique /// @digest + accessTokenExpiresAt DateTime? + + refreshToken String? @unique /// @digest + refreshTokenExpiresAt DateTime? + + scopes Json @default("[]") /// [OAuthScopes] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [applicationId]) +} + +// --- +// --- +// --- + +enum Visibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +// --- +// --- +// --- + +model Lock { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + blueprints Blueprint[] + bots Bot[] + datasets Dataset[] + skillsets Skillset[] + files File[] + secrets Secret[] + policies Policy[] + portals Portal[] + + // connections: integrations + triggerIntegrations TriggerIntegration[] + widgetIntegrations WidgetIntegration[] + slackIntegrations SlackIntegration[] + discordIntegrations DiscordIntegration[] + microsoftteamsIntegrations MicrosoftteamsIntegration[] + googlechatIntegrations GooglechatIntegration[] + whatsappIntegrations WhatsappIntegration[] + messengerIntegrations MessengerIntegration[] + instagramIntegrations InstagramIntegration[] + telegramIntegrations TelegramIntegration[] + twilioIntegrations TwilioIntegration[] + githubIntegrations GithubIntegration[] + emailIntegrations EmailIntegration[] + sitemapIntegrations SitemapIntegration[] + notionIntegrations NotionIntegration[] + supportIntegrations SupportIntegration[] + extractIntegrations ExtractIntegration[] + mcpserverIntegrations McpserverIntegration[] + skillserverIntegrations SkillserverIntegration[] +} + +// --- +// --- +// --- + +model Context { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + payload Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) + @@index(fields: [datasetId]) + @@index(fields: [skillsetId]) + @@index(fields: [contactId]) +} + +// --- +// --- +// --- + +enum BlueprintVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Blueprint { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + visibility BlueprintVisibility @default(private) + + config Json? /// [JsonRecord] + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: hub + hubBlueprintPage HubBlueprintPage? + + // connections: resources + bots Bot[] /// @resource + datasets Dataset[] /// @resource + skillsets Skillset[] /// @resource + abilities Ability[] /// @resource + secrets Secret[] /// @resource + files File[] /// @resource + spaces Space[] /// @resource + portals Portal[] /// @resource + policies Policy[] /// @resource + tasks Task[] /// @resource + + // connections: special + contexts Context[] + + // connections: oauth + oAuthConnections OAuthConnection[] /// @resource + + // connections: integrations + triggerIntegrations TriggerIntegration[] /// @resource + widgetIntegrations WidgetIntegration[] /// @resource + slackIntegrations SlackIntegration[] /// @resource + discordIntegrations DiscordIntegration[] /// @resource + microsoftteamsIntegrations MicrosoftteamsIntegration[] /// @resource + googlechatIntegrations GooglechatIntegration[] /// @resource + whatsappIntegrations WhatsappIntegration[] /// @resource + messengerIntegrations MessengerIntegration[] /// @resource + instagramIntegrations InstagramIntegration[] /// @resource + telegramIntegrations TelegramIntegration[] /// @resource + twilioIntegrations TwilioIntegration[] /// @resource + avatarIntegrations AvatarIntegration[] /// @resource + anamIntegrations AnamIntegration[] /// @resource + recallIntegrations RecallIntegration[] /// @resource + githubIntegrations GithubIntegration[] /// @resource + emailIntegrations EmailIntegration[] /// @resource + supportIntegrations SupportIntegration[] /// @resource + extractIntegrations ExtractIntegration[] /// @resource + sitemapIntegrations SitemapIntegration[] /// @resource + notionIntegrations NotionIntegration[] /// @resource + mcpserverIntegrations McpserverIntegration[] /// @resource + skillserverIntegrations SkillserverIntegration[] /// @resource + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +enum BotVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Bot { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + backstory String @default("") + + model String @default("") + + privacy Boolean @default(false) + moderation Boolean @default(false) + + visibility BotVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: hub + hubBotPage HubBotPage? + + // connections: objects + conversations Conversation[] + tasks Task[] + ratings Rating[] + memories Memory[] + + // connections: resources + abilities Ability[] + policies Policy[] + + // connections: special + contexts Context[] + + // connections: integrations + triggerIntegrations TriggerIntegration[] + widgetIntegrations WidgetIntegration[] + slackIntegrations SlackIntegration[] + discordIntegrations DiscordIntegration[] + microsoftteamsIntegrations MicrosoftteamsIntegration[] + googlechatIntegrations GooglechatIntegration[] + whatsappIntegrations WhatsappIntegration[] + messengerIntegrations MessengerIntegration[] + instagramIntegrations InstagramIntegration[] + telegramIntegrations TelegramIntegration[] + twilioIntegrations TwilioIntegration[] + anamIntegrations AnamIntegration[] + avatarIntegrations AvatarIntegration[] + recallIntegrations RecallIntegration[] + githubIntegrations GithubIntegration[] + emailIntegrations EmailIntegration[] + supportIntegrations SupportIntegration[] + extractIntegrations ExtractIntegration[] + + // connections: connections + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [datasetId]) + @@index(fields: [skillsetId]) + @@index(fields: [id, datasetId]) +} + +// --- +// --- +// --- + +// @idea A/B Testing / Experiments +// +// This is a proposed design for A/B testing bots. The idea is that an +// Experiment acts as a "router" that distributes traffic across multiple +// bot variants based on configurable weights. +// +// How it works: +// 1. User creates multiple bots (each bot is effectively a "version") +// 2. User creates an Experiment with ExperimentVariants pointing to those bots +// 3. Integration (Widget, Slack, etc.) references the experimentId +// 4. Session creation selects a variant based on weights and routes to that bot +// 5. Conversation records experimentId + experimentVariantId for analytics +// +// Precedence logic for integrations: +// - If experimentId is set and experiment is active → use experiment routing +// - Otherwise fall back to botId +// +// Example: +// Experiment "New Prompt Test" +// ├─ Variant "Control" (50%) → Bot A +// └─ Variant "Friendly Tone" (50%) → Bot B +// +// Analytics can then compare ratings, engagement, etc. per variant. +// +// Schema: +// +// enum ExperimentStatus { +// draft // not yet running +// active // accepting traffic +// paused // temporarily stopped +// completed // finished, winner declared +// } +// +// model Experiment { +// id String @id @default(cuid()) +// +// // owner +// userId String +// user User @relation(fields: [userId], references: [id], onDelete: Cascade) +// +// // relationships +// blueprintId String? +// blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) +// +// // fields +// name String @default("") +// description String @default("") +// +// status ExperimentStatus @default(draft) +// +// startedAt DateTime? +// completedAt DateTime? +// +// // optional: reference to winning variant after experiment concludes +// winnerVariantId String? +// +// meta Json? /// [Meta] +// +// // timestamps +// createdAt DateTime @default(now()) +// updatedAt DateTime @updatedAt +// +// // connections +// variants ExperimentVariant[] +// conversations Conversation[] +// +// // indexes +// @@index(fields: [userId, status, createdAt(sort: Desc)]) +// @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) +// } +// +// model ExperimentVariant { +// id String @id @default(cuid()) +// +// // relationships +// experimentId String +// experiment Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade) +// botId String +// bot Bot @relation(fields: [botId], references: [id], onDelete: Cascade) +// +// // fields +// name String @default("") // e.g., "Control", "Variant A", "Friendly Tone" +// weight Int @default(1) // relative weight (not percentage) - traffic is distributed proportionally +// isControl Boolean @default(false) +// +// meta Json? /// [Meta] +// +// // timestamps +// createdAt DateTime @default(now()) +// updatedAt DateTime @updatedAt +// +// // connections +// conversations Conversation[] +// +// // indexes +// @@index(fields: [experimentId]) +// @@index(fields: [botId]) +// } +// +// Additional changes needed: +// +// 1. Add to integrations (WidgetIntegration, SlackIntegration, etc.): +// experimentId String? +// experiment Experiment? @relation(fields: [experimentId], references: [id], onDelete: SetNull) +// +// 2. Add to Conversation: +// experimentId String? +// experiment Experiment? @relation(fields: [experimentId], references: [id], onDelete: SetNull) +// experimentVariantId String? +// experimentVariant ExperimentVariant? @relation(fields: [experimentVariantId], references: [id], onDelete: SetNull) +// +// 3. Add to Bot: +// experimentVariants ExperimentVariant[] + +// --- +// --- +// --- + +enum DatasetVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Dataset { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + reranker String? + + recordMaxTokens Int? + + searchMinScore Float? + searchMaxRecords Int? + searchMaxTokens Int? + + separators String? + + matchInstruction String? // do not tempt to set to empty string + mismatchInstruction String? // do not tempt to set to empty string + + visibility DatasetVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: hub + hubDatasetPage HubDatasetPage? + + // connections: objects + conversations Conversation[] + + // connections: resources + bots Bot[] + files DatasetFileAttachment[] + + // connections: special + contexts Context[] + + // connections: integrations + sitemapIntegrations SitemapIntegration[] + notionIntegrations NotionIntegration[] + + // connections: many-to-many + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) +} + +// --- +// --- +// --- + +enum SkillsetVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Skillset { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + visibility SkillsetVisibility @default(private) + + // lifecycle state - toggle the whole skillset on/off without deleting it + state ResourceState @default(enabled) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: conversation + conversations Conversation[] + + // connections: resources + bots Bot[] + abilities Ability[] + + // connections: special + contexts Context[] + + // connections: integrations + mcpserverIntegrations McpserverIntegration[] + skillserverIntegrations SkillserverIntegration[] + + // connections: hub + hubSkillsetPage HubSkillsetPage? + + // connections: many-to-many + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) +} + +model Ability { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + // @note the resource an ability is linked to (what it acts on), as opposed + // to the owner/container relations above + linkedSecretId String? + linkedSecret Secret? @relation(fields: [linkedSecretId], references: [id], onDelete: SetNull) + linkedFileId String? + linkedFile File? @relation(fields: [linkedFileId], references: [id], onDelete: SetNull) + linkedBotId String? + linkedBot Bot? @relation(fields: [linkedBotId], references: [id], onDelete: SetNull) + linkedSpaceId String? + linkedSpace Space? @relation(fields: [linkedSpaceId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + instruction String @default("") + + // lifecycle state - toggle the ability on/off without deleting it + state ResourceState @default(enabled) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [blueprintId]) + @@index(fields: [skillsetId]) +} + +// --- +// --- +// --- + +enum FileVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model File { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + visibility FileVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + abilities Ability[] + datasets DatasetFileAttachment[] + widgetIntegrations WidgetIntegrationFileAttachment[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +enum SecretKind { + shared + personal +} + +enum SecretType { + plain + basic + bearer + jwt + oauth + template + reference +} + +enum SecretVisibility { + // @note the order is important because of the default value is the first one + + private + protected + public +} + +model Secret { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + kind SecretKind @default(shared) + + type SecretType @default(plain) + + value String? /// @encrypted + + config Json? /// [SecretConfig] @encrypted @see prisma/post-generate-zod.js + + visibility SecretVisibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + abilities Ability[] + + // connections: secret values + secretValues SecretValue[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) +} + +model SecretValue { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + secretId String? + secret Secret? @relation(fields: [secretId], references: [id], onDelete: Cascade) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + value String /// @encrypted + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, secretId, contactId]) + // indexes: other + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +model Portal { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + slug String @unique + + config Json? /// [PortalConfig] @see prisma/post-generate-zod.js + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +enum Trigger { + never + + automatic +} + +// --- + +enum Schedule { + never + + quarterhourly + halfhourly + hourly + + twicedaily + daily + + twiceweekly + weekly + + twicemonthly + monthly +} + +// --- + +enum SyncStatus { + pending + synced + error +} + +// --- +// --- +// --- + +model TriggerIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + secret String /// @encrypted + + authenticate Boolean @default(true) + + sessionDuration Float? + + schedule String? + timezone String? + + lastTriggerAt DateTime? + nextTriggerAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [schedule, lastTriggerAt]) + @@index(fields: [nextTriggerAt]) + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model WidgetIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + theme String? + + layout String? + + title String? + + intro String? + + initial String? + + placeholder String? + + origin String? + + sessionDuration Float? + + language String? + + plugins String? + + stream Boolean @default(true) + + verbose Boolean @default(true) + + tools Boolean @default(false) + + unfurl Boolean @default(true) + + math Boolean @default(false) + + carousel Boolean @default(false) + + form Boolean @default(false) + + attachments Boolean @default(false) + + autoScroll Boolean @default(true) + + startFirst Boolean @default(false) + + contactCollection Boolean @default(false) + + exportConversation Boolean @default(true) + restartConversation Boolean @default(true) + + maximize Boolean @default(true) + + messagePeek Boolean @default(true) + + voiceIn Boolean @default(false) + voiceOut Boolean @default(false) + + poweredBy Boolean @default(true) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: resources + files WidgetIntegrationFileAttachment[] + + // connections: hub + hubWidgetPage HubWidgetPage? + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model SlackIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + signingSecret String? /// @encrypted + + botToken String? /// @encrypted + userToken String? /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + references Boolean @default(false) + + ratings Boolean @default(false) + + visibleMessages Int? + + autoRespond String? + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model DiscordIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + appId String? + botToken String? /// @encrypted + publicKey String? + + handle String? + + ephemeral Boolean? + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model MicrosoftteamsIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + botFrameworkAppId String? + botFrameworkAppSecret String? /// @encrypted + tenantId String? + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model GooglechatIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + serviceAccountKey String? /// @encrypted + + projectNumber String? // @note Google Cloud project number used to verify the JWT audience on incoming events + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + autoRespond String? + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model WhatsappIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + verifyToken String /// @encrypted + + appSecret String? /// @encrypted + + phoneNumberId String? + + accessToken String? /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model MessengerIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + verifyToken String /// @encrypted + + accessToken String? /// @encrypted + + // @note the customer's own Meta APP secret - the key X-Hub-Signature-256 + // callbacks are signed with. Optional: without it callbacks are accepted + // unverified, with a logged bypass. + appSecret String? /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model InstagramIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + verifyToken String /// @encrypted + + accessToken String? /// @encrypted + + // @note the customer's own Meta APP secret - the key X-Hub-Signature-256 + // callbacks are signed with. Optional: without it callbacks are accepted + // unverified, with a logged bypass. + appSecret String? /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model TelegramIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + botToken String? /// @encrypted + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model TwilioIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + accountSid String? + authToken String? /// @encrypted + + voice String? + + contactCollection Boolean @default(false) + + sessionDuration Float? + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model AnamIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + apiKey String? /// @encrypted + + personaId String? + + visibility Visibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model AvatarIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + visibility Visibility @default(private) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model RecallIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + apiKey String? /// @encrypted + + // @note the signing secret of the Recall webhook endpoint (Svix, `whsec_...`). + // Optional: without it status callbacks are accepted unverified, with a logged + // bypass. + webhookSecret String? /// @encrypted + + region String? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model GithubIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + // this integration's GitHub App identity + credentials. Each integration is + // its own GitHub App. The installation id is NOT stored: it rides in every + // event payload and is combined with the App key to mint a token to reply. + appId String? // the GitHub App id (public-ish; signs the App JWT as `iss`) + privateKey String? /// @encrypted - the App's RSA private key (PEM) + webhookSecret String? /// @encrypted - validates x-hub-signature-256 HMAC-SHA256 + + contactCollection Boolean @default(false) + + sessionDuration Float? + + // @note unlike the other integrations this defaults to `@collaborators` and + // not `*`: an installed App hears from everyone who can comment, which on a + // public repository is every GitHub account. Rows predating this field were + // backfilled to `*` to preserve their behaviour, so the column default only + // governs new integrations. See lib/github.validation.ts + allowFrom String? @default("@collaborators") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model EmailIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + contactCollection Boolean @default(false) + + sessionDuration Float? + + attachments Boolean @default(false) + + allowFrom String? @default("*") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model SitemapIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + url String? + + glob String? + + selectors String? + + javascript Boolean? + + expiresIn Float? + + syncStatus SyncStatus @default(pending) + syncSchedule Schedule @default(never) + + lastSyncedAt DateTime @default(now()) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [syncSchedule, lastSyncedAt]) // used to find sitemap integrations to sync +} + +// --- +// --- +// --- + +model NotionIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + token String /// @encrypted + + expiresIn Float? + + syncStatus SyncStatus @default(pending) + syncSchedule Schedule @default(never) + + lastSyncedAt DateTime @default(now()) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [syncSchedule, lastSyncedAt]) // used to find notion integrations to sync +} + +// --- +// --- +// --- + +model SupportIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + email String? + + trigger Trigger? @default(automatic) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId]) + @@index(fields: [userId, botId]) +} + +// --- +// --- +// --- + +model ExtractIntegrationItem { + id String @id @default(cuid()) + + // relationships + extractIntegrationId String + extractIntegration ExtractIntegration @relation(fields: [extractIntegrationId], references: [id], onDelete: Cascade) + conversationId String + conversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade) + + // fields + data Json? /// [JsonRecord] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([extractIntegrationId, conversationId]) + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [extractIntegrationId, createdAt(sort: Desc)]) // used for listing +} + +model ExtractIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + schema Json? /// [JsonRecord] + + request String? @default("") + + model String? + + trigger Trigger? @default(automatic) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: items + items ExtractIntegrationItem[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId]) +} + +// --- +// --- +// --- + +model McpserverIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + oAuthConnectionId String? + oAuthConnection OAuthConnection? @relation(fields: [oAuthConnectionId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + accessToken String /// @encrypted + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model SkillserverIntegration { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + accessToken String /// @encrypted + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +enum PolicyType { + // @note the order is important because of the default value is the first one + + retention + usage +} + +model Policy { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + lockId String? + lock Lock? @relation(fields: [lockId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: Cascade) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + type PolicyType @default(retention) + + // lifecycle state - toggle the policy on/off without deleting it + state ResourceState @default(enabled) + + config Json? /// [PolicyConfig] @see prisma/post-generate-zod.js + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) + @@index(fields: [userId, type]) + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [botId, type]) +} + +// --- +// --- +// --- + +model HubBotPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + botId String @unique + bot Bot @relation(fields: [botId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubDatasetPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + datasetId String @unique + dataset Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubSkillsetPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + skillsetId String @unique + skillset Skillset @relation(fields: [skillsetId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubBlueprintPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String @unique + blueprint Blueprint @relation(fields: [blueprintId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + slug String? @unique + icon String? + rank Int? + + shareLog Boolean @default(false) + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model HubWidgetPage { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + widgetId String @unique + widget WidgetIntegration @relation(fields: [widgetId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + slug String? @unique + icon String? + rank Int? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +// @note UseType was migrated from an enum to a plain String to avoid requiring +// database migrations every time a new model is added. Valid use type values +// are now derived from the model config at runtime. See lib/usage.types.js. + +model Usage { + id String @id @default(cuid()) + + // owner + // @note usage records must survive user deletion with userId intact + userId String + user User @relation("UserUsage", fields: [userId], references: [id], onDelete: Restrict) + + // parent + // @note parent usage records must survive user deletion with parentUserId intact + parentUserId String? + parentUser User? @relation("ParentUserUsage", fields: [parentUserId], references: [id], onDelete: Restrict) + + // relationships: shallow + conversationId String? + messageId String? + taskId String? + contactId String? + blueprintId String? + botId String? + datasetId String? + skillsetId String? + abilityId String? + + // fields + type String + + count Int + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [type, createdAt(sort: Desc)]) // used for listing + @@index(fields: [userId, type, createdAt(sort: Desc)]) + @@index(fields: [userId, botId, createdAt(sort: Desc)]) // used by bot usage stats query + @@index(fields: [parentUserId, type, createdAt(sort: Desc)]) // used for parent account usage aggregation +} + +// --- +// --- +// --- + +model EventLog { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships: shallow + conversationId String? + taskId String? + contactId String? + spaceId String? + blueprintId String? + botId String? + datasetId String? + recordId String? + skillsetId String? + abilityId String? + fileId String? + secretId String? + portalId String? + policyId String? + widgetIntegrationId String? + slackIntegrationId String? + discordIntegrationId String? + microsoftteamsIntegrationId String? @map("teamsIntegrationId") + googlechatIntegrationId String? + whatsappIntegrationId String? + messengerIntegrationId String? + instagramIntegrationId String? + telegramIntegrationId String? + twilioIntegrationId String? + githubIntegrationId String? + emailIntegrationId String? + sitemapIntegrationId String? + notionIntegrationId String? + triggerIntegrationId String? + supportIntegrationId String? + extractIntegrationId String? + mcpserverIntegrationId String? + skillserverIntegrationId String? + // @note the event log tracks conversational integrations only; anam, + // avatar and recall are excluded by design (they emit no events). If that + // ever changes, add the three `*IntegrationId` columns here, the matching + // entries in the event log/metric list+export whitelists and the + // EventLogItem type, then regenerate the client. + webhookId String? + taskExecutionId String? + triggerExecutionId String? + + // fields + name String @default("") + description String @default("") + + type String + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, type, createdAt(sort: Desc)]) +} + +model EventMetric { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships: shallow + conversationId String? + taskId String? + contactId String? + spaceId String? + blueprintId String? + botId String? + datasetId String? + recordId String? + skillsetId String? + abilityId String? + fileId String? + secretId String? + portalId String? + policyId String? + widgetIntegrationId String? + slackIntegrationId String? + discordIntegrationId String? + microsoftteamsIntegrationId String? @map("teamsIntegrationId") + googlechatIntegrationId String? + whatsappIntegrationId String? + messengerIntegrationId String? + instagramIntegrationId String? + telegramIntegrationId String? + twilioIntegrationId String? + githubIntegrationId String? + emailIntegrationId String? + sitemapIntegrationId String? + notionIntegrationId String? + triggerIntegrationId String? + supportIntegrationId String? + extractIntegrationId String? + mcpserverIntegrationId String? + skillserverIntegrationId String? + // @note the event log tracks conversational integrations only; anam, + // avatar and recall are excluded by design (they emit no events). If that + // ever changes, add the three `*IntegrationId` columns here, the matching + // entries in the event log/metric list+export whitelists and the + // EventLogItem type, then regenerate the client. + webhookId String? + + // fields + name String @default("") + description String @default("") + + type String + + value Float? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, type, createdAt(sort: Desc)]) +} + +// --- + +model AuditLog { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + conversationId String? + taskId String? + contactId String? + spaceId String? + blueprintId String? + botId String? + datasetId String? + recordId String? + skillsetId String? + abilityId String? + fileId String? + secretId String? + portalId String? + policyId String? + webhookId String? + sessionId String? + + // fields + name String @default("") + description String @default("") + + action String + + oldValues Json? /// [JsonRecord] + newValues Json? /// [JsonRecord] + + ipAddress String? + userAgent String? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: delete + @@index(fields: [createdAt]) // used for purging old records + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, action, createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +model Contact { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + preferences String? + + fingerprint String @default(cuid()) + + email String? + phone String? + nick String? + + verifiedAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: objects + conversations Conversation[] + tasks Task[] + ratings Rating[] + memories Memory[] + spaces Space[] + contexts Context[] + + // connections: secret values + secretValues SecretValue[] + + // indexes: unique + @@unique([userId, fingerprint]) + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [userId, email]) +} + +// --- + +model Conversation { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: SetNull) + spaceId String? + space Space? @relation(fields: [spaceId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + taskId String? + task Task? @relation(fields: [taskId], references: [id], onDelete: SetNull) + datasetId String? + dataset Dataset? @relation(fields: [datasetId], references: [id], onDelete: SetNull) + + skillsetId String? + skillset Skillset? @relation(fields: [skillsetId], references: [id], onDelete: SetNull) + + // fields + name String @default("") + description String @default("") + + backstory String? @default("") + + model String? @default("") + + privacy Boolean? @default(false) + moderation Boolean? @default(false) + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: objects + messages Message[] + ratings Rating[] + + // connections: tasks + taskExecutions TaskExecution[] + + // connections: extractIntegrationItems + extractIntegrationItems ExtractIntegrationItem[] + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) // used for listing + @@index(fields: [userId, contactId, createdAt(sort: Desc)]) // used for listing + @@index(fields: [id, botId]) + @@index(fields: [id, datasetId]) + @@index(fields: [id, skillsetId]) + @@index(fields: [expiresAt]) // used for cleaning up expired conversations + @@index(fields: [createdAt]) // used for cleaning up empty conversations + @@index(fields: [datasetId]) + @@index(fields: [skillsetId]) + @@index(fields: [botId]) + @@index(fields: [contactId, createdAt(sort: Desc)]) // used by conversation fetch in actions + @@index(fields: [spaceId, createdAt(sort: Desc)]) // used by conversation fetch in actions + @@index(fields: [taskId, createdAt(sort: Desc)]) // used by task fetch in actions + @@index(fields: [contactId, taskId, createdAt(sort: Desc)]) // used by task fetch in actions +} + +// --- + +enum MessageType { + // stable types + + user // a message from the user + bot // a message from the bot + reasoning // a message that carries reasoning such as a thought process + context // a message that carries some context such as additional information + instruction // a message that carries some instruction such as a command + backstory // a message that describes the backstory of the bot + activity // a message that describes an activity + checkpoint // a message that describes a compact summary of the conversation + // notification // a message that describes a notification +} + +model Message { + id String @id @default(cuid()) + + // relationships + conversationId String + conversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + type MessageType + + text String + + nps Int? + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: objects + ratings Rating[] + + // connections: tasks + taskExecutionsAsStart TaskExecution[] @relation("TaskExecutionStart") + taskExecutionsAsEnd TaskExecution[] @relation("TaskExecutionEnd") + + // indexes: other + @@index(fields: [conversationId, type, createdAt(sort: Desc), id(sort: Desc)]) // supports reverse-chronological engine reads within one conversation and message type; id breaks timestamp ties + @@index(fields: [conversationId, type, createdAt(sort: Desc)]) // specifically used in /api/v1/conversation/[conversationId]/send to get messages fast + @@index(fields: [conversationId, createdAt(sort: Desc)]) // @note all messages are almost always pulled in reverse order +} + +// --- + +model Rating { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: SetNull) + conversationId String? + conversation Conversation? @relation(fields: [conversationId], references: [id], onDelete: SetNull) + messageId String? + message Message? @relation(fields: [messageId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + value Int + + reason String? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [botId]) + @@index(fields: [conversationId]) + @@index(fields: [messageId]) +} + +// --- + +enum TaskStatus { + idle // not currently running + running // actively executing + canceled // execution was canceled before completion +} + +enum TaskOutcome { + pending // never run yet + success // last run completed successfully + failure // last run failed or was incomplete +} + +model TaskExecution { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + taskId String + task Task @relation(fields: [taskId], references: [id], onDelete: Cascade) + conversationId String? + conversation Conversation? @relation(fields: [conversationId], references: [id], onDelete: SetNull) + startMessageId String? + startMessage Message? @relation("TaskExecutionStart", fields: [startMessageId], references: [id], onDelete: SetNull) + endMessageId String? + endMessage Message? @relation("TaskExecutionEnd", fields: [endMessageId], references: [id], onDelete: SetNull) + + // fields + name String @default("") + description String @default("") + + status TaskStatus @default(idle) + outcome TaskOutcome @default(pending) + + completedAt DateTime? + + resumeAt DateTime? + + keepAliveUntil DateTime? + + summary String? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [taskId, createdAt(sort: Desc)]) + // indexes: reaper (stalled sweep over running executions) + @@index(fields: [status, keepAliveUntil]) + // indexes: foreign keys + @@index(fields: [userId]) + @@index(fields: [taskId]) + @@index(fields: [conversationId]) + @@index(fields: [startMessageId]) + @@index(fields: [endMessageId]) +} + +// --- + +model Task { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: SetNull) + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + status TaskStatus @default(idle) + outcome TaskOutcome @default(pending) + + sessionDuration Float? + + maxIterations Int? + maxTime Float? + maxCalls Int? + + schedule String? + timezone String? + + nextRunAt DateTime? + lastRunAt DateTime? + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: tasks + taskExecutions TaskExecution[] + + // connections: objects + conversations Conversation[] + + // indexes: listing + @@index(fields: [userId, createdAt(sort: Desc), id(sort: Desc)]) // used for cursor pagination in list endpoint + // indexes: other + @@index(fields: [userId]) + @@index(fields: [blueprintId]) + @@index(fields: [schedule, lastRunAt]) + @@index(fields: [nextRunAt]) +} + +// --- + +model Memory { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: Cascade) + botId String? + bot Bot? @relation(fields: [botId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + text String + + expiresAt DateTime? + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [botId]) +} + +// --- + +model Space { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // relationships + contactId String? + contact Contact? @relation(fields: [contactId], references: [id], onDelete: SetNull) + + blueprintId String? + blueprint Blueprint? @relation(fields: [blueprintId], references: [id], onDelete: SetNull) + + // ref + alias String? + + // fields + name String @default("") + description String @default("") + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // connections: conversation + conversations Conversation[] + + // connections: resources + abilities Ability[] + + // connections: publishing + sites SpaceSite[] + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId, createdAt(sort: Desc)]) + @@index(fields: [blueprintId]) +} + +// --- + +model SpaceSite { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // attachment (many sites per space) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + + // ref + alias String? + + // basic information + name String @default("") + description String @default("") + + // host binding + slug String @unique + + // serving config + prefix String? + index String @default("index.html") + notFound String @default("404.html") + + // meta and others + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: unique + @@unique([userId, alias]) // for alias lookup by user + // indexes: other + @@index(fields: [userId]) + @@index(fields: [spaceId]) +} + +// --- +// --- +// --- + +// @note `Token` is the API token (auth credential); LLM token counting is +// `usage` and `limits` vocabulary. Both keep the word by decision - the two +// never share a type or a table, and prose distinguishes "API token" from +// "usage tokens". + +model Token { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + token String @unique /// @digest + + config Json? /// [TokenConfig] @see prisma/post-generate-zod.js + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [createdAt(sort: Desc)]) // used for listing + @@index(fields: [userId]) +} + +// --- +// --- +// --- + +model Webhook { + id String @id @default(cuid()) + + // owner + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + // fields + name String @default("") + description String @default("") + + request String @default("") + + events String @default("") + + secret String @unique /// @encrypted + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@index(fields: [userId]) + @@index(fields: [createdAt(sort: Desc)]) +} + +// --- +// --- +// --- + +enum DatasetFileAttachmentType { + source +} + +model DatasetFileAttachment { + // relationships + datasetId String + dataset Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade) + fileId String + file File @relation(fields: [fileId], references: [id], onDelete: Cascade) + + // fields + type DatasetFileAttachmentType + + meta Json? /// [Meta] + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@id([datasetId, fileId]) + @@index(fields: [fileId]) // because of @relation + @@index(fields: [datasetId, createdAt(sort: Desc)]) // used for listing + @@index(fields: [datasetId, type]) +} + +// --- + +enum WidgetIntegrationFileAttachmentType { + bar + user + bot + button +} + +model WidgetIntegrationFileAttachment { + // relationships + widgetIntegrationId String + widgetIntegration WidgetIntegration @relation(fields: [widgetIntegrationId], references: [id], onDelete: Cascade) + fileId String + file File @relation(fields: [fileId], references: [id], onDelete: Cascade) + + // fields + type WidgetIntegrationFileAttachmentType + + // timestamps + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // indexes: other + @@id([widgetIntegrationId, type]) + @@index(fields: [fileId]) // because of @relation +} diff --git a/packages/db/prisma/sql/breakdownTotalContactsWithConversationsOverPeriod.sql b/packages/db/prisma/sql/breakdownTotalContactsWithConversationsOverPeriod.sql new file mode 100644 index 0000000..cba8623 --- /dev/null +++ b/packages/db/prisma/sql/breakdownTotalContactsWithConversationsOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(c.createdAt) AS date, + COUNT(DISTINCT c.contactId) AS total +FROM Conversation c +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY DATE(c.createdAt) +ORDER BY date ASC diff --git a/packages/db/prisma/sql/breakdownTotalConversationsOverPeriod.sql b/packages/db/prisma/sql/breakdownTotalConversationsOverPeriod.sql new file mode 100644 index 0000000..16f2b42 --- /dev/null +++ b/packages/db/prisma/sql/breakdownTotalConversationsOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(c.createdAt) AS date, + COUNT(c.id) AS total +FROM Conversation c +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY DATE(c.createdAt) +ORDER BY date ASC diff --git a/packages/db/prisma/sql/breakdownTotalMessagesOfTypeOverPeriod.sql b/packages/db/prisma/sql/breakdownTotalMessagesOfTypeOverPeriod.sql new file mode 100644 index 0000000..45a966a --- /dev/null +++ b/packages/db/prisma/sql/breakdownTotalMessagesOfTypeOverPeriod.sql @@ -0,0 +1,16 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The message type ('user', 'bot', 'activity') +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT + DATE(m.createdAt) AS date, + COUNT(m.id) AS total +FROM Message m +JOIN Conversation c ON m.conversationId = c.id +WHERE c.userId = ? + AND m.type = ? + AND c.contactId IS NOT NULL + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY DATE(m.createdAt) +ORDER BY date ASC diff --git a/packages/db/prisma/sql/breakdownTotalMessagesOverPeriod.sql b/packages/db/prisma/sql/breakdownTotalMessagesOverPeriod.sql new file mode 100644 index 0000000..c67aae1 --- /dev/null +++ b/packages/db/prisma/sql/breakdownTotalMessagesOverPeriod.sql @@ -0,0 +1,14 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(m.createdAt) AS date, + COUNT(m.id) AS total +FROM Message m +JOIN Conversation c ON m.conversationId = c.id +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY DATE(m.createdAt) +ORDER BY date ASC diff --git a/packages/db/prisma/sql/breakdownTotalRatingsOverPeriod.sql b/packages/db/prisma/sql/breakdownTotalRatingsOverPeriod.sql new file mode 100644 index 0000000..b923cd8 --- /dev/null +++ b/packages/db/prisma/sql/breakdownTotalRatingsOverPeriod.sql @@ -0,0 +1,14 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT + DATE(createdAt) as date, + COUNT(*) as total, + COUNT(CASE WHEN value > 0 THEN 1 END) as thumbsUp, + COUNT(CASE WHEN value < 0 THEN 1 END) as thumbsDown +FROM Rating +WHERE userId = ? + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/breakdownTotalUsageTokensOverPeriod.sql b/packages/db/prisma/sql/breakdownTotalUsageTokensOverPeriod.sql new file mode 100644 index 0000000..c312f22 --- /dev/null +++ b/packages/db/prisma/sql/breakdownTotalUsageTokensOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + DATE(u.createdAt) AS date, + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +WHERE u.userId = ? + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ? +GROUP BY DATE(u.createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/getAverageMessagesOfTypeOverPeriod.sql b/packages/db/prisma/sql/getAverageMessagesOfTypeOverPeriod.sql new file mode 100644 index 0000000..18c984f --- /dev/null +++ b/packages/db/prisma/sql/getAverageMessagesOfTypeOverPeriod.sql @@ -0,0 +1,15 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:messageType The type of the message (e.g., 'bot') +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT COALESCE(AVG(msg_count), 0) AS average +FROM ( + SELECT m.conversationId, COUNT(*) AS msg_count + FROM Message m + JOIN Conversation c ON c.id = m.conversationId + WHERE c.userId = ? + AND m.type = ? + AND m.createdAt >= ? + AND m.createdAt <= ? + GROUP BY m.conversationId +) AS messages_per_conversation_in_period; diff --git a/packages/db/prisma/sql/getBotConversationCountByDay.sql b/packages/db/prisma/sql/getBotConversationCountByDay.sql new file mode 100644 index 0000000..0d876f3 --- /dev/null +++ b/packages/db/prisma/sql/getBotConversationCountByDay.sql @@ -0,0 +1,12 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:botId The ID of the bot +-- @param {DateTime} $3:fromDate Inclusive start of the period +-- @param {DateTime} $4:toDate Inclusive end of the period +SELECT DATE(createdAt) AS date, COUNT(*) AS total +FROM Conversation +WHERE userId = ? + AND botId = ? + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/getBotConversationsWithUserMessageCount.sql b/packages/db/prisma/sql/getBotConversationsWithUserMessageCount.sql new file mode 100644 index 0000000..bc9feb2 --- /dev/null +++ b/packages/db/prisma/sql/getBotConversationsWithUserMessageCount.sql @@ -0,0 +1,16 @@ +-- @param {String} $1:userMessageType The message type that marks a user turn +-- @param {String} $2:userId The ID of the user +-- @param {String} $3:botId The ID of the bot +-- @param {DateTime} $4:fromDate Inclusive start of the period +-- @param {DateTime} $5:toDate Inclusive end of the period +SELECT COUNT(*) AS total +FROM ( + SELECT c.id + FROM Conversation c + JOIN Message m ON m.conversationId = c.id AND m.type = ? + WHERE c.userId = ? + AND c.botId = ? + AND c.createdAt >= ? + AND c.createdAt <= ? + GROUP BY c.id +) AS conversationsWithUserMessages; diff --git a/packages/db/prisma/sql/getBotMessageCountByDay.sql b/packages/db/prisma/sql/getBotMessageCountByDay.sql new file mode 100644 index 0000000..4243a0e --- /dev/null +++ b/packages/db/prisma/sql/getBotMessageCountByDay.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:botId The ID of the bot +-- @param {DateTime} $3:fromDate Inclusive start of the period +-- @param {DateTime} $4:toDate Inclusive end of the period +SELECT DATE(m.createdAt) AS date, COUNT(*) AS total +FROM Message m +JOIN Conversation c ON c.id = m.conversationId +WHERE c.userId = ? + AND c.botId = ? + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY DATE(m.createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/getBotSingleTurnConversationCount.sql b/packages/db/prisma/sql/getBotSingleTurnConversationCount.sql new file mode 100644 index 0000000..74f9adf --- /dev/null +++ b/packages/db/prisma/sql/getBotSingleTurnConversationCount.sql @@ -0,0 +1,17 @@ +-- @param {String} $1:userMessageType The message type that marks a user turn +-- @param {String} $2:userId The ID of the user +-- @param {String} $3:botId The ID of the bot +-- @param {DateTime} $4:fromDate Inclusive start of the period +-- @param {DateTime} $5:toDate Inclusive end of the period +SELECT COUNT(*) AS total +FROM ( + SELECT c.id + FROM Conversation c + LEFT JOIN Message m ON m.conversationId = c.id AND m.type = ? + WHERE c.userId = ? + AND c.botId = ? + AND c.createdAt >= ? + AND c.createdAt <= ? + GROUP BY c.id + HAVING COUNT(m.id) = 1 +) AS singleTurnConvs; diff --git a/packages/db/prisma/sql/getBotUsageStats.sql b/packages/db/prisma/sql/getBotUsageStats.sql new file mode 100644 index 0000000..226b040 --- /dev/null +++ b/packages/db/prisma/sql/getBotUsageStats.sql @@ -0,0 +1,22 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:botId The ID of the bot +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_BASE_TOKEN' THEN count + ELSE 0 + END), 0) AS totalTokens, + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_CONVERSATION' THEN count + ELSE 0 + END), 0) AS totalConversations, + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_MESSAGE' THEN count + ELSE 0 + END), 0) AS totalMessages +FROM Usage +WHERE userId = ? + AND botId = ? + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db/prisma/sql/getConversationUsageStats.sql b/packages/db/prisma/sql/getConversationUsageStats.sql new file mode 100644 index 0000000..0afa09b --- /dev/null +++ b/packages/db/prisma/sql/getConversationUsageStats.sql @@ -0,0 +1,18 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:conversationId The ID of the conversation +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_BASE_TOKEN' THEN count + ELSE 0 + END), 0) AS totalTokens, + COALESCE(SUM(CASE + WHEN type = 'CHATBOTKIT_MESSAGE' THEN count + ELSE 0 + END), 0) AS totalMessages +FROM Usage +WHERE userId = ? + AND conversationId = ? + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db/prisma/sql/getDailyNegativeRatingCount.sql b/packages/db/prisma/sql/getDailyNegativeRatingCount.sql new file mode 100644 index 0000000..a374304 --- /dev/null +++ b/packages/db/prisma/sql/getDailyNegativeRatingCount.sql @@ -0,0 +1,11 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Inclusive start of the period +-- @param {DateTime} $3:toDate Inclusive end of the period +SELECT DATE(createdAt) AS date, COUNT(*) AS total +FROM Rating +WHERE userId = ? + AND value < 0 + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/getEventMetricSeriesOverPeriod.sql b/packages/db/prisma/sql/getEventMetricSeriesOverPeriod.sql new file mode 100644 index 0000000..a920e11 --- /dev/null +++ b/packages/db/prisma/sql/getEventMetricSeriesOverPeriod.sql @@ -0,0 +1,12 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The event metric type +-- @param {DateTime} $3:fromDate Inclusive start of the period (DateTime) +SELECT + DATE(createdAt) AS date, + COALESCE(SUM(value), 0) AS total +FROM EventMetric +WHERE userId = ? + AND type = ? + AND createdAt >= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/getTotalAbilitiesForUser.sql b/packages/db/prisma/sql/getTotalAbilitiesForUser.sql new file mode 100644 index 0000000..a478fd6 --- /dev/null +++ b/packages/db/prisma/sql/getTotalAbilitiesForUser.sql @@ -0,0 +1,7 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Ability a +JOIN Skillset s ON a.skillsetId = s.id +JOIN User u ON s.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalBotsForUser.sql b/packages/db/prisma/sql/getTotalBotsForUser.sql new file mode 100644 index 0000000..2cf95ec --- /dev/null +++ b/packages/db/prisma/sql/getTotalBotsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Bot b +JOIN User u ON b.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalContacts.sql b/packages/db/prisma/sql/getTotalContacts.sql new file mode 100644 index 0000000..3f89622 --- /dev/null +++ b/packages/db/prisma/sql/getTotalContacts.sql @@ -0,0 +1,4 @@ +-- @param {String} $1:userId The ID of the user +SELECT count(*) as total +FROM Contact +WHERE userId = ?; diff --git a/packages/db/prisma/sql/getTotalContactsWithConversationsOverPeriod.sql b/packages/db/prisma/sql/getTotalContactsWithConversationsOverPeriod.sql new file mode 100644 index 0000000..ca20908 --- /dev/null +++ b/packages/db/prisma/sql/getTotalContactsWithConversationsOverPeriod.sql @@ -0,0 +1,13 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM ( + SELECT c.contactId + FROM Conversation c + WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? + GROUP BY c.contactId +) AS distinct_contacts_with_conversations_in_period; diff --git a/packages/db/prisma/sql/getTotalConversationsOverPeriod.sql b/packages/db/prisma/sql/getTotalConversationsOverPeriod.sql new file mode 100644 index 0000000..91e1662 --- /dev/null +++ b/packages/db/prisma/sql/getTotalConversationsOverPeriod.sql @@ -0,0 +1,8 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM Conversation c +WHERE c.userId = ? + AND c.createdAt >= ? + AND c.createdAt <= ? diff --git a/packages/db/prisma/sql/getTotalConversationsWithNumberedMessagesOverPeriod.sql b/packages/db/prisma/sql/getTotalConversationsWithNumberedMessagesOverPeriod.sql new file mode 100644 index 0000000..a1c74ff --- /dev/null +++ b/packages/db/prisma/sql/getTotalConversationsWithNumberedMessagesOverPeriod.sql @@ -0,0 +1,15 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:minMessages Minimum number of messages per conversation in the period +SELECT COUNT(*) AS total +FROM ( + SELECT m.conversationId + FROM Message m + JOIN Conversation c ON c.id = m.conversationId + WHERE c.userId = ? + AND m.createdAt >= ? + AND m.createdAt <= ? + GROUP BY m.conversationId + HAVING COUNT(*) >= ? +) AS conversations_with_min_messages_in_period; diff --git a/packages/db/prisma/sql/getTotalDatasetsForUser.sql b/packages/db/prisma/sql/getTotalDatasetsForUser.sql new file mode 100644 index 0000000..4b37715 --- /dev/null +++ b/packages/db/prisma/sql/getTotalDatasetsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Dataset d +JOIN User u ON d.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalFilesForUser.sql b/packages/db/prisma/sql/getTotalFilesForUser.sql new file mode 100644 index 0000000..7f90e5c --- /dev/null +++ b/packages/db/prisma/sql/getTotalFilesForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM File f +JOIN User u ON f.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalMessagesOfTypeOverPeriod.sql b/packages/db/prisma/sql/getTotalMessagesOfTypeOverPeriod.sql new file mode 100644 index 0000000..62da574 --- /dev/null +++ b/packages/db/prisma/sql/getTotalMessagesOfTypeOverPeriod.sql @@ -0,0 +1,11 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:messageType The type of the message +-- @param {DateTime} $3:fromDate Start of the period (DateTime) +-- @param {DateTime} $4:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM Message m +JOIN Conversation c ON c.id = m.conversationId +WHERE c.userId = ? + AND m.type = ? + AND m.createdAt >= ? + AND m.createdAt <= ?; diff --git a/packages/db/prisma/sql/getTotalMessagesOverPeriod.sql b/packages/db/prisma/sql/getTotalMessagesOverPeriod.sql new file mode 100644 index 0000000..b67c687 --- /dev/null +++ b/packages/db/prisma/sql/getTotalMessagesOverPeriod.sql @@ -0,0 +1,9 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT COUNT(*) AS total +FROM Message m +JOIN Conversation c ON c.id = m.conversationId +WHERE c.userId = ? + AND m.createdAt >= ? + AND m.createdAt <= ?; diff --git a/packages/db/prisma/sql/getTotalPoliciesForUser.sql b/packages/db/prisma/sql/getTotalPoliciesForUser.sql new file mode 100644 index 0000000..4bcf107 --- /dev/null +++ b/packages/db/prisma/sql/getTotalPoliciesForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Policy p +JOIN User u ON p.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalPortalsForUser.sql b/packages/db/prisma/sql/getTotalPortalsForUser.sql new file mode 100644 index 0000000..63b09e3 --- /dev/null +++ b/packages/db/prisma/sql/getTotalPortalsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Portal p +JOIN User u ON p.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalRatingsOverPeriod.sql b/packages/db/prisma/sql/getTotalRatingsOverPeriod.sql new file mode 100644 index 0000000..159ba8d --- /dev/null +++ b/packages/db/prisma/sql/getTotalRatingsOverPeriod.sql @@ -0,0 +1,8 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT count(*) as total +FROM Rating +WHERE userId = ? + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db/prisma/sql/getTotalSkillsetsForUser.sql b/packages/db/prisma/sql/getTotalSkillsetsForUser.sql new file mode 100644 index 0000000..cf28f88 --- /dev/null +++ b/packages/db/prisma/sql/getTotalSkillsetsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Skillset s +JOIN User u ON s.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalTeamMembersForUser.sql b/packages/db/prisma/sql/getTotalTeamMembersForUser.sql new file mode 100644 index 0000000..8295fc1 --- /dev/null +++ b/packages/db/prisma/sql/getTotalTeamMembersForUser.sql @@ -0,0 +1,7 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM TeamMembership tm +JOIN Team t ON tm.teamId = t.id +JOIN User u ON t.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalTeamsForUser.sql b/packages/db/prisma/sql/getTotalTeamsForUser.sql new file mode 100644 index 0000000..564fd21 --- /dev/null +++ b/packages/db/prisma/sql/getTotalTeamsForUser.sql @@ -0,0 +1,6 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:effectiveUserId The ID of the user +SELECT COUNT(*) as count +FROM Team t +JOIN User u ON t.userId = u.id +WHERE u.parentId = ? OR u.id = ?; diff --git a/packages/db/prisma/sql/getTotalThumbsDownOverPeriod.sql b/packages/db/prisma/sql/getTotalThumbsDownOverPeriod.sql new file mode 100644 index 0000000..3bae679 --- /dev/null +++ b/packages/db/prisma/sql/getTotalThumbsDownOverPeriod.sql @@ -0,0 +1,9 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT count(*) as total +FROM Rating +WHERE userId = ? + AND value <= 0 + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db/prisma/sql/getTotalThumbsUpOverPeriod.sql b/packages/db/prisma/sql/getTotalThumbsUpOverPeriod.sql new file mode 100644 index 0000000..75769a7 --- /dev/null +++ b/packages/db/prisma/sql/getTotalThumbsUpOverPeriod.sql @@ -0,0 +1,9 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +SELECT count(*) as total +FROM Rating +WHERE userId = ? + AND value > 0 + AND createdAt >= ? + AND createdAt <= ?; diff --git a/packages/db/prisma/sql/getTotalUsageTokensOverPeriod.sql b/packages/db/prisma/sql/getTotalUsageTokensOverPeriod.sql new file mode 100644 index 0000000..4e6b89c --- /dev/null +++ b/packages/db/prisma/sql/getTotalUsageTokensOverPeriod.sql @@ -0,0 +1,10 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +SELECT + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +WHERE u.userId = ? + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ?; diff --git a/packages/db/prisma/sql/getTotalUsersForUser.sql b/packages/db/prisma/sql/getTotalUsersForUser.sql new file mode 100644 index 0000000..5b87daf --- /dev/null +++ b/packages/db/prisma/sql/getTotalUsersForUser.sql @@ -0,0 +1,4 @@ +-- @param {String} $1:userId The ID of the user +SELECT COUNT(*) as count +FROM User u +WHERE u.parentId = ?; diff --git a/packages/db/prisma/sql/getUsageCountByTypeOverPeriod.sql b/packages/db/prisma/sql/getUsageCountByTypeOverPeriod.sql new file mode 100644 index 0000000..6472805 --- /dev/null +++ b/packages/db/prisma/sql/getUsageCountByTypeOverPeriod.sql @@ -0,0 +1,14 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The usage type +-- @param {DateTime} $3:fromDate Inclusive start of the period +-- @param {DateTime} $4:toDate Inclusive end of the period +SELECT + DATE(createdAt) AS date, + COALESCE(SUM(count), 0) AS total +FROM Usage +WHERE userId = ? + AND type = ? + AND createdAt >= ? + AND createdAt <= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/getUsageCountByTypeSince.sql b/packages/db/prisma/sql/getUsageCountByTypeSince.sql new file mode 100644 index 0000000..1bdf8b3 --- /dev/null +++ b/packages/db/prisma/sql/getUsageCountByTypeSince.sql @@ -0,0 +1,12 @@ +-- @param {String} $1:userId The ID of the user +-- @param {String} $2:type The usage type +-- @param {DateTime} $3:since Inclusive start of the period +SELECT + DATE(createdAt) AS date, + COALESCE(SUM(count), 0) AS total +FROM Usage +WHERE userId = ? + AND type = ? + AND createdAt >= ? +GROUP BY DATE(createdAt) +ORDER BY date ASC; diff --git a/packages/db/prisma/sql/listContacts.sql b/packages/db/prisma/sql/listContacts.sql new file mode 100644 index 0000000..2836206 --- /dev/null +++ b/packages/db/prisma/sql/listContacts.sql @@ -0,0 +1,7 @@ +-- @param {String} $1:userId The ID of the user +-- @param {Int} $2:limit The maximum number of contacts to return (optional) +SELECT id, name, description, email, nick, meta, createdAt +FROM Contact +WHERE userId = ? +ORDER BY createdAt DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listContactsWithConversationsOverPeriod.sql b/packages/db/prisma/sql/listContactsWithConversationsOverPeriod.sql new file mode 100644 index 0000000..6315e7b --- /dev/null +++ b/packages/db/prisma/sql/listContactsWithConversationsOverPeriod.sql @@ -0,0 +1,23 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of contacts to return (optional) +SELECT + co.id, + co.name, + co.description, + co.email, + co.nick, + co.meta, + co.createdAt, + COUNT(c.id) AS _countValue, + 'conversation' AS _countType +FROM Conversation c +JOIN Contact co ON c.contactId = co.id +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY co.id, co.name, co.description, co.email, co.nick +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listContactsWithMessagesOverPeriod.sql b/packages/db/prisma/sql/listContactsWithMessagesOverPeriod.sql new file mode 100644 index 0000000..fd3e718 --- /dev/null +++ b/packages/db/prisma/sql/listContactsWithMessagesOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of contacts to return (optional) +SELECT + co.id, + co.name, + co.description, + co.email, + co.nick, + co.meta, + co.createdAt, + COUNT(m.id) AS _countValue, + 'message' AS _countType +FROM Message m +JOIN Conversation c ON m.conversationId = c.id +JOIN Contact co ON c.contactId = co.id +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND m.createdAt >= ? + AND m.createdAt <= ? +GROUP BY co.id, co.name, co.description, co.email, co.nick +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listContactsWithRatingsOverPeriod.sql b/packages/db/prisma/sql/listContactsWithRatingsOverPeriod.sql new file mode 100644 index 0000000..1347c6a --- /dev/null +++ b/packages/db/prisma/sql/listContactsWithRatingsOverPeriod.sql @@ -0,0 +1,26 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +-- @param {Int} $4:limit The maximum number of contacts to return +SELECT + c.id, + c.name, + c.description, + c.email, + c.nick, + c.meta, + c.createdAt, + COUNT(CASE WHEN r.value > 0 THEN 1 END) as _upvoteCount, + COUNT(CASE WHEN r.value < 0 THEN 1 END) as _downvoteCount, + COUNT(r.id) as _countValue, + 'rating' as _countType +FROM Contact c +LEFT JOIN Rating r ON c.id = r.contactId + AND r.userId = ? + AND r.createdAt >= ? + AND r.createdAt <= ? +WHERE c.userId = r.userId +GROUP BY c.id, c.name, c.description, c.email, c.nick, c.meta, c.createdAt +HAVING _countValue > 0 +ORDER BY _countValue DESC, _upvoteCount DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listConversationsOverPeriod.sql b/packages/db/prisma/sql/listConversationsOverPeriod.sql new file mode 100644 index 0000000..c85e953 --- /dev/null +++ b/packages/db/prisma/sql/listConversationsOverPeriod.sql @@ -0,0 +1,25 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of conversations to return (optional) +SELECT + c.id, + co.id AS contactId, + co.name, + co.email, + co.nick, + co.description, + co.meta, + c.createdAt, + COUNT(m.id) AS _countValue, + 'message' AS _countType +FROM Conversation c +JOIN Contact co ON c.contactId = co.id +LEFT JOIN Message m ON c.id = m.conversationId +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY c.id, c.createdAt, co.id, co.name, co.email, co.nick, co.description +ORDER BY _countValue DESC, c.createdAt DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listConversationsWithNumberedMessagesOverPeriod.sql b/packages/db/prisma/sql/listConversationsWithNumberedMessagesOverPeriod.sql new file mode 100644 index 0000000..8949a8e --- /dev/null +++ b/packages/db/prisma/sql/listConversationsWithNumberedMessagesOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:messageCount Minimum number of messages for follow-ups +-- @param {Int} $5:limit The maximum number of conversations to return (optional) +SELECT + c.id, + c.createdAt, + co.name, + co.email, + co.nick, + co.description, + COUNT(m.id) AS messageCount +FROM Conversation c +JOIN Contact co ON c.contactId = co.id +LEFT JOIN Message m ON c.id = m.conversationId +WHERE c.userId = ? + AND c.contactId IS NOT NULL + AND c.createdAt >= ? + AND c.createdAt <= ? +GROUP BY c.id, c.createdAt, co.name, co.email, co.nick, co.description +HAVING COUNT(m.id) >= ? +ORDER BY messageCount DESC, c.createdAt DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listEventLogsOfTypeActionsGroupedByTypeOverPeriod.sql b/packages/db/prisma/sql/listEventLogsOfTypeActionsGroupedByTypeOverPeriod.sql new file mode 100644 index 0000000..b46093b --- /dev/null +++ b/packages/db/prisma/sql/listEventLogsOfTypeActionsGroupedByTypeOverPeriod.sql @@ -0,0 +1,18 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of action types to return (optional) +SELECT + el.type, + el.name, + el.description, + COUNT(el.id) AS _countValue, + 'action' AS _countType +FROM EventLog el +WHERE el.userId = ? + AND el.type LIKE 'action.%' + AND el.createdAt >= ? + AND el.createdAt <= ? +GROUP BY el.type, el.name, el.description +ORDER BY _countValue DESC, el.type ASC +LIMIT ?; diff --git a/packages/db/prisma/sql/listTopBotsByTokenUsageOverPeriod.sql b/packages/db/prisma/sql/listTopBotsByTokenUsageOverPeriod.sql new file mode 100644 index 0000000..3e2cfe8 --- /dev/null +++ b/packages/db/prisma/sql/listTopBotsByTokenUsageOverPeriod.sql @@ -0,0 +1,19 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of results to return +SELECT + u.botId AS id, + b.name AS name, + b.description AS description, + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +LEFT JOIN Bot b ON u.botId = b.id +WHERE u.userId = ? + AND u.botId IS NOT NULL + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ? +GROUP BY u.botId, b.name, b.description +ORDER BY total DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listTopContactsByTokenUsageOverPeriod.sql b/packages/db/prisma/sql/listTopContactsByTokenUsageOverPeriod.sql new file mode 100644 index 0000000..4852c9c --- /dev/null +++ b/packages/db/prisma/sql/listTopContactsByTokenUsageOverPeriod.sql @@ -0,0 +1,19 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:fromDate Start of the period (DateTime) +-- @param {DateTime} $3:toDate End of the period (DateTime) +-- @param {Int} $4:limit The maximum number of results to return +SELECT + u.contactId AS id, + c.name AS name, + c.description AS description, + COALESCE(SUM(u.count), 0) AS total +FROM Usage u +LEFT JOIN Contact c ON u.contactId = c.id +WHERE u.userId = ? + AND u.contactId IS NOT NULL + AND u.type LIKE '%_TOKEN' + AND u.createdAt >= ? + AND u.createdAt <= ? +GROUP BY u.contactId, c.name, c.description +ORDER BY total DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listTopDownvotersOverPeriod.sql b/packages/db/prisma/sql/listTopDownvotersOverPeriod.sql new file mode 100644 index 0000000..f37a07c --- /dev/null +++ b/packages/db/prisma/sql/listTopDownvotersOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +-- @param {Int} $4:limit The maximum number of contacts to return +SELECT + c.id, + c.name, + c.description, + c.email, + c.nick, + c.meta, + c.createdAt, + COUNT(CASE WHEN r.value < 0 THEN 1 END) as _countValue, + 'downvote' as _countType +FROM Contact c +LEFT JOIN Rating r ON c.id = r.contactId + AND r.userId = ? + AND r.createdAt >= ? + AND r.createdAt <= ? +WHERE c.userId = r.userId +GROUP BY c.id, c.name, c.description, c.email, c.nick, c.meta, c.createdAt +HAVING _countValue > 0 +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db/prisma/sql/listTopUpvotersOverPeriod.sql b/packages/db/prisma/sql/listTopUpvotersOverPeriod.sql new file mode 100644 index 0000000..4c8ac91 --- /dev/null +++ b/packages/db/prisma/sql/listTopUpvotersOverPeriod.sql @@ -0,0 +1,24 @@ +-- @param {String} $1:userId The ID of the user +-- @param {DateTime} $2:startDate The start date for the period +-- @param {DateTime} $3:endDate The end date for the period +-- @param {Int} $4:limit The maximum number of contacts to return +SELECT + c.id, + c.name, + c.description, + c.email, + c.nick, + c.meta, + c.createdAt, + COUNT(CASE WHEN r.value > 0 THEN 1 END) as _countValue, + 'upvote' as _countType +FROM Contact c +LEFT JOIN Rating r ON c.id = r.contactId + AND r.userId = ? + AND r.createdAt >= ? + AND r.createdAt <= ? +WHERE c.userId = r.userId +GROUP BY c.id, c.name, c.description, c.email, c.nick, c.meta, c.createdAt +HAVING _countValue > 0 +ORDER BY _countValue DESC +LIMIT ?; diff --git a/packages/db/prisma/zod-generator.config.json b/packages/db/prisma/zod-generator.config.json new file mode 100644 index 0000000..eec0afa --- /dev/null +++ b/packages/db/prisma/zod-generator.config.json @@ -0,0 +1,24 @@ +{ + "mode": "custom", + "pureModels": true, + "pureModelsLean": true, + "pureModelsIncludeRelations": false, + "dateTimeStrategy": "date", + "emit": { + "enums": true, + "objects": false, + "crud": false, + "results": false, + "pureModels": true, + "variants": false + }, + "naming": { + "pureModel": { + "filePattern": "{model}.ts", + "schemaSuffix": "Model", + "typeSuffix": "", + "exportNamePattern": "{Model}Model", + "legacyAliases": false + } + } +} diff --git a/packages/db/scripts/derive.js b/packages/db/scripts/derive.js new file mode 100644 index 0000000..3e16f47 --- /dev/null +++ b/packages/db/scripts/derive.js @@ -0,0 +1,16 @@ +// @note pulls this implementation's schema from the spec. The spec does not +// know who implements it - the dependency arrow points this way on purpose. +import { derive, renderSqlite } from '@chatbotkit-dev/db-spec/derive' + +import path from 'node:path' +import url from 'node:url' + +await derive({ + prismaDir: path.join( + path.dirname(url.fileURLToPath(import.meta.url)), + '..', + 'prisma' + ), + + render: renderSqlite, +}) diff --git a/packages/db/scripts/generate.js b/packages/db/scripts/generate.js new file mode 100644 index 0000000..2591d8e --- /dev/null +++ b/packages/db/scripts/generate.js @@ -0,0 +1,51 @@ +/** + * @file generate.js + * + * Generates this package's Prisma client, fully self-contained. + * + * TypedSQL typechecks the queries in prisma/sql against a live database, so a + * throwaway file database is pushed first - which is the whole point of the + * SQLite default: nothing to provision, nothing to reach. + */ +import { execSync } from 'node:child_process' +import fs from 'node:fs/promises' +import { createRequire } from 'node:module' +import path from 'node:path' +import url from 'node:url' + +const ROOT = path.join(path.dirname(url.fileURLToPath(import.meta.url)), '..') + +const run = (command) => + execSync(command, { + cwd: ROOT, + stdio: 'inherit', + + // @note the generators (json, zod, pothos) are resolved off PATH by the + // prisma CLI, so the package's own bin directory has to be on it + env: { + ...process.env, + + // @note generation always runs against its own throwaway database, never + // whatever PRISMA_DATABASE_URL happens to point at + PRISMA_DATABASE_URL: `file:${path.join(ROOT, 'prisma', '.dev.db')}`, + + PATH: `${path.join(ROOT, 'node_modules', '.bin')}:${process.env.PATH}`, + }, + }) + +const databasePath = path.join(ROOT, 'prisma', '.dev.db') + +await fs.rm(databasePath, { force: true }) +await fs.writeFile(databasePath, '') + +run('prisma db push --accept-data-loss') +run('prisma generate --sql') + +// @note the zod generator needs the shared post-processing pass - it fixes the +// generated files (a shadowed `Record` type) and recreates the compatibility +// re-exports. Shared with every db module, so it lives in the spec. +run( + `node ${createRequire(import.meta.url).resolve( + '@chatbotkit-dev/db-spec/scripts/post-generate-zod.js' + )} ${path.join(ROOT, 'prisma')}` +) diff --git a/packages/db/src/constraints.ts b/packages/db/src/constraints.ts new file mode 100644 index 0000000..522a676 --- /dev/null +++ b/packages/db/src/constraints.ts @@ -0,0 +1,48 @@ +/** + * @file constraints.ts + * + * The column size limits this engine imposes, in bytes. + * + * @note SQLite's own, not MySQL's. Deriving strips the native types, so + * `@db.VarChar(191)`, `@db.Text` and `@db.MediumText` are all a plain TEXT + * column here, and SQLite does not enforce a declared VARCHAR length at all. + * What is left is one engine-wide ceiling, `SQLITE_MAX_LENGTH`, which the three + * constants below therefore share. They are deliberately not the numbers the + * MySQL implementation exports - that is the point of each implementation + * declaring its own. + * + * @note bytes, not characters. Callers measure encoded length - a multi-byte + * character costs more than one. + * + * @note the ceiling is a build-time option, not a format limit. 1e9 is the + * default every stock build ships with, including the one better-sqlite3 + * bundles; a build that lowered it would need this file lowered to match. + * + * @note a consequence worth knowing: this database accepts values the MySQL + * implementation would reject, so a row written here need not fit there. Data + * that has to survive the move must be held to the stricter limits by whatever + * moves it - these constants describe the engine, and cannot describe both. + */ + +/** + * `SQLITE_MAX_LENGTH` - the max size of any TEXT or BLOB value. + */ +const SQLITE_MAX_LENGTH = 1000000000 + +/** + * The max length of what the blueprint declares `@db.VarChar(191)`, stored here + * as an unconstrained TEXT column. + */ +export const MAX_DB_STRING_BYTES_LENGTH = SQLITE_MAX_LENGTH + +/** + * The max length of what the blueprint declares `@db.Text`, stored here as an + * unconstrained TEXT column. + */ +export const MAX_DB_TEXT_BYTES_LENGTH = SQLITE_MAX_LENGTH + +/** + * The max length of what the blueprint declares `@db.MediumText`, stored here + * as an unconstrained TEXT column. + */ +export const MAX_DB_MEDIUMTEXT_BYTES_LENGTH = SQLITE_MAX_LENGTH diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts new file mode 100644 index 0000000..399a4e7 --- /dev/null +++ b/packages/db/src/index.ts @@ -0,0 +1,86 @@ +// @note the community database: SQLite in a file. +// +// This package is the proof that the platform's data layer runs somewhere a +// laptop can be: the schema is derived from the blueprint in +// `@chatbotkit-dev/db-spec`, the client is generated against a file database +// this package creates itself, and the 48 analytics queries typecheck and run +// against it unchanged. +// +// It is not the platform's client yet - the shim in `platform/prisma` still +// generates in place. Wiring this in behind that seam is the next step, and +// deliberately a separate one. + +import { PrismaBetterSqlite3 } from '@prisma/adapter-better-sqlite3' + +import { PrismaClient } from '../prisma/generated/prisma/client' + +export { Prisma, PrismaClient } from '../prisma/generated/prisma/client' + +export * as sql from '../prisma/generated/prisma/sql' + +/** + * @note resolved on first use, never at import - the convention every module + * follows, so importing this package never requires it to be configured. + * + * @throws when the url is missing or not a file: url, naming what to set + */ +function getUrl(): string { + const url = process.env.PRISMA_DATABASE_URL + + if (!url || !url.startsWith('file:')) { + throw new Error( + 'PRISMA_DATABASE_URL must be a file: url for the SQLite database, e.g. file:./data/cbk.db' + ) + } + + return url +} + +/** + * Constructs a raw client. + * + * @note raw on purpose, matching the contract every database module exports: + * the platform's own extensions (audit, cache, retry, methods) are platform + * behaviour and are applied by the platform over this, not baked in here. The + * platform also owns the lifecycle of the one shared instance, which is why + * this constructs rather than caches. + * + * @throws when the url is missing or not a file: url, naming what to set + */ +export function createInstance() { + return new PrismaClient({ + adapter: new PrismaBetterSqlite3({ url: getUrl() }), + }) +} + +let client: PrismaClient | undefined + +export function getClient(): PrismaClient { + if (!client) { + client = createInstance() + } + + return client +} + +/** + * @note exported for tests, which need a fresh client per case. + */ +export function resetClient(): void { + client = undefined +} + +/** + * @throws when the database cannot be opened or queried, naming what to set. + */ +export async function assertConfigured(): Promise { + try { + await getClient().$queryRaw`SELECT 1` + } catch (error) { + throw new Error( + `the SQLite database at ${process.env.PRISMA_DATABASE_URL} could not be opened, so nothing can be stored or read: ${ + error instanceof Error ? error.message : String(error) + }` + ) + } +} diff --git a/packages/db/tsconfig.json b/packages/db/tsconfig.json new file mode 100644 index 0000000..fa98df2 --- /dev/null +++ b/packages/db/tsconfig.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node" + ], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js", + "./prisma/generated/**/*.ts", + "./scripts/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/debug/README.md b/packages/debug/README.md new file mode 100644 index 0000000..862e22d --- /dev/null +++ b/packages/debug/README.md @@ -0,0 +1,11 @@ +# @chatbotkit-dev/debug + +Namespaced logging, assertions and spans. + +`debug('...').log('key')` emits only when the key is enabled by the active +configuration. The built-in default is driven by the `DEBUG_KEYS`, `WARN_KEYS` +and `ERROR_KEYS` environment variables; a deployment with opinions about its +own subsystems supplies a full `DebugConfig` through `configure()` at boot. +Spans go to [`@chatbotkit-dev/observability`](../observability). + +Extracted from `platform/lib/debug.ts`. diff --git a/packages/debug/jest.config.js b/packages/debug/jest.config.js new file mode 100644 index 0000000..8a86699 --- /dev/null +++ b/packages/debug/jest.config.js @@ -0,0 +1,19 @@ +// @note CommonJS transform: these tests use `jest.mock` hoisting and the `jest` +// global, neither of which is available under the ESM preset. + +export default { + preset: 'ts-jest', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', + + transform: { + '^.+\\.[jt]sx?$': [ + 'ts-jest', + { + useESM: false, + // @note transpile only. Type checking is the `check` script's job. + tsconfig: { module: 'commonjs', esModuleInterop: true, allowJs: true }, + }, + ], + }, +} diff --git a/packages/debug/package.json b/packages/debug/package.json new file mode 100644 index 0000000..f5796ab --- /dev/null +++ b/packages/debug/package.json @@ -0,0 +1,42 @@ +{ + "name": "@chatbotkit-dev/debug", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "@chatbotkit-dev/env": "workspace:*", + "@chatbotkit-dev/observability": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/debug/src/index.test.js b/packages/debug/src/index.test.js new file mode 100644 index 0000000..28ffedb --- /dev/null +++ b/packages/debug/src/index.test.js @@ -0,0 +1,187 @@ +import { assert, debug } from './index' + +describe('assert', () => { + it('should not throw an error when the test is true', () => { + expect(() => { + assert(true, 'This should not throw an error') + }).not.toThrow() + }) + + it('should throw an error when the test is false', () => { + expect(() => { + assert(false, 'This should throw an error') + }).toThrow('This should throw an error') + }) +}) + +describe('debug logging', () => { + let consoleSpy + + beforeEach(() => { + consoleSpy = jest.spyOn(console, 'debug').mockImplementation(() => {}) + }) + + afterEach(() => { + consoleSpy.mockRestore() + }) + + it('should serialize nested objects properly in logs', () => { + const nestedObject = { + level1: { + level2: { + level3: { + value: 'test', + }, + }, + }, + } + + debug(nestedObject).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + // should contain the nested value, not [Object] + expect(loggedArg).toContain('test') + expect(loggedArg).toContain('level3') + }) + + it('should limit depth of very deep objects', () => { + // create an object deeper than maxDepth (10 in development) + const deepObject = { + l1: { + l2: { + l3: { + l4: { + l5: { + l6: { + l7: { + l8: { + l9: { + l10: { + l11: { + deep: 'value', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + debug(deepObject).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + // at depth 10, deeper objects should be [Object] + expect(loggedArg).toContain('[Object]') + }) + + it('should handle circular references', () => { + const circularObj = { name: 'test' } + + circularObj.self = circularObj + + debug(circularObj).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + expect(loggedArg).toContain('[Circular]') + expect(loggedArg).toContain('test') + }) + + it('should handle arrays with depth limiting', () => { + const deepArray = { + items: [ + { + l1: { + l2: { + l3: { + l4: { + l5: { + l6: { + l7: { + l8: { + l9: { l10: { tooDeep: 'value' } }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + } + + debug(deepArray).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + // should show [Object] for very deep arrays + expect(loggedArg).toContain('[Object]') + }) + + it('should truncate long strings', () => { + debug({ audio: 'a'.repeat(3000) }).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + expect(loggedArg).toContain('[String(3000)]') + expect(loggedArg.length).toBeLessThan(2500) + }) + + it('should summarize binary data', () => { + debug({ buffer: Buffer.alloc(4096) }).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + expect(loggedArg).toContain('[Buffer(4096)]') + expect(loggedArg).not.toContain('"0":') + }) + + it('should summarize class instances', () => { + class CustomRuntimeObject { + constructor() { + this._events = {} + this._buffer = Buffer.alloc(4096) + } + } + + debug({ socket: new CustomRuntimeObject() }).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + expect(loggedArg).toContain('[CustomRuntimeObject]') + expect(loggedArg).not.toContain('_buffer') + }) + + it('should safely summarize invalid dates', () => { + debug({ createdAt: new Date('invalid') }).log() + + expect(consoleSpy).toHaveBeenCalled() + + const loggedArg = consoleSpy.mock.calls[0][1] + + expect(loggedArg).toContain('Invalid Date') + }) +}) diff --git a/packages/debug/src/index.ts b/packages/debug/src/index.ts new file mode 100644 index 0000000..aff1479 --- /dev/null +++ b/packages/debug/src/index.ts @@ -0,0 +1,635 @@ +import { isDevelopment, isTest } from '@chatbotkit-dev/env' +import observability from '@chatbotkit-dev/observability' + +import { ok } from 'assert' + +/** + * A map of log key patterns to whether they are enabled. Patterns may end in + * `*` to match a namespace, for example `conversation.*`. + */ +export type LogKeys = Record + +export interface DebugConfig { + /** Enables all debug output regardless of key. */ + debug: boolean + + /** Enables all error output regardless of key. */ + error: boolean + + /** Enables all warning output regardless of key. */ + warn: boolean + + log: { + debug: LogKeys + error: LogKeys + warn: LogKeys + } +} + +export type KnownKeys = string + +// @note always stringify objects to ensure consistent logging across +// environments and to properly serialize nested objects (avoids [Object] in +// logs) +const USE_SAFE_STRINGIFY = true + +// @note maximum depth for safeStringify +const DEFAULT_MAX_DEPTH = isDevelopment ? 10 : 5 + +const MAX_STRING_LENGTH = isDevelopment ? 2_048 : 1_024 +const MAX_ARRAY_LENGTH = isDevelopment ? 100 : 50 +const MAX_OBJECT_KEYS = isDevelopment ? 100 : 50 + +// --- sensitive-value redaction --- + +// @note outside development every log line is scrubbed of credential-shaped +// material before it is serialized. In development logs stay verbatim - the +// local email sign-in flow deliberately prints its code to the server log, +// and a developer machine is the one place full values are worth more than +// they cost. Persistence call sites (event metadata and similar) must not +// rely on this gate and should call redact() themselves, which always runs. + +export const REDACTED = '[REDACTED]' + +// key fragments that mark a value as a credential wherever the key appears +const SENSITIVE_KEY_FRAGMENTS = [ + 'auth', + 'cookie', + 'password', + 'passwd', + 'secret', + 'token', + 'credential', + 'signature', + 'apikey', + 'api-key', + 'api_key', + 'privatekey', + 'private-key', + 'private_key', +] + +// keys that contain a sensitive fragment but name a location, shape, or +// bookkeeping fact rather than a value ('tokenUrl', 'accessTokenExpiresAt', +// 'authorName', 'tokenType', 'tokens'/'maxTokens' usage counts) +const SAFE_KEY_SUFFIXES = [ + 'url', + 'uri', + 'endpoint', + 'id', + 'at', + 'name', + 'type', + 'method', + 'count', + 'length', + 'size', + 'hint', + 'mask', + 'kind', + 'mode', + 'tokens', +] + +/** + * Whether a key names credential-shaped material. Case-insensitive; a key + * counts when it contains a sensitive fragment and does not end in a suffix + * that marks it as a location or bookkeeping fact. + */ +export function isSensitiveKey(key: string): boolean { + const normalized = key.toLowerCase() + + if ( + !SENSITIVE_KEY_FRAGMENTS.some((fragment) => normalized.includes(fragment)) + ) { + return false + } + + return !SAFE_KEY_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) +} + +/** + * Scrubs credential-shaped material out of a string regardless of the key it + * sits under: authorization scheme credentials ("Bearer x", "Basic y"), URL + * userinfo passwords, and sensitive query-string parameters (tokens, keys, + * codes, signatures, secrets, state), and slash-delimited configuration + * fields such as custom model credentials. + */ +export function redactString(value: string): string { + return value + .replace( + /\b(bearer|basic|digest)\s+[a-z0-9._~+/=-]+/gi, + (match, scheme) => `${scheme} ${REDACTED}` + ) + .replace(/(\/\/[^/\s@:]+):([^/\s@]+)@/g, `$1:${REDACTED}@`) + .replace( + /([?&](?:key|code|sig|state|[\w-]*(?:token|secret|password|passwd|signature|credential|auth|api[_-]?key)[\w-]*)=)[^&\s"']*/gi, + `$1${REDACTED}` + ) + .replace( + /((?:^|\/)(?:key|code|sig|state|token|secret|password|passwd|passphrase|authorization|auth|credentials?|signature|api[_-]?key|private[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|oauth[_-]?token|client[_-]?secret)=)[^/\s"']*/gi, + `$1${REDACTED}` + ) +} + +const MAX_REDACT_DEPTH = 8 + +function redactValue( + value: unknown, + forceAll: boolean, + depth: number +): unknown { + if (typeof value === 'string') { + return forceAll ? REDACTED : redactString(value) + } + + if (typeof value === 'number' || typeof value === 'bigint') { + return forceAll ? REDACTED : value + } + + if (typeof value !== 'object' || value === null) { + return value + } + + // @note beyond this depth the value is passed through untouched - the + // serializer's own depth limit sits lower, so nothing this deep is printed + // in any detail anyway, and the cap keeps cycles finite + + if (depth >= MAX_REDACT_DEPTH) { + return value + } + + if (Array.isArray(value)) { + return value.map((item) => redactValue(item, forceAll, depth + 1)) + } + + const prototype = Object.getPrototypeOf(value) + + if (prototype !== Object.prototype && prototype !== null) { + // @note class instances (Date, Error, Map, buffers) are summarized by the + // serializer rather than walked here + return value + } + + const result: Record = {} + + for (const [k, v] of Object.entries(value)) { + result[k] = redactValue(v, forceAll || isSensitiveKey(k), depth + 1) + } + + return result +} + +/** + * Deeply redacts credential-shaped material from a value: every string or + * number under a sensitive key becomes '[REDACTED]' (shape preserved), and + * every other string is scrubbed with redactString(). Always active - use + * this at persistence boundaries (event metadata, stored records) where the + * development-mode logging exemption must not apply. + */ +export function redact(value: unknown): unknown { + return redactValue(value, false, 0) +} + +// @note test runs can use injected staging credentials, so only interactive +// local development may bypass log redaction + +const REDACT_LOGS = !isDevelopment || isTest + +function prepareLogArg(arg: unknown): unknown { + if (typeof arg === 'string') { + return REDACT_LOGS ? redactString(arg) : arg + } + + return safeStringify(REDACT_LOGS ? redact(arg) : arg, 2) +} + +/** + * Safely stringify an object, handling circular references and limiting depth. + */ +function safeStringify( + obj: unknown, + indent?: number, + maxDepth: number = DEFAULT_MAX_DEPTH +): string { + const seen = new WeakSet() + + function summarizeString(value: string): string { + if (value.length <= MAX_STRING_LENGTH) { + return value + } + + return `${value.slice(0, MAX_STRING_LENGTH)}...[String(${value.length})]` + } + + function summarizeBinary(value: ArrayBufferView): string { + const constructorName = value.constructor?.name || 'ArrayBufferView' + + return `[${constructorName}(${value.byteLength})]` + } + + function isPlainObject(value: object): boolean { + const prototype = Object.getPrototypeOf(value) + + return prototype === Object.prototype || prototype === null + } + + function summarizeObject(value: object): unknown { + if (value instanceof Date) { + return Number.isNaN(value.getTime()) + ? value.toString() + : value.toISOString() + } + + if (value instanceof Error) { + return { + name: value.name, + message: value.message, + stack: summarizeString(value.stack || ''), + } + } + + if (value instanceof RegExp) { + return value.toString() + } + + if (value instanceof Map || value instanceof Set) { + return `[${value.constructor.name}(${value.size})]` + } + + return `[${value.constructor?.name || 'Object'}]` + } + + function processValue(value: unknown, currentDepth: number): unknown { + if (typeof value === 'string') { + return summarizeString(value) + } + + // handle primitives + + if (typeof value !== 'object' || value === null) { + return value + } + + if (ArrayBuffer.isView(value)) { + return summarizeBinary(value) + } + + if (value instanceof ArrayBuffer) { + return `[ArrayBuffer(${value.byteLength})]` + } + + // handle circular references + + if (seen.has(value)) { + return '[Circular]' + } + + seen.add(value) + + // handle max depth + + if (currentDepth >= maxDepth) { + if (Array.isArray(value)) { + return `[Array(${value.length})]` + } + + return '[Object]' + } + + // handle arrays + + if (Array.isArray(value)) { + const result = value + .slice(0, MAX_ARRAY_LENGTH) + .map((item) => processValue(item, currentDepth + 1)) + + if (value.length > MAX_ARRAY_LENGTH) { + result.push(`...[Array(${value.length})]`) + } + + return result + } + + if (!isPlainObject(value)) { + return summarizeObject(value) + } + + // handle objects + + const result: Record = {} + const entries = Object.entries(value) + + for (const [k, v] of entries.slice(0, MAX_OBJECT_KEYS)) { + result[k] = processValue(v, currentDepth + 1) + } + + if (entries.length > MAX_OBJECT_KEYS) { + result.__truncated = `...[Object(${entries.length})]` + } + + return result + } + + const processed = processValue(obj, 0) + + return JSON.stringify(processed, null, indent) +} + +function toKeys(value: string | undefined): LogKeys { + return Object.fromEntries( + (value || '') + .split(',') + .map((key) => key.trim()) + .filter(Boolean) + .map((key) => [key, true]) + ) +} + +// @note the built-in configuration enables nothing by default and is driven +// entirely by the environment. A deployment with opinions about its own +// subsystems supplies its own key map through configure(), typically at boot. + +function defaultConfig(): DebugConfig { + return { + debug: false, + error: false, + warn: false, + + log: { + debug: toKeys(process.env.DEBUG_KEYS), + error: toKeys(process.env.ERROR_KEYS), + warn: toKeys(process.env.WARN_KEYS), + }, + } +} + +type WildcardKeys = Record + +let activeConfig: DebugConfig | null = null +let wildcardKeys: WildcardKeys | null = null + +function getConfig(): DebugConfig { + if (!activeConfig) { + activeConfig = defaultConfig() + } + + return activeConfig +} + +/** + * Replaces the active debug configuration. Key matching is resolved against + * the current configuration on every call, so this can run at any point during + * boot and takes effect for all subsequent logging. + */ +export function configure(config: DebugConfig): void { + activeConfig = config + wildcardKeys = null +} + +function getWildcardKeys(): WildcardKeys { + if (!wildcardKeys) { + wildcardKeys = Object.entries(getConfig().log).reduce( + (acc, [category, value]) => { + if (value.all) { + acc[category] = true + } else { + const keys = Object.entries(value) + .filter(([, e]) => e) + .map(([k]) => (k.endsWith('*') ? k.slice(0, -1) : false)) + .filter((k): k is string => typeof k === 'string') + + acc[category] = keys + } + + return acc + }, + {} as WildcardKeys + ) + } + + return wildcardKeys +} + +function hasKey(key: string, category: keyof DebugConfig['log']): boolean { + const config = getConfig() + const wildcards = getWildcardKeys() + + return ( + config.log[category].all || + config.log[category][key] || + (Array.isArray(wildcards[category]) && + wildcards[category].length > 0 && + (wildcards[category] as string[]).some((k) => key.startsWith(k))) + ) +} + +export interface DebugResult { + log: (key?: KnownKeys) => DebugResult + trace: () => DebugResult +} + +export function print(...args: unknown[]): void { + // eslint-disable-next-line + console.log(...args) +} + +export function log(...args: unknown[]): void { + // eslint-disable-next-line + console.log('*', ...(USE_SAFE_STRINGIFY ? args.map(prepareLogArg) : args)) +} + +export function debuglog(...args: unknown[]): void { + // eslint-disable-next-line + ;(console.debug || console.log)( + ...(USE_SAFE_STRINGIFY ? args.map(prepareLogArg) : args) + ) + + if (process.env.TRACE_DEBUG) { + try { + // eslint-disable-next-line + console.trace?.(...args) + } catch { + // @note for whatever reason it could fail + } + } +} + +/** + * Prints a debug message to the console if the DEBUG environment variable is + * set. + */ +export function debug(...args: unknown[]): DebugResult { + if (!!process.env.DEBUG || getConfig().debug) { + debuglog('*', ...args) + } + + return { + log(key?: KnownKeys): DebugResult { + if (!key || hasKey(key, 'debug')) { + debuglog(key ? `[${key}]` : '*', ...args) + } + + return this + }, + + trace(): DebugResult { + try { + // eslint-disable-next-line + console.trace?.(...args) + } catch { + // @note for whatever reason it could fail + } + + return this + }, + } +} + +export function errorlog(...args: unknown[]): void { + // eslint-disable-next-line + ;(console.error || console.log)( + ...(USE_SAFE_STRINGIFY ? args.map(prepareLogArg) : args) + ) + + if (process.env.TRACE_ERROR) { + try { + // eslint-disable-next-line + console.trace?.(...args) + } catch { + // @note for whatever reason it could fail + } + } +} + +export function error(...args: unknown[]): DebugResult { + errorlog('*', ...args) + + return { + log(key?: KnownKeys): DebugResult { + if (!key || hasKey(key, 'error')) { + errorlog(key ? `[${key}]` : '*', ...args) + } + + return this + }, + + trace(): DebugResult { + try { + // eslint-disable-next-line + console.trace?.(...args) + } catch { + // @note for whatever reason it could fail + } + + return this + }, + } +} + +export function warnlog(...args: unknown[]): void { + // eslint-disable-next-line + ;(console.warn || console.log)( + ...(USE_SAFE_STRINGIFY ? args.map(prepareLogArg) : args) + ) + + if (process.env.TRACE_WARN) { + try { + // eslint-disable-next-line + console.trace?.(...args) + } catch { + // @note for whatever reason it could fail + } + } +} + +export function warn(...args: unknown[]): DebugResult { + if (!!process.env.WARN || getConfig().warn) { + warnlog('*', ...args) + } + + return { + log(key?: KnownKeys): DebugResult { + if (!key || hasKey(key, 'warn')) { + warnlog(key ? `[${key}]` : '*', ...args) + } + + return this + }, + + trace(): DebugResult { + try { + // eslint-disable-next-line + console.trace?.(...args) + } catch { + // @note for whatever reason it could fail + } + + return this + }, + } +} + +export function exit(...args: unknown[]): never { + if (args.length) { + // eslint-disable-next-line + ;(console.error || console.log)(...args) + } + + process.exit(1) +} + +export function assert(test: unknown, message: string): void { + ok(test, message) +} + +export function fassert(test: () => unknown, message: string): void { + ok(test(), message) +} + +/** + * @deprecated + * @throws {Error} Always throws an error indicating unreachable code was reached + */ +export function unreachable(test: never): never { + throw new Error(`Unreachable code reached: ${test}`) +} + +interface SpanOptions { + name: string + op?: string +} + +interface Span { + finish: () => void + setAttribute: (name: string, value: unknown) => void +} + +export function createSpan({ name, op }: SpanOptions): Span { + return observability.startSpan({ name, op }) +} + +export function span(options: { name: string }, fn: () => void): void { + const span = createSpan(options) + + try { + fn() + } finally { + span.finish() + } +} + +export async function spanAsync( + options: { name: string }, + fn: () => Promise +): Promise { + const span = createSpan(options) + + try { + await fn() + } finally { + span.finish() + } +} + +export default debug diff --git a/packages/debug/src/redact.test.js b/packages/debug/src/redact.test.js new file mode 100644 index 0000000..ebe0848 --- /dev/null +++ b/packages/debug/src/redact.test.js @@ -0,0 +1,244 @@ +/** + * Tests for the sensitive-value redaction layer. + * + * The production log gate is exercised by mocking the environment as + * non-development BEFORE the module loads - the gate is computed at import + * time. The redact()/redactString()/isSensitiveKey() primitives are + * environment-independent and are tested directly. + */ + +jest.mock('@chatbotkit-dev/env', () => ({ + isDevelopment: true, + isStaging: false, + isProduction: false, + isTest: true, +})) + +import { REDACTED, isSensitiveKey, log, redact, redactString } from './index' + +describe('isSensitiveKey', () => { + it.each([ + 'authorization', + 'Authorization', + 'proxy-authorization', + 'cookie', + 'set-cookie', + 'password', + 'clientSecret', + 'client_secret', + 'accessToken', + 'access_token', + 'refresh_token', + 'x-api-key', + 'apiKey', + 'x-hub-signature', + 'x-twilio-signature', + 'credentials', + 'privateKey', + 'x-secret-header', + 'oauthToken', + ])('treats %s as sensitive', (key) => { + expect(isSensitiveKey(key)).toBe(true) + }) + + it.each([ + 'tokenUrl', + 'authorizationUrl', + 'accessTokenExpiresAt', + 'tokenType', + 'authorName', + 'method', + 'url', + 'status', + 'email', + 'identifier', + 'body', + 'conversationId', + 'tokenEndpoint', + 'tokens', + 'maxTokens', + 'promptTokens', + ])('treats %s as not sensitive', (key) => { + expect(isSensitiveKey(key)).toBe(false) + }) +}) + +describe('redactString', () => { + it('redacts bearer scheme credentials', () => { + expect(redactString('Bearer sk-abc123.def')).toBe(`Bearer ${REDACTED}`) + expect(redactString('sending Authorization: Bearer eyJx.yy.zz now')).toBe( + `sending Authorization: Bearer ${REDACTED} now` + ) + }) + + it('redacts basic scheme credentials', () => { + expect(redactString('Basic dXNlcjpwYXNz')).toBe(`Basic ${REDACTED}`) + }) + + it('redacts url userinfo passwords', () => { + expect(redactString('https://user:hunter2@example.com/path')).toBe( + `https://user:${REDACTED}@example.com/path` + ) + }) + + it('redacts sensitive query parameters', () => { + expect( + redactString('https://example.com/cb?code=abc123&state=xyz&next=1') + ).toBe(`https://example.com/cb?code=${REDACTED}&state=${REDACTED}&next=1`) + + expect( + redactString('https://example.com/x?access_token=t1&api_key=t2') + ).toBe(`https://example.com/x?access_token=${REDACTED}&api_key=${REDACTED}`) + + expect(redactString('https://s3.example.com/f?sig=AAA&expires=9')).toBe( + `https://s3.example.com/f?sig=${REDACTED}&expires=9` + ) + }) + + it('redacts credentials embedded in slash-delimited model strings', () => { + expect( + redactString( + 'custom/name=gpt-4o/provider=openai/credentials=sk-live-123/maxTokens=1000' + ) + ).toBe( + `custom/name=gpt-4o/provider=openai/credentials=${REDACTED}/maxTokens=1000` + ) + }) + + it('does not redact look-alike parameter names', () => { + expect(redactString('https://example.com/?monkey=1&decode=2')).toBe( + 'https://example.com/?monkey=1&decode=2' + ) + }) + + it('leaves ordinary strings alone', () => { + expect(redactString('fetching the dataset list')).toBe( + 'fetching the dataset list' + ) + }) +}) + +describe('redact', () => { + it('redacts values under sensitive keys and preserves shape', () => { + expect( + redact({ + method: 'POST', + url: 'https://example.com/hook', + headers: { + 'content-type': 'application/json', + authorization: 'Bearer sk-live-123', + 'x-api-key': 'ak-42', + }, + }) + ).toEqual({ + method: 'POST', + url: 'https://example.com/hook', + headers: { + 'content-type': 'application/json', + authorization: REDACTED, + 'x-api-key': REDACTED, + }, + }) + }) + + it('redacts every leaf under a sensitive branch, including arrays', () => { + expect( + redact({ + secrets: [{ value: 'aaa', label: 'first' }, 'raw-token'], + }) + ).toEqual({ + secrets: [{ value: REDACTED, label: REDACTED }, REDACTED], + }) + }) + + it('preserves booleans and null under sensitive keys', () => { + expect(redact({ hasClientSecret: true, refreshToken: null })).toEqual({ + hasClientSecret: true, + refreshToken: null, + }) + }) + + it('redacts numbers under sensitive keys but not elsewhere', () => { + expect(redact({ pinToken: 123456, status: 200 })).toEqual({ + pinToken: REDACTED, + status: 200, + }) + }) + + it('leaves token usage counts alone', () => { + expect( + redact({ tokens: 205, model: 'gpt-5.4-mini', type: 'input', debit: 11 }) + ).toEqual({ tokens: 205, model: 'gpt-5.4-mini', type: 'input', debit: 11 }) + }) + + it('scrubs credential-shaped strings under non-sensitive keys', () => { + expect( + redact({ note: 'call used Bearer abc123', location: 'https://x.y/z' }) + ).toEqual({ + note: `call used Bearer ${REDACTED}`, + location: 'https://x.y/z', + }) + }) + + it('survives circular references', () => { + const value = { name: 'a' } + + value.self = value + + expect(() => redact(value)).not.toThrow() + }) +}) + +describe('non-interactive log output', () => { + it('redacts sensitive keys in logged objects', () => { + const spy = jest.spyOn(console, 'log').mockImplementation(() => {}) + + try { + log('request sent', { + url: 'https://example.com', + headers: { authorization: 'Bearer sk-live-9' }, + }) + + const output = spy.mock.calls.flat().join(' ') + + expect(output).toContain(REDACTED) + expect(output).not.toContain('sk-live-9') + expect(output).toContain('https://example.com') + } finally { + spy.mockRestore() + } + }) + + it('scrubs bare string arguments', () => { + const spy = jest.spyOn(console, 'log').mockImplementation(() => {}) + + try { + log('token was Bearer abc.def.ghi') + + const output = spy.mock.calls.flat().join(' ') + + expect(output).not.toContain('abc.def.ghi') + expect(output).toContain(REDACTED) + } finally { + spy.mockRestore() + } + }) + + it('scrubs model credentials nested under a non-sensitive key', () => { + const spy = jest.spyOn(console, 'log').mockImplementation(() => {}) + + try { + log('parsing language model', { + model: + 'custom/name=gpt-4o/provider=openai/credentials=sk-live-9/maxTokens=1000', + }) + + const output = spy.mock.calls.flat().join(' ') + + expect(output).not.toContain('sk-live-9') + expect(output).toContain(`credentials=${REDACTED}`) + } finally { + spy.mockRestore() + } + }) +}) diff --git a/packages/debug/tsconfig.json b/packages/debug/tsconfig.json new file mode 100644 index 0000000..72b364c --- /dev/null +++ b/packages/debug/tsconfig.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node", + "jest" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/email-spec/README.md b/packages/email-spec/README.md new file mode 100644 index 0000000..b5c3b17 --- /dev/null +++ b/packages/email-spec/README.md @@ -0,0 +1,28 @@ +# @chatbotkit-dev/email-spec + +The **contract** for outbound email. Types only. + +Separate from [`@chatbotkit-dev/email`](../email) because that package is +swappable: a deployment replaces it with its own implementation through a pnpm +override, and an implementation cannot import the package it replaces. + +## The two kinds of mail + +`sendEmailNotification` is the deployment writing to its own user: trial +notices, login links, limit warnings. The sending identity belongs to the +deployment, so the implementation owns the from address and the caller only +chooses where replies go. + +`sendEmailAction` is an agent or integration writing to a third party. The +caller may supply the sending mailbox, because the message comes from an address +the deployment does not necessarily own, and may carry a `messageId` to thread +against inbound mail. + +They are separate functions rather than one function with a flag because they +usually want separate sending domains and separate sending reputations. + +## What is deliberately absent + +Vendor concepts. Tracking flags, suppression groups, list-management bypasses +and sending domains are all decisions an implementation makes per purpose. A +platform caller says what kind of mail it is sending and nothing more. diff --git a/packages/email-spec/package.json b/packages/email-spec/package.json new file mode 100644 index 0000000..eb53b76 --- /dev/null +++ b/packages/email-spec/package.json @@ -0,0 +1,34 @@ +{ + "name": "@chatbotkit-dev/email-spec", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "true" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "eslint": "^9.0.0", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/email-spec/src/index.ts b/packages/email-spec/src/index.ts new file mode 100644 index 0000000..cbb412a --- /dev/null +++ b/packages/email-spec/src/index.ts @@ -0,0 +1,220 @@ +// @note the contract for outbound email. Implementations decide the sending +// identity, the delivery vendor, tracking and suppression behaviour. None of +// that appears here, because none of it is the platform's concern. +// +// There are two kinds of outbound mail and they differ in who the message is +// from, not merely in content: +// +// notification - the deployment writing to its own user. Sent from the +// deployment's identity, so the implementation owns the from +// address and the caller only chooses where replies go. +// +// action - an agent or integration writing to a third party. The +// caller may supply the sending identity, because the message +// is from a mailbox the deployment does not own. +// +// transport - the deployment writing to its own user, but as an identity it +// hosts on someone else's behalf: a whitelabel partner's +// domain, a portal's own host. The caller names that identity +// and gets something it can send through repeatedly. +// +// They are separate entry points rather than a flag because they usually want +// separate sending domains, reputations and often separate vendors - the +// identity a deployment hosts for a partner is verified wherever that partner's +// domain is verified, which need not be where the deployment's own mail goes. + +/** + * A fully rendered message body. Callers render before handing off. + */ +export interface EmailContent { + text: string + html: string +} + +/** + * Mail the deployment sends to its own user. + */ +export interface NotificationEmail { + to: string + subject: string + + /** + * The rendered body, both parts. Callers render before handing off: whether a + * body is markdown, HTML or plain text is something the caller knows and the + * provider should not have to guess. + */ + content: EmailContent + + /** + * Where replies should go, when this particular message needs them to go + * somewhere specific. + * + * @note omit it and the implementation uses its own reply address. Where a + * deployment's notifications should be replied to is the implementation's + * business, not the caller's, so the platform does not carry that address. + */ + replyTo?: string + + /** + * Marks mail the user must receive regardless of their subscription + * preferences: login links, invitations, security notices. + * + * @note this is a property of the message, not a delivery setting. An + * implementation decides what it means - typically suppressing tracking and + * bypassing list management - but the platform only states that the message + * is essential. + */ + essential?: boolean +} + +/** + * Mail an agent or integration sends to a third party. + */ +export interface ActionEmail { + to: string + subject: string + + /** + * The rendered body, both parts. See NotificationEmail.content. + */ + content: EmailContent + + /** + * The sending mailbox. Absent means the implementation supplies its own + * action identity. + */ + from?: string + + /** Display name for the sending mailbox. */ + name?: string + + replyTo?: string + + /** + * Threads the message against an existing conversation, for replies to + * inbound mail. + */ + messageId?: string +} + +/** + * A fully rendered message, ready to deliver. See NotificationEmail.content for + * why both parts are the caller's job. + */ +export interface EmailTransportMessage { + to: string + subject: string + text: string + html: string +} + +/** + * Delivers mail sent as one particular identity, which was fixed when the + * transport was created. + */ +export interface EmailTransport { + send(message: EmailTransportMessage): Promise +} + +/** + * An attachment carried by an inbound message, already extracted from the + * vendor's payload format. + */ +export interface InboundEmailAttachment { + name: string + size: number + type: string + data: ArrayBuffer +} + +/** + * An inbound message addressed to an email integration, normalized out of + * whatever payload format the implementation's inbound vendor delivers. + */ +export interface InboundEmail { + /** + * The integration whose inbox received the message - the implementation + * recognizes its own integration addresses and extracts the id. + */ + integrationId: string + + /** The integration inbox address the message arrived on. */ + to: string + + fromName?: string + fromEmail: string + + subject: string + + text?: string + html?: string + + /** Raw transport headers, for message-id / in-reply-to threading. */ + headers?: string + + senderIp?: string + + attachments: InboundEmailAttachment[] +} + +export interface EmailProvider { + sendEmailNotification(email: NotificationEmail): Promise + sendEmailAction(email: ActionEmail): Promise + + /** + * Composes the inbox address of an email integration. This is sending + * identity, so the implementation owns the address scheme: the hosted + * domain, the routing, and the inbound recognition are all its business. + * Implementations without a hosted domain derive a deterministic address + * from the deployment's site URL. + */ + formatIntegrationInbox(integrationId: string): string + + /** + * Mints a fresh RFC 5322 message-id for mail sent from an integration + * inbox. Sending identity again: the id-right part is the implementation's + * domain. The caller keeps the value for reply threading (replies echo it + * in In-Reply-To), which is why it is minted up front rather than by the + * delivery vendor. + */ + formatIntegrationMessageId(integrationId: string): string + + /** + * Parses an inbound-mail webhook payload into a normalized message, or + * returns null when the payload is not recognized or is not addressed to + * an integration inbox. The payload format belongs to whichever inbound + * vendor the implementation uses - the application never sees it. An + * implementation without inbound delivery logs and returns null. + */ + parseInboundEmail(form: FormData): Promise + + /** + * Creates a transport that sends as `source`, a full RFC 5322 address such as + * `Login `. + * + * @note this is how a deployment sends as an identity it hosts for someone + * else - a whitelabel partner, a portal on its own domain. Neither of the two + * functions above fits: a notification comes from the deployment's own + * identity, and an action comes from a mailbox the deployment does not own at + * all. Here the deployment does own the mail, but not the name on it. + * + * The sending domain has to be verified with whichever vendor the + * implementation delivers through, and which vendor that is - along with the + * credential it needs - is the implementation's business. Callers hold the + * returned transport and call `send`. + */ + createEmailTransport(source: string): EmailTransport + + /** + * Throws when this provider is not usable with the current configuration. + * + * @note an implementation resolves its own configuration lazily, so nothing + * that merely imports it needs that configuration present. This is how a + * deployment gets the guarantee back: it calls this where its environment is + * loaded - in its test suite or at startup - and finds out then rather than + * when a user fails to receive a login link. + * + * An implementation needing no configuration should resolve. + */ + assertConfigured(): Promise +} diff --git a/packages/email-spec/tsconfig.json b/packages/email-spec/tsconfig.json new file mode 100644 index 0000000..f12aa0f --- /dev/null +++ b/packages/email-spec/tsconfig.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2019" + ], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/email/README.md b/packages/email/README.md new file mode 100644 index 0000000..3ff9d97 --- /dev/null +++ b/packages/email/README.md @@ -0,0 +1,23 @@ +# @chatbotkit-dev/email + +The **community email provider**. It does not deliver mail: it writes a line to +the console describing what would have been sent, so a deployment runs and is +observable without an email vendor configured. + +The message body is deliberately never logged. Notification mail routinely +carries login links, and action mail carries conversation content. + +## Providing your own + +Replace this package at install time with one satisfying +[`@chatbotkit-dev/email-spec`](../email-spec): + +```yaml +# Root pnpm-workspace.yaml +overrides: + '@chatbotkit-dev/email': npm:your-email-implementation@* +``` + +An implementation owns the sending identity, the delivery vendor, tracking and +suppression. See the spec package for why notification and action mail are +separate functions. diff --git a/packages/email/jest.config.js b/packages/email/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/email/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/email/package.json b/packages/email/package.json new file mode 100644 index 0000000..6c2b057 --- /dev/null +++ b/packages/email/package.json @@ -0,0 +1,42 @@ +{ + "name": "@chatbotkit-dev/email", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + }, + "dependencies": { + "@chatbotkit-dev/email-spec": "workspace:*", + "@types/node": "^24.0.0" + } +} diff --git a/packages/email/src/index.test.js b/packages/email/src/index.test.js new file mode 100644 index 0000000..13ce64f --- /dev/null +++ b/packages/email/src/index.test.js @@ -0,0 +1,67 @@ +import provider, { sendEmailAction, sendEmailNotification } from './index' + +describe('community email provider', () => { + let logged + + // eslint-disable-next-line no-console + const original = console.log + + beforeEach(() => { + logged = [] + + // eslint-disable-next-line no-console + console.log = (...args) => { + logged.push(args.map(String).join(' ')) + } + }) + + afterEach(() => { + // eslint-disable-next-line no-console + console.log = original + }) + + it('satisfies the provider contract', () => { + expect(typeof provider.sendEmailNotification).toBe('function') + expect(typeof provider.sendEmailAction).toBe('function') + }) + + it('reports a notification without delivering it', async () => { + await sendEmailNotification({ + to: 'user@example.com', + subject: 'Your trial has started', + content: { text: 'hello', html: '

hello

' }, + }) + + expect(logged).toHaveLength(1) + expect(logged[0]).toContain('[email:notification]') + expect(logged[0]).toContain('user@example.com') + }) + + it('reports an action without delivering it', async () => { + await sendEmailAction({ + to: 'third@example.com', + subject: 'Re: your enquiry', + content: { text: 'hello', html: '

hello

' }, + from: 'agent@example.com', + }) + + expect(logged).toHaveLength(1) + expect(logged[0]).toContain('[email:action]') + }) + + it('logs the text body framed as mail, which is the only delivery there is', async () => { + await sendEmailNotification({ + to: 'user@example.com', + subject: 'Sign in', + content: { + text: 'your code:\n123456', + html: '123456', + }, + }) + + expect(logged.join('\n')).toContain('│ your code:') + expect(logged.join('\n')).toContain('│ 123456') + expect(logged.join('\n')).toContain('┌') + expect(logged.join('\n')).toContain('└') + }) +}) diff --git a/packages/email/src/index.ts b/packages/email/src/index.ts new file mode 100644 index 0000000..a5c7f7f --- /dev/null +++ b/packages/email/src/index.ts @@ -0,0 +1,112 @@ +import type { + ActionEmail, + EmailProvider, + EmailTransport, + InboundEmail, + NotificationEmail, +} from '@chatbotkit-dev/email-spec' + +export type * from '@chatbotkit-dev/email-spec' + +// @note the community implementation does not deliver mail. It writes what it +// would have sent to the console, text body included, so a deployment runs and +// stays usable without an email vendor configured - the console IS delivery +// here: sign-in codes and invitations reach the operator nowhere else. Replace +// this package to deliver for real (and to keep bodies out of logs). + +// @note the body is framed with an open left rail rather than a closed box: +// every line stands alone, so long URLs never break the frame and per-line +// log timestamps do not mangle it +function describe( + kind: string, + email: { to: string; subject: string }, + text: string +): void { + const rule = '─'.repeat(50) + + const body = text + .trim() + .split('\n') + .map((line) => `│ ${line}`) + .join('\n') + + // eslint-disable-next-line no-console + console.log( + `[email:${kind}] to=${email.to} subject=${JSON.stringify(email.subject)} (not delivered: no email provider configured)\n┌${rule}\n${body}\n└${rule}` + ) +} + +export async function sendEmailNotification( + email: NotificationEmail +): Promise { + describe('notification', email, email.content.text) +} + +export async function sendEmailAction(email: ActionEmail): Promise { + describe('action', email, email.content.text) +} + +/** + * @note the community implementation delivers nothing, so a transport is the + * same console line as anything else - with the identity it would have sent as, + * because that is the whole point of asking for one. + */ +export function createEmailTransport(source: string): EmailTransport { + return { + async send({ to, subject, text }) { + describe(`transport from=${source}`, { to, subject }, text) + }, + } +} + +/** + * @note the community provider needs no configuration, so there is nothing that + * can be misconfigured. + */ +export async function assertConfigured(): Promise { + // pass +} + +// @note the community implementation hosts no sending domain, so integration +// inboxes derive deterministically from the deployment's site URL - the +// address scheme an operator routes when they configure real email +function integrationHostname(): string { + try { + return new URL(process.env.SITE_URL ?? '').hostname || 'localhost' + } catch { + return 'localhost' + } +} + +export function formatIntegrationInbox(integrationId: string): string { + return `${integrationId}@integration.${integrationHostname()}` +} + +export function formatIntegrationMessageId(_integrationId: string): string { + return `<${crypto.randomUUID()}@integration.${integrationHostname()}>` +} + +// @note no inbound vendor means no inbound mail - describe and decline, the +// same posture as outbound delivery above +export async function parseInboundEmail( + _form: FormData +): Promise { + // eslint-disable-next-line no-console + console.log( + '[email:inbound] inbound message ignored (not parsed: no email provider configured)' + ) + + return null +} + +const provider: EmailProvider = { + sendEmailNotification, + sendEmailAction, + createEmailTransport, + assertConfigured, + formatIntegrationInbox, + formatIntegrationMessageId, + parseInboundEmail, +} + +export default provider diff --git a/packages/email/tsconfig.json b/packages/email/tsconfig.json new file mode 100644 index 0000000..53607f2 --- /dev/null +++ b/packages/email/tsconfig.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "types": [ + "node" + ], + "lib": [ + "DOM", + "DOM.Iterable", + "ES2021" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "composite": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/encoding/README.md b/packages/encoding/README.md new file mode 100644 index 0000000..a99d948 --- /dev/null +++ b/packages/encoding/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/encoding diff --git a/packages/encoding/jest.config.js b/packages/encoding/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/encoding/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/encoding/package.json b/packages/encoding/package.json new file mode 100644 index 0000000..9b8cb8f --- /dev/null +++ b/packages/encoding/package.json @@ -0,0 +1,35 @@ +{ + "name": "@chatbotkit-dev/encoding", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/encoding/src/index.test.ts b/packages/encoding/src/index.test.ts new file mode 100644 index 0000000..d7178ff --- /dev/null +++ b/packages/encoding/src/index.test.ts @@ -0,0 +1,27 @@ +import { getEncoding } from './index' + +describe('getEncoding', () => { + it('should detect empty buffer', () => { + const buffer = new Uint8Array([]) + + expect(getEncoding(buffer)).toBe('utf8') + }) + + it('should detect buffer with space', () => { + const buffer = new Uint8Array([0x20]) // Space character + + expect(getEncoding(buffer)).toBe('utf8') + }) + + it('should detect utf8', () => { + const buffer = new Uint8Array([0xe2, 0x82, 0xac]) // € (euro sign) + + expect(getEncoding(buffer)).toBe('utf8') + }) + + it('should detect binary', () => { + const buffer = new Uint8Array([0xff, 0x00, 0x01]) // Non-UTF8 bytes + + expect(getEncoding(buffer)).toBe('binary') + }) +}) diff --git a/packages/encoding/src/index.ts b/packages/encoding/src/index.ts new file mode 100644 index 0000000..2469d4b --- /dev/null +++ b/packages/encoding/src/index.ts @@ -0,0 +1,200 @@ +// @note copied from https://github.com/bevry/istextorbinary because it does not +// compiled correctly + +export interface EncodingOpts { + /** Defaults to 24 */ + chunkLength?: number + + /** If not provided, will check the start, beginning, and end */ + chunkBegin?: number +} + +function getChunkBegin(buf: Uint8Array, chunkBegin: number) { + // If it's the beginning, just return. + if (chunkBegin === 0) { + return 0 + } + + if (!isLaterByteOfUtf8(buf[chunkBegin])) { + return chunkBegin + } + + let begin = chunkBegin - 3 + + if (begin >= 0) { + if (isFirstByteOf4ByteChar(buf[begin])) { + return begin + } + } + + begin = chunkBegin - 2 + + if (begin >= 0) { + if ( + isFirstByteOf4ByteChar(buf[begin]) || + isFirstByteOf3ByteChar(buf[begin]) + ) { + return begin + } + } + + begin = chunkBegin - 1 + + if (begin >= 0) { + // Is it a 4-byte, 3-byte utf8 character? + if ( + isFirstByteOf4ByteChar(buf[begin]) || + isFirstByteOf3ByteChar(buf[begin]) || + isFirstByteOf2ByteChar(buf[begin]) + ) { + return begin + } + } + + return -1 +} + +function getChunkEnd(buf: Uint8Array, chunkEnd: number) { + // If it's the end, just return. + if (chunkEnd === buf.byteLength) { + return chunkEnd + } + + let index = chunkEnd - 3 + + if (index >= 0) { + if (isFirstByteOf4ByteChar(buf[index])) { + return chunkEnd + 1 + } + } + + index = chunkEnd - 2 + + if (index >= 0) { + if (isFirstByteOf4ByteChar(buf[index])) { + return chunkEnd + 2 + } + + if (isFirstByteOf3ByteChar(buf[index])) { + return chunkEnd + 1 + } + } + + index = chunkEnd - 1 + + if (index >= 0) { + if (isFirstByteOf4ByteChar(buf[index])) { + return chunkEnd + 3 + } + + if (isFirstByteOf3ByteChar(buf[index])) { + return chunkEnd + 2 + } + + if (isFirstByteOf2ByteChar(buf[index])) { + return chunkEnd + 1 + } + } + + return chunkEnd +} + +function isFirstByteOf4ByteChar(byte: number) { + // eslint-disable-next-line no-bitwise + return byte >> 3 === 30 // 11110xxx? +} + +function isFirstByteOf3ByteChar(byte: number) { + // eslint-disable-next-line no-bitwise + return byte >> 4 === 14 // 1110xxxx? +} + +function isFirstByteOf2ByteChar(byte: number) { + // eslint-disable-next-line no-bitwise + return byte >> 5 === 6 // 110xxxxx? +} + +function isLaterByteOfUtf8(byte: number) { + // eslint-disable-next-line no-bitwise + return byte >> 6 === 2 // 10xxxxxx? +} + +export function getEncoding( + buffer: Uint8Array | null, + opts?: EncodingOpts +): 'utf8' | 'binary' | null { + // Check + if (!buffer) { + return null + } + + // Prepare + const textEncoding = 'utf8' + const binaryEncoding = 'binary' + const chunkLength = opts?.chunkLength ?? 24 + let chunkBegin = opts?.chunkBegin ?? 0 + + // Discover + if (opts?.chunkBegin == null) { + // Start + let encoding = getEncoding(buffer, { chunkLength, chunkBegin }) + + if (encoding === textEncoding) { + // Middle + chunkBegin = Math.max(0, Math.floor(buffer.byteLength / 2) - chunkLength) + encoding = getEncoding(buffer, { + chunkLength, + chunkBegin, + }) + + if (encoding === textEncoding) { + // End + chunkBegin = Math.max(0, buffer.byteLength - chunkLength) + encoding = getEncoding(buffer, { + chunkLength, + chunkBegin, + }) + } + } + + // Return + return encoding + } else { + // Extract + chunkBegin = getChunkBegin(buffer, chunkBegin) + + if (chunkBegin === -1) { + return binaryEncoding + } + + const chunkEnd = getChunkEnd( + buffer, + Math.min(buffer.byteLength, chunkBegin + chunkLength) + ) + + if (chunkEnd > buffer.byteLength) { + return binaryEncoding + } + + const decoder = new TextDecoder(textEncoding, { + fatal: false, + }) + + const contentChunkUTF8 = decoder.decode(buffer).slice(chunkBegin, chunkEnd) + + // Detect encoding + for (let i = 0; i < contentChunkUTF8.length; ++i) { + const charCode = contentChunkUTF8.charCodeAt(i) + + if (charCode === 65533 || charCode <= 8) { + // 8 and below are control characters (e.g. backspace, null, eof, etc.) + // 65533 is the unknown character + // console.log(charCode, contentChunkUTF8[i]) + return binaryEncoding + } + } + + // Return + return textEncoding + } +} diff --git a/packages/encoding/tsconfig.json b/packages/encoding/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/encoding/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/env/README.md b/packages/env/README.md new file mode 100644 index 0000000..68458c1 --- /dev/null +++ b/packages/env/README.md @@ -0,0 +1,8 @@ +# @chatbotkit-dev/env + +Which environment the process is running in. + +Exports `isDevelopment`, `isStaging`, `isProduction` and `isTest`, resolved once +from the provider-neutral `NODE_ENV` and `TARGET_ENV` variables. + +Extracted from `platform/lib/env.ts`. Zero dependencies. diff --git a/packages/env/jest.config.js b/packages/env/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/env/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/env/package.json b/packages/env/package.json new file mode 100644 index 0000000..3011a84 --- /dev/null +++ b/packages/env/package.json @@ -0,0 +1,38 @@ +{ + "name": "@chatbotkit-dev/env", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/env/src/index.test.js b/packages/env/src/index.test.js new file mode 100644 index 0000000..ca9f7ca --- /dev/null +++ b/packages/env/src/index.test.js @@ -0,0 +1,94 @@ +import { jest } from '@jest/globals' + +const ENV_KEYS = [ + 'NODE_ENV', + 'TARGET_ENV', + 'VERCEL_ENV', + 'VERCEL_URL', + 'NEXT_PUBLIC_VERCEL_URL', +] + +function setEnv(name, value) { + if (value === undefined) { + delete process.env[name] + + return + } + + process.env[name] = value +} + +async function withEnvironment(environment, fn) { + const previousEnv = Object.fromEntries( + ENV_KEYS.map((name) => [name, process.env[name]]) + ) + + try { + for (const name of ENV_KEYS) { + setEnv(name, environment[name]) + } + + jest.resetModules() + + await jest.isolateModulesAsync(async () => { + await fn(await import('./index')) + }) + } finally { + for (const [name, value] of Object.entries(previousEnv)) { + setEnv(name, value) + } + + jest.resetModules() + } +} + +describe('environment identity', () => { + it('identifies test from NODE_ENV', async () => { + await withEnvironment({ NODE_ENV: 'test' }, async (environment) => { + expect(environment.isTest).toBe(true) + expect(environment.isDevelopment).toBe(true) + expect(environment.isStaging).toBe(false) + expect(environment.isProduction).toBe(false) + }) + }) + + it('identifies staging from TARGET_ENV', async () => { + await withEnvironment( + { NODE_ENV: 'production', TARGET_ENV: 'staging' }, + async (environment) => { + expect(environment.isTest).toBe(false) + expect(environment.isDevelopment).toBe(false) + expect(environment.isStaging).toBe(true) + expect(environment.isProduction).toBe(false) + } + ) + }) + + it('identifies production when TARGET_ENV is absent', async () => { + await withEnvironment( + { NODE_ENV: 'production' }, + async (environment) => { + expect(environment.isTest).toBe(false) + expect(environment.isDevelopment).toBe(false) + expect(environment.isStaging).toBe(false) + expect(environment.isProduction).toBe(true) + } + ) + }) + + it('does not derive environment identity from Vercel variables', async () => { + await withEnvironment( + { + NODE_ENV: 'production', + VERCEL_ENV: 'preview', + VERCEL_URL: 'preview.example.com', + }, + async (environment) => { + expect(environment.isStaging).toBe(false) + expect(environment.isProduction).toBe(true) + expect(environment).not.toHaveProperty('isOnVercel') + expect(environment).not.toHaveProperty('isOnVercelPreview') + } + ) + }) +}) diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts new file mode 100644 index 0000000..f345541 --- /dev/null +++ b/packages/env/src/index.ts @@ -0,0 +1,27 @@ +import { ok } from 'assert' + +export const isTest: boolean = + process.env.NODE_ENV === 'test' || process.env.TARGET_ENV === 'test' + +export const isDevelopment: boolean = + isTest || + process.env.NODE_ENV === 'development' || + process.env.TARGET_ENV === 'development' + +export const isStaging: boolean = + !isTest && + process.env.NODE_ENV === 'production' && + process.env.TARGET_ENV === 'staging' + +export const isProduction: boolean = + !isTest && + process.env.NODE_ENV === 'production' && + (process.env.TARGET_ENV === 'production' || !process.env.TARGET_ENV) + +/** + * Inline assertions to ensure the environment is correctly set. + */ +ok(isDevelopment || isStaging || isProduction, 'unknown environment') +ok(!(isDevelopment && isStaging), 'multiple environments detected') +ok(!(isDevelopment && isProduction), 'multiple environments detected') +ok(!(isStaging && isProduction), 'multiple environments detected') diff --git a/packages/env/tsconfig.json b/packages/env/tsconfig.json new file mode 100644 index 0000000..72b364c --- /dev/null +++ b/packages/env/tsconfig.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node", + "jest" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/errors/README.md b/packages/errors/README.md new file mode 100644 index 0000000..0adea55 --- /dev/null +++ b/packages/errors/README.md @@ -0,0 +1,12 @@ +# @chatbotkit-dev/errors + +The platform's error taxonomy. + +`SystemError` and its subclasses, and the reporting helpers that decide which errors are worth +reporting. The distinction matters: `captureException` filters on this taxonomy, so an error that +does not extend it is classified differently. + +Reports through [`@chatbotkit-dev/observability`](../observability), so it does not know which error +tracker, if any, a deployment runs. + +Extracted from `platform/lib/error.js`. diff --git a/packages/errors/jest.config.js b/packages/errors/jest.config.js new file mode 100644 index 0000000..8a86699 --- /dev/null +++ b/packages/errors/jest.config.js @@ -0,0 +1,19 @@ +// @note CommonJS transform: these tests use `jest.mock` hoisting and the `jest` +// global, neither of which is available under the ESM preset. + +export default { + preset: 'ts-jest', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', + + transform: { + '^.+\\.[jt]sx?$': [ + 'ts-jest', + { + useESM: false, + // @note transpile only. Type checking is the `check` script's job. + tsconfig: { module: 'commonjs', esModuleInterop: true, allowJs: true }, + }, + ], + }, +} diff --git a/packages/errors/package.json b/packages/errors/package.json new file mode 100644 index 0000000..6ee0818 --- /dev/null +++ b/packages/errors/package.json @@ -0,0 +1,42 @@ +{ + "name": "@chatbotkit-dev/errors", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "@chatbotkit-dev/observability": "workspace:*", + "joi": "^17.7.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/errors/src/index.test.js b/packages/errors/src/index.test.js new file mode 100644 index 0000000..eea8d3e --- /dev/null +++ b/packages/errors/src/index.test.js @@ -0,0 +1,904 @@ +/* eslint-disable @typescript-eslint/no-require-imports */ +import { UnexpectedStateError, captureUnexpectedState } from './index' + +jest.mock('@chatbotkit-dev/observability', () => ({ + __esModule: true, + + default: { + captureException: jest.fn(), + captureMessage: jest.fn(), + setTag: jest.fn(), + }, +})) + +describe('UnexpectedStateError', () => { + it('should create error with message only', () => { + const message = 'Test error message' + const error = new UnexpectedStateError(message) + + expect(error.message).toBe(message) + expect(error.data).toBeUndefined() + expect(error).toBeInstanceOf(Error) + expect(error).toBeInstanceOf(UnexpectedStateError) + }) + + it('should create error with message and context', () => { + const message = 'Test error message' + const context = { key: 'value' } + const error = new UnexpectedStateError(message, context) + + expect(error.message).toBe(message) + expect(error.data).toEqual(context) + expect(error).toBeInstanceOf(Error) + expect(error).toBeInstanceOf(UnexpectedStateError) + }) + + it('should handle null context', () => { + const message = 'Test error message' + const error = new UnexpectedStateError(message, null) + + expect(error.message).toBe(message) + expect(error.data).toBeUndefined() + }) + + it('should handle undefined context', () => { + const message = 'Test error message' + const error = new UnexpectedStateError(message, undefined) + + expect(error.message).toBe(message) + expect(error.data).toBeUndefined() + }) + + it('should handle string context', () => { + const message = 'Test error message' + const context = 'string context' + const error = new UnexpectedStateError(message, context) + + expect(error.message).toBe(message) + expect(error.data).toEqual(context) + }) + + it('should handle complex object context', () => { + const message = 'Test error message' + + const context = { + module: 'test.js', + function: 'testFunction', + details: { key: 'value', nested: { prop: 'test' } }, + } + + const error = new UnexpectedStateError(message, context) + + expect(error.message).toBe(message) + expect(error.data).toEqual(context) + expect(error.data.module).toBe('test.js') + expect(error.data.function).toBe('testFunction') + expect(error.data.details.nested.prop).toBe('test') + }) +}) + +describe('captureUnexpectedState observability integration', () => { + const observability = jest.requireMock('@chatbotkit-dev/observability') + .default + const mockCaptureException = jest.mocked(observability.captureException) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should send context data to the reporter when provided', async () => { + const message = 'Test error with context' + + const context = { + userId: '123', + action: 'testAction', + requestId: 'req-456', + } + + await captureUnexpectedState(message, context) + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + + const [error, reportedContext] = mockCaptureException.mock.calls[0] + + // verify the error is an UnexpectedStateError with correct message + + expect(error).toBeInstanceOf(UnexpectedStateError) + expect(error.message).toBe(message) + expect(error.data).toEqual(context) + + // verify the context is passed to the reporter + + expect(reportedContext).toEqual(context) + }) + + it('should send string context to the reporter', async () => { + const message = 'Test error with string context' + const context = 'debug information string' + + await captureUnexpectedState(message, context) + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + + const [error, reportedContext] = mockCaptureException.mock.calls[0] + + expect(error).toBeInstanceOf(UnexpectedStateError) + expect(error.message).toBe(message) + expect(error.data).toBe(context) + expect(reportedContext).toBe(context) + }) + + it('should send complex nested context to the reporter', async () => { + const message = 'Test error with complex context' + + const context = { + request: { + method: 'POST', + url: '/api/test', + headers: { 'content-type': 'application/json' }, + body: { field: 'value' }, + }, + user: { + id: 'user-123', + email: 'test@example.com', + }, + metadata: { + timestamp: '2025-08-13T10:00:00Z', + version: '1.0.0', + }, + } + + await captureUnexpectedState(message, context) + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + + const [error, reportedContext] = mockCaptureException.mock.calls[0] + + expect(error).toBeInstanceOf(UnexpectedStateError) + expect(error.message).toBe(message) + expect(error.data).toEqual(context) + expect(reportedContext).toEqual(context) + + // verify nested structure is preserved + + expect(reportedContext.request.method).toBe('POST') + expect(reportedContext.user.id).toBe('user-123') + expect(reportedContext.metadata.version).toBe('1.0.0') + }) + + it('should send undefined to the reporter when no context provided', async () => { + const message = 'Test error without context' + + await captureUnexpectedState(message) + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + + const [error, reportedContext] = mockCaptureException.mock.calls[0] + + expect(error).toBeInstanceOf(UnexpectedStateError) + expect(error.message).toBe(message) + expect(error.data).toBeUndefined() + expect(reportedContext).toBeUndefined() + }) + + it('should send undefined to the reporter when null context provided', async () => { + const message = 'Test error with null context' + + await captureUnexpectedState(message, null) + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + + const [error, reportedContext] = mockCaptureException.mock.calls[0] + + expect(error).toBeInstanceOf(UnexpectedStateError) + expect(error.message).toBe(message) + expect(error.data).toBeUndefined() + expect(reportedContext).toBeUndefined() + }) + + it('should send undefined to the reporter when undefined context provided', async () => { + const message = 'Test error with undefined context' + + await captureUnexpectedState(message, undefined) + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + + const [error, reportedContext] = mockCaptureException.mock.calls[0] + + expect(error).toBeInstanceOf(UnexpectedStateError) + expect(error.message).toBe(message) + expect(error.data).toBeUndefined() + expect(reportedContext).toBeUndefined() + }) + + it('should handle reporter errors gracefully', async () => { + const message = 'Test error with the reporter failure' + const context = { key: 'value' } + + // mock the reporter to throw an error + + mockCaptureException.mockRejectedValueOnce( + new Error('the reporter service unavailable') + ) + + // this should not throw even if the reporter fails + + await expect( + captureUnexpectedState(message, context) + ).resolves.toBeUndefined() + + expect(mockCaptureException).toHaveBeenCalledTimes(1) + }) +}) + +describe('captureUnexpectedState backward compatibility', () => { + it('should accept message-only calls without throwing', async () => { + // this should not throw an error + + await expect( + captureUnexpectedState('Test message') + ).resolves.toBeUndefined() + }) + + it('should accept message and context calls without throwing', async () => { + // this should not throw an error + + await expect( + captureUnexpectedState('Test message', { key: 'value' }) + ).resolves.toBeUndefined() + }) + + it('should accept message and string context calls without throwing', async () => { + // this should not throw an error + + await expect( + captureUnexpectedState('Test message', 'string context') + ).resolves.toBeUndefined() + }) + + it('should accept message and null context calls without throwing', async () => { + // this should not throw an error + + await expect( + captureUnexpectedState('Test message', null) + ).resolves.toBeUndefined() + }) + + it('should accept message and undefined context calls without throwing', async () => { + // this should not throw an error + + await expect( + captureUnexpectedState('Test message', undefined) + ).resolves.toBeUndefined() + }) +}) + +describe('captureException excluded errors', () => { + const observability = jest.requireMock('@chatbotkit-dev/observability') + .default + const mockCaptureException = jest.mocked(observability.captureException) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should not send BotInputError to the reporter', async () => { + const { BotInputError, captureException } = await import('./index') + + const error = new BotInputError( + 'Invalid input for ability "test": missing required field' + ) + + await captureException(error) + + // @note BotInputError should be excluded from the reporter logging + expect(mockCaptureException).not.toHaveBeenCalled() + }) + + it('should not send UserInputError to the reporter', async () => { + const { UserInputError, captureException } = await import('./index') + + const error = new UserInputError('Invalid user input') + + await captureException(error) + + expect(mockCaptureException).not.toHaveBeenCalled() + }) + + it('should not send UserAuthError to the reporter', async () => { + const { UserAuthError, captureException } = await import('./index') + + const error = new UserAuthError('Authentication failed') + + await captureException(error) + + expect(mockCaptureException).not.toHaveBeenCalled() + }) + + it('should send other errors to the reporter', async () => { + const { captureException } = await import('./index') + + const error = new Error('Some other error') + + await captureException(error) + + expect(mockCaptureException).toHaveBeenCalledWith(error, undefined) + }) + + it('should not send ObservationError to the reporter', async () => { + const { ObservationError, captureException } = await import('./index') + + const error = new ObservationError( + 'skillset action returned large response', + { tokenCount: 150000 } + ) + + await captureException(error) + + // @note ObservationError should be excluded from the reporter logging + expect(mockCaptureException).not.toHaveBeenCalled() + }) + + it('should not send ContentModerationError to the reporter', async () => { + const { ContentModerationError, captureException } = await import('./index') + + const error = new ContentModerationError('Inappropriate content (400)') + + await captureException(error) + + // @note content moderation rejections are expected provider behaviour + expect(mockCaptureException).not.toHaveBeenCalled() + }) + + it('should not send UnexpectedStateError via captureObservation to the reporter', async () => { + const { captureObservation } = await import('./index') + + await captureObservation('Chunked handler aborted', { runCount: 3 }) + + // @note captureObservation creates ObservationError which should not go to the reporter + expect(mockCaptureException).not.toHaveBeenCalled() + }) +}) + +describe('captureObservation the reporter opt-in', () => { + const observability = jest.requireMock('@chatbotkit-dev/observability') + .default + const mockCaptureException = jest.mocked(observability.captureException) + const mockCaptureMessage = jest.mocked(observability.captureMessage) + + beforeEach(() => { + jest.clearAllMocks() + jest.spyOn(console, 'log').mockImplementation() + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + + it('reports an opted-in observation to the reporter as a message, not an exception', async () => { + const { captureObservation } = await import('./index') + + const context = { + event: 'runaway_text_run_detected', + repeatedPhrase: 'let me call lint', + repeatCount: 6, + } + + await captureObservation('runaway text run detected', context, { + sentry: true, + level: 'warning', + }) + + expect(mockCaptureException).not.toHaveBeenCalled() + expect(mockCaptureMessage).toHaveBeenCalledTimes(1) + + const [message, captureContext] = mockCaptureMessage.mock.calls[0] + + expect(message).toBe('runaway text run detected') + expect(captureContext.level).toBe('warning') + expect(captureContext.extra).toEqual(context) + // @note grouped by the observation event so related stuck runs cluster + expect(captureContext.tags.observation).toBe('runaway_text_run_detected') + expect(captureContext.fingerprint).toEqual([ + 'observation', + 'runaway_text_run_detected', + ]) + }) + + it('defaults the the reporter level to warning when not specified', async () => { + const { captureObservation } = await import('./index') + + await captureObservation('call limit max reached', { event: 'x' }, { + sentry: true, + }) + + expect(mockCaptureMessage.mock.calls[0][1].level).toBe('warning') + }) + + it('keeps non-opted-in observations out of the reporter entirely', async () => { + const { captureObservation } = await import('./index') + + await captureObservation('Slow function call', { ms: 1200 }) + + expect(mockCaptureException).not.toHaveBeenCalled() + expect(mockCaptureMessage).not.toHaveBeenCalled() + }) +}) + +describe('captureError console.trace behavior', () => { + let consoleTraceSpy + + beforeEach(() => { + consoleTraceSpy = jest.spyOn(console, 'trace').mockImplementation() + jest.spyOn(console, 'error').mockImplementation() + }) + + afterEach(() => { + consoleTraceSpy.mockRestore() + jest.restoreAllMocks() + }) + + it('should call console.trace when error has stack', async () => { + const { captureError } = await import('./index') + + const error = new Error('Test error with stack') + + await captureError(error) + + expect(consoleTraceSpy).toHaveBeenCalledWith(error) + }) + + it('should not call console.trace when error lacks stack', async () => { + const { captureError } = await import('./index') + + const error = { message: 'Error without stack' } + + await captureError(error) + + expect(consoleTraceSpy).not.toHaveBeenCalled() + }) + + it('should not call console.trace for undefined error', async () => { + const { captureError } = await import('./index') + + await captureError(undefined) + + expect(consoleTraceSpy).not.toHaveBeenCalled() + }) + + it('should not call console.trace for null error', async () => { + const { captureError } = await import('./index') + + await captureError(null) + + expect(consoleTraceSpy).not.toHaveBeenCalled() + }) + + it('should not call console.trace for ObservationError', async () => { + const { captureError, ObservationError } = await import('./index') + + const error = new ObservationError('Test observation', { data: 'test' }) + + await captureError(error) + + // @note ObservationError should not reach console.trace as it's filtered out + expect(consoleTraceSpy).not.toHaveBeenCalled() + }) +}) + +describe('captureException console.trace behavior', () => { + let consoleTraceSpy + + beforeEach(() => { + consoleTraceSpy = jest.spyOn(console, 'trace').mockImplementation() + jest.spyOn(console, 'error').mockImplementation() + }) + + afterEach(() => { + consoleTraceSpy.mockRestore() + jest.restoreAllMocks() + }) + + it('should call console.trace when error has stack', async () => { + const { captureException } = await import('./index') + + const error = new Error('Test error with stack') + + await captureException(error) + + expect(consoleTraceSpy).toHaveBeenCalledWith(error) + }) + + it('should not call console.trace when error lacks stack', async () => { + const { captureException } = await import('./index') + + const error = { message: 'Error without stack' } + + await captureException(error) + + expect(consoleTraceSpy).not.toHaveBeenCalled() + }) +}) + +describe('errorToErrorResponse', () => { + it('should handle undefined error', () => { + const { errorToErrorResponse } = require('./index') + + const result = errorToErrorResponse(undefined) + + expect(result).toEqual({ + code: 'GENERIC_ERROR', + message: 'An unknown error occurred', + }) + }) + + it('should handle null error', () => { + const { errorToErrorResponse } = require('./index') + + const result = errorToErrorResponse(null) + + expect(result).toEqual({ + code: 'GENERIC_ERROR', + message: 'An unknown error occurred', + }) + }) + + it('should handle Error instance', () => { + const { errorToErrorResponse } = require('./index') + + const error = new Error('Test error') + const result = errorToErrorResponse(error) + + expect(result).toEqual({ + code: 'GENERIC_ERROR', + message: 'Test error', + }) + }) + + it('should handle SystemError instance', () => { + const { errorToErrorResponse, SystemError } = require('./index') + + const error = new SystemError('Test system error', 'TEST_CODE') + const result = errorToErrorResponse(error) + + expect(result).toEqual({ + code: 'TEST_CODE', + message: 'Test system error', + }) + }) + + it('should handle string error', () => { + const { errorToErrorResponse } = require('./index') + + const result = errorToErrorResponse('String error message') + + expect(result).toEqual({ + code: 'GENERIC_ERROR', + message: 'String error message', + }) + }) +}) + +describe('errorToSafeErrorResponse', () => { + it('should expose SafeError details', () => { + const { SafeError, errorToSafeErrorResponse } = require('./index') + + const error = new SafeError('Visible message', 'VISIBLE_CODE') + const result = errorToSafeErrorResponse(error) + + expect(result).toEqual({ + code: 'VISIBLE_CODE', + message: 'Visible message', + }) + }) + + it('should hide regular Error details', () => { + const { errorToSafeErrorResponse } = require('./index') + + const result = errorToSafeErrorResponse(new Error('Internal detail')) + + expect(result).toEqual({ + code: 'GENERIC_ERROR', + message: 'Something went wrong', + }) + }) + + it('should hide SystemError details unless explicitly safe', () => { + const { SystemError, errorToSafeErrorResponse } = require('./index') + + const error = new SystemError('Internal system detail', 'INTERNAL_CODE') + const result = errorToSafeErrorResponse(error) + + expect(result).toEqual({ + code: 'GENERIC_ERROR', + message: 'Something went wrong', + }) + }) +}) + +describe('ContentModerationError', () => { + it('is a SafeError carrying the CONTENT_MODERATION code and no data', () => { + const { + ContentModerationError, + SafeError, + CONTENT_MODERATION_ERROR_CODE, + } = require('./index') + + const error = new ContentModerationError('Inappropriate content (400)') + + expect(error).toBeInstanceOf(SafeError) + expect(error.code).toBe(CONTENT_MODERATION_ERROR_CODE) + expect(error.message).toBe('Inappropriate content (400)') + expect(error.data).toBeUndefined() + }) + + it('does not leak any data when serialized', () => { + const { ContentModerationError } = require('./index') + + const serialized = JSON.parse( + JSON.stringify(new ContentModerationError('blocked (400)')) + ) + + // @note message is non-enumerable on Error, code is the only own prop + expect(serialized).toEqual({ code: 'CONTENT_MODERATION' }) + expect(serialized).not.toHaveProperty('data') + expect(serialized).not.toHaveProperty('body') + }) + + it('is exposed through errorToSafeErrorResponse', () => { + const { ContentModerationError, errorToSafeErrorResponse } = require('./index') + + const result = errorToSafeErrorResponse( + new ContentModerationError('blocked (400)') + ) + + expect(result).toEqual({ + code: 'CONTENT_MODERATION', + message: 'blocked (400)', + }) + }) +}) + +describe('isContentModerationError', () => { + it('detects ContentModerationError instances', () => { + const { ContentModerationError, isContentModerationError } = require('./index') + + expect(isContentModerationError(new ContentModerationError('x'))).toBe(true) + }) + + it('detects errors carrying the CONTENT_MODERATION code', () => { + const { SystemError, isContentModerationError } = require('./index') + + expect( + isContentModerationError(new SystemError('x', 'CONTENT_MODERATION')) + ).toBe(true) + }) + + it('returns false for unrelated errors and nullish values', () => { + const { SystemError, isContentModerationError } = require('./index') + + expect(isContentModerationError(new SystemError('x', 'VR_BAD_REQUEST'))).toBe( + false + ) + expect(isContentModerationError(new Error('x'))).toBe(false) + expect(isContentModerationError(null)).toBe(false) + expect(isContentModerationError(undefined)).toBe(false) + }) +}) + +describe('extractCauseChain', () => { + it('returns undefined when there is no cause', () => { + const { extractCauseChain } = require('./index') + + expect(extractCauseChain(new Error('terminated'))).toBeUndefined() + expect(extractCauseChain(null)).toBeUndefined() + expect(extractCauseChain(undefined)).toBeUndefined() + expect(extractCauseChain('a raw string')).toBeUndefined() + }) + + it('summarizes a single undici-style cause (name, message, code)', () => { + const { extractCauseChain } = require('./index') + + const cause = Object.assign(new Error('other side closed'), { + code: 'UND_ERR_SOCKET', + }) + + const error = new Error('terminated', { cause }) + + expect(extractCauseChain(error)).toEqual([ + { name: 'Error', message: 'other side closed', code: 'UND_ERR_SOCKET' }, + ]) + }) + + it('walks a nested cause chain', () => { + const { extractCauseChain } = require('./index') + + const root = Object.assign(new Error('ECONNRESET'), { code: 'ECONNRESET' }) + const mid = new Error('socket hang up', { cause: root }) + const top = new Error('terminated', { cause: mid }) + + expect(extractCauseChain(top)).toEqual([ + { name: 'Error', message: 'socket hang up', code: undefined }, + { name: 'Error', message: 'ECONNRESET', code: 'ECONNRESET' }, + ]) + }) + + it('caps the chain at the maximum depth', () => { + const { extractCauseChain } = require('./index') + + let error = new Error('root') + + for (let i = 0; i < 8; i++) { + error = new Error(`level-${i}`, { cause: error }) + } + + expect(extractCauseChain(error)).toHaveLength(5) + }) + + it('does not loop forever on a circular cause chain', () => { + const { extractCauseChain } = require('./index') + + const a = new Error('a') + const b = new Error('b', { cause: a }) + + a.cause = b // @note circular + + expect(extractCauseChain(b)).toHaveLength(2) + }) + + it('handles a string cause', () => { + const { extractCauseChain } = require('./index') + + const error = new Error('wrap') + + error.cause = 'raw string reason' + + expect(extractCauseChain(error)).toEqual([ + { name: undefined, message: 'raw string reason', code: undefined }, + ]) + }) +}) + +describe('captureException cause context', () => { + const observability = jest.requireMock('@chatbotkit-dev/observability') + .default + const mockCaptureException = jest.mocked(observability.captureException) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('attaches the cause chain under extra.cause', async () => { + const { captureException } = require('./index') + + const cause = Object.assign(new Error('other side closed'), { + code: 'UND_ERR_SOCKET', + }) + + const error = new Error('terminated', { cause }) + + await captureException(error) + + const [captured, context] = mockCaptureException.mock.calls[0] + + expect(captured).toBe(error) + expect(context).toEqual({ + extra: { + cause: [ + { + name: 'Error', + message: 'other side closed', + code: 'UND_ERR_SOCKET', + }, + ], + }, + }) + }) + + it('passes undefined context when there is no cause and no data', async () => { + const { captureException } = require('./index') + + const error = new Error('plain') + + await captureException(error) + + expect(mockCaptureException).toHaveBeenCalledWith(error, undefined) + }) + + it('merges the cause into existing error.data without clobbering it', async () => { + const { SystemError, captureException } = require('./index') + + const cause = Object.assign(new Error('boom'), { code: 'ECONNRESET' }) + + const error = new SystemError('wrapped', 'GENERIC_ERROR', { + foo: 'bar', + extra: { existing: true }, + }) + + error.cause = cause + + await captureException(error) + + const [, context] = mockCaptureException.mock.calls[0] + + expect(context).toEqual({ + foo: 'bar', + extra: { + existing: true, + cause: [{ name: 'Error', message: 'boom', code: 'ECONNRESET' }], + }, + }) + }) +}) + +describe('errorToSystemError cause preservation', () => { + const observability = jest.requireMock('@chatbotkit-dev/observability') + .default + const mockCaptureException = jest.mocked(observability.captureException) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('preserves the original error as the cause when wrapping', () => { + const { SystemError, errorToSystemError } = require('./index') + + const original = Object.assign(new Error('terminated'), { + code: 'UND_ERR_SOCKET', + }) + + const wrapped = errorToSystemError(original) + + expect(wrapped).toBeInstanceOf(SystemError) + expect(wrapped).not.toBe(original) + expect(wrapped.cause).toBe(original) + }) + + it('returns an existing SystemError untouched (no cause added)', () => { + const { SystemError, errorToSystemError } = require('./index') + + const sys = new SystemError('x', 'CODE') + + expect(errorToSystemError(sys)).toBe(sys) + expect(sys.cause).toBeUndefined() + }) + + it('surfaces the preserved cause to the reporter after wrapping', async () => { + const { errorToSystemError, captureException } = require('./index') + + const original = Object.assign(new Error('terminated'), { + code: 'UND_ERR_SOCKET', + }) + + await captureException(errorToSystemError(original)) + + const [, context] = mockCaptureException.mock.calls[0] + + expect(context.extra.cause).toEqual([ + { name: 'Error', message: 'terminated', code: 'UND_ERR_SOCKET' }, + ]) + }) + + it('keeps the cause non-enumerable so it cannot leak via serialization', () => { + const { errorToSystemError } = require('./index') + + const original = Object.assign(new Error('terminated'), { + code: 'UND_ERR_SOCKET', + requestBody: 'super-secret', + }) + + const wrapped = errorToSystemError(original) + + // @note readable for extractCauseChain/the reporter... + expect(wrapped.cause).toBe(original) + + // @note ...but invisible to enumeration and JSON serialization, so an + // accidental raw serialization of the SystemError can never expose the + // underlying error to the client. + expect(Object.prototype.propertyIsEnumerable.call(wrapped, 'cause')).toBe( + false + ) + expect(Object.keys(wrapped)).not.toContain('cause') + expect(JSON.stringify(wrapped)).not.toContain('super-secret') + }) +}) diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts new file mode 100644 index 0000000..a8d84f4 --- /dev/null +++ b/packages/errors/src/index.ts @@ -0,0 +1,714 @@ +// @ts-check +import observability from '@chatbotkit-dev/observability' + +import joi from 'joi' + +/** + * @todo use from response if possible + */ +export const GENERIC_ERROR_CODE = 'GENERIC_ERROR' +export const TIMEOUT_ERROR_CODE = 'TIMEOUT' +export const BAD_REQUEST_ERROR_CODE = 'BAD_REQUEST' +export const CONFLICT_REQUEST_ERROR_CODE = 'CONFLICT' +export const NOT_FOUND_ERROR_CODE = 'NOT_FOUND' +export const NOT_AUTHENTICATED_ERROR_CODE = 'NOT_AUTHENTICATED' +export const CONTENT_MODERATION_ERROR_CODE = 'CONTENT_MODERATION' + +/** + * Represents any error that is thrown by the system. + */ +/** + * A value that was thrown. JavaScript lets anything be thrown, and the helpers + * below exist precisely to make sense of whatever arrived - an `Error`, a + * string, a plain object from a foreign SDK - so the parameter type is `any` + * on purpose, declared once here rather than at every signature. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type Thrown = any + +export class SystemError extends Error { + public code: string + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public data: any + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(message: string, code: string, data?: any) { + super(message) + + this.code = code + this.data = data + } +} + +/** + * Represents an error that is composed of multiple errors. + */ +export class CompositeError extends SystemError { + public errors: Error[] + + constructor(message: string, code: string, errors: Error[]) { + super(message, code) + + this.errors = errors + } +} + +/** + * Represents an error that is safe to show to the user. + */ +export class SafeError extends SystemError { + constructor(message: string, code: string) { + super(message, code) + } +} + +/** + * Represents an error that is not safe to show to the user. + */ +export class UnsafeError extends SystemError { + constructor(message: string, code: string) { + super(message, code) + } +} + +/** + * Represents an error that is related to the user input. + */ +export class UserInputError extends SafeError { + constructor(message: string) { + super(message, BAD_REQUEST_ERROR_CODE) + } +} + +/** + * Represents an error that is related to the user authentication. + */ +export class UserAuthError extends SafeError { + constructor(message: string) { + super(message, NOT_AUTHENTICATED_ERROR_CODE) + } +} + +/** + * Represents an error that is related to a user resource not being found. + */ +export class UserResourceNotFoundError extends SafeError { + constructor(message: string) { + super(message, NOT_FOUND_ERROR_CODE) + } +} + +/** + * Represents an error that is related to bot input. + */ +export class SafeInputError extends SafeError { + constructor(message: string) { + super(message, BAD_REQUEST_ERROR_CODE) + } +} + +/** + * Represents an error that is related to bot input. + */ +export class BotInputError extends SafeError { + constructor(message: string) { + super(message, BAD_REQUEST_ERROR_CODE) + } +} + +/** + * Represents a rejection by a provider-side content moderation / safety filter. + * + * @note These are not malformed requests - the provider refused to process the + * input on policy grounds. They are expected provider behaviour rather than + * bugs, so they are SafeError (the message is meaningful to the caller) and are + * excluded from Sentry bug tracking. They deliberately carry no `data` payload + * so the offending request body is never serialized to a client. + */ +export class ContentModerationError extends SafeError { + constructor(message: string) { + super(message, CONTENT_MODERATION_ERROR_CODE) + } +} + +/** + * Returns true if the error represents a provider content moderation rejection. + * + */ +export function isContentModerationError(error: Thrown): boolean { + return ( + error instanceof ContentModerationError || + (!!error && error.code === CONTENT_MODERATION_ERROR_CODE) + ) +} + +/** + * Represents an error that is related to user configuration. + */ +export class UserConfigError extends UnsafeError { + constructor(message: string) { + super(message, BAD_REQUEST_ERROR_CODE) + } +} + +/** + * Represents an error that is related to the admin authentication + */ +export class AdminAuthError extends UnsafeError { + constructor(message: string) { + super(message, NOT_AUTHENTICATED_ERROR_CODE) + } +} + +/** + * + */ +export class ObservationError extends Error { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public data?: any + + /** + * Whether this observation should additionally be reported. Observations are + * log-only by default; high-signal ones (e.g. stuck-run detections) opt in so + * they are searchable and carry their context. + */ + public sentry: boolean + + /** + * Severity to use when the observation is reported. + */ + public level: 'info' | 'warning' | 'error' + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(message: string, data?: any) { + super(message) + + if (data !== undefined && data !== null) { + this.data = data + } + + this.sentry = false + this.level = 'warning' + } +} + +/** + */ +export class UnexpectedStateError extends Error { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public data?: any + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(message: string, data?: any) { + super(message) + + if (data !== undefined && data !== null) { + this.data = data + } + } +} + +/** + */ +export const KNOWN_PRISMA_ERRORS = [ + // prisma + // @see https://www.prisma.io/docs/reference/api-reference/error-reference#prismaclientvalidationerror + + 'PrismaClientKnownRequestError', + 'PrismaClientUnknownRequestError', + 'PrismaClientRustPanicError', + 'PrismaClientInitializationError', + 'PrismaClientValidationError', +] + +/** + */ +export const KNOWN_TIMEOUT_ERRORS = [ + // timeout + + 'TimeoutError', +] + +/** + */ +export const KNOWN_SUBSCRIPTION_ERRORS = [ + // limits + + 'Limits reached', + 'You have exceeded your allocated database limits: database/dataset', + 'You have exceeded your allocated database limits: database/record', + 'You have exceeded your allocated database limits: database/skillset', + 'You have exceeded your allocated database limits: database/ability', + 'You have exceeded your allocated database limits: database/file', +] + +/** + * @returns boolean + */ +export function errorIn(error: Error, collection: string[]) { + if (!error || typeof error !== 'object') { + return false + } + + return collection.includes(error.name) || collection.includes(error.message) +} + +export function isKnownError(error: Error|string): boolean { + if (typeof error === 'string') { + error = new Error(error) + } + + for (const collection of [ + KNOWN_PRISMA_ERRORS, + KNOWN_TIMEOUT_ERRORS, + KNOWN_SUBSCRIPTION_ERRORS, + ]) { + if (errorIn(error, collection)) { + return true + } + } + + return false +} + +/** + * Converts any error to a standardized error response format, which includes a + * code and a message. + * + */ +export function errorToErrorResponse(error: Thrown): { + code: string + message: string +} { + // @note handle undefined/null errors explicitly to avoid confusing trace logs + + if (error === undefined || error === null) { + void captureUnexpectedState('Undefined or null error encountered') + + return { code: GENERIC_ERROR_CODE, message: 'An unknown error occurred' } + } + + switch (true) { + case error instanceof SystemError: { + return { code: error.code, message: error.message.toString() } + } + + case error instanceof joi.ValidationError: { + return { code: BAD_REQUEST_ERROR_CODE, message: error.message.toString() } + } + + case errorIn(error, KNOWN_PRISMA_ERRORS): { + if (error.code === 'P2002') { + return { + code: CONFLICT_REQUEST_ERROR_CODE, + message: 'Unique constraint violation', + } + } else { + return { code: GENERIC_ERROR_CODE, message: 'System error occurred' } + } + } + + case errorIn(error, KNOWN_TIMEOUT_ERRORS): { + return { code: TIMEOUT_ERROR_CODE, message: 'Response timeout' } + } + + case error instanceof Error: { + return { code: GENERIC_ERROR_CODE, message: error.message.toString() } + } + + case typeof error === 'string': { + return { code: GENERIC_ERROR_CODE, message: error } + } + + default: { + return { code: GENERIC_ERROR_CODE, message: 'Something went wrong' } + } + } +} + +/** + * Converts an error to a public error response. Only SafeError messages are + * exposed; all other errors are intentionally collapsed to a generic response. + * + */ +export function errorToSafeErrorResponse(error: Thrown): { + code: string + message: string +} { + if (error instanceof SafeError) { + return errorToErrorResponse(error) + } + + return { + code: GENERIC_ERROR_CODE, + message: 'Something went wrong', + } +} + +/** + * Converts an error response to a SystemError instance. + */ +export function errorResponseToError( + errorResponse: { code: string; message: string }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data?: any +): SystemError { + return new SystemError(errorResponse.message, errorResponse.code, data) +} + +/** + * Converts any error to a SystemError instance. + * + */ +export function errorToSystemError(error: Thrown, data?: unknown): SystemError { + if (error instanceof SystemError) { + return error + } + + const systemError = errorResponseToError(errorToErrorResponse(error), data) + + // @note preserve the original error as the `cause` so its underlying detail + // (e.g. an undici "terminated" whose real reason lives on its own `cause`) + // survives normalization and can be surfaced to Sentry via + // `extractCauseChain`. Defined non-enumerable - matching native `Error.cause` + // semantics - so that even an accidental raw serialization of the + // `SystemError` (e.g. `JSON.stringify`) can never leak the underlying error + // to the client. Client-facing responses only ever expose `{code, message}` + // via `errorToErrorResponse`/`errorToSafeErrorResponse`. + + if (error instanceof Error) { + Object.defineProperty(systemError, 'cause', { + value: error, + enumerable: false, + configurable: true, + writable: true, + }) + } + + return systemError +} + +export async function logError(e: Thrown): Promise { + if (process.env.NODE_ENV !== 'production') { + // eslint-disable-next-line + console.error(e) + } +} + +export async function debugError(e: Thrown): Promise { + if (!!process.env.DEBUG) { + // eslint-disable-next-line + console.error(e) + } +} + +export function setTag(name: string, value: string) { + observability.setTag(name, value) +} + +/** + * Handles an `ObservationError`: always logs it, and additionally reports it to + * Sentry as a (non-exception) message when the observation opted in via + * `sentry`. Observations are not bugs, so they are sent with their own severity + * and a stable fingerprint (keyed on the observation `event`) so related stuck + * runs group together and carry the supporting context needed to troubleshoot. + * + */ +function reportObservation(e: ObservationError): void { + // eslint-disable-next-line + console.log(`ObservationError: ${e.message}`) + + if (!e.sentry) { + return + } + + const event = e.data?.event + + try { + observability.captureMessage(e.message, { + level: e.level || 'warning', + tags: { observation: event || 'observation' }, + fingerprint: ['observation', event || e.message], + extra: e.data || undefined, + }) + } catch (err) { + // eslint-disable-next-line + console.error(err) + } +} + +const MAX_CAUSE_DEPTH = 5 + +/** + * Walks the `cause` chain of an error and returns a compact, serializable + * summary of each link. + * + * This is what lets us see *why* an otherwise opaque error happened. A Node + * `fetch` (undici) failure, for example, surfaces only as `Error: terminated` + * while the real reason - `ECONNRESET`, "other side closed", a body timeout - + * lives on `error.cause`. We attach this chain to the Sentry event server-side + * only (as `extra.cause`); it is never emitted to the client, which only ever + * sees `{code, message}` via `errorToErrorResponse`/`errorToSafeErrorResponse`. + * + */ +export function extractCauseChain( + error: Thrown +): + | Array<{ name?: string; message?: string; code?: string }> + | undefined { + /** @type {Array<{name?: string, message?: string, code?: string}>} */ + const chain: { + name: string | undefined + message: string | undefined + code: string | undefined + }[] = [] + + const seen = new Set() + + let current = error?.cause + + while ( + current !== undefined && + current !== null && + chain.length < MAX_CAUSE_DEPTH && + !seen.has(current) + ) { + seen.add(current) + + chain.push({ + name: typeof current.name === 'string' ? current.name : undefined, + + message: + typeof current.message === 'string' + ? current.message + : typeof current === 'string' + ? current + : undefined, + + code: + current.code !== undefined && current.code !== null + ? String(current.code) + : undefined, + }) + + current = typeof current === 'object' ? current.cause : undefined + } + + return chain.length > 0 ? chain : undefined +} + +/** + * Builds the Sentry capture context for an error, merging any existing + * `error.data` context with the serialized `cause` chain (under `extra.cause`). + * + * When the error has no cause, the original `error.data` is returned untouched + * - preserving reference identity and the prior capture behaviour. + * + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function buildCaptureContext(e: Thrown): any { + const cause = extractCauseChain(e) + + const data = e?.data + + if (!cause) { + return data ?? undefined + } + + // @note merge the cause into a structured `extra` without clobbering any + // context the error already carries. If `data` is not a plain object (e.g. a + // raw string passed as context) keep it intact alongside the cause. + + if (data && typeof data === 'object' && !Array.isArray(data)) { + return { + ...data, + + extra: { + ...(data.extra && typeof data.extra === 'object' ? data.extra : {}), + + cause, + }, + } + } + + return { + extra: data === undefined ? { cause } : { cause, data }, + } +} + +export async function captureError(e: Thrown): Promise { + // @note it is important to exclude some errors from being logged or even sent + // to Sentry, because it is a user-facing error that contains sensitive + // information + + if (e instanceof UserAuthError) { + // eslint-disable-next-line + console.log(`UserAuthError: ${e.message}`) + + return + } + + if (e instanceof UserInputError) { + // eslint-disable-next-line + console.log(`UserInputError: ${e.message}`) + + return + } + + if (e instanceof BotInputError) { + // eslint-disable-next-line + console.log(`BotInputError: ${e.message}`) + + return + } + + if (e instanceof UserConfigError) { + // eslint-disable-next-line + console.log(`UserConfigError: ${e.message}`) + + return + } + + // @note content moderation rejections are expected provider behaviour, not + // bugs - log them but keep them out of Sentry + if (e instanceof ContentModerationError) { + // eslint-disable-next-line + console.log(`ContentModerationError: ${e.message}`) + + return + } + + // @note ObservationError is for logging/analysis, not bug tracking - but an + // observation may opt in to Sentry (as a message, not an exception) + if (e instanceof ObservationError) { + reportObservation(e) + + return + } + + // eslint-disable-next-line + console.error(e) + + // @note only trace if error has a stack property to avoid "Trace: undefined" + if (e?.stack) { + // eslint-disable-next-line + console.trace(e) + } + + try { + await observability.captureException(e, buildCaptureContext(e)) + } catch (e) { + // eslint-disable-next-line + console.error(e) + } +} + +export async function captureException(e: Thrown): Promise { + // @note it is important to exclude some errors from being logged or even sent + // to Sentry, because it is a user-facing error that contains sensitive + // information + + if (e instanceof UserAuthError) { + // eslint-disable-next-line + console.log(`UserAuthError: ${e.message}`) + + return + } + + if (e instanceof UserInputError) { + // eslint-disable-next-line + console.log(`UserInputError: ${e.message}`) + + return + } + + if (e instanceof BotInputError) { + // eslint-disable-next-line + console.log(`BotInputError: ${e.message}`) + + return + } + + if (e instanceof UserConfigError) { + // eslint-disable-next-line + console.log(`UserConfigError: ${e.message}`) + + return + } + + // @note content moderation rejections are expected provider behaviour, not + // bugs - log them but keep them out of Sentry + if (e instanceof ContentModerationError) { + // eslint-disable-next-line + console.log(`ContentModerationError: ${e.message}`) + + return + } + + // @note ObservationError is for logging/analysis, not bug tracking - but an + // observation may opt in to Sentry (as a message, not an exception) + if (e instanceof ObservationError) { + reportObservation(e) + + return + } + + // eslint-disable-next-line + console.error(e) + + // @note only trace if error has a stack property to avoid "Trace: undefined" + if (e?.stack) { + // eslint-disable-next-line + console.trace(e) + } + + try { + await observability.captureException(e, buildCaptureContext(e)) + } catch (e) { + // eslint-disable-next-line + console.error(e) + } +} + +export async function captureInputError(e: Thrown, data: unknown): Promise { + // eslint-disable-next-line + console.error(e) + + // @note only trace if error has a stack property to avoid "Trace: undefined" + if (e?.stack) { + // eslint-disable-next-line + console.trace(e) + } + + try { + await observability.captureException(e, { extra: { data } }) + } catch (e) { + // eslint-disable-next-line + console.error(e) + } +} + +export async function captureObservation( + message: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + context?: any, + options: { sentry?: boolean; level?: 'info' | 'warning' | 'error' } = {} +): Promise { + const error = new ObservationError(message, context) + + if (options.sentry) { + error.sentry = true + error.level = options.level || 'warning' + } + + await captureError(error) +} + +export async function captureUnexpectedState( + message: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + context?: any +): Promise { + await captureError(new UnexpectedStateError(message, context)) +} diff --git a/packages/errors/tsconfig.json b/packages/errors/tsconfig.json new file mode 100644 index 0000000..72b364c --- /dev/null +++ b/packages/errors/tsconfig.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node", + "jest" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/eslint-config/index.js b/packages/eslint-config/index.js new file mode 100644 index 0000000..70b8f0c --- /dev/null +++ b/packages/eslint-config/index.js @@ -0,0 +1,202 @@ +module.exports = { + extends: ['plugin:@typescript-eslint/recommended', 'prettier'], + parser: '@typescript-eslint/parser', + parserOptions: { + project: true, + }, + plugins: ['@typescript-eslint', 'unused-imports', 'jsdoc'], + rules: { + 'no-console': 'error', + 'newline-before-return': 'error', + 'no-cond-assign': ['error', 'always'], + 'no-return-assign': ['error', 'always'], + curly: ['error', 'all'], + 'no-throw-literal': 'error', + 'space-before-blocks': 'error', + 'space-before-function-paren': [ + 'error', + { + named: 'never', + anonymous: 'always', + asyncArrow: 'always', + }, + ], + 'no-multi-spaces': 'error', + 'spaced-comment': ['error', 'always', { exceptions: ['*', '-', '+'] }], + 'no-multiple-empty-lines': ['error', { max: 1, maxEOF: 1 }], + 'padding-line-between-statements': [ + 'error', + { blankLine: 'always', prev: '*', next: 'return' }, + { blankLine: 'always', prev: 'return', next: '*' }, + // -- + { blankLine: 'always', prev: '*', next: 'for' }, + { blankLine: 'always', prev: 'for', next: '*' }, + { blankLine: 'always', prev: '*', next: 'while' }, + { blankLine: 'always', prev: 'while', next: '*' }, + { blankLine: 'always', prev: '*', next: 'do' }, + { blankLine: 'always', prev: 'do', next: '*' }, + // --- + { blankLine: 'always', prev: '*', next: 'if' }, + { blankLine: 'always', prev: 'if', next: '*' }, + { blankLine: 'always', prev: '*', next: 'switch' }, + { blankLine: 'always', prev: 'switch', next: '*' }, + // { blankLine: 'always', prev: '*', next: 'case' }, + // { blankLine: 'always', prev: 'case', next: '*' }, + { blankLine: 'always', prev: '*', next: 'break' }, + // --- + { blankLine: 'always', prev: 'break', next: '*' }, + { blankLine: 'always', prev: '*', next: 'continue' }, + { blankLine: 'always', prev: 'continue', next: '*' }, + // --- + { blankLine: 'always', prev: '*', next: 'class' }, + { blankLine: 'always', prev: 'class', next: '*' }, + { blankLine: 'always', prev: '*', next: 'function' }, + { blankLine: 'always', prev: 'function', next: '*' }, + // --- + { blankLine: 'always', prev: '*', next: 'try' }, + { blankLine: 'always', prev: 'try', next: '*' }, + { blankLine: 'always', prev: '*', next: 'throw' }, + { blankLine: 'always', prev: 'throw', next: '*' }, + // --- + { + blankLine: 'any', + prev: ['const', 'let', 'var'], + next: ['const', 'let', 'var'], + }, + { + blankLine: 'always', + prev: ['const', 'let', 'var'], + next: [ + 'expression', + 'block', + 'block-like', + 'if', + 'switch', + 'for', + 'while', + 'do', + 'try', + 'return', + 'throw', + 'function', + 'class', + ], + }, + { + blankLine: 'always', + prev: [ + 'expression', + 'block', + 'block-like', + 'if', + 'switch', + 'for', + 'while', + 'do', + 'try', + 'return', + 'throw', + 'function', + 'class', + ], + next: ['const', 'let', 'var'], + }, + // --- + // { + // blankLine: 'never', + // prev: 'singleline-const', + // next: 'singleline-const', + // }, + // { + // blankLine: 'never', + // prev: 'singleline-let', + // next: 'singleline-let', + // }, + // { + // blankLine: 'never', + // prev: 'singleline-var', + // next: 'singleline-var', + // }, + // --- + { + blankLine: 'always', + prev: ['block', 'block-like'], + next: ['multiline-const', 'multiline-let', 'multiline-var'], + }, + { + blankLine: 'always', + prev: ['multiline-const', 'multiline-let', 'multiline-var'], + next: ['block', 'block-like'], + }, + ], + '@typescript-eslint/no-wrapper-object-types': 'warn', + '@typescript-eslint/no-unused-vars': 'off', + 'no-unused-vars': 'off', + '@typescript-eslint/no-unused-expressions': 'off', + '@typescript-eslint/ban-ts-comment': 'off', + 'import/no-anonymous-default-export': ['off'], + '@typescript-eslint/no-floating-promises': 'error', + '@typescript-eslint/consistent-type-imports': 'error', + 'jsdoc/require-throws': 'error', + yoda: 'error', + 'unused-imports/no-unused-imports': ['error', { caughtErrors: 'none' }], + 'unused-imports/no-unused-vars': [ + 'warn', + { + vars: 'all', + varsIgnorePattern: '^_', + args: 'after-used', + argsIgnorePattern: '^_', + }, + ], + 'no-restricted-globals': [ + 'error', + { + name: 'fetch', + message: + 'Please import fetch explicitly (e.g., from "@/lib/fetch") instead of using the global fetch.', + }, + ], + }, + env: { + browser: true, + node: true, + es6: true, + }, + ignorePatterns: ['node_modules/'], + overrides: [ + { + files: [ + '**/*.test.js', + '**/*.test.jsx', + '**/*.test.ts', + '**/*.test.tsx', + '**/*.utest.js', + '**/*.utest.jsx', + '**/*.utest.ts', + '**/*.utest.tsx', + '**/*.itest.js', + '**/*.itest.jsx', + '**/*.itest.ts', + '**/*.itest.tsx', + ], + env: { + jest: true, + }, + }, + { + files: ['**/*.jsx'], + extends: ['plugin:@typescript-eslint/disable-type-checked'], + rules: { + '@typescript-eslint/consistent-type-imports': 'off', + }, + }, + { + files: ['**/*.js'], + extends: ['plugin:@typescript-eslint/disable-type-checked'], + rules: { + '@typescript-eslint/consistent-type-imports': 'off', + }, + }, + ], +} diff --git a/packages/eslint-config/package.json b/packages/eslint-config/package.json new file mode 100644 index 0000000..8554366 --- /dev/null +++ b/packages/eslint-config/package.json @@ -0,0 +1,36 @@ +{ + "name": "@chatbotkit-dev/eslint-config", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "commonjs", + "exports": { + ".": "./index.js" + }, + "scripts": { + "build": "true", + "check": "true", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "true", + "test": "true" + }, + "access": "restricted", + "dependencies": { + "@typescript-eslint/eslint-plugin": "^8.56.0", + "@typescript-eslint/parser": "^8.56.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-jsdoc": "^63.3.2", + "eslint-plugin-unused-imports": "^4.4.1" + }, + "peerDependencies": { + "eslint": "^9.0.0" + }, + "devDependencies": { + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5" + } +} diff --git a/packages/fetch/README.md b/packages/fetch/README.md new file mode 100644 index 0000000..391e85c --- /dev/null +++ b/packages/fetch/README.md @@ -0,0 +1,11 @@ +# @chatbotkit-dev/fetch + +The platform's fetch client. + +`fetchPlusPlus` is `withRetry(withTimeout(fetch))`: five attempts by default with exponential +backoff, a thirty second timeout, and retries on failed responses as well as thrown errors. + +Use this rather than reimplementing a retry loop. An earlier hand rolled copy in the email provider +silently used three attempts with no backoff and no retry on failed responses. + +Extracted from `platform/lib/fetch.js`. diff --git a/packages/fetch/jest.config.js b/packages/fetch/jest.config.js new file mode 100644 index 0000000..8b4f6e4 --- /dev/null +++ b/packages/fetch/jest.config.js @@ -0,0 +1,19 @@ +// @note CommonJS transform: these tests use `jest.mock` hoisting, which is not +// available under ESM. + +export default { + preset: 'ts-jest', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', + + transform: { + '^.+\\.[jt]sx?$': [ + 'ts-jest', + { + useESM: false, + // @note transpile only. Type checking is the `check` script's job. + tsconfig: { module: 'commonjs', esModuleInterop: true, allowJs: true }, + }, + ], + }, +} diff --git a/packages/fetch/package.json b/packages/fetch/package.json new file mode 100644 index 0000000..b5a15bd --- /dev/null +++ b/packages/fetch/package.json @@ -0,0 +1,43 @@ +{ + "name": "@chatbotkit-dev/fetch", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "scripts": { + "build": "tsc6 --emitDeclarationOnly", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js", + "test": "jest" + }, + "access": "restricted", + "types": "./types/src/index.d.ts", + "dependencies": { + "@chatbotkit-dev/debug": "workspace:*", + "@chatbotkit-dev/errors": "workspace:*", + "@chatbotkit-dev/http-codes": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/fetch/src/index.test.js b/packages/fetch/src/index.test.js new file mode 100644 index 0000000..bcd2bd0 --- /dev/null +++ b/packages/fetch/src/index.test.js @@ -0,0 +1,2009 @@ +import { + CONTEXT_MODEL_TAG, + FETCH_PHASE_RESPONSE_BODY, + FETCH_PHASE_RESPONSE_HEADERS, + FETCH_PHASE_TAG, + FetchError, + TIMEOUT_ERROR_NAME, + anySignal, + download, + fetch, + getFetchError, + isBodyStallTimeout, + jsonl, + withBodyTimeout, + withLimit, + withRetry, + withTimeout, +} from './index' + +jest.mock('@chatbotkit-dev/http-codes', () => ({ + statusToMessageMap: { + 400: 'Bad Request', + 401: 'Unauthorized', + 404: 'Not Found', + 500: 'Internal Server Error', + 502: 'Bad Gateway', + 503: 'Service Unavailable', + }, + statusToCodeMap: { + 400: 'BAD_REQUEST', + 401: 'UNAUTHORIZED', + 404: 'NOT_FOUND', + 500: 'INTERNAL_SERVER_ERROR', + 502: 'BAD_GATEWAY', + 503: 'SERVICE_UNAVAILABLE', + }, +})) + +describe('withTimeout', () => { + it('must timeout', async () => { + const fetch = withTimeout( + async (url, { signal }) => { + return new Promise((resolve, reject) => { + // We expect to receive an abort event from the timeout handler, which + // we use to reject the promise. + + signal.addEventListener('abort', () => { + reject(new Error(signal.reason)) + }) + }) + }, + { timeout: 1000 } + ) + + await expect(async () => { + await fetch() + }).rejects.toThrow() + }) + + it('attaches diagnostics (model, url, timeout budget) to the TimeoutError', async () => { + const fetch = withTimeout( + async (url, { signal }) => { + // @note mirror undici: a timed-out fetch rejects with the abort reason + // (the TimeoutError we passed to abort), not a wrapped Error + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason)) + }) + }, + { timeout: 5 } + ) + + let error + + try { + await fetch('https://gateway.example/v1/chat/completions', { + meta: { model: 'test-model' }, + }) + } catch (e) { + error = e + } + + expect(error?.name).toBe(TIMEOUT_ERROR_NAME) + expect(error?.data?.tags?.[CONTEXT_MODEL_TAG]).toBe('test-model') + expect(error?.data?.tags?.[FETCH_PHASE_TAG]).toBe( + FETCH_PHASE_RESPONSE_HEADERS + ) + // @note a header-phase timeout is retried at the fetch layer, so the + // streaming recogniser must NOT claim it as a (downstream) body stall + expect(isBodyStallTimeout(error)).toBe(false) + expect(error?.data?.extra?.fetch?.url).toBe( + 'https://gateway.example/v1/chat/completions' + ) + expect(error?.data?.extra?.fetch?.timeoutMs).toBe(5) + expect(error?.data?.extra?.fetch?.model).toBe('test-model') + }) + + if (process.env.SLOW_TESTS) { + it('httpstat: must timeout', async () => { + let error + + const f = withTimeout(fetch, { timeout: 5000 }) + + try { + await f('https://httpstat.us/504?sleep=60000') + } catch (e) { + error = e + } + + expect(error).toBeTruthy() + }) + } +}) + +describe('withBodyTimeout', () => { + const encoder = new TextEncoder() + + /** + * Build an ok Response that streams the given chunks and closes immediately. + * + * @param {string[]} chunks + * @param {{ status?: number }} [opts] + * @returns {Response} + */ + function makeStreamingResponse(chunks, { status = 200 } = {}) { + const stream = new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue(encoder.encode(chunk)) + } + + controller.close() + }, + }) + + return new Response(stream, { status }) + } + + /** + * Build an ok Response whose headers are delivered but whose body never emits + * a byte and never closes - a stalled upstream. + * + * @returns {Response} + */ + function makeStallingResponse() { + const stream = new ReadableStream({ + start() { + // @note never enqueue, never close + }, + }) + + return new Response(stream, { status: 200 }) + } + + /** + * @param {Response} response + * @returns {Promise} + */ + async function readAll(response) { + const decoder = new TextDecoder() + + let out = '' + + // @ts-ignore - body is async-iterable (polyfilled in @/lib/fetch) + for await (const chunk of response.body) { + out += decoder.decode(chunk) + } + + return out + } + + it('times out and attributes the stall when the body goes silent', async () => { + jest.useFakeTimers() + + try { + const fetch = withBodyTimeout(async () => makeStallingResponse(), { + bodyTimeout: 1000, + }) + + const response = await fetch( + 'https://gateway.example/v1/chat/completions', + { meta: { model: 'test-model' } } + ) + + let settled = false + + // @note consume in the background; it must reject once the idle timer fires + const consume = (async () => { + try { + await readAll(response) + + return null + } catch (e) { + return e + } finally { + settled = true + } + })() + + // @note just shy of the deadline: still streaming, nothing settled + await jest.advanceTimersByTimeAsync(999) + + expect(settled).toBe(false) + + // @note crossing the deadline fires the idle timer + await jest.advanceTimersByTimeAsync(1) + + const error = await consume + + expect(settled).toBe(true) + expect(error?.name).toBe(TIMEOUT_ERROR_NAME) + expect(error?.data?.tags?.[FETCH_PHASE_TAG]).toBe(FETCH_PHASE_RESPONSE_BODY) + expect(error?.data?.tags?.[CONTEXT_MODEL_TAG]).toBe('test-model') + // @note the streaming layer keys its retry decision off exactly this + // error shape, so assert the recogniser agrees - this ties the producer + // (withBodyTimeout) to the consumer (isBodyStallTimeout) end to end + expect(isBodyStallTimeout(error)).toBe(true) + expect(error?.data?.extra?.fetch?.bodyTimeoutMs).toBe(1000) + expect(error?.data?.extra?.fetch?.url).toBe( + 'https://gateway.example/v1/chat/completions' + ) + } finally { + jest.useRealTimers() + } + }) + + it('resets the idle timer on each chunk so a steady stream outlives the timeout', async () => { + jest.useFakeTimers() + + try { + const gap = 800 + + const bodyTimeout = 1000 + + const count = 5 + + let i = 0 + + // @note deliver one chunk per pull, each `gap` (< bodyTimeout) after the + // previous read. Total elapsed ((count + 1) * gap = 4800ms) far exceeds + // bodyTimeout, so only a correct per-chunk reset keeps the stream alive; a + // broken reset would fire at t=1000ms (mid second gap) and reject. + // @note pull returns a promise that settles only after the delayed + // delivery, so the stream waits between pulls instead of re-invoking pull + // synchronously (which would schedule overlapping timers and double-close) + const source = new ReadableStream({ + pull(controller) { + return new Promise((resolve) => { + if (i < count) { + const value = String(i++) + + setTimeout(() => { + controller.enqueue(encoder.encode(value)) + + resolve() + }, gap) + } else { + setTimeout(() => { + controller.close() + + resolve() + }, gap) + } + }) + }, + }) + + const fetch = withBodyTimeout( + async () => new Response(source, { status: 200 }), + { bodyTimeout } + ) + + const response = await fetch('https://gateway.example') + + const collected = readAll(response) + + await jest.advanceTimersByTimeAsync((count + 1) * gap + 50) + + await expect(collected).resolves.toBe('01234') + } finally { + jest.useRealTimers() + } + }) + + it('preserves status, statusText, and headers on the wrapped response', async () => { + const original = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('hi')) + + controller.close() + }, + }), + { + status: 206, + statusText: 'Partial Content', + headers: { 'content-type': 'text/event-stream', 'x-trace': 'abc' }, + } + ) + + const fetch = withBodyTimeout(async () => original, { bodyTimeout: 1000 }) + + const response = await fetch('https://gateway.example') + + // @note an ok response IS wrapped (new object), but its metadata must carry + // over verbatim - downstream relies on status and content-type + expect(response).not.toBe(original) + expect(response.status).toBe(206) + expect(response.statusText).toBe('Partial Content') + expect(response.headers.get('content-type')).toBe('text/event-stream') + expect(response.headers.get('x-trace')).toBe('abc') + + await expect(readAll(response)).resolves.toBe('hi') + }) + + it('does not wrap non-ok responses', async () => { + const original = makeStreamingResponse(['boom'], { status: 500 }) + + const fetch = withBodyTimeout(async () => original, { bodyTimeout: 10 }) + + await expect(fetch('https://gateway.example')).resolves.toBe(original) + }) + + it('is a pass-through when the timeout is disabled', async () => { + const original = makeStreamingResponse(['x']) + + const fetch = withBodyTimeout(async () => original, { bodyTimeout: 0 }) + + await expect(fetch('https://gateway.example')).resolves.toBe(original) + }) + + it('lets a per-call bodyTimeout override the default', async () => { + jest.useFakeTimers() + + try { + // @note default disables the guard; the per-call option must re-enable it + const fetch = withBodyTimeout(async () => makeStallingResponse(), { + bodyTimeout: 0, + }) + + const response = await fetch('https://gateway.example', { + bodyTimeout: 500, + }) + + const consume = (async () => { + try { + await readAll(response) + + return null + } catch (e) { + return e + } + })() + + await jest.advanceTimersByTimeAsync(500) + + const error = await consume + + expect(error?.name).toBe('TimeoutError') + expect(error?.data?.extra?.fetch?.bodyTimeoutMs).toBe(500) + } finally { + jest.useRealTimers() + } + }) + + it('cancels the upstream body when the consumer stops early', async () => { + let cancelled = false + + const encoder = new TextEncoder() + + const source = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('one')) + + // @note never close - the consumer will break out before the end + }, + + cancel() { + cancelled = true + }, + }) + + const fetch = withBodyTimeout( + async () => new Response(source, { status: 200 }), + { bodyTimeout: 1000 } + ) + + const response = await fetch('https://gateway.example') + + let first + + // @ts-ignore - body is async-iterable (polyfilled in @/lib/fetch) + for await (const chunk of response.body) { + first = new TextDecoder().decode(chunk) + + break + } + + // @note let the cancel propagate through guarded -> source + await new Promise((resolve) => setTimeout(resolve, 10)) + + expect(first).toBe('one') + expect(cancelled).toBe(true) + }) + + it('propagates a mid-stream source error unchanged (not masked as a timeout)', async () => { + const encoder = new TextEncoder() + + let delivered = false + + // @note deliver one chunk, then error on the next pull. Erroring + // synchronously right after enqueue would reset the queue (spec) and drop + // the chunk, so we split it across two pulls to mimic a real mid-stream cut. + const source = new ReadableStream({ + pull(controller) { + if (!delivered) { + controller.enqueue(encoder.encode('partial')) + + delivered = true + + return + } + + controller.error(new Error('terminated-ish')) + }, + }) + + const fetch = withBodyTimeout( + async () => new Response(source, { status: 200 }), + { bodyTimeout: 1000 } + ) + + const response = await fetch('https://gateway.example') + + let collected = '' + + let error + + try { + // @ts-ignore - body is async-iterable (polyfilled in @/lib/fetch) + for await (const chunk of response.body) { + collected += new TextDecoder().decode(chunk) + } + } catch (e) { + error = e + } + + expect(collected).toBe('partial') + expect(error?.name).not.toBe(TIMEOUT_ERROR_NAME) + expect(isBodyStallTimeout(error)).toBe(false) + expect(error?.message).toBe('terminated-ish') + }) +}) + +describe('isBodyStallTimeout', () => { + it('matches a body-phase TimeoutError', () => { + expect( + isBodyStallTimeout({ + name: TIMEOUT_ERROR_NAME, + data: { tags: { [FETCH_PHASE_TAG]: FETCH_PHASE_RESPONSE_BODY } }, + }) + ).toBe(true) + }) + + it('rejects a header-phase TimeoutError (retried at the fetch layer)', () => { + expect( + isBodyStallTimeout({ + name: TIMEOUT_ERROR_NAME, + data: { tags: { [FETCH_PHASE_TAG]: FETCH_PHASE_RESPONSE_HEADERS } }, + }) + ).toBe(false) + }) + + it('rejects a non-timeout error that happens to carry the body-phase tag', () => { + expect( + isBodyStallTimeout({ + name: 'FetchError', + data: { tags: { [FETCH_PHASE_TAG]: FETCH_PHASE_RESPONSE_BODY } }, + }) + ).toBe(false) + }) + + it('rejects a bare error, null, and undefined without throwing', () => { + expect(isBodyStallTimeout(new Error('boom'))).toBe(false) + expect(isBodyStallTimeout(null)).toBe(false) + expect(isBodyStallTimeout(undefined)).toBe(false) + }) +}) + +describe('withRetry', () => { + it('must retry without timeouts', async () => { + let count = 0 + + const fetch = withRetry( + async () => { + return new Promise((resolve, reject) => { + count += 1 + + reject(new Error(`Error`)) + }) + }, + { retries: 5, retryDelay: 1 } + ) + + await expect(async () => { + await fetch() + }).rejects.toThrow() + + expect(count).toEqual(6) + }) + + it('must retry timeouts', async () => { + let count = 0 + + const fetch = withRetry( + withTimeout( + async (url, { signal }) => { + return new Promise((resolve, reject) => { + count += 1 + + // We expect to receive an abort event from the timeout handler, + // which we use to reject the promise. + + signal.addEventListener('abort', () => { + reject(new Error(signal.reason)) + }) + }) + }, + { timeout: 1 } + ), + { retries: 5, retryDelay: 1, retryTimeout: true } + ) + + await expect(async () => { + await fetch() + }).rejects.toThrow() + + expect(count).toEqual(6) + }) + + it('records attempt count and elapsed time after exhausting timeout retries', async () => { + let count = 0 + + const fetch = withRetry( + withTimeout( + async (url, { signal }) => { + return new Promise((_resolve, reject) => { + count += 1 + + signal.addEventListener('abort', () => reject(signal.reason)) + }) + }, + { timeout: 1 } + ), + { retries: 3, retryDelay: 1, retryTimeout: true } + ) + + let error + + try { + await fetch('https://gateway.example/v1/chat/completions', { + meta: { model: 'test-model' }, + }) + } catch (e) { + error = e + } + + expect(count).toEqual(4) // 1 initial + 3 retries + + expect(error?.name).toBe('TimeoutError') + expect(error?.data?.tags?.['fetch.outcome']).toBe('timeout') + expect(error?.data?.tags?.['fetch.attempts']).toBe('4') + expect(error?.data?.extra?.fetchRetry?.attempts).toBe(4) + expect(error?.data?.extra?.fetchRetry?.maxAttempts).toBe(4) + expect(typeof error?.data?.extra?.fetchRetry?.elapsedMs).toBe('number') + + // @note the model context set by withTimeout survives the retry annotation + expect(error?.data?.tags?.['context.model']).toBe('test-model') + }) + + it('records a single attempt when retryTimeout is disabled', async () => { + let count = 0 + + const fetch = withRetry( + withTimeout( + async (url, { signal }) => { + return new Promise((_resolve, reject) => { + count += 1 + + signal.addEventListener('abort', () => reject(signal.reason)) + }) + }, + { timeout: 1 } + ), + { retries: 5, retryDelay: 1, retryTimeout: false } + ) + + let error + + try { + await fetch('https://gateway.example/v1/chat/completions') + } catch (e) { + error = e + } + + expect(count).toEqual(1) // not retried because retryTimeout is false + expect(error?.data?.tags?.['fetch.attempts']).toBe('1') + expect(error?.data?.extra?.fetchRetry?.attempts).toBe(1) + }) + + if (process.env.SLOW_TESTS) { + it('httpstat: must retry timeouts', async () => { + const f = withRetry(withTimeout(fetch, { timeout: 5000 }), { + retries: 5, + retryDelay: 1, + retryTimeout: true, + }) + + let error + + try { + await f('https://httpstat.us/504?sleep=60000') + } catch (e) { + error = e + } + + expect(error).toBeTruthy() + }) + } +}) + +describe('jsonl', () => { + it('should parse JSON lines correctly', async () => { + const input = '{"name": "Alice"}\n{"name": "Bob"}\n{"name": "Charlie"}' + + const body = new ReadableStream({ + async start(controller) { + controller.enqueue(new TextEncoder().encode(input)) + controller.close() + }, + }) + + const result = [] + + for await (const obj of jsonl(body)) { + result.push(obj) + } + + expect(result).toEqual([ + { name: 'Alice' }, + { name: 'Bob' }, + { name: 'Charlie' }, + ]) + }) + + it('should parse JSON lines correctly when last line does not end with newline character', async () => { + const input = + '{"name": "Alice"}\n{"name": "Bob"}\n{"name": "Charlie"}\n{"name": "Dave"}' + + const body = new ReadableStream({ + async start(controller) { + controller.enqueue(new TextEncoder().encode(input)) + controller.close() + }, + }) + + const result = [] + + for await (const obj of jsonl(body)) { + result.push(obj) + } + + expect(result).toEqual([ + { name: 'Alice' }, + { name: 'Bob' }, + { name: 'Charlie' }, + { name: 'Dave' }, + ]) + }) +}) + +describe('getFetchError', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe('with valid JSON error response', () => { + it('should create FetchError with nested error object', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + error: { + message: 'Custom error message', + code: 'CUSTOM_ERROR_CODE', + }, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result).toBeInstanceOf(FetchError) + expect(result.message).toBe('Custom error message') + expect(result.code).toBe('CUSTOM_ERROR_CODE') + expect(result.meta).toBeUndefined() + }) + + it('should create FetchError with flat error object', async () => { + const mockResponse = { + ok: false, + status: 404, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Resource not found', + code: 'NOT_FOUND_RESOURCE', + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result).toBeInstanceOf(FetchError) + expect(result.message).toBe('Resource not found') + expect(result.code).toBe('NOT_FOUND_RESOURCE') + }) + + it('should prefer nested error over flat error', async () => { + const mockResponse = { + ok: false, + status: 500, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Flat message', + code: 'FLAT_CODE', + error: { + message: 'Nested message', + code: 'NESTED_CODE', + }, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Nested message') + expect(result.code).toBe('NESTED_CODE') + }) + + it('should handle partial nested error objects', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Flat message', + code: 'FLAT_CODE', + error: { + message: 'Nested message only', + // no code in nested error + }, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Nested message only') + expect(result.code).toBe('FLAT_CODE') + }) + + it('should surface a string `error` field', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest + .fn() + .mockResolvedValue(JSON.stringify({ error: 'The value is invalid' })), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('The value is invalid') + expect(result.code).toBe('BAD_REQUEST') + }) + + it('should surface detail from an `errors` array', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + errors: ['person_seniorities: c_level not allowed'], + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toContain('c_level not allowed') + expect(result.code).toBe('BAD_REQUEST') + }) + + it('should prefer a flat `message` over a string `error` and `errors` array', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Primary message', + error: 'secondary error string', + errors: ['secondary detail'], + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Primary message') + }) + }) + + describe('with invalid JSON response', () => { + it('should fallback to status mappings when JSON parsing fails', async () => { + const mockResponse = { + ok: false, + status: 404, + text: jest.fn().mockResolvedValue('Not Found - invalid JSON'), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Not Found') + expect(result.code).toBe('NOT_FOUND') + }) + + it('should fallback to 500 mappings for unknown status codes', async () => { + const mockResponse = { + ok: false, + status: 999, // unknown status + text: jest.fn().mockResolvedValue('Unknown error'), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Internal Server Error') + expect(result.code).toBe('INTERNAL_SERVER_ERROR') + }) + + it('should handle empty response text', async () => { + const mockResponse = { + ok: false, + status: 500, + text: jest.fn().mockResolvedValue(''), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Internal Server Error') + expect(result.code).toBe('INTERNAL_SERVER_ERROR') + }) + + it('should handle null response text', async () => { + const mockResponse = { + ok: false, + status: 502, + text: jest.fn().mockResolvedValue(null), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Bad Gateway') + expect(result.code).toBe('BAD_GATEWAY') + }) + }) + + describe('with meta parameter', () => { + it('should pass meta parameter to FetchError constructor', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Test error', + code: 'TEST_CODE', + }) + ), + } + + const meta = { url: 'https://api.example.com', requestId: 'req-123' } + + const result = await getFetchError(mockResponse, meta) + + expect(result.name).toBe( + 'FetchError({"url":"https://api.example.com","requestId":"req-123"})' + ) + }) + + it('should handle undefined meta parameter', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Test error', + code: 'TEST_CODE', + }) + ), + } + + const result = await getFetchError(mockResponse, undefined) + + expect(result.name).toBe('FetchError') + }) + + it('should handle null meta parameter', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: 'Test error', + code: 'TEST_CODE', + }) + ), + } + + const result = await getFetchError(mockResponse, null) + + expect(result.name).toBe('FetchError') + }) + }) + + describe('fallback behavior', () => { + it('should fallback through all message options correctly', async () => { + const mockResponse = { + ok: false, + status: 401, + text: jest.fn().mockResolvedValue(JSON.stringify({})), // empty JSON + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Unauthorized') + expect(result.code).toBe('UNAUTHORIZED') + }) + + it('should fallback to 500 when all status lookups fail', async () => { + const mockResponse = { + ok: false, + status: 999, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + // no message or code fields + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Internal Server Error') + expect(result.code).toBe('INTERNAL_SERVER_ERROR') + }) + + it('should handle null and undefined values in JSON gracefully', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: null, + code: undefined, + error: { + message: undefined, + code: null, + }, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Bad Request') + expect(result.code).toBe('BAD_REQUEST') + }) + }) + + describe('edge cases', () => { + it('should handle response.text() throwing an error', async () => { + const mockResponse = { + ok: false, + status: 500, + text: jest.fn().mockRejectedValue(new Error('Failed to read response')), + } + + await expect(getFetchError(mockResponse)).rejects.toThrow( + 'Failed to read response' + ) + }) + + it('should handle very large JSON responses', async () => { + const largeMessage = 'x'.repeat(10000) + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + message: largeMessage, + code: 'LARGE_ERROR', + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe(largeMessage) + expect(result.code).toBe('LARGE_ERROR') + }) + + it('should handle JSON with unexpected structure', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + error: 'not an object', + message: ['array', 'instead', 'of', 'string'], + code: 123, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('array,instead,of,string') + // @note numeric json codes are ignored in favor of string codes from statusToCodeMap + expect(result.code).toBe('BAD_REQUEST') + }) + + it('should handle deeply nested error objects', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + error: { + nested: { + message: 'Deep message', + code: 'DEEP_CODE', + }, + message: 'Shallow message', + code: 'SHALLOW_CODE', + }, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Shallow message') + expect(result.code).toBe('SHALLOW_CODE') + }) + + it('should normalize google api numeric error codes to string codes', async () => { + // google apis return numeric codes in the error body (e.g. {"error": {"code": 404, "message": "..."}}) + // these must map to string codes so isUnknownError() recognizes them as known errors + const mockResponse = { + ok: false, + status: 404, + text: jest.fn().mockResolvedValue( + JSON.stringify({ + error: { + code: 404, + message: 'Requested entity was not found.', + status: 'NOT_FOUND', + }, + }) + ), + } + + const result = await getFetchError(mockResponse) + + expect(result.message).toBe('Requested entity was not found.') + // @note must be the string 'NOT_FOUND', not the number 404, so isUnknownError() works + expect(result.code).toBe('NOT_FOUND') + }) + }) + + describe('assertion behavior', () => { + it('should assert that response is not ok', async () => { + const mockResponse = { + ok: false, + status: 400, + text: jest.fn().mockResolvedValue('{}'), + } + + const result = await getFetchError(mockResponse) + + expect(result).toBeInstanceOf(FetchError) + expect(result.message).toBe('Bad Request') + }) + + // @todo add test for when response.ok is true - should fail assertion + test('should throw assertion error when response is ok', async () => { + const mockResponse = { + ok: true, + status: 200, + text: jest.fn().mockResolvedValue('{}'), + } + + await expect(getFetchError(mockResponse)).rejects.toThrow('Response ok') + }) + }) + + describe('function integration', () => { + it('should work correctly with debug logging enabled', async () => { + const mockResponse = { + ok: false, + status: 404, + text: jest.fn().mockResolvedValue('{"message":"Not found"}'), + } + + const result = await getFetchError(mockResponse) + + expect(result).toBeInstanceOf(FetchError) + expect(result.message).toBe('Not found') + expect(result.code).toBe('NOT_FOUND') + }) + }) +}) + +describe('download', () => { + /** + * Helper to create a mock ReadableStream + * @param {Uint8Array[]} chunks + */ + function createMockStream(chunks) { + let index = 0 + + return { + getReader: () => ({ + read: async () => { + if (index >= chunks.length) { + return { done: true, value: undefined } + } + + return { done: false, value: chunks[index++] } + }, + releaseLock: jest.fn(), + }), + } + } + + describe('basic functionality', () => { + it('should download response within size limit', async () => { + const testData = new TextEncoder().encode('Hello, World!') + const mockResponse = { + body: createMockStream([testData]), + } + + const maxSize = 100 // 100 bytes + const result = await download(mockResponse, maxSize) + + expect(result).toHaveProperty('byteLength') + expect(result.byteLength).toBe(testData.length) + + // Verify the content + const decoder = new TextDecoder() + + expect(decoder.decode(result)).toBe('Hello, World!') + }) + + it('should handle multiple small chunks', async () => { + const chunks = [] + + for (let i = 0; i < 10; i++) { + chunks.push(new TextEncoder().encode(`chunk${i}`)) + } + + const mockResponse = { + body: createMockStream(chunks), + } + + const maxSize = 1024 + const result = await download(mockResponse, maxSize) + + expect(result).toHaveProperty('byteLength') + + const decoder = new TextDecoder() + const text = decoder.decode(result) + + expect(text).toBe( + 'chunk0chunk1chunk2chunk3chunk4chunk5chunk6chunk7chunk8chunk9' + ) + }) + }) + + describe('size limit enforcement', () => { + it('should truncate when single chunk exceeds limit', async () => { + const largeData = new TextEncoder().encode( + 'This is a very long string that exceeds the limit' + ) + const mockResponse = { + body: createMockStream([largeData]), + } + + const maxSize = 10 // Only allow 10 bytes + const result = await download(mockResponse, maxSize) + + expect(result).toHaveProperty('byteLength') + expect(result.byteLength).toBe(maxSize) + + const decoder = new TextDecoder() + + expect(decoder.decode(result)).toBe('This is a ') + }) + + it('should stop reading when cumulative size exceeds limit', async () => { + const chunk1 = new TextEncoder().encode('Hello, ') // 7 bytes + const chunk2 = new TextEncoder().encode('World!') // 6 bytes + const chunk3 = new TextEncoder().encode(' Extra data') // Should not be included + const mockResponse = { + body: createMockStream([chunk1, chunk2, chunk3]), + } + + const maxSize = 10 // Only allow 10 bytes total + const result = await download(mockResponse, maxSize) + + expect(result).toHaveProperty('byteLength') + expect(result.byteLength).toBe(maxSize) + + const decoder = new TextDecoder() + + expect(decoder.decode(result)).toBe('Hello, Wor') // 7 + 3 = 10 bytes + }) + + it('should handle exact boundary match', async () => { + const chunk1 = new Uint8Array(50) + const chunk2 = new Uint8Array(50) + const chunk3 = new Uint8Array(10) // Should not be included + const mockResponse = { + body: createMockStream([chunk1, chunk2, chunk3]), + } + + const maxSize = 100 + const result = await download(mockResponse, maxSize) + + expect(result.byteLength).toBe(maxSize) + }) + + it('should handle chunk that partially exceeds limit', async () => { + const chunk1 = new Uint8Array(300 * 1024) // 300 KB + const chunk2 = new Uint8Array(300 * 1024) // 300 KB - will be truncated + const mockResponse = { + body: createMockStream([chunk1, chunk2]), + } + + const maxSize = 0.5 * 1024 * 1024 // 0.5 MB (512 KB) + const result = await download(mockResponse, maxSize) + + expect(result.byteLength).toBe(maxSize) + }) + + it('should release reader lock when stopping at limit', async () => { + const chunk1 = new Uint8Array(60) + const chunk2 = new Uint8Array(60) // Will be truncated + const releaseLock = jest.fn() + + const mockResponse = { + body: { + getReader: () => { + let index = 0 + const chunks = [chunk1, chunk2] + + return { + read: async () => { + if (index >= chunks.length) { + return { done: true, value: undefined } + } + + return { done: false, value: chunks[index++] } + }, + releaseLock, + } + }, + }, + } + + const maxSize = 100 + const result = await download(mockResponse, maxSize) + + expect(result.byteLength).toBe(maxSize) + expect(releaseLock).toHaveBeenCalled() + }) + }) + + describe('edge cases', () => { + it('should handle empty response', async () => { + const mockResponse = { + body: createMockStream([]), + } + + const maxSize = 100 + const result = await download(mockResponse, maxSize) + + expect(result).toHaveProperty('byteLength') + expect(result.byteLength).toBe(0) + }) + + it('should return empty body when the response body is missing', async () => { + const mockResponse = { + body: null, + } + + const maxSize = 100 + + const result = await download(mockResponse, maxSize) + + expect(result).toBeInstanceOf(ArrayBuffer) + expect(result.byteLength).toBe(0) + }) + + it('should handle zero max size', async () => { + const testData = new TextEncoder().encode('test') + const mockResponse = { + body: createMockStream([testData]), + } + + const maxSize = 0 + const result = await download(mockResponse, maxSize) + + // With zero max size, should return empty buffer + expect(result.byteLength).toBe(0) + }) + + it('should handle very large max size', async () => { + const testData = new TextEncoder().encode('small data') + const mockResponse = { + body: createMockStream([testData]), + } + + const maxSize = Number.MAX_SAFE_INTEGER + const result = await download(mockResponse, maxSize) + + expect(result).toHaveProperty('byteLength') + expect(result.byteLength).toBe(testData.length) + }) + + it('should properly combine chunks in correct order', async () => { + const chunks = [ + new TextEncoder().encode('ABC'), + new TextEncoder().encode('DEF'), + new TextEncoder().encode('GHI'), + ] + + const mockResponse = { + body: createMockStream(chunks), + } + + const maxSize = 100 + const result = await download(mockResponse, maxSize) + + const decoder = new TextDecoder() + + expect(decoder.decode(result)).toBe('ABCDEFGHI') + }) + }) +}) + +describe('withLimit', () => { + /** + * Helper to create a mock fetch that returns a response with the given data + * + * @param {string|Uint8Array} data + * @param {number} [contentLength] + */ + function createMockFetch(data, contentLength) { + return async () => { + const buffer = + typeof data === 'string' ? new TextEncoder().encode(data) : data + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(buffer) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'text/plain', + }) + + if (contentLength !== undefined) { + headers.set('content-length', contentLength.toString()) + } + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + } + + describe('basic functionality', () => { + it('should pass through when no maxSize is specified', async () => { + const mockFetch = createMockFetch('Hello, World!') + const limitedFetch = withLimit(mockFetch) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('Hello, World!') + expect(response.headers.has('X-Content-Truncated')).toBe(false) + }) + + it('should pass through when maxSize is 0', async () => { + const mockFetch = createMockFetch('Hello, World!') + const limitedFetch = withLimit(mockFetch, { maxSize: 0 }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('Hello, World!') + expect(response.headers.has('X-Content-Truncated')).toBe(false) + }) + + it('should pass through when maxSize is Infinity', async () => { + const mockFetch = createMockFetch('Hello, World!') + const limitedFetch = withLimit(mockFetch, { maxSize: Infinity }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('Hello, World!') + expect(response.headers.has('X-Content-Truncated')).toBe(false) + }) + + it('should not truncate when content is within limit', async () => { + const mockFetch = createMockFetch('Hello!', 6) + const limitedFetch = withLimit(mockFetch, { maxSize: 100 }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('Hello!') + expect(response.headers.get('X-Content-Truncated')).toBe(null) + expect(response.headers.get('content-length')).toBe('6') + }) + + it('should preserve bodyless response statuses', async () => { + const limitedFetch = withLimit( + async () => new Response(null, { status: 204 }), + { maxSize: 100 } + ) + + const response = await limitedFetch('http://example.com') + + expect(response.status).toBe(204) + await expect(response.text()).resolves.toBe('') + }) + }) + + describe('truncation behavior', () => { + it('should truncate when content exceeds limit', async () => { + const mockFetch = createMockFetch( + 'This is a long message that will be truncated', + 46 + ) + const limitedFetch = withLimit(mockFetch, { maxSize: 10 }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('This is a ') + expect(text.length).toBe(10) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + expect(response.headers.get('X-Content-Original-Size')).toBe('46') + expect(response.headers.get('content-length')).toBe('10') + }) + + it('should truncate large binary data', async () => { + const largeData = new Uint8Array(1024 * 1024) // 1 MB + const mockFetch = createMockFetch(largeData, 1024 * 1024) + const limitedFetch = withLimit(mockFetch, { maxSize: 512 * 1024 }) // 512 KB + + const response = await limitedFetch('http://example.com') + const buffer = await response.arrayBuffer() + + expect(buffer.byteLength).toBe(512 * 1024) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + expect(response.headers.get('X-Content-Original-Size')).toBe( + (1024 * 1024).toString() + ) + expect(response.headers.get('content-length')).toBe( + (512 * 1024).toString() + ) + }) + + it('should handle exact boundary match', async () => { + const mockFetch = createMockFetch('Exact!', 6) + const limitedFetch = withLimit(mockFetch, { maxSize: 6 }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('Exact!') + + // @note at exact boundary, it's considered truncated because we reached + // the limit + + expect(response.headers.get('X-Content-Truncated')).toBe('true') + expect(response.headers.get('content-length')).toBe('6') + }) + }) + + describe('header handling', () => { + it('should preserve original headers', async () => { + const mockFetch = async () => { + const headers = new Headers({ + 'content-type': 'application/json', + 'x-custom-header': 'custom-value', + 'content-length': '50', + }) + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('Short')) + controller.close() + }, + }) + + return new Response(stream, { + status: 200, + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { maxSize: 100 }) + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('application/json') + expect(response.headers.get('x-custom-header')).toBe('custom-value') + expect(response.headers.get('content-length')).toBe('5') + }) + + it('should handle missing content-length in original response', async () => { + const mockFetch = createMockFetch('Test data without content-length') + const limitedFetch = withLimit(mockFetch, { maxSize: 10 }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text).toBe('Test data ') + expect(response.headers.get('X-Content-Truncated')).toBe('true') + expect(response.headers.get('X-Content-Original-Size')).toBe('unknown') + expect(response.headers.get('content-length')).toBe('10') + }) + + it('should preserve response status and statusText', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('Error message')) + controller.close() + }, + }) + + return new Response(stream, { + status: 404, + statusText: 'Not Found', + headers: new Headers({ + 'content-type': 'text/plain', + 'content-length': '13', + }), + }) + } + + const limitedFetch = withLimit(mockFetch, { maxSize: 5 }) + const response = await limitedFetch('http://example.com') + + expect(response.status).toBe(404) + expect(response.statusText).toBe('Not Found') + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + }) + + describe('options handling', () => { + it('should use default maxSize from decorator', async () => { + const mockFetch = createMockFetch('This is a test message', 22) + const limitedFetch = withLimit(mockFetch, { maxSize: 10 }) + + const response = await limitedFetch('http://example.com') + const text = await response.text() + + expect(text.length).toBe(10) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should override default maxSize with option', async () => { + const mockFetch = createMockFetch('This is a test message', 22) + const limitedFetch = withLimit(mockFetch, { maxSize: 100 }) + + const response = await limitedFetch('http://example.com', { maxSize: 10 }) + const text = await response.text() + + expect(text.length).toBe(10) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should allow disabling limit with option', async () => { + const mockFetch = createMockFetch('This is a test message', 22) + const limitedFetch = withLimit(mockFetch, { maxSize: 10 }) + + const response = await limitedFetch('http://example.com', { + maxSize: Infinity, + }) + const text = await response.text() + + expect(text).toBe('This is a test message') + expect(response.headers.has('X-Content-Truncated')).toBe(false) + }) + }) + + describe('content-type handling', () => { + it('should preserve content-type in blob', async () => { + const mockFetch = async () => { + const headers = new Headers({ + 'content-type': 'application/pdf', + 'content-length': '100', + }) + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(50)) + controller.close() + }, + }) + + return new Response(stream, { + status: 200, + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { maxSize: 20 }) + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('application/pdf') + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should default to application/octet-stream when content-type is missing', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('Data')) + controller.close() + }, + }) + + return new Response(stream, { + status: 200, + headers: new Headers(), + }) + } + + const limitedFetch = withLimit(mockFetch, { maxSize: 100 }) + const response = await limitedFetch('http://example.com') + + const blob = await response.blob() + + expect(blob.type).toBe('application/octet-stream') + }) + }) + + describe('content type remapping', () => { + it('should remap content type when truncated with contentTypeRemap option', async () => { + const mockFetch = createMockFetch( + 'This is a long JSON response that will be truncated', + 52 + ) + + const jsonMockFetch = async () => { + const response = await mockFetch() + const headers = new Headers(response.headers) + + headers.set('content-type', 'application/json') + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }) + } + + const limitedFetch = withLimit(jsonMockFetch, { + maxSize: 10, + contentTypeRemap: { + 'application/json': 'text/plain', + }, + }) + + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('text/plain') + expect(response.headers.get('X-Content-Original-Type')).toBe( + 'application/json' + ) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should remap PNG to octet-stream when truncated', async () => { + const largeData = new Uint8Array(100) + + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(largeData) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'image/png', + 'content-length': '100', + }) + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { + maxSize: 10, + contentTypeRemap: { + 'image/png': 'application/octet-stream', + }, + }) + + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe( + 'application/octet-stream' + ) + expect(response.headers.get('X-Content-Original-Type')).toBe('image/png') + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should not remap content type when not truncated', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('Short')) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'application/json', + 'content-length': '5', + }) + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { + maxSize: 100, + contentTypeRemap: { + 'application/json': 'text/plain', + }, + }) + + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('application/json') + expect(response.headers.has('X-Content-Original-Type')).toBe(false) + expect(response.headers.get('X-Content-Truncated')).toBe(null) + }) + + it('should keep original content type when no remap is provided', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('This is a long message') + ) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'application/x-random', + 'content-length': '22', + }) + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { + maxSize: 10, + }) + + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('application/x-random') + expect(response.headers.has('X-Content-Original-Type')).toBe(false) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should keep original content type when truncated but not in remap', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('This is a long message') + ) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'application/xml', + 'content-length': '22', + }) + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { + maxSize: 10, + contentTypeRemap: { + 'application/json': 'text/plain', + }, + }) + + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('application/xml') + expect(response.headers.has('X-Content-Original-Type')).toBe(false) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should handle content type with charset parameters', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('This is a long message') + ) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'application/json; charset=utf-8', + 'content-length': '22', + }) + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { + maxSize: 10, + contentTypeRemap: { + 'application/json': 'text/plain', + }, + }) + + const response = await limitedFetch('http://example.com') + + expect(response.headers.get('content-type')).toBe('text/plain') + expect(response.headers.get('X-Content-Original-Type')).toBe( + 'application/json; charset=utf-8' + ) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + + it('should support option-level contentTypeRemap override', async () => { + const mockFetch = async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('This is a long message') + ) + controller.close() + }, + }) + + const headers = new Headers({ + 'content-type': 'application/json', + 'content-length': '22', + }) + + return new Response(stream, { + status: 200, + statusText: 'OK', + headers, + }) + } + + const limitedFetch = withLimit(mockFetch, { + maxSize: 10, + contentTypeRemap: { + 'application/json': 'text/plain', + }, + }) + + const response = await limitedFetch('http://example.com', { + contentTypeRemap: { + 'application/json': 'application/octet-stream', + }, + }) + + expect(response.headers.get('content-type')).toBe( + 'application/octet-stream' + ) + expect(response.headers.get('X-Content-Original-Type')).toBe( + 'application/json' + ) + expect(response.headers.get('X-Content-Truncated')).toBe('true') + }) + }) +}) + +describe('anySignal', () => { + it('should return a signal that is not any of the input signals when one is pre-aborted', () => { + const ac = new AbortController() + + ac.abort('already done') + + const result = anySignal([ac.signal]) + + // @note the bug: anySignal returns the original input signal instead of + // controller.signal when given a pre-aborted signal + expect(result).not.toBe(ac.signal) + expect(result.aborted).toBe(true) + expect(result.reason).toBe('already done') + }) + + it('should return a consistent signal type regardless of pre-aborted input position', () => { + const ac1 = new AbortController() + const ac2 = new AbortController() + + ac2.abort('reason2') + + const result = anySignal([ac1.signal, ac2.signal]) + + expect(result).not.toBe(ac1.signal) + expect(result).not.toBe(ac2.signal) + expect(result.aborted).toBe(true) + expect(result.reason).toBe('reason2') + }) + + it('should return a non-aborted signal for empty array', () => { + const result = anySignal([]) + + expect(result.aborted).toBe(false) + }) + + it('should skip null and undefined entries', () => { + const result = anySignal([null, undefined, null]) + + expect(result.aborted).toBe(false) + }) + + it('should abort when any input signal aborts later', () => { + const ac1 = new AbortController() + const ac2 = new AbortController() + + const result = anySignal([ac1.signal, ac2.signal]) + + expect(result.aborted).toBe(false) + + ac1.abort('reason1') + + expect(result.aborted).toBe(true) + expect(result.reason).toBe('reason1') + }) + + it('should propagate abort reason from the second signal', () => { + const ac1 = new AbortController() + const ac2 = new AbortController() + + const result = anySignal([ac1.signal, ac2.signal]) + + ac2.abort('reason2') + + expect(result.aborted).toBe(true) + expect(result.reason).toBe('reason2') + }) +}) diff --git a/packages/fetch/src/index.ts b/packages/fetch/src/index.ts new file mode 100644 index 0000000..849d789 --- /dev/null +++ b/packages/fetch/src/index.ts @@ -0,0 +1,1176 @@ +import debug, { assert, fassert } from '@chatbotkit-dev/debug' +import { SystemError } from '@chatbotkit-dev/errors' +import type { Thrown } from '@chatbotkit-dev/errors' +import { + FAILURE_CODE_HEADER_NAME, + LIMITS_REACHED_CODE, + statusToCodeMap, + statusToMessageMap, +} from '@chatbotkit-dev/http-codes' + +// @note `withNextCache` passes Next.js's `next` request option, which Next +// declares by augmenting RequestInit globally. Outside the application that +// augmentation is not in scope, so it is reproduced here - in the entry point, +// so consumers that type-check this package from source see it too. + +declare global { + interface RequestInit { + next?: { + tags?: string[] + revalidate?: number | false + } + } +} + +export const ABORT_ERROR_NAME = 'AbortError' +export const TIMEOUT_ERROR_NAME = 'TimeoutError' + +export const FETCH_PHASE_TAG = 'fetch.phase' +export const FETCH_PHASE_RESPONSE_HEADERS = 'response-headers' +export const FETCH_PHASE_RESPONSE_BODY = 'response-body' + +export const CONTEXT_MODEL_TAG = 'context.model' + +export const DEFAULT_TIMEOUT = 30000 + +export const DEFAULT_RETRIES = 5 +export const DEFAULT_RETRY_DELAY = 250 +export const DEFAULT_RETRY_TIMEOUT = false + +export const HEADER_CONTENT_TRUNCATED = 'x-content-truncated' +export const HEADER_CONTENT_ORIGINAL_SIZE = 'x-content-original-size' +export const HEADER_CONTENT_ORIGINAL_TYPE = 'x-content-original-type' + +const globalObject = typeof global !== 'undefined' ? global : globalThis + +// we need to polyfill the ReadableStream for chrome and Safari +{ + if ( + typeof globalObject !== 'undefined' && + typeof globalObject.ReadableStream === 'function' && + // @ts-ignore + typeof globalObject.ReadableStream.prototype[Symbol.asyncIterator] !== + 'function' + ) { + // @ts-ignore + globalObject.ReadableStream.prototype[Symbol.asyncIterator] = function () { + const reader = this.getReader() + + return { + next: () => reader.read(), + return: () => { + reader.releaseLock() + + return Promise.resolve({ done: true }) + }, + } + } + } +} + +/** + * Represents an error that occurs during a fetch operation. + */ +export class FetchError extends SystemError { + constructor(message: string, code: string, meta?: Record) { + super(message, code) + + this.name = meta + ? `FetchError(${JSON.stringify(meta || {})})` + : 'FetchError' + } +} + +/** + * Represents an error that occurs when a fetch operation is aborted. + */ +export class AbortError extends SystemError { + constructor(message?: string) { + super(message || ABORT_ERROR_NAME, ABORT_ERROR_NAME) + + this.name = ABORT_ERROR_NAME + } +} + +/** + * Represents an error that occurs when a fetch operation times out. + */ +export class TimeoutError extends SystemError { + /** + * `data` is Sentry capture context (e.g. `{ tags, extra }`) carried through to + * `captureError` so timeouts arrive with the url, timeout budget, model and + * retry stats attached instead of a bare `TimeoutError`. + */ + constructor(message?: string, data?: unknown) { + super(message || TIMEOUT_ERROR_NAME, TIMEOUT_ERROR_NAME, data) + + this.name = TIMEOUT_ERROR_NAME + } +} + +/** + * Merge Sentry capture context (`tags`/`extra`) into an error's `data` field + * (read by `buildCaptureContext` in `@/lib/error`) without clobbering anything + * already there. Used by the fetch wrappers to annotate the error they throw + * with diagnostics (url, timeout budget, attempt count, elapsed time) so the + * Sentry event explains *why* and *how hard* a request failed. + */ +function annotateCaptureData( + error: Thrown, + { + tags, + extra, + }: { tags?: Record; extra?: Record } = {} +): Thrown { + if (!error || typeof error !== 'object') { + return error + } + + const existing = + error.data && typeof error.data === 'object' && !Array.isArray(error.data) + ? error.data + : undefined + + // @note if the error already carried non-object `data` (e.g. a raw string), + // preserve it under extra rather than dropping it + + const preserved = + error.data !== undefined && !existing + ? { originalData: error.data } + : undefined + + error.data = { + ...(existing || {}), + + tags: { ...(existing?.tags || {}), ...(tags || {}) }, + + extra: { ...(existing?.extra || {}), ...preserved, ...(extra || {}) }, + } + + return error +} + +/** + * Fetches a resource from the network. + * + * @throws + */ +export function fetch( + url: string | URL, + init?: RequestInit +): Promise { + debug(`fetching`, { url, init }).log('fetch.fetch') + + fassert(() => { + let href + + if (typeof url === 'object' && url !== null && 'href' in url) { + href = url.href + } else { + href = url + } + + return /^(?:https?:\/\/|data:|blob:|\/)/i.test(href || '') + }, `url ${url} is not fetchable`) + + const nativeFetch = + typeof globalObject !== 'undefined' ? globalObject.fetch : undefined + + if (!nativeFetch) { + throw new Error(`No suitable fetch implementation found`) + } + + return nativeFetch(url, init) +} + +/** + * Expose a JSONL stream as an async iterable. + */ +export async function* jsonl( + body: ReadableStream & { + [Symbol.asyncIterator](): AsyncIterator + } +): AsyncGenerator> { + try { + const decoder = new TextDecoder() + + let previous = '' + + for await (const chunk of body) { + previous += decoder.decode(chunk) + + while (true) { + const eolIndex = previous.indexOf('\n') + + if (eolIndex < 0) { + break + } + + const line = previous.slice(0, eolIndex + 1) + + if (line) { + yield JSON.parse(line) + } + + previous = previous.slice(eolIndex + 1) + } + } + + if (previous.trim().length > 0) { + yield JSON.parse(previous) + } + } catch (error: Thrown) { + if (error?.name !== ABORT_ERROR_NAME) { + throw error + } + } +} + +/** + * Downloads a response up to the specified size limit. Consumes the stream and + * returns data up to maxSize bytes. * Does not throw on size limit - simply + * stops reading at the limit. + */ +export async function download( + response: Response, + maxSize: number +): Promise { + debug(`downloading with size limit`, { maxSize }).log('fetch.download') + + if (!response.body) { + return new ArrayBuffer(0) + } + + const reader = response.body.getReader() + + const chunks: Uint8Array[] = [] + + let receivedLength = 0 + + try { + while (true) { + const { done, value } = await reader.read() + + if (done) { + break + } + + const remainingSpace = maxSize - receivedLength + + if (value.length > remainingSpace) { + if (remainingSpace > 0) { + chunks.push(value.slice(0, remainingSpace)) + + receivedLength += remainingSpace + + debug(`chunk truncated to fit limit`, { + chunkSize: value.length, + taken: remainingSpace, + receivedLength, + }).log('fetch.download') + } + + break + } + + chunks.push(value) + + receivedLength += value.length + + debug(`received chunk`, { chunkSize: value.length, receivedLength }).log( + 'fetch.download' + ) + } + } finally { + reader.releaseLock() + } + + debug(`download complete`, { receivedLength }).log('fetch.download') + + const allChunks = new Uint8Array(receivedLength) + + let position = 0 + + for (const chunk of chunks) { + allChunks.set(chunk, position) + + position += chunk.length + } + + return allChunks.buffer +} + +/** + * Check if the response indicates an error and if so, create a FetchError. + */ +export async function getFetchError( + response: Response, + meta?: Record +): Promise { + assert(!response.ok, `Response ok`) + + const status = response.status + const text = await response.text() + + debug(`fetch error`, { status, text }).log('fetch.getFetchError') + + let json + + try { + json = JSON.parse(text) + } catch { + json = { + message: statusToMessageMap[status], + code: statusToCodeMap[status], + } + } + + // @note prefer string codes from statusToCodeMap over numeric codes from json bodies, + // since numeric codes (e.g. google api returns 404 as a number) are not recognized + // by isUnknownError() which checks against string codes only + const codeFromJson = + (typeof json?.error?.code === 'string' && json.error.code) || + (typeof json?.code === 'string' && json.code) || + null + + // @note surface upstream validation detail held in less common shapes (a + // string `error`, or an `errors` array) so callers - e.g. agents - can see + // exactly what was rejected and self-correct, instead of a generic status + // message + + return new FetchError( + json?.error?.message || + json?.message || + (typeof json?.error === 'string' ? json.error : undefined) || + (Array.isArray(json?.errors) && json.errors.length + ? JSON.stringify(json.errors) + : undefined) || + statusToMessageMap[status] || + statusToMessageMap[500], + codeFromJson || statusToCodeMap[status] || statusToCodeMap[500], + meta + ) +} + +/** + * Returns an AbortSignal that resolves when any of the provided signals abort. + */ +export function anySignal( + signals: (AbortSignal | null | undefined)[] +): AbortSignal { + const controller = new AbortController() + + for (const signal of signals) { + if (!signal) { + continue + } + + if (signal.aborted) { + controller.abort(signal.reason) + + return controller.signal + } + + signal.addEventListener('abort', () => controller.abort(signal.reason), { + signal: controller.signal, + }) + } + + return controller.signal +} + +export type FetchFn = ( + url: string | URL, + options?: RequestInit & T +) => Promise + +export type withDebugOptions = object + +/** + * @todo move to @chatbotkit/fetch sdk + */ +export function withDebug( + fetch: FetchFn, + defaultOptions?: withDebugOptions +): FetchFn { + debug(`with debug`, { defaultOptions }).log('fetch.withDebug') + + return async function fetchWithDebug( + url: string | URL, + options?: RequestInit & withDebugOptions + ): Promise { + debug(`fetching`, { url, options }).log('fetch.withDebug.fetchWithDebug') + + const response = await fetch(url, { ...options }) + + debug(`fetched`, { url, options, response }).log( + 'fetch.withDebug.fetchWithDebug' + ) + + return response + } +} + +export type withInitOptions = { + duplex?: 'half' +} + +/** + * @todo move to @chatbotkit/fetch sdk + */ +export function withInit( + fetch: FetchFn, + defaultOptions?: withInitOptions +): FetchFn { + debug(`with init`, { defaultOptions }).log('fetch.withInit') + + return async function fetchWithInit( + url: string | URL, + options?: RequestInit & withInitOptions + ): Promise { + debug(`fetching`, { url, options, defaultOptions }).log( + 'fetch.withInit.fetchWithInit' + ) + + const response = await fetch(url, { ...options, ...defaultOptions }) + + debug(`fetched`, { url, options, response }).log( + 'fetch.withInit.fetchWithInit' + ) + + return response + } +} + +export type withCacheOptions = { + ttl?: number +} + +/** + * @todo move to @chatbotkit/fetch sdk + */ +export function withCache( + fetch: FetchFn, + defaultOptions?: withCacheOptions +): FetchFn { + debug(`with cache`, { defaultOptions }).log('fetch.withCache') + + const cache = new Map() + + return async function fetchWithCache( + url: string | URL, + options?: RequestInit & withCacheOptions + ): Promise { + const ttl = options?.ttl ?? defaultOptions?.ttl ?? 60000 + + const key = JSON.stringify([url, options]) + + const cachedEntry = cache.get(key) + + const currentTime = Date.now() + + if (cachedEntry && currentTime - cachedEntry.timestamp < ttl) { + return cachedEntry.response.clone() + } + + const response = await fetch(url, { ...options }) + + if (response.ok) { + // @todo make this work because it doesn't + // cache.set(key, { + // timestamp: currentTime, + // response: response.clone(), + // }) + } + + return response + } +} + +export type withNextCacheOptions = { + tags?: string[] + ttl?: number +} + +/** + * @todo move to @chatbotkit/fetch sdk + */ +export function withNextCache( + fetch: FetchFn, + defaultOptions?: withNextCacheOptions +): FetchFn { + debug(`with next cache`, { defaultOptions }).log('fetch.withNextCache') + + return async function fetchWithNextCache( + url: string | URL, + options?: RequestInit & withNextCacheOptions + ): Promise { + const tags = options?.tags ?? defaultOptions?.tags ?? [] + const ttl = options?.ttl ?? defaultOptions?.ttl ?? 60000 + + const response = await fetch(url, { + ...options, + + cache: 'force-cache', + + next: { + tags, + + revalidate: Math.round(ttl / 1000), + }, + }) + + return response + } +} + +/** + * Diagnostic identifiers for a fetch (e.g. the model being called). Ignored by + * the network layer; surfaced onto any `TimeoutError`'s Sentry context so the + * event records which upstream stalled. + */ +export type FetchMeta = { model?: string } & Record + +export type withTimeoutOptions = { + timeout?: number + meta?: FetchMeta +} + +/** + * Add timeout capabilities to any fetch implementation. + * + * @todo move to @chatbotkit/fetch sdk + */ +export function withTimeout( + fetch: FetchFn, + defaultOptions?: withTimeoutOptions +): FetchFn { + debug(`with timeout`, { defaultOptions }).log('fetch.withTimeout') + + return async function fetchWithTimeout( + url: string | URL, + options?: RequestInit & withTimeoutOptions + ): Promise { + const timeout = + options?.timeout ?? defaultOptions?.timeout ?? DEFAULT_TIMEOUT + + debug(`fetching with timeout`, { url, timeout }).log( + 'fetch.withTimeout.fetchWithTimeout' + ) + + // @note the timeout only covers time-to-response-headers - `fetch()` + // resolves once headers arrive and the body is streamed afterwards, outside + // this guard. A `TimeoutError` here therefore means the upstream never + // started responding within the budget (e.g. a stalled/overloaded gateway), + // NOT that generation was slow. We attach that context so the Sentry event + // says exactly what timed out instead of a bare `TimeoutError`. + + const meta = options?.meta + + const makeTimeoutError = () => + new TimeoutError(undefined, { + tags: { + [FETCH_PHASE_TAG]: FETCH_PHASE_RESPONSE_HEADERS, + + ...(meta?.model ? { [CONTEXT_MODEL_TAG]: String(meta.model) } : {}), + }, + + extra: { + fetch: { + url: typeof url === 'string' ? url : url?.toString?.(), + + timeoutMs: timeout, + + ...(meta || {}), + }, + }, + }) + + let signal + let handler + + let isTimeOutAbort = false + + if (timeout > 0 && timeout !== Infinity) { + const abortController = new AbortController() + + // @todo use AbortSignal.timeout(n) when widely supported, right now there + // in fact little to no support with known bugs in Chrome + + handler = setTimeout(() => { + debug(`aborting fetch`, { url }).log( + 'fetch.withTimeout.fetchWithTimeout' + ) + + isTimeOutAbort = true + + abortController.abort(makeTimeoutError()) + }, timeout) + + // @todo use AbortSignal.any([]) when widely supported, right now most + // implementation simply do not have it + + signal = options?.signal + ? anySignal([abortController.signal, options.signal]) + : abortController.signal + } else { + signal = options?.signal + } + + let response + + try { + response = await fetch(url, { + ...options, + + signal, + }) + } catch (error: Thrown) { + // @note we have a problem because some implementation (Chrome) do not + // correctly transfer the real reason for the abort i.e. the timeout + // error, so we need to check if we have raised a timeout above and if so + // we need to throw the correct error + + if ([error?.name, error?.message].includes(ABORT_ERROR_NAME)) { + if (isTimeOutAbort) { + throw makeTimeoutError() + } + } + + throw error + } finally { + clearTimeout(handler) + } + + return response + } +} + +export type withBodyTimeoutOptions = { + bodyTimeout?: number + meta?: FetchMeta +} + +/** + * Guard the *body* (post-headers) phase of a streaming response against a + * stalled upstream. {@link withTimeout} only covers time-to-response-headers - + * once headers arrive the body is streamed afterwards, outside that guard, so a + * gateway that returns headers and then goes silent (sends no tokens) hangs + * until undici's ~300s default body timeout. That surfaces as a bare + * `TypeError: terminated` (cause `UND_ERR_BODY_TIMEOUT`) with no model + * attribution, burns the whole turn for 0 tokens, and leaves it "incomplete". + * + * This wrapper caps the gap between body chunks - including time-to-first-chunk. + * If no chunk arrives within `bodyTimeout` ms it cancels the underlying body + * (releasing the socket) and surfaces the same annotated {@link TimeoutError} + * that {@link withTimeout} throws, so the stall is attributable in Sentry + * (`context.model`, `fetch.phase: response-body`) instead of opaque. The timer + * resets on every received chunk, so a slow-but-steady stream is never killed. + * + * @todo move to @chatbotkit/fetch sdk + */ +export function withBodyTimeout( + fetch: FetchFn, + defaultOptions?: withBodyTimeoutOptions +): FetchFn { + debug(`with body timeout`, { defaultOptions }).log('fetch.withBodyTimeout') + + return async function fetchWithBodyTimeout( + url: string | URL, + options?: RequestInit & withBodyTimeoutOptions + ): Promise { + const bodyTimeout = options?.bodyTimeout ?? defaultOptions?.bodyTimeout ?? 0 + + const response = await fetch(url, options) + + // @note nothing to guard: disabled, no body, or an error response the caller + // will read in full (e.g. to extract the upstream error message) rather than + // stream. Wrapping a non-ok body would arm a timer on a stream the caller may + // never iterate. + + if ( + !bodyTimeout || + bodyTimeout === Infinity || + !response.body || + !response.ok + ) { + return response + } + + const meta = options?.meta ?? defaultOptions?.meta + + const makeTimeoutError = () => + new TimeoutError(undefined, { + tags: { + [FETCH_PHASE_TAG]: FETCH_PHASE_RESPONSE_BODY, + + ...(meta?.model ? { [CONTEXT_MODEL_TAG]: String(meta.model) } : {}), + }, + + extra: { + fetch: { + url: typeof url === 'string' ? url : url?.toString?.(), + + bodyTimeoutMs: bodyTimeout, + + ...(meta || {}), + }, + }, + }) + + const source = response.body + + // @note hoisted so `cancel` can tear down through the reader - `start` + // locks `source` with this reader, and a locked stream cannot be cancelled + // directly (it throws), so the cancel path must go through the reader. + let reader + + const guarded = new ReadableStream({ + start(controller) { + reader = source.getReader() + + let timer + + let settled = false + + // @note run every terminal controller transition exactly once, clearing + // the idle timer first. Wrapped in try/catch because a consumer that + // cancels mid-stream can move the controller to a closed state before we + // observe it - closing/erroring it again would otherwise throw. + + const settle = (transition) => { + if (settled) { + return + } + + settled = true + + if (timer) { + clearTimeout(timer) + + timer = undefined + } + + try { + transition() + } catch { + // @note controller already closed/errored (e.g. consumer cancelled) + } + } + + const pump = async () => { + try { + while (true) { + timer = setTimeout(() => { + const error = makeTimeoutError() + + // @note release the upstream socket, then surface the timeout to + // the consumer's read/`for await` + + void reader.cancel(error).catch(() => {}) + + settle(() => controller.error(error)) + }, bodyTimeout) + + const { done, value } = await reader.read() + + if (timer) { + clearTimeout(timer) + + timer = undefined + } + + if (settled) { + return + } + + if (done) { + settle(() => controller.close()) + + return + } + + controller.enqueue(value) + } + } catch (error: Thrown) { + settle(() => controller.error(error)) + } + } + + void pump() + }, + + cancel(reason) { + // @note cancel through the reader (it holds the lock on `source`). + // Swallow rejections: an already-errored/cancelled reader rejects here, + // and a consumer that stopped early does not care about the outcome. + return reader?.cancel(reason).catch(() => {}) + }, + }) + + return new Response(guarded, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }) + } +} + +/** + * Recognises the one error {@link withBodyTimeout} throws: a `TimeoutError` + * tagged `fetch.phase: response-body`. Lives here, next to the wrapper that + * produces it and keyed off the same {@link FETCH_PHASE_TAG} / + * {@link FETCH_PHASE_RESPONSE_BODY} constants, so the producer and this + * recogniser cannot drift. + * + * It exists because a body-phase stall is raised *while the streaming body is + * being consumed* - downstream of {@link withRetry}, which has already returned + * the (headers-ok) response and so never retries it. The streaming layer uses + * this to decide a stall is safe to re-issue (header-phase timeouts, already + * retried at the fetch layer, and real provider errors are deliberately + * excluded). + */ +export function isBodyStallTimeout(error: Thrown): boolean { + return ( + error?.name === TIMEOUT_ERROR_NAME && + error?.data?.tags?.[FETCH_PHASE_TAG] === FETCH_PHASE_RESPONSE_BODY + ) +} + +export type withRetryOptions = { + retries?: number + retryDelay?: number + retryTimeout?: boolean + meta?: FetchMeta +} + +/** + * Add retry capabilities to any fetch implementation. + * + * @todo move to @chatbotkit/fetch sdk + */ +export function withRetry( + fetch: FetchFn, + defaultOptions?: withRetryOptions +): FetchFn { + debug(`with retry`, { defaultOptions }).log('fetch.withRetry') + + return async function fetchWithRetry( + url: string | URL, + options?: RequestInit & withRetryOptions + ): Promise { + const maxRetries = + options?.retries ?? defaultOptions?.retries ?? DEFAULT_RETRIES + + const retryTimeout = + options?.retryTimeout ?? + defaultOptions?.retryTimeout ?? + DEFAULT_RETRY_TIMEOUT + + // @note attempts run in a loop rather than via recursion, so the stats we + // report on a final failure (how many attempts, how long they took) live in + // plain local state instead of being threaded through `options`. `attempt` + // is the 1-based number of the attempt currently running; `retriesLeft` + // counts down; `retryDelay` doubles each retry (exponential backoff). + + const startedAt = Date.now() + + let retriesLeft = maxRetries + + let retryDelay = + options?.retryDelay ?? defaultOptions?.retryDelay ?? DEFAULT_RETRY_DELAY + + let attempt = 0 + + /** + * Annotate the about-to-be-thrown error with retry diagnostics for Sentry. + */ + const annotateRetry = (error) => + annotateCaptureData(error, { + tags: { + 'fetch.attempts': String(attempt), + + 'fetch.outcome': [error?.name, error?.message].includes( + TIMEOUT_ERROR_NAME + ) + ? 'timeout' + : 'error', + }, + + extra: { + fetchRetry: { + url: typeof url === 'string' ? url : url?.toString?.(), + + attempts: attempt, + + maxAttempts: maxRetries + 1, + + elapsedMs: Date.now() - startedAt, + + retryTimeout, + }, + }, + }) + + while (true) { + attempt += 1 + + debug(`fetching with retry`, { + url, + retries: retriesLeft, + retryDelay, + retryTimeout, + }).log('fetch.withRetry.fetchWithRetry') + + let response + + try { + response = await fetch(url, { ...options }) + + if (!response.ok) { + debug(`response not ok`, { + url, + status: response.status, + statusText: response.statusText, + }).log('fetch.withRetry.fetchWithRetry') + + switch (response.status) { + // we always attempt to retry if it is one of the following error + // codes as long as we do not also have a corner case + + case 429: + // as a special case, we return 429 when we have exceeded account + + if ( + response.headers.get(FAILURE_CODE_HEADER_NAME) === + LIMITS_REACHED_CODE + ) { + return response + } + + case 500: + case 502: + case 503: + case 504: + // Previously we were not retrying status code 503 which meant that + // some OpenAI requests were failing due to 503 being treated + // similar to 429 when the model is overloaded. + + // @note we don't use getFetchError here because we don't want to + // read the response body, we just want to throw an error with the + // status code and the URL + + throw new FetchError( + `Fetch failed with status ${response.status} (${response.statusText})`, + statusToCodeMap[response.status] || statusToCodeMap[500], + { + url: new URL(url).href, + options: options, + } + ) + + // by default we return the response as is + + default: + return response + } + } + + return response + } catch (error: Thrown) { + debug(`fetch error`, { url, error }).log( + 'fetch.withRetry.fetchWithRetry' + ) + + // @note never retry a timeout when the caller opted out + + if ( + [error?.name, error?.message].includes(TIMEOUT_ERROR_NAME) && + !retryTimeout + ) { + debug(`not retrying timeout`, { url, error }).log( + 'fetch.withRetry.fetchWithRetry' + ) + + throw annotateRetry(error) + } + + // @note out of retries - surface the last bad response if we have one, + // otherwise throw the (annotated) error + + if (retriesLeft === 0) { + debug(`no retries left`, { url, error, retries: retriesLeft }).log( + 'fetch.withRetry.fetchWithRetry' + ) + + if (response) { + return response + } + + throw annotateRetry(error) + } + + debug(`sleeping`, { retryDelay }).log('fetch.withRetry.fetchWithRetry') + + await new Promise((resolve) => setTimeout(resolve, retryDelay)) + + retriesLeft -= 1 + retryDelay *= 2 + + debug(`retrying fetch`, { url, retries: retriesLeft, retryDelay }).log( + 'fetch.withRetry.fetchWithRetry' + ) + } + } + } +} + +export type withLimitOptions = { + maxSize?: number + contentTypeRemap?: Record +} + +/** + * Add response size limiting to any fetch implementation. Downloads response up + * to maxSize bytes and adds a header to indicate truncation. + * + * @todo move to @chatbotkit/fetch sdk + */ +export function withLimit( + fetch: FetchFn, + defaultOptions?: withLimitOptions +): FetchFn { + debug(`with limit`, { defaultOptions }).log('fetch.withLimit') + + return async function fetchWithLimit( + url: string | URL, + options?: RequestInit & withLimitOptions + ): Promise { + const maxSize = options?.maxSize ?? defaultOptions?.maxSize + + if (!maxSize || maxSize <= 0 || maxSize === Infinity) { + return await fetch(url, options) + } + + const contentTypeRemap = options?.contentTypeRemap ?? + defaultOptions?.contentTypeRemap ?? { + 'application/json': 'text/plain', + 'application/yaml': 'text/plain', + 'application/xml': 'text/plain', + 'text/xml': 'text/plain', + 'text/html': false, + 'application/xhtml+xml': 'text/html', + 'text/csv': false, + } + + debug(`fetching with size limit`, { url, maxSize, contentTypeRemap }).log( + 'fetch.withLimit.fetchWithLimit' + ) + + const response = await fetch(url, options) + + const contentLength = response.headers.get('content-length') + + const expectedSize = contentLength ? parseInt(contentLength, 10) : null + + const willTruncate = + expectedSize !== null && !isNaN(expectedSize) && expectedSize > maxSize + + debug(`response received`, { + contentLength, + expectedSize, + willTruncate, + }).log('fetch.withLimit.fetchWithLimit') + + const buffer = await download(response, maxSize) + + debug(`download complete`, { bufferSize: buffer.byteLength }).log( + 'fetch.withLimit.fetchWithLimit' + ) + + const wasTruncated = buffer.byteLength >= maxSize + + const originalContentType = + response.headers.get('content-type') || 'application/octet-stream' + + let contentType = originalContentType + + if (wasTruncated) { + if (contentTypeRemap) { + const mainType = originalContentType.split(';')[0].trim() + + switch (true) { + case mainType in contentTypeRemap: { + contentType = + contentTypeRemap[mainType] === false + ? mainType + : contentTypeRemap[mainType] + + break + } + + case originalContentType in contentTypeRemap: { + contentType = + contentTypeRemap[originalContentType] === false + ? originalContentType + : contentTypeRemap[originalContentType] + + break + } + } + } else { + contentType = 'application/octet-stream' + } + } + + const blob = new Blob([buffer], { + type: contentType, + }) + + const newHeaders = new Headers(response.headers) + + if (wasTruncated) { + newHeaders.set(HEADER_CONTENT_TRUNCATED, 'true') + newHeaders.set( + HEADER_CONTENT_ORIGINAL_SIZE, + expectedSize?.toString() || 'unknown' + ) + + if (contentType !== originalContentType) { + newHeaders.set('content-type', contentType) + + newHeaders.set(HEADER_CONTENT_ORIGINAL_TYPE, originalContentType) + } + } + + newHeaders.set('content-length', buffer.byteLength.toString()) + + const responseBody = [204, 205, 304].includes(response.status) ? null : blob + + const limitedResponse = new Response(responseBody, { + status: response.status, + statusText: response.statusText, + headers: newHeaders, + }) + + debug(`response limited`, { + originalSize: expectedSize, + limitedSize: buffer.byteLength, + truncated: wasTruncated, + originalContentType, + contentType, + }).log('fetch.withLimit.fetchWithLimit') + + return limitedResponse + } +} + +// @note the composition erases the inner wrapper's options from the inferred +// type: withRetry only knows it returns a FetchFn. The +// assertion restores what a caller can actually pass. + +export const fetchPlusPlus = withRetry(withTimeout(fetch)) as FetchFn< + withTimeoutOptions & withRetryOptions +> + +export default fetch diff --git a/packages/fetch/tsconfig.json b/packages/fetch/tsconfig.json new file mode 100644 index 0000000..72b364c --- /dev/null +++ b/packages/fetch/tsconfig.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "noEmit": true, + "composite": true, + "target": "es2021", + "module": "esnext", + "moduleResolution": "bundler", + "lib": [ + "DOM", + "ES2021" + ], + "types": [ + "node", + "jest" + ], + "allowJs": true, + "checkJs": false, + "declaration": true, + "strict": false, + "strictNullChecks": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.js" + ], + "exclude": [ + "node_modules", + "**/*.test.js" + ] +} diff --git a/packages/file-csv/README.md b/packages/file-csv/README.md new file mode 100644 index 0000000..a4e184d --- /dev/null +++ b/packages/file-csv/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-csv diff --git a/packages/file-csv/data/test01.csv b/packages/file-csv/data/test01.csv new file mode 100644 index 0000000..c1c37fa --- /dev/null +++ b/packages/file-csv/data/test01.csv @@ -0,0 +1,2 @@ +column1,column2,column3 +cell1,cell2,cell3 \ No newline at end of file diff --git a/packages/file-csv/jest.config.js b/packages/file-csv/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-csv/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-csv/package.json b/packages/file-csv/package.json new file mode 100644 index 0000000..632701f --- /dev/null +++ b/packages/file-csv/package.json @@ -0,0 +1,42 @@ +{ + "name": "@chatbotkit-dev/file-csv", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./parse": { + "import": "./src/parse.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "csv-parse": "^5.6.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-csv/src/index.ts b/packages/file-csv/src/index.ts new file mode 100644 index 0000000..395a310 --- /dev/null +++ b/packages/file-csv/src/index.ts @@ -0,0 +1,17 @@ +import { csv2blocks } from './parse' + +export { csv2blocks } + +interface Chunk { + text: string + meta: Record +} + +export async function* chunk(blob: Blob): AsyncGenerator { + for (const block of csv2blocks(await blob.text())) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-csv/src/parse.test.ts b/packages/file-csv/src/parse.test.ts new file mode 100644 index 0000000..5d861b4 --- /dev/null +++ b/packages/file-csv/src/parse.test.ts @@ -0,0 +1,21 @@ +import { csv2blocks } from './parse' + +import fs from 'node:fs' + +describe('csv2blocks', () => { + test('csv must produce blocks', () => { + expect(csv2blocks('1,2,3\na,b,c')).toEqual(['1: a\n2: b\n3: c']) + expect(csv2blocks('1,2,3\na,b,c\nx,y,z')).toEqual([ + '1: a\n2: b\n3: c', + '1: x\n2: y\n3: z', + ]) + }) + + test('should return text', async () => { + const data = fs.readFileSync('./data/test01.csv') + + const blocks = await csv2blocks(new Uint8Array(data)) + + expect(blocks).toEqual(['column1: cell1\ncolumn2: cell2\ncolumn3: cell3']) + }) +}) diff --git a/packages/file-csv/src/parse.ts b/packages/file-csv/src/parse.ts new file mode 100644 index 0000000..c449b8f --- /dev/null +++ b/packages/file-csv/src/parse.ts @@ -0,0 +1,26 @@ +import { parse } from 'csv-parse/sync' + +export function csv2blocks(input: string | Uint8Array): string[] { + const records = parse(input as string | Buffer, { + columns: true, + skip_empty_lines: true, + relax_quotes: true, + relax_column_count: true, + }) + + return records.map((record: Record) => { + return Object.entries(record) + .map(([name, value]) => { + name = name.replace(/\s+/g, ' ').trim() + value = value.replace(/\s+/g, ' ').trim() + + if (!name || !value) { + return + } + + return `${name}: ${value}` + }) + .filter((b) => b) + .join('\n') + }) +} diff --git a/packages/file-csv/tsconfig.json b/packages/file-csv/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-csv/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-docx/README.md b/packages/file-docx/README.md new file mode 100644 index 0000000..c2475c7 --- /dev/null +++ b/packages/file-docx/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-docx diff --git a/packages/file-docx/data/test01.docx b/packages/file-docx/data/test01.docx new file mode 100644 index 0000000..c206e7a --- /dev/null +++ b/packages/file-docx/data/test01.docx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c67b51a17dbca2cfc24a0756d560166f90af966515ceb217d42eb306e36d519 +size 7388 diff --git a/packages/file-docx/jest.config.js b/packages/file-docx/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-docx/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-docx/package.json b/packages/file-docx/package.json new file mode 100644 index 0000000..c1dc6d7 --- /dev/null +++ b/packages/file-docx/package.json @@ -0,0 +1,43 @@ +{ + "name": "@chatbotkit-dev/file-docx", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./parse": { + "import": "./src/parse.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/gpt": "workspace:*", + "officeparser": "7.8.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-docx/src/index.ts b/packages/file-docx/src/index.ts new file mode 100644 index 0000000..ae98913 --- /dev/null +++ b/packages/file-docx/src/index.ts @@ -0,0 +1,41 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +import { docx2text } from './parse' + +export { docx2text } from './parse' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = await docx2text(new Uint8Array(await blob.arrayBuffer())) + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-docx/src/parse.test.ts b/packages/file-docx/src/parse.test.ts new file mode 100644 index 0000000..3b6a0a1 --- /dev/null +++ b/packages/file-docx/src/parse.test.ts @@ -0,0 +1,13 @@ +import { docx2text } from './parse' + +import fs from 'node:fs' + +describe('pptx2text', () => { + test('should return text', async () => { + const data = fs.readFileSync('./data/test01.docx') + + const text = await docx2text(data) + + expect(text.trim()).toEqual('HELLO WORLD') + }) +}) diff --git a/packages/file-docx/src/parse.ts b/packages/file-docx/src/parse.ts new file mode 100644 index 0000000..0901be7 --- /dev/null +++ b/packages/file-docx/src/parse.ts @@ -0,0 +1,7 @@ +import { parseOffice } from 'officeparser' + +export async function docx2text(buffer: Uint8Array): Promise { + const result = await parseOffice(Buffer.from(buffer)) + + return result.toText() +} diff --git a/packages/file-docx/tsconfig.json b/packages/file-docx/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-docx/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-html/README.md b/packages/file-html/README.md new file mode 100644 index 0000000..bf8d1c1 --- /dev/null +++ b/packages/file-html/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-html diff --git a/packages/file-html/jest.config.js b/packages/file-html/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-html/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-html/package.json b/packages/file-html/package.json new file mode 100644 index 0000000..fa2df20 --- /dev/null +++ b/packages/file-html/package.json @@ -0,0 +1,44 @@ +{ + "name": "@chatbotkit-dev/file-html", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./parse": { + "import": "./src/parse.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/gpt": "workspace:*", + "html-to-text": "^9.0.5" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/html-to-text": "^9.0.4", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-html/src/index.ts b/packages/file-html/src/index.ts new file mode 100644 index 0000000..262e49a --- /dev/null +++ b/packages/file-html/src/index.ts @@ -0,0 +1,41 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +import { html2text } from './parse' + +export { html2text, validateSelectors } from './parse' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = html2text(await blob.text()) + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-html/src/parse.test.ts b/packages/file-html/src/parse.test.ts new file mode 100644 index 0000000..89f84ec --- /dev/null +++ b/packages/file-html/src/parse.test.ts @@ -0,0 +1,132 @@ +import { html2text, validateSelectors } from './parse' + +describe('html2text', () => { + it('must correctly get the text from html', () => { + expect(html2text('
test
')).toEqual('test') + }) + + it('must correctly get non-empty text from html with selectors', () => { + expect( + html2text('
main
footer
', { + selectors: 'main', + }) + ).toEqual('main') + }) + + it('must correctly get empty text from html with selectors', () => { + expect( + html2text('
main
footer
', { + selectors: 'article', + }) + ).toEqual('') + }) + + it('must correctly get empty text from html with non-existing selectors', () => { + expect( + html2text('
main
footer
', { + selectors: 'skip', + }) + ).toEqual('') + }) + + it('must correctly get text but also skip some elements such as nav', () => { + expect( + html2text( + '
main
footer
' + ) + ).toEqual('main') + }) + + it('must correctly get text but also skip element that have role of navigation', () => { + expect( + html2text( + '
nav
main
footer
' + ) + ).toEqual('main') + }) + + it('must be able to preserve image urls', () => { + expect( + html2text( + 'image' + ) + ).toEqual('image [https://example.com/image.png]') + }) + + describe('data url handling', () => { + it('must skip data urls in links by default', () => { + expect( + html2text( + 'click here' + ) + ).toEqual('') + }) + + it('must skip data urls in images by default', () => { + expect( + html2text( + 'inline image' + ) + ).toEqual('') + }) + + it('must include data urls when includeDataUrls is true', () => { + expect( + html2text( + 'click here', + { includeDataUrls: true } + ) + ).toEqual('click here [data:text/plain;base64,SGVsbG8=]') + }) + + it('must include data urls in images when includeDataUrls is true', () => { + expect( + html2text( + 'inline image', + { includeDataUrls: true } + ) + ).toEqual('inline image [data:image/png;base64,iVBORw0KGgo=]') + }) + }) + + it.skip('must be able to preserve video urls', () => { + expect( + html2text( + '' + ) + ).toEqual('video [https://example.com/video.mp4]') + }) + + // @note we have turn off this because it could be causing issues + + it.skip('must correctly skip footer classes', () => { + expect( + html2text( + '
footer
main
footer
' + ) + ).toEqual('main') + }) + + it('test harness 001', () => { + const html = `

Launched in 2023, McLuck is a new sweeps cash casino that has taken the US by storm.

` + + const expected = `Launched in 2023, McLuck [https://casinos.com/us/mcluck-social-casino] is a new sweeps cash casino that has taken the US by storm.` + + const result = html2text(html, { + url: 'https://casinos.com/', + }) + + expect(result).toEqual(expected) + }) +}) + +describe('validateSelectors', () => { + it('must correctly validate selectors', () => { + expect(validateSelectors('html, body')).toEqual({ valid: true }) + expect(validateSelectors('html, body, jsonl')).toEqual({ valid: true }) + expect(validateSelectors('body div')).toEqual({ + valid: false, + message: 'Unsupported selector kind: combinator', + }) + }) +}) diff --git a/packages/file-html/src/parse.ts b/packages/file-html/src/parse.ts new file mode 100644 index 0000000..61b5c19 --- /dev/null +++ b/packages/file-html/src/parse.ts @@ -0,0 +1,196 @@ +import { compile } from 'html-to-text' + +export const DEFAULT_SELECTORS = ['article', 'main', 'body'] + +export function html2text( + html: string, + options?: { + url?: string + selectors?: string | string[] + includeDataUrls?: boolean + } +): string { + const { url, selectors, includeDataUrls = false } = options || {} + + let preferredSelectors: string[] = [] + + // add the selectors + { + let theseSelectors + + if (typeof selectors === 'string') { + theseSelectors = selectors + .split(',') + .map((i) => i.trim()) + .filter((i) => i) + } else if (Array.isArray(selectors)) { + theseSelectors = selectors + } + + theseSelectors = theseSelectors + ?.map((selector) => selector.trim()) + .filter(Boolean) + .filter((selector) => !selector.startsWith('@')) + + if (theseSelectors && theseSelectors.length) { + preferredSelectors.unshift(...theseSelectors) + } else { + preferredSelectors.unshift(...DEFAULT_SELECTORS) + } + } + + // normalize selectors + { + preferredSelectors = Array.from( + new Set( + preferredSelectors.map((selector) => selector.trim()).filter(Boolean) + ) + ) + } + + // skip helper + + function getSkipTagSelectorFor( + type: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + defaultValue?: any + ): { + selector: string + format: 'skip' + }[] { + return selectors?.includes(`@skiptag-${type}`) + ? [{ selector: type, format: 'skip' }] + : defaultValue + ? [defaultValue] + : [] + } + + // convert + + const convert = compile({ + wordwrap: false, + + baseElements: { + selectors: preferredSelectors, + + returnDomByDefault: false, + }, + + selectors: [ + // data urls + ...(includeDataUrls + ? [] + : [ + { selector: '[href^="data:"]', format: 'skip' }, + { selector: '[src^="data:"]', format: 'skip' }, + ]), + // links + ...getSkipTagSelectorFor('a', { + selector: 'a', + options: { + ignoreHref: false, + noAnchorUrl: true, + hideLinkHrefIfSameAsText: true, + baseUrl: url, + }, + }), + // media + ...getSkipTagSelectorFor('img'), + ...getSkipTagSelectorFor('audio'), + ...getSkipTagSelectorFor('video'), + { selector: 'object', format: 'skip' }, + { selector: 'canvas', format: 'skip' }, + // visual + ...getSkipTagSelectorFor('hr'), + // navigation + { selector: 'nav', format: 'skip' }, + { selector: 'header', format: 'skip' }, + { selector: 'footer', format: 'skip' }, + // roles + { selector: '[role="alert"]', format: 'skip' }, + { selector: '[role="alertdialog"]', format: 'skip' }, + { selector: '[role="application"]', format: 'skip' }, + { selector: '[role="banner"]', format: 'skip' }, + { selector: '[role="button"]', format: 'skip' }, + { selector: '[role="checkbox"]', format: 'skip' }, + { selector: '[role="combobox"]', format: 'skip' }, + { selector: '[role="command"]', format: 'skip' }, + { selector: '[role="dialog"]', format: 'skip' }, + { selector: '[role="form"]', format: 'skip' }, + { selector: '[role="input"]', format: 'skip' }, + { selector: '[role="menu"]', format: 'skip' }, + { selector: '[role="navigation"]', format: 'skip' }, + { selector: '[role="radio"]', format: 'skip' }, + { selector: '[role="radiogroup"]', format: 'skip' }, + { selector: '[role="range"]', format: 'skip' }, + { selector: '[role="scrollbar"]', format: 'skip' }, + { selector: '[role="search"]', format: 'skip' }, + { selector: '[role="searchbox"]', format: 'skip' }, + { selector: '[role="slider"]', format: 'skip' }, + { selector: '[role="spinbutton"]', format: 'skip' }, + { selector: '[role="status"]', format: 'skip' }, + { selector: '[role="suggestion"]', format: 'skip' }, + { selector: '[role="switch"]', format: 'skip' }, + // @note disabled because these could be useful content + // { selector: '[role="tab"]', format: 'skip' }, + // { selector: '[role="tabpanel"]', format: 'skip' }, + { selector: '[role="textbox"]', format: 'skip' }, + { selector: '[role="timer"]', format: 'skip' }, + { selector: '[role="toolbar"]', format: 'skip' }, + { selector: '[role="tooltip"]', format: 'skip' }, + { selector: '[role="widget"]', format: 'skip' }, + // interaction + { selector: 'form', format: 'skip' }, + { selector: 'button', format: 'skip' }, + { selector: 'input', format: 'skip' }, + { selector: 'textarea', format: 'skip' }, + { selector: 'dialog', format: 'skip' }, + // frames + { selector: 'iframe', format: 'skip' }, + { selector: 'frame', format: 'skip' }, + { selector: 'frameset', format: 'skip' }, + // functional + { selector: 'meta', format: 'skip' }, + { selector: 'script', format: 'skip' }, + { selector: 'noscript', format: 'skip' }, + // other + // @note disabled because these are sometimes causing issues + // @todo perhaps we should first try with these and if empty response try without them + // { selector: '[class*="header"]', format: 'skip' }, + // { selector: '[class*="footer"]', format: 'skip' }, + // { selector: '[class*="menu"]', format: 'skip' }, + // { selector: '[class*="breadcrumbs"]', format: 'skip' }, + ], + }) + + let text = convert(html) + + // remove double new lines + + text = text.replace(/\n+/g, '\n') + + // remove double spaces + + text = text.replace(/\s+/g, ' ') + + return text +} + +export function stripHtml(html: string): string { + return html2text(`
${html}
`, { selectors: ['main'] }) +} + +export function validateSelectors(selectors: string | string[]): { + valid: boolean + message?: string +} { + try { + html2text('', { selectors }) + } catch (e) { + const errorMessage = e instanceof Error ? e.message : String(e) + + return { valid: false, message: errorMessage.trim() } + } + + return { valid: true } +} diff --git a/packages/file-html/tsconfig.json b/packages/file-html/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-html/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-json/README.md b/packages/file-json/README.md new file mode 100644 index 0000000..a5bff8a --- /dev/null +++ b/packages/file-json/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-json diff --git a/packages/file-json/jest.config.js b/packages/file-json/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-json/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-json/package.json b/packages/file-json/package.json new file mode 100644 index 0000000..970b517 --- /dev/null +++ b/packages/file-json/package.json @@ -0,0 +1,39 @@ +{ + "name": "@chatbotkit-dev/file-json", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "js-yaml": "^4.1.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/js-yaml": "^4.0.9", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-json/src/index.test.ts b/packages/file-json/src/index.test.ts new file mode 100644 index 0000000..1d8db57 --- /dev/null +++ b/packages/file-json/src/index.test.ts @@ -0,0 +1,172 @@ +import { chunk, split } from './index' + +describe('split', () => { + it('should return null for invalid JSON', () => { + expect(split('invalid')).toBeNull() + }) + + it('should return a Chunk with content and meta for valid JSON', () => { + const line = '{"name": "John", "age": 30}' + const result = split(line) + + expect(result).toEqual({ + text: 'name: John\nage: 30', + meta: { age: 30 }, + }) + }) + + it('should not move ints represented as strings to meta', () => { + const line = '{"name": "John", "age": "30"}' + const result = split(line) + + expect(result).toEqual({ + text: "name: John\nage: '30'", + meta: {}, + }) + }) + + it('should move non-alphanumeric fields to meta', () => { + const line = '{"name": "John", "_age": 30}' + const result = split(line) + + expect(result).toEqual({ + text: 'name: John', + meta: { age: 30 }, + }) + }) + + it('should move boolean fields to meta', () => { + const line = '{"name": "John", "active": true}' + const result = split(line) + + expect(result).toEqual({ + text: 'name: John\nactive: true', + meta: { active: true }, + }) + }) + + it('should return empty content for empty object', () => { + const line = '{}' + const result = split(line) + + expect(result).toEqual({ + text: '', + meta: {}, + }) + }) + + it('should return empty content for empty array', () => { + const line = '[]' + const result = split(line) + + expect(result).toEqual({ + text: '', + meta: {}, + }) + }) +}) + +describe('chunk', () => { + it('should yield Chunks for valid JSON array', async () => { + const data = '[{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]' + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + { text: 'name: Jane\nage: 25', meta: { age: 25 } }, + ]) + }) + + it('should yield a Chunk for valid JSON object', async () => { + const data = '{"name": "John", "age": 30}' + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([{ text: 'name: John\nage: 30', meta: { age: 30 } }]) + }) + + it('should yield a Chunk for each property array of objects in JSON', async () => { + const data = + '{"people": [{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]}' + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + { text: 'name: Jane\nage: 25', meta: { age: 25 } }, + ]) + }) + + it('should not yield a Chunk for array properties that are not objects', async () => { + const data = '{"people": [1, 2, 3]}' + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([ + { + text: 'people:\n - 1\n - 2\n - 3', + meta: {}, + }, + ]) + }) + + it('should not yield anything for invalid JSON', async () => { + const data = 'invalid' + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([]) + }) + + it('should yield a single Chunk for a multi-key JSON object', async () => { + const data = '{"name": "John", "city": "NYC"}' + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([{ text: 'name: John\ncity: NYC', meta: {} }]) + }) + + it('should not throw for JSON object with single null property', async () => { + const data = JSON.stringify({ items: null }) + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/json' }) + )) { + chunks.push(c) + } + + expect(chunks.length).toBeGreaterThanOrEqual(0) + }) +}) diff --git a/packages/file-json/src/index.ts b/packages/file-json/src/index.ts new file mode 100644 index 0000000..50440f2 --- /dev/null +++ b/packages/file-json/src/index.ts @@ -0,0 +1,108 @@ +import jsYaml from 'js-yaml' + +interface Chunk { + text: string + meta: Record +} + +export function split(input: string): Chunk | null { + let data + + try { + data = JSON.parse(input) + } catch { + return null + } + + const meta: Record = {} + + // any field that starts with non-alphanumeric character should go into the meta data object + + for (const key in data) { + if (!/^[a-zA-Z]/.test(key)) { + meta[key.replace(/^([^a-zA-Z0-9]+)(.*)$/, '$2')] = data[key] + + delete data[key] + } + } + + // any field that is a number or a boolean should go into the meta data object + + for (const key in data) { + if (typeof data[key] === 'number' || typeof data[key] === 'boolean') { + meta[key] = data[key] + } + } + + // the final content is trimmed + + let text = jsYaml.dump(data, { lineWidth: -1 }).trim() + + // if the content is empty object or array, it should be an empty string + + text = text === '{}' || text === '[]' ? '' : text + + return { + text: text, + meta: meta, + } +} + +export async function* chunk(blob: Blob): AsyncGenerator { + let json + + try { + const data = await blob.text() + + json = JSON.parse(data) + } catch { + return + } + + // handle arrays + { + if (Array.isArray(json)) { + for (const item of json) { + const result = split(JSON.stringify(item)) + + if (result) { + yield result + } + } + + return + } + } + + // handle objects with a single property that is an array + { + if (Object.keys(json).length === 1) { + const firstValue = json[Object.keys(json)[0]] + + if (Array.isArray(firstValue) && typeof firstValue[0] === 'object' && firstValue[0] !== null) { + for (const item of firstValue) { + const result = split(JSON.stringify(item)) + + if (result) { + yield result + } + } + + return + } else { + yield { + text: jsYaml.dump(json, { lineWidth: -1 }).trim(), + meta: {}, + } + + return + } + } + } + + const result = split(JSON.stringify(json)) + + if (result) { + yield result + } +} diff --git a/packages/file-json/tsconfig.json b/packages/file-json/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/file-json/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-jsonl/README.md b/packages/file-jsonl/README.md new file mode 100644 index 0000000..33c5c77 --- /dev/null +++ b/packages/file-jsonl/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-jsonl diff --git a/packages/file-jsonl/jest.config.js b/packages/file-jsonl/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-jsonl/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-jsonl/package.json b/packages/file-jsonl/package.json new file mode 100644 index 0000000..e1edcfe --- /dev/null +++ b/packages/file-jsonl/package.json @@ -0,0 +1,38 @@ +{ + "name": "@chatbotkit-dev/file-jsonl", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/file-json": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-jsonl/src/index.test.ts b/packages/file-jsonl/src/index.test.ts new file mode 100644 index 0000000..7bdd03e --- /dev/null +++ b/packages/file-jsonl/src/index.test.ts @@ -0,0 +1,101 @@ +import { chunk, split } from './index' + +describe('split', () => { + it('should return null for invalid JSON', () => { + const result = split('invalid json') + + expect(result).toBeNull() + }) + + it('should split valid JSON string into content and meta', () => { + const line = JSON.stringify({ + name: 'John', + age: 30, + }) + const result = split(line) + + expect(result).toEqual({ + text: 'name: John\nage: 30', + meta: { age: 30 }, + }) + }) + + it('should move non-alphanumeric fields to meta', () => { + const line = '{"name": "John", "_age": 30}' + const result = split(line) + + expect(result).toEqual({ + text: 'name: John', + meta: { age: 30 }, + }) + }) + + it('should handle JSON with only string fields', () => { + const line = JSON.stringify({ name: 'John', city: 'New York' }) + const result = split(line) + + expect(result).toEqual({ + text: 'name: John\ncity: New York', + meta: {}, + }) + }) + + it('should handle JSON with only non-string fields', () => { + const line = JSON.stringify({ age: 30, active: true }) + const result = split(line) + + expect(result).toEqual({ + text: 'age: 30\nactive: true', + meta: { age: 30, active: true }, + }) + }) +}) + +describe('chunk', () => { + it('should yield chunks for each valid JSON line', async () => { + const data = + JSON.stringify({ name: 'John' }) + '\n' + JSON.stringify({ age: 30 }) + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/jsonl' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([ + { text: 'name: John', meta: {} }, + { text: 'age: 30', meta: { age: 30 } }, + ]) + }) + + it('should skip empty lines', async () => { + const data = + JSON.stringify({ name: 'John' }) + '\n\n' + JSON.stringify({ age: 30 }) + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/jsonl' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([ + { text: 'name: John', meta: {} }, + { text: 'age: 30', meta: { age: 30 } }, + ]) + }) + + it('should handle all invalid JSON lines', async () => { + const data = 'invalid json' + JSON.stringify({ name: 'John' }) + const chunks = [] + + for await (const c of chunk( + new Blob([data], { type: 'application/jsonl' }) + )) { + chunks.push(c) + } + + expect(chunks).toEqual([]) + }) +}) diff --git a/packages/file-jsonl/src/index.ts b/packages/file-jsonl/src/index.ts new file mode 100644 index 0000000..e75e381 --- /dev/null +++ b/packages/file-jsonl/src/index.ts @@ -0,0 +1,32 @@ +import { split } from '@chatbotkit-dev/file-json' + +export { split } from '@chatbotkit-dev/file-json' + +interface Chunk { + text: string + meta: Record +} + +export async function* chunk(blob: Blob): AsyncGenerator { + let lines + + try { + const data = await blob.text() + + lines = data.split('\n') + } catch { + return + } + + for (const line of lines) { + if (!line) { + continue + } + + const item = split(line) + + if (item) { + yield item + } + } +} diff --git a/packages/file-jsonl/tsconfig.json b/packages/file-jsonl/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/file-jsonl/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-md/README.md b/packages/file-md/README.md new file mode 100644 index 0000000..742bf46 --- /dev/null +++ b/packages/file-md/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-md diff --git a/packages/file-md/jest.config.js b/packages/file-md/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-md/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-md/package.json b/packages/file-md/package.json new file mode 100644 index 0000000..46889ce --- /dev/null +++ b/packages/file-md/package.json @@ -0,0 +1,39 @@ +{ + "name": "@chatbotkit-dev/file-md", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/gpt": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-md/src/index.test.ts b/packages/file-md/src/index.test.ts new file mode 100644 index 0000000..6374dc4 --- /dev/null +++ b/packages/file-md/src/index.test.ts @@ -0,0 +1,5 @@ +describe('pass', () => { + it('pass', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/file-md/src/index.ts b/packages/file-md/src/index.ts new file mode 100644 index 0000000..2387111 --- /dev/null +++ b/packages/file-md/src/index.ts @@ -0,0 +1,37 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = await blob.text() + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-md/tsconfig.json b/packages/file-md/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-md/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-pdf/README.md b/packages/file-pdf/README.md new file mode 100644 index 0000000..6fa441f --- /dev/null +++ b/packages/file-pdf/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-pdf diff --git a/packages/file-pdf/data/test01.pdf b/packages/file-pdf/data/test01.pdf new file mode 100644 index 0000000..49e3a9e Binary files /dev/null and b/packages/file-pdf/data/test01.pdf differ diff --git a/packages/file-pdf/jest.config.js b/packages/file-pdf/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-pdf/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-pdf/package.json b/packages/file-pdf/package.json new file mode 100644 index 0000000..0006e60 --- /dev/null +++ b/packages/file-pdf/package.json @@ -0,0 +1,44 @@ +{ + "name": "@chatbotkit-dev/file-pdf", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./parse": { + "import": "./src/parse.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/gpt": "workspace:*", + "@napi-rs/canvas": "file:../../stubs/napi-rs-canvas", + "unpdf": "^1.4.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-pdf/src/index.ts b/packages/file-pdf/src/index.ts new file mode 100644 index 0000000..cea54d3 --- /dev/null +++ b/packages/file-pdf/src/index.ts @@ -0,0 +1,41 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +import { pdf2text } from './parse' + +export { pdf2pages, pdf2text } from './parse' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = await pdf2text(new Uint8Array(await blob.arrayBuffer())) + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-pdf/src/parse.test.ts b/packages/file-pdf/src/parse.test.ts new file mode 100644 index 0000000..963b5c0 --- /dev/null +++ b/packages/file-pdf/src/parse.test.ts @@ -0,0 +1,13 @@ +import { pdf2text } from './parse' + +import fs from 'node:fs' + +describe('pptx2text', () => { + test('should return text', async () => { + const data = fs.readFileSync('./data/test01.pdf') + + const text = await pdf2text(new Uint8Array(data)) + + expect(text.trim()).toEqual('HELLO WORLD') + }) +}) diff --git a/packages/file-pdf/src/parse.ts b/packages/file-pdf/src/parse.ts new file mode 100644 index 0000000..54f21e2 --- /dev/null +++ b/packages/file-pdf/src/parse.ts @@ -0,0 +1,15 @@ +import { extractText, getDocumentProxy } from 'unpdf' + +export async function pdf2pages(data: Uint8Array): Promise { + const document = await getDocumentProxy(data) + + const { text: pages } = await extractText(document) + + return pages +} + +export async function pdf2text(data: Uint8Array): Promise { + const pages = await pdf2pages(data) + + return pages.join('\n\n') +} diff --git a/packages/file-pdf/tsconfig.json b/packages/file-pdf/tsconfig.json new file mode 100644 index 0000000..73969da --- /dev/null +++ b/packages/file-pdf/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ESNext"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-pptx/README.md b/packages/file-pptx/README.md new file mode 100644 index 0000000..7584ebb --- /dev/null +++ b/packages/file-pptx/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-pptx diff --git a/packages/file-pptx/data/test01.pptx b/packages/file-pptx/data/test01.pptx new file mode 100644 index 0000000..cd094b2 --- /dev/null +++ b/packages/file-pptx/data/test01.pptx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdad238875e6402f7a90fa3216d15581c55a10f98dd4e704c90bc581206d463c +size 34431 diff --git a/packages/file-pptx/jest.config.js b/packages/file-pptx/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-pptx/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-pptx/package.json b/packages/file-pptx/package.json new file mode 100644 index 0000000..bb91520 --- /dev/null +++ b/packages/file-pptx/package.json @@ -0,0 +1,43 @@ +{ + "name": "@chatbotkit-dev/file-pptx", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./parse": { + "import": "./src/parse.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/gpt": "workspace:*", + "officeparser": "7.8.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-pptx/src/index.ts b/packages/file-pptx/src/index.ts new file mode 100644 index 0000000..283b729 --- /dev/null +++ b/packages/file-pptx/src/index.ts @@ -0,0 +1,41 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +import { pptx2text } from './parse' + +export { pptx2text } from './parse' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = await pptx2text(new Uint8Array(await blob.arrayBuffer())) + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-pptx/src/parse.test.ts b/packages/file-pptx/src/parse.test.ts new file mode 100644 index 0000000..da9e10f --- /dev/null +++ b/packages/file-pptx/src/parse.test.ts @@ -0,0 +1,13 @@ +import { pptx2text } from './parse' + +import fs from 'node:fs' + +describe('pptx2text', () => { + test('should return text', async () => { + const data = fs.readFileSync('./data/test01.pptx') + + const text = await pptx2text(data) + + expect(text.trim()).toEqual('HELLO\nWORLD') + }) +}) diff --git a/packages/file-pptx/src/parse.ts b/packages/file-pptx/src/parse.ts new file mode 100644 index 0000000..0cb4475 --- /dev/null +++ b/packages/file-pptx/src/parse.ts @@ -0,0 +1,7 @@ +import { parseOffice } from 'officeparser' + +export async function pptx2text(buffer: Uint8Array): Promise { + const result = await parseOffice(Buffer.from(buffer)) + + return result.toText() +} diff --git a/packages/file-pptx/tsconfig.json b/packages/file-pptx/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-pptx/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-txt/README.md b/packages/file-txt/README.md new file mode 100644 index 0000000..0ad3ef0 --- /dev/null +++ b/packages/file-txt/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-txt diff --git a/packages/file-txt/jest.config.js b/packages/file-txt/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-txt/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-txt/package.json b/packages/file-txt/package.json new file mode 100644 index 0000000..1c8c9e7 --- /dev/null +++ b/packages/file-txt/package.json @@ -0,0 +1,39 @@ +{ + "name": "@chatbotkit-dev/file-txt", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/file-json": "workspace:*", + "@chatbotkit-dev/gpt": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-txt/src/index.test.ts b/packages/file-txt/src/index.test.ts new file mode 100644 index 0000000..6374dc4 --- /dev/null +++ b/packages/file-txt/src/index.test.ts @@ -0,0 +1,5 @@ +describe('pass', () => { + it('pass', () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/file-txt/src/index.ts b/packages/file-txt/src/index.ts new file mode 100644 index 0000000..2387111 --- /dev/null +++ b/packages/file-txt/src/index.ts @@ -0,0 +1,37 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = await blob.text() + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-txt/tsconfig.json b/packages/file-txt/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-txt/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-xlsx/README.md b/packages/file-xlsx/README.md new file mode 100644 index 0000000..129a03d --- /dev/null +++ b/packages/file-xlsx/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-xlsx diff --git a/packages/file-xlsx/data/test01.xlsx b/packages/file-xlsx/data/test01.xlsx new file mode 100644 index 0000000..c631a35 --- /dev/null +++ b/packages/file-xlsx/data/test01.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:211778744fbe178229fa8ee0e2d12b5a408ac1e20313c9b23db2cd6524cc5c35 +size 4740 diff --git a/packages/file-xlsx/jest.config.js b/packages/file-xlsx/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-xlsx/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-xlsx/package.json b/packages/file-xlsx/package.json new file mode 100644 index 0000000..8211b0f --- /dev/null +++ b/packages/file-xlsx/package.json @@ -0,0 +1,43 @@ +{ + "name": "@chatbotkit-dev/file-xlsx", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./parse": { + "import": "./src/parse.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/gpt": "workspace:*", + "officeparser": "7.8.0" + }, + "devDependencies": { + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/node": "^24.0.0", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-xlsx/src/index.ts b/packages/file-xlsx/src/index.ts new file mode 100644 index 0000000..7f4dba2 --- /dev/null +++ b/packages/file-xlsx/src/index.ts @@ -0,0 +1,41 @@ +import { split } from '@chatbotkit-dev/gpt' +import { splitTextRecursiveByTokens } from '@chatbotkit-dev/gpt/text-splitter' + +import { xlsx2text } from './parse' + +export { xlsx2text } from './parse' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const text = await xlsx2text(new Uint8Array(await blob.arrayBuffer())) + + // @note when separators are provided use recursive character splitting with + // token-based length which matches the original Python service behavior + const blocks = options.separators + ? splitTextRecursiveByTokens(text, { + chunkSize: options.size || 512, + chunkOverlap: options.overlap || 0, + separators: options.separators, + }) + : split(text, options.size || 512, options.overlap || 0) + + for (const block of blocks) { + yield { + text: block, + meta: {}, + } + } +} diff --git a/packages/file-xlsx/src/parse.test.ts b/packages/file-xlsx/src/parse.test.ts new file mode 100644 index 0000000..bac68bf --- /dev/null +++ b/packages/file-xlsx/src/parse.test.ts @@ -0,0 +1,15 @@ +import { xlsx2text } from './parse' + +import fs from 'node:fs' + +describe('xlsx2text', () => { + test('should return text', async () => { + const data = fs.readFileSync('./data/test01.xlsx') + + const text = await xlsx2text(data) + + expect(text.trim()).toEqual( + 'column1\ncolumn2\ncolumn3\ncell1\ncell2\ncell3' + ) + }) +}) diff --git a/packages/file-xlsx/src/parse.ts b/packages/file-xlsx/src/parse.ts new file mode 100644 index 0000000..99ffb05 --- /dev/null +++ b/packages/file-xlsx/src/parse.ts @@ -0,0 +1,7 @@ +import { parseOffice } from 'officeparser' + +export async function xlsx2text(buffer: Uint8Array): Promise { + const result = await parseOffice(Buffer.from(buffer)) + + return result.toText() +} diff --git a/packages/file-xlsx/tsconfig.json b/packages/file-xlsx/tsconfig.json new file mode 100644 index 0000000..3d5f905 --- /dev/null +++ b/packages/file-xlsx/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest", "node"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file-yaml/README.md b/packages/file-yaml/README.md new file mode 100644 index 0000000..e96913c --- /dev/null +++ b/packages/file-yaml/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file-yaml diff --git a/packages/file-yaml/jest.config.js b/packages/file-yaml/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file-yaml/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file-yaml/package.json b/packages/file-yaml/package.json new file mode 100644 index 0000000..e1d5881 --- /dev/null +++ b/packages/file-yaml/package.json @@ -0,0 +1,40 @@ +{ + "name": "@chatbotkit-dev/file-yaml", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/file-json": "workspace:*", + "js-yaml": "^4.1.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/js-yaml": "^4.0.9", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file-yaml/src/index.test.ts b/packages/file-yaml/src/index.test.ts new file mode 100644 index 0000000..44a842c --- /dev/null +++ b/packages/file-yaml/src/index.test.ts @@ -0,0 +1,82 @@ +import { chunk, split } from './index' + +async function collect(data: string, type = 'text/yaml') { + const chunks = [] + + for await (const c of chunk(new Blob([data], { type }))) { + chunks.push(c) + } + + return chunks +} + +describe('split', () => { + it('should be re-exported from the json chunker', () => { + expect(typeof split).toBe('function') + + expect(split('{"name": "John", "age": 30}')).toEqual({ + text: 'name: John\nage: 30', + meta: { age: 30 }, + }) + }) +}) + +describe('chunk', () => { + it('should yield a chunk per item for a top level list of maps', async () => { + const data = '- name: John\n age: 30\n- name: Jane\n age: 25' + + expect(await collect(data)).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + { text: 'name: Jane\nage: 25', meta: { age: 25 } }, + ]) + }) + + it('should yield a single chunk for a multi-key map', async () => { + const data = 'name: John\nage: 30' + + expect(await collect(data)).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + ]) + }) + + it('should yield a chunk per item for a single array property map', async () => { + const data = + 'people:\n - name: John\n age: 30\n - name: Jane\n age: 25' + + expect(await collect(data)).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + { text: 'name: Jane\nage: 25', meta: { age: 25 } }, + ]) + }) + + it('should round-trip a nested map into yaml text', async () => { + const data = "name: John\naddress:\n city: NYC\n zip: '10001'" + + expect(await collect(data)).toEqual([ + { + text: "name: John\naddress:\n city: NYC\n zip: '10001'", + meta: {}, + }, + ]) + }) + + it('should also accept json (yaml is a json superset)', async () => { + const data = '{"name": "John", "age": 30}' + + expect(await collect(data, 'application/yaml')).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + ]) + }) + + it('should yield nothing for an empty document', async () => { + expect(await collect('')).toEqual([]) + }) + + it('should yield nothing for a null document', async () => { + expect(await collect('null')).toEqual([]) + }) + + it('should yield nothing for malformed yaml', async () => { + expect(await collect('key: [unclosed')).toEqual([]) + }) +}) diff --git a/packages/file-yaml/src/index.ts b/packages/file-yaml/src/index.ts new file mode 100644 index 0000000..d3ff927 --- /dev/null +++ b/packages/file-yaml/src/index.ts @@ -0,0 +1,85 @@ +import { split } from '@chatbotkit-dev/file-json' + +import jsYaml from 'js-yaml' + +// @note yaml is structurally a superset of json so we reuse the exact same +// record splitting as the json chunker (which itself already emits yaml via +// js-yaml). the only difference is that we parse the document with yaml. + +export { split } from '@chatbotkit-dev/file-json' + +interface Chunk { + text: string + meta: Record +} + +export async function* chunk(blob: Blob): AsyncGenerator { + let doc: unknown + + try { + const data = await blob.text() + + doc = jsYaml.load(data) + } catch { + return + } + + // @note empty or explicitly null documents have nothing to chunk + + if (doc === null || doc === undefined) { + return + } + + // handle arrays + { + if (Array.isArray(doc)) { + for (const item of doc) { + const result = split(JSON.stringify(item)) + + if (result) { + yield result + } + } + + return + } + } + + // handle objects with a single property that is an array + { + if (typeof doc === 'object' && Object.keys(doc).length === 1) { + const record = doc as Record + + const firstValue = record[Object.keys(record)[0]] + + if ( + Array.isArray(firstValue) && + typeof firstValue[0] === 'object' && + firstValue[0] !== null + ) { + for (const item of firstValue) { + const result = split(JSON.stringify(item)) + + if (result) { + yield result + } + } + + return + } else { + yield { + text: jsYaml.dump(doc, { lineWidth: -1 }).trim(), + meta: {}, + } + + return + } + } + } + + const result = split(JSON.stringify(doc)) + + if (result) { + yield result + } +} diff --git a/packages/file-yaml/tsconfig.json b/packages/file-yaml/tsconfig.json new file mode 100644 index 0000000..54214b6 --- /dev/null +++ b/packages/file-yaml/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ES2019"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/file/README.md b/packages/file/README.md new file mode 100644 index 0000000..b664839 --- /dev/null +++ b/packages/file/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/file diff --git a/packages/file/jest.config.js b/packages/file/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/file/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/file/package.json b/packages/file/package.json new file mode 100644 index 0000000..3ac5a84 --- /dev/null +++ b/packages/file/package.json @@ -0,0 +1,55 @@ +{ + "name": "@chatbotkit-dev/file", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./index2": { + "import": "./src/index2.ts" + }, + "./support": { + "import": "./src/support.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/file-csv": "workspace:*", + "@chatbotkit-dev/file-docx": "workspace:*", + "@chatbotkit-dev/file-html": "workspace:*", + "@chatbotkit-dev/file-json": "workspace:*", + "@chatbotkit-dev/file-jsonl": "workspace:*", + "@chatbotkit-dev/file-md": "workspace:*", + "@chatbotkit-dev/file-pdf": "workspace:*", + "@chatbotkit-dev/file-pptx": "workspace:*", + "@chatbotkit-dev/file-txt": "workspace:*", + "@chatbotkit-dev/file-xlsx": "workspace:*", + "@chatbotkit-dev/file-yaml": "workspace:*" + }, + "devDependencies": { + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "@types/js-yaml": "^4.0.9", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/file/src/index.test.ts b/packages/file/src/index.test.ts new file mode 100644 index 0000000..93429cf --- /dev/null +++ b/packages/file/src/index.test.ts @@ -0,0 +1,35 @@ +import { canChunkContentType, chunk } from './index' + +const yamlContentTypes = [ + 'application/yaml', + 'application/x-yaml', + 'text/yaml', + 'text/x-yaml', +] + +async function collect(data: string, type: string) { + const chunks = [] + + for await (const c of chunk(new Blob([data], { type }), {})) { + chunks.push(c) + } + + return chunks +} + +describe('yaml content type support', () => { + it.each(yamlContentTypes)('chunks a %s document as core', async (type) => { + // @note core (no experimental flag) — yaml is treated like json + expect(canChunkContentType(type)).toBe(true) + + expect(await collect('name: John\nage: 30', type)).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + ]) + }) + + it('still throws for a genuinely unsupported content type', async () => { + await expect(collect('hello', 'text/vnd.unknown')).rejects.toThrow( + 'Unsupported content type text/vnd.unknown' + ) + }) +}) diff --git a/packages/file/src/index.ts b/packages/file/src/index.ts new file mode 100644 index 0000000..206e000 --- /dev/null +++ b/packages/file/src/index.ts @@ -0,0 +1,109 @@ +import { chunk as chunkCsv } from '@chatbotkit-dev/file-csv' +import { chunk as chunkHtml } from '@chatbotkit-dev/file-html' +import { chunk as chunkJson } from '@chatbotkit-dev/file-json' +import { chunk as chunkJsonl } from '@chatbotkit-dev/file-jsonl' +import { chunk as chunkMd } from '@chatbotkit-dev/file-md' +import { chunk as chunkTxt } from '@chatbotkit-dev/file-txt' +import { chunk as chunkYaml } from '@chatbotkit-dev/file-yaml' + +import type { CoreContentType, ExperimentalContentType } from './support' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] + experimental?: boolean +} + +function withoutOptions(fn: (blob: Blob) => AsyncGenerator) { + return (blob: Blob, options: Options) => { + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + options + + return fn(blob) + } +} + +function getBlobContentType(blob: Blob): string { + return blob.type.toLowerCase().split(';')[0].trim() +} + +const coreFns: Record< + CoreContentType, + (blob: Blob, options: Options) => AsyncGenerator +> = { + 'text/csv': withoutOptions(chunkCsv), + 'application/json': withoutOptions(chunkJson), + 'application/jsonl': withoutOptions(chunkJsonl), + 'application/yaml': withoutOptions(chunkYaml), + 'application/x-yaml': withoutOptions(chunkYaml), + 'text/yaml': withoutOptions(chunkYaml), + 'text/x-yaml': withoutOptions(chunkYaml), +} + +const experimentalFns: Record< + ExperimentalContentType, + ((blob: Blob, options: Options) => AsyncGenerator) | null +> = { + 'text/plain': chunkTxt, + 'text/markdown': chunkMd, + 'text/html': chunkHtml, + // @note the reason these are disabled for now is because we use the this + // module inside components that are loaded in the edge environment and + // unfortunately the document APIs are not supported well + 'application/pdf': null, + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + null, + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': + null, + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': null, +} + +export function getChunkFunction( + blob: Blob, + options?: Options +): ((blob: Blob, options: Options) => AsyncGenerator) | null { + const type = getBlobContentType(blob) + + let fn = type in coreFns ? coreFns[type as CoreContentType] : null + + if (options?.experimental) { + fn ??= + type in experimentalFns + ? experimentalFns[type as ExperimentalContentType] + : null + } + + return fn +} + +export function canChunk(blob: Blob, options?: Options): boolean { + return !!getChunkFunction(blob, options) +} + +export function canChunkContentType( + contentType: string, + options?: Options +): boolean { + return !!getChunkFunction(new Blob([], { type: contentType }), options) +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const fn = getChunkFunction(blob, options) + + if (!fn) { + throw new Error(`Unsupported content type ${getBlobContentType(blob)}`) + } + + for await (const chunk of fn(blob, options)) { + yield chunk + } +} diff --git a/packages/file/src/index2.test.ts b/packages/file/src/index2.test.ts new file mode 100644 index 0000000..993dcf3 --- /dev/null +++ b/packages/file/src/index2.test.ts @@ -0,0 +1,44 @@ +import { canChunkContentType, chunk } from './index2' + +const yamlContentTypes = [ + 'application/yaml', + 'application/x-yaml', + 'text/yaml', + 'text/x-yaml', +] + +async function collect(data: string, type: string) { + const chunks = [] + + for await (const c of chunk(new Blob([data], { type }), {})) { + chunks.push(c) + } + + return chunks +} + +describe('yaml content type support', () => { + // @note this is the map used by the /api/auxiliary/dataset/chunk endpoint; + // before yaml was wired in, a text/yaml url served through chunkUrl threw + // "Unsupported content type text/yaml" + + it.each(yamlContentTypes)('chunks a %s document as core', async (type) => { + expect(canChunkContentType(type)).toBe(true) + + expect(await collect('name: John\nage: 30', type)).toEqual([ + { text: 'name: John\nage: 30', meta: { age: 30 } }, + ]) + }) + + it('normalises a content type with charset params', async () => { + expect( + await collect('name: John\nage: 30', 'text/yaml; charset=utf-8') + ).toEqual([{ text: 'name: John\nage: 30', meta: { age: 30 } }]) + }) + + it('still throws for a genuinely unsupported content type', async () => { + await expect(collect('hello', 'text/vnd.unknown')).rejects.toThrow( + 'Unsupported content type text/vnd.unknown' + ) + }) +}) diff --git a/packages/file/src/index2.ts b/packages/file/src/index2.ts new file mode 100644 index 0000000..ec84c86 --- /dev/null +++ b/packages/file/src/index2.ts @@ -0,0 +1,111 @@ +import { chunk as chunkCsv } from '@chatbotkit-dev/file-csv' +import { chunk as chunkDocx } from '@chatbotkit-dev/file-docx' +import { chunk as chunkHtml } from '@chatbotkit-dev/file-html' +import { chunk as chunkJson } from '@chatbotkit-dev/file-json' +import { chunk as chunkJsonl } from '@chatbotkit-dev/file-jsonl' +import { chunk as chunkMd } from '@chatbotkit-dev/file-md' +import { chunk as chunkPdf } from '@chatbotkit-dev/file-pdf' +import { chunk as chunkPptx } from '@chatbotkit-dev/file-pptx' +import { chunk as chunkTxt } from '@chatbotkit-dev/file-txt' +import { chunk as chunkXlsx } from '@chatbotkit-dev/file-xlsx' +import { chunk as chunkYaml } from '@chatbotkit-dev/file-yaml' + +import type { CoreContentType, ExperimentalContentType } from './support' + +interface Chunk { + text: string + meta: Record +} + +interface Options { + size?: number + overlap?: number + separators?: string[] + experimental?: boolean +} + +function withoutOptions(fn: (blob: Blob) => AsyncGenerator) { + return (blob: Blob, options: Options) => { + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + options + + return fn(blob) + } +} + +function getBlobContentType(blob: Blob): string { + return blob.type.toLowerCase().split(';')[0].trim() +} + +const coreFns: Record< + CoreContentType, + (blob: Blob, options: Options) => AsyncGenerator +> = { + 'text/csv': withoutOptions(chunkCsv), + 'application/json': withoutOptions(chunkJson), + 'application/jsonl': withoutOptions(chunkJsonl), + 'application/yaml': withoutOptions(chunkYaml), + 'application/x-yaml': withoutOptions(chunkYaml), + 'text/yaml': withoutOptions(chunkYaml), + 'text/x-yaml': withoutOptions(chunkYaml), +} + +const experimentalFns: Record< + ExperimentalContentType, + (blob: Blob, options: Options) => AsyncGenerator +> = { + 'text/plain': chunkTxt, + 'text/markdown': chunkMd, + 'text/html': chunkHtml, + 'application/pdf': chunkPdf, + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + chunkDocx, + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': + chunkPptx, + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': + chunkXlsx, +} + +export function getChunkFunction( + blob: Blob, + options?: Options +): ((blob: Blob, options: Options) => AsyncGenerator) | null { + const type = getBlobContentType(blob) + + let fn = type in coreFns ? coreFns[type as CoreContentType] : null + + if (options?.experimental) { + fn ??= + type in experimentalFns + ? experimentalFns[type as ExperimentalContentType] + : null + } + + return fn +} + +export function canChunk(blob: Blob, options?: Options): boolean { + return !!getChunkFunction(blob, options) +} + +export function canChunkContentType( + contentType: string, + options?: Options +): boolean { + return !!getChunkFunction(new Blob([], { type: contentType }), options) +} + +export async function* chunk( + blob: Blob, + options: Options +): AsyncGenerator { + const fn = getChunkFunction(blob, options) + + if (!fn) { + throw new Error(`Unsupported content type ${getBlobContentType(blob)}`) + } + + for await (const chunk of fn(blob, options)) { + yield chunk + } +} diff --git a/packages/file/src/support.ts b/packages/file/src/support.ts new file mode 100644 index 0000000..4f183d9 --- /dev/null +++ b/packages/file/src/support.ts @@ -0,0 +1,36 @@ +export const coreContentTypes = [ + 'text/csv', + 'application/json', + 'application/jsonl', + 'application/yaml', + 'application/x-yaml', + 'text/yaml', + 'text/x-yaml', +] as const + +export const experimentalContentTypes = [ + 'text/plain', + 'text/markdown', + 'text/html', + 'application/pdf', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +] as const + +export type CoreContentType = (typeof coreContentTypes)[number] +export type ExperimentalContentType = (typeof experimentalContentTypes)[number] + +export type SupportedContentType = + Experimental extends true + ? CoreContentType | ExperimentalContentType + : CoreContentType + +export function getSupportedContentTypes(options?: { + experimental?: E +}): SupportedContentType[] { + return [ + ...coreContentTypes, + ...(options?.experimental ? experimentalContentTypes : []), + ] as SupportedContentType[] +} diff --git a/packages/file/tsconfig.json b/packages/file/tsconfig.json new file mode 100644 index 0000000..cb51e43 --- /dev/null +++ b/packages/file/tsconfig.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "isolatedModules": true, + "rootDir": ".", + "types": ["jest"], + "noEmit": true, + "composite": true, + "target": "es2019", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["DOM", "ESNext"], + "allowJs": true, + "checkJs": true, + "declaration": true, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["./src/**/*.ts", "./src/**/*.js"], + "exclude": ["node_modules"] +} diff --git a/packages/gpt/README.md b/packages/gpt/README.md new file mode 100644 index 0000000..4ee1ab6 --- /dev/null +++ b/packages/gpt/README.md @@ -0,0 +1 @@ +# @chatbotkit-dev/gpt diff --git a/packages/gpt/jest.config.js b/packages/gpt/jest.config.js new file mode 100644 index 0000000..e5bd08d --- /dev/null +++ b/packages/gpt/jest.config.js @@ -0,0 +1,5 @@ +export default { + preset: 'ts-jest/presets/js-with-ts-esm', + roots: ['/src'], + testEnvironment: '@chatbotkit-dev/jest-jsdom', +} diff --git a/packages/gpt/package.json b/packages/gpt/package.json new file mode 100644 index 0000000..41782e6 --- /dev/null +++ b/packages/gpt/package.json @@ -0,0 +1,42 @@ +{ + "name": "@chatbotkit-dev/gpt", + "version": "0.0.0", + "license": "Apache-2.0", + "private": true, + "type": "module", + "exports": { + ".": { + "import": "./src/index.ts" + }, + "./text-splitter": { + "import": "./src/text-splitter.ts" + } + }, + "scripts": { + "build": "true", + "check": "tsc6 --noEmit --incremental", + "clean": "run-s clean:*", + "clean:01-tsbuildinfo": "rimraf *.tsbuildinfo", + "clean:02-dist": "rimraf dist", + "clean:03-node_modules": "rimraf node_modules", + "format": "true", + "lint": "eslint src --ext .ts,.js,.tsx,.jsx", + "test": "NODE_OPTIONS=--experimental-vm-modules jest" + }, + "access": "restricted", + "dependencies": { + "@types/node": "^24.0.0", + "gpt-tokenizer": "^2.9.0" + }, + "devDependencies": { + "@chatbotkit-dev/eslint-config": "workspace:*", + "@chatbotkit-dev/jest-jsdom": "workspace:*", + "@types/jest": "^29.5.11", + "eslint": "^9.0.0", + "jest": "^29", + "npm-run-all2": "^9.0.3", + "rimraf": "^5.0.5", + "ts-jest": "^29.4.12", + "typescript": "npm:@typescript/typescript6@^6.0.2" + } +} diff --git a/packages/gpt/src/index.test.ts b/packages/gpt/src/index.test.ts new file mode 100644 index 0000000..d19c66c --- /dev/null +++ b/packages/gpt/src/index.test.ts @@ -0,0 +1,519 @@ +import { + ELLIPSIS, + SEP, + adjoinTokens, + batchTokens, + connectText, + countFrequencies, + getBytePairEncodingFrequencies, + getBytePairEncodings, + getTextTokens, + separateText, + slice, + split, + splitTextBlocks, + tokenizeTextBlocks, + trimTextBlockL, + trimTextBlockR, +} from './index' + +function toa(it: Iterable): T[] { + const array = [] + + for (const i of it) { + array.push(i) + } + + return array +} + +describe('getBytePairEncodings', () => { + it('returns correct tokens', () => { + expect(getBytePairEncodings('abc xyz')).toEqual([13997, 41611]) + expect(getBytePairEncodings('abc xyz').length).toEqual( + getTextTokens('abc xyz').length + ) + }) + + it('handles special tokens without throwing', () => { + // @note these special tokens previously caused "Disallowed special token found" errors + expect(() => getBytePairEncodings('<|im_end|>')).not.toThrow() + expect(() => getBytePairEncodings('<|im_start|>')).not.toThrow() + expect(() => getBytePairEncodings('<|endoftext|>')).not.toThrow() + expect(() => getBytePairEncodings('Hello <|im_end|> world')).not.toThrow() + }) + + it('returns tokens for text containing special tokens', () => { + const tokens = getBytePairEncodings('<|im_end|>') + + expect(Array.isArray(tokens)).toBe(true) + expect(tokens.length).toBeGreaterThan(0) + }) +}) + +describe('getBytePairEncodingFrequencies', () => { + it('returns the correct frequencies', () => { + expect(getBytePairEncodingFrequencies('abc xyz')).toEqual({ + 13997: 1, + 41611: 1, + }) + expect(getBytePairEncodingFrequencies('test')).toEqual({ 1985: 1 }) + expect(getBytePairEncodingFrequencies('train train train')).toEqual({ + 10613: 1, + 5542: 2, + }) + }) +}) + +describe('getTextTokens', () => { + it('returns correct tokens', async () => { + expect(getTextTokens('abc xyz')).toEqual(['abc', ' xyz']) + expect(getTextTokens('abc xyz').length).toEqual( + getBytePairEncodings('abc xyz').length + ) + }) + + it('handles special tokens without throwing', () => { + // @note these special tokens previously caused "Disallowed special token found" errors + expect(() => getTextTokens('<|im_end|>')).not.toThrow() + expect(() => getTextTokens('<|im_start|>')).not.toThrow() + expect(() => getTextTokens('<|endoftext|>')).not.toThrow() + expect(() => getTextTokens('Hello <|im_end|> world')).not.toThrow() + }) + + it('returns tokens for text containing special tokens', () => { + const tokens = getTextTokens('<|im_end|>') + + expect(Array.isArray(tokens)).toBe(true) + expect(tokens.length).toBeGreaterThan(0) + }) +}) + +describe('adjoinTokens', () => { + it('must concat tokens basic cases', () => { + expect(adjoinTokens([], [], [], 123, 123)).toEqual([]) + expect(adjoinTokens([], ['a'], [], 123, 123)).toEqual(['a']) + expect(adjoinTokens(['l'], ['a'], [], 123, 123)).toEqual(['l', 'a']) + expect(adjoinTokens(['l'], ['a'], ['r'], 123, 123)).toEqual(['l', 'a', 'r']) + }) + + it('must concat tokens left side trim cases', () => { + expect(adjoinTokens(['a', 'b', 'c', 'd', 'e'], ['-'], [], 123, 3)).toEqual([ + ELLIPSIS, + 'd', + 'e', + '-', + ]) + expect( + adjoinTokens(['a', 'b', 'c', 'd', 'e'], ['-'], [], 123, 3, '') + ).toEqual(['c', 'd', 'e', '-']) + }) + + it('must concat tokens right side trim cases', () => { + expect(adjoinTokens([], ['-'], ['v', 'w', 'x', 'y', 'z'], 123, 3)).toEqual([ + '-', + 'v', + 'w', + ELLIPSIS, + ]) + expect( + adjoinTokens([], ['-'], ['v', 'w', 'x', 'y', 'z'], 123, 3, '') + ).toEqual(['-', 'v', 'w', 'x']) + }) + + it('must concat tokens equal site cases', () => { + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 123, + 3 + ) + ).toEqual([ELLIPSIS, 'd', 'e', '-', 'v', 'w', ELLIPSIS]) + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 123, + 3, + '' + ) + ).toEqual(['c', 'd', 'e', '-', 'v', 'w', 'x']) + }) + + it('must concat tokens odd cases', () => { + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 1, + 3 + ) + ).toEqual(['-']) + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 1, + 3, + '' + ) + ).toEqual(['-']) + + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 2, + 3 + ) + ).toEqual(['-']) + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 2, + 3, + '' + ) + ).toEqual(['-']) + + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 3, + 3 + ) + ).toEqual(['e', '-', 'v']) + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 3, + 3, + '' + ) + ).toEqual(['e', '-', 'v']) + + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 4, + 3 + ) + ).toEqual(['e', '-', 'v']) + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 4, + 3, + '' + ) + ).toEqual(['e', '-', 'v']) + + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 5, + 3 + ) + ).toEqual([ELLIPSIS, 'e', '-', 'v', ELLIPSIS]) + expect( + adjoinTokens( + ['a', 'b', 'c', 'd', 'e'], + ['-'], + ['v', 'w', 'x', 'y', 'z'], + 5, + 3, + '' + ) + ).toEqual(['d', 'e', '-', 'v', 'w']) + }) +}) + +describe('separateText', () => { + it('must correctly separate the text', () => { + expect( + connectText( + separateText( + 'Hello.\nThis is a long text.\n\nSome parts of it should be on sep lines.', + ['\n\n'] + ), + '<-|->' + ) + ).toEqual( + 'Hello.\nThis is a long text.<-|->Some parts of it should be on sep lines.' + ) + + expect( + connectText( + separateText( + 'Hello.\nThis is a long text.\n\nSome parts of it should be on sep lines.\n\nAnd more lines.', + ['\n\n'] + ), + '<-|->' + ) + ).toEqual( + 'Hello.\nThis is a long text.<-|->Some parts of it should be on sep lines.<-|->And more lines.' + ) + + expect( + connectText( + separateText( + 'Hello.\nThis is a long text.\n\n\n\nSome parts of it should be on sep lines.', + ['\n\n'] + ), + '<-|->' + ) + ).toEqual( + 'Hello.\nThis is a long text.<-|->Some parts of it should be on sep lines.' + ) + }) +}) + +describe('tokenizeTextBlocks', () => { + it('must correctly tokenize text blocks', () => { + expect(toa(tokenizeTextBlocks(['a']))).toEqual(['a']) + expect(toa(tokenizeTextBlocks(['a', 'b']))).toEqual(['a', SEP, 'b']) + expect(toa(tokenizeTextBlocks(['hello world', 'hello world']))).toEqual([ + 'hello', + ' world', + SEP, + 'hello', + ' world', + ]) + }) +}) + +describe('batchTokens', () => { + it('must correctly batch tokens', () => { + expect(toa(batchTokens(['a', 'b', 'c', 'd', 'e'], 1))).toEqual([ + ['a', 'b', ELLIPSIS], + [ELLIPSIS, 'c', ELLIPSIS], + [ELLIPSIS, 'd', ELLIPSIS], + [ELLIPSIS, 'e'], + ]) + expect(toa(batchTokens(['a', 'b', 'c', 'd', 'e'], 2))).toEqual([ + ['a', 'b', ELLIPSIS], + [ELLIPSIS, 'c', ELLIPSIS], + [ELLIPSIS, 'd', ELLIPSIS], + [ELLIPSIS, 'e'], + ]) + expect(toa(batchTokens(['a', 'b', 'c', 'd', 'e'], 3))).toEqual([ + ['a', 'b', ELLIPSIS], + [ELLIPSIS, 'c', ELLIPSIS], + [ELLIPSIS, 'd', ELLIPSIS], + [ELLIPSIS, 'e'], + ]) + expect(toa(batchTokens(['a', 'b', 'c', 'd', 'e'], 4))).toEqual([ + ['a', 'b', 'c', ELLIPSIS], + [ELLIPSIS, 'd', 'e'], + ]) + expect(toa(batchTokens(['a', 'b', 'c', 'd', 'e'], 10))).toEqual([ + ['a', 'b', 'c', 'd', 'e'], + ]) + }) +}) + +describe('trimTextBlockL', () => { + it('must trim l', () => { + expect(trimTextBlockL('')).toEqual('') + expect(trimTextBlockL('.')).toEqual('.') + expect(trimTextBlockL('..')).toEqual('..') + expect(trimTextBlockL('...')).toEqual('...') + expect(trimTextBlockL('....')).toEqual('...') + expect(trimTextBlockL('.....')).toEqual('....') + }) +}) + +describe('trimTextBlockR', () => { + it('must trim r', () => { + expect(trimTextBlockR('')).toEqual('') + expect(trimTextBlockR('.')).toEqual('.') + expect(trimTextBlockR('..')).toEqual('..') + expect(trimTextBlockR('...')).toEqual('...') + expect(trimTextBlockR('....')).toEqual('...') + expect(trimTextBlockR('.....')).toEqual('....') + }) +}) + +describe('splitTextBlocks', () => { + it('must correctly split text blocks', () => { + expect( + toa( + splitTextBlocks( + 'Hello.\nThis is a long text.\n\nSome parts of it should be on sep lines.', + 100, + 5, + ['\n\n'] + ) + ) + ).toEqual([ + 'Hello.\nThis is a long text. Some parts of it...', + '... long text. Some parts of it should be on sep lines.', + ]) + + expect( + toa( + splitTextBlocks( + 'Hello.\nThis is a long text.\n\nSome parts of it should be on sep lines.\n\nAnd more lines.', + 100, + 3, + ['\n\n'] + ) + ) + ).toEqual([ + 'Hello.\nThis is a long text. Some parts...', + '... Some parts of it should be on sep lines. And more...', + '... And more lines.', + ]) + }) +}) + +describe('slice', () => { + it('must correctly slice the text', () => { + expect(slice('this is a test', 1)).toEqual(' is a test') + expect(slice('this is a test', 1, -1)).toEqual(' is a') + }) + + it('correctly slices the text between start and stop tokens', () => { + expect(slice('this is a test', 1, 3)).toEqual(' is a') + expect(slice('hello world', 0, -1)).toEqual('hello') + }) + + it('slices to the end if stop token is omitted', () => { + expect(slice('this is a test', 2)).toEqual(' a test') + }) + + it('handles negative indices correctly', () => { + expect(slice('this is a test', -3, -1)).toEqual(' is a') + }) + + it('returns an empty string if startToken equals stopToken', () => { + expect(slice('this is a test', 2, 2)).toEqual('') + }) + + it('handles out of bounds indices gracefully', () => { + expect(slice('this is a test', 10, 20)).toEqual('') + expect(slice('this is a test', -20, -10)).toEqual('') + }) + + it('returns an empty string if startToken is greater than stopToken', () => { + expect(slice('this is a test', 3, 1)).toEqual('') + }) + + it('handles empty input strings', () => { + expect(slice('', 0, 1)).toEqual('') + }) + + it('returns the entire string if both startToken and stopToken are out of bounds', () => { + expect(slice('hello world', -100, 100)).toEqual('hello world') + }) +}) + +describe('split', () => { + it('returns an empty array for empty input', () => { + expect(split('', 5)).toEqual([]) + }) + + it('returns single word as chunk irrespective of maxTokens or overlapTokens', () => { + expect(split('hello', 1, 0)).toEqual(['hello']) + expect(split('hello', 10, 2)).toEqual(['hello']) + }) + + it('returns entire input as a single chunk when maxTokens exceeds number of words', () => { + expect(split('hello world', 5)).toEqual(['hello world']) + }) + + it('throws error when overlapTokens is equal to maxTokens', () => { + expect(() => split('hello world', 5, 5)).toThrow( + 'overlapTokens must be less than maxTokens' + ) + }) + + it('throws error when overlapTokens is negative', () => { + expect(() => split('hello world', 5, -1)).toThrow( + 'overlapTokens must be greater than or equal to 0' + ) + }) + + it('handles fewer tokens than maxTokens without issue', () => { + expect(split('one two', 5)).toEqual(['one two']) + }) + + it('must correctly split the text', () => { + expect(split('The quick brown fox jumps over the lazy dog', 3)).toEqual([ + 'The quick brown', + ' fox jumps over', + ' the lazy dog', + ]) + expect(split('The quick brown fox jumps over the lazy dog', 3, 1)).toEqual([ + 'The quick brown', + ' brown fox jumps', + ' jumps over the', + ' the lazy dog', + ]) + expect( + split( + 'The quick brown fox jumps over the lazy dog and feels as if he were in the seventh heaven', + 3, + 1 + ) + ).toEqual([ + 'The quick brown', + ' brown fox jumps', + ' jumps over the', + ' the lazy dog', + ' dog and feels', + ' feels as if', + ' if he were', + ' were in the', + ' the seventh heaven', + ]) + }) + + it('must correctly spit out single chunk', () => { + expect(split('The quick brown fox jumps over the lazy dog', 400)).toEqual([ + 'The quick brown fox jumps over the lazy dog', + ]) + }) +}) + +describe('countFrequencies', () => { + it('returns an empty object if given an empty array', () => { + expect(countFrequencies([])).toEqual({}) + }) + + it('returns the correct frequency counts for a simple array', () => { + const arr = [1, 2, 2, 3, 3, 3] + + expect(countFrequencies(arr)).toEqual({ 1: 1, 2: 2, 3: 3 }) + }) + + it('returns the correct frequency counts for an array with string elements', () => { + const arr = ['foo', 'bar', 'foo', 'baz', 'baz'] + + expect(countFrequencies(arr)).toEqual({ foo: 2, bar: 1, baz: 2 }) + }) + + it('returns the correct frequency counts for an array with null and undefined elements', () => { + const arr = [null, undefined, null, undefined, null, undefined] + + expect(countFrequencies(arr)).toEqual({ null: 3, undefined: 3 }) + }) +}) diff --git a/packages/gpt/src/index.ts b/packages/gpt/src/index.ts new file mode 100644 index 0000000..723507e --- /dev/null +++ b/packages/gpt/src/index.ts @@ -0,0 +1,484 @@ +import { ok as assert } from 'assert' +import { decodeGenerator, encode } from 'gpt-tokenizer' + +// should be cl100k_base + +// --- +// --- +// --- + +export const SEP = null +export const ELLIPSIS = '...' + +type Model = string + +// --- +// --- +// --- + +export function countFrequencies(arr: T[]): Record { + const frequency: Record = {} + + for (const elem of arr) { + const key = `${elem}` + + if (typeof frequency[key] === 'undefined') { + frequency[key] = 1 + } else { + frequency[key]++ + } + } + + return frequency +} + +export function getBytePairEncodings( + text: string, + _model: Model = 'gpt-4' +): number[] { + // @todo deal with the model + + // @note allow all special tokens to prevent crashes when user input contains tokens like <|im_end|> + const result = encode(text, { allowedSpecial: 'all' }) + + return result +} + +export function getBytePairEncodingFrequencies( + text: string, + model: Model = 'gpt-4' +): Record { + const result = countFrequencies(getBytePairEncodings(text, model)) + + return result +} + +export function getTextTokens(text: string, _model: Model = 'gpt-4'): string[] { + // @todo deal with the model + + const tokens: string[] = [] + + // @note allow all special tokens to prevent crashes when user input contains tokens like <|im_end|> + for (const token of decodeGenerator( + encode(text, { allowedSpecial: 'all' }) + )) { + tokens.push(token) + } + + return tokens +} + +export function getTextTokensLength( + text: string, + model: Model = 'gpt-4' +): number { + const result = getBytePairEncodings(text, model) + + return result.length +} + +// --- +// --- +// --- + +/** + * @deprecated + */ +export function adjoinTokens( + prevTokens: string[], + tokens: string[], + nextTokens: string[], + maxTokens: number, + overlapTokens: number, + ellipsis: string = ELLIPSIS +): string[] { + // @note This function is not designed to accurately slice the tokens hence we + // use whichever is the maximum tokens size. Why is this? Well, where should + // we cut from? It is not clearly defined, thus we do not do it. The maxTokens + // should be always greater than or equal to tokens.length. + + maxTokens = Math.max(tokens.length, maxTokens) + + if (maxTokens === tokens.length) { + return tokens + } + + // @note The overlapTokens cannot be negative thus we always peg it to zero to + // avoid any future problems. + + overlapTokens = Math.max(0, overlapTokens) + + if (overlapTokens == 0) { + return tokens + } + + // @note Here we calculate how to trim the prev and next tokens. We need to + // carefully consider if we should trim both side or one side only. + + let trimByL + let trimByR + + switch (true) { + case !!prevTokens.length && !!nextTokens.length: + trimByL = trimByR = Math.floor((maxTokens - tokens.length) / 2) + + break + + case !!prevTokens.length && !nextTokens.length: + trimByL = maxTokens - tokens.length + trimByR = 0 + + break + + case !prevTokens.length && !!nextTokens.length: + trimByL = 0 + trimByR = maxTokens - tokens.length + + break + + default: + trimByL = 0 + trimByR = 0 + + break + } + + // @note Bot the left and right trim cannot exceed overlapTokens or the tokens + // length. We take whatever value is the lowest. The assumption is that with + // all of the previous steps in order we cannot get negative values or zero. + + trimByL = Math.min(trimByL, overlapTokens, prevTokens.length) + trimByR = Math.min(trimByR, overlapTokens, nextTokens.length) + + // @note Calculate the trim and the ellipsis on the left side. Not only the + // trim value must be positive but also less than prevTokens length. + + if (trimByL && trimByL <= prevTokens.length) { + let ellipsisL + + if (ellipsis && trimByL > 1 && prevTokens.length > trimByL) { + ellipsisL = ellipsis + } + + prevTokens = prevTokens.slice(-trimByL) + + if (ellipsisL) { + prevTokens[0] = ellipsis + } + } else { + prevTokens = [] + } + + // @note Calculate the trim and the ellipsis on the right side. Not only the + // trim value must be positive but also less than nextTokens length. + + if (trimByR && trimByR <= nextTokens.length) { + let ellipsisR + + if (ellipsis && trimByR > 1 && nextTokens.length > trimByR) { + ellipsisR = ellipsis + } + + nextTokens = nextTokens.slice(0, trimByR) + + if (ellipsisR) { + nextTokens[nextTokens.length - 1] = ellipsis + } + } else { + nextTokens = [] + } + + // @note Build the final list tokens. + + return [...prevTokens, ...tokens, ...nextTokens] +} + +// --- +// --- +// --- + +/** + * Recursively split a text into text blocks. + * + * @deprecated + */ +export function* separateText( + text: string, + separators: string[] = [] +): Generator { + separators = separators.slice(0) + + const sep = separators.shift() + + if (!sep) { + yield text + + return + } + + const sections = text.split(sep) + + do { + const section = sections.shift() + + if (section) { + yield* separateText(section, separators) + } + } while (sections.length) +} + +/** + * Connect text blocks into contingent text. + * + * @deprecated + */ +export function connectText( + iterator: Iterable, + connector: string = '\n\n' +): string { + const tokens: string[] = [] + + for (const text of iterator) { + tokens.push(text) + } + + return tokens.join(connector) +} + +/** + * Split text blocks into a vector of tokens. SEP is a special tokens that + * indicates start of a block. + * + * @deprecated + */ +export function* tokenizeTextBlocks( + blocks: Iterable +): Generator { + let isFirst = true + + for (const block of blocks) { + if (isFirst) { + isFirst = false + } else { + yield SEP + } + + yield* getTextTokens(block) + } +} + +/** + * Yield batches of maxTokens for vector of tokens. + * + * @deprecated + */ +export function* batchTokens( + tokens: Iterable, + maxTokens: number, + ellipsis: string = ELLIPSIS +): Generator { + maxTokens = Math.max(maxTokens, ellipsis ? 3 : 1) + + let batch: string[] = [] + + for (const token of tokens) { + if (token === SEP) { + if (batch.length) { + batch.push(' ') + + yield batch + + batch = [] + } + } else { + batch.push(token) + + if (batch.length === maxTokens) { + let last + + if (ellipsis) { + last = batch.pop() + + batch.push(ellipsis) + } + + yield batch + + batch = [] + + if (ellipsis) { + batch.push(ellipsis) + } + + if (last) { + batch.push(last) + } + } + } + } + + if (batch.length) { + yield batch + } +} + +// --- +// --- +// --- + +/** + * @deprecated + */ +export function trimTextBlockL( + block: string, + ellipsis: string = ELLIPSIS +): string { + if ( + ellipsis && + block !== ellipsis && + block[0] === ellipsis[0] && + block.slice(1, ellipsis.length + 1) === ellipsis + ) { + block = block.slice(1) + } + + return block +} + +/** + * @deprecated + */ +export function trimTextBlockR( + block: string, + ellipsis: string = ELLIPSIS +): string { + if ( + ellipsis && + block !== ellipsis && + block[block.length - 1] === ellipsis[ellipsis.length - 1] && + block.slice(-ellipsis.length - 1, -1) === ellipsis + ) { + block = block.slice(0, -1) + } + + return block +} + +/** + * @deprecated + */ +export function trimTextBlock( + block: string, + ellipsis: string = ELLIPSIS +): string { + return trimTextBlockL(trimTextBlockR(block, ellipsis), ellipsis) +} + +/** + * @deprecated use split instead + */ +export function* splitTextBlocks( + text: string, + maxTokens: number, + overlapTokens: number, + separators: string[] = [], + ellipsis: string = ELLIPSIS +): Generator { + let prevBatch: string[] = [] + + const itr = batchTokens( + tokenizeTextBlocks(separateText(text, separators)), + maxTokens, + ellipsis + ) + + for (const thisBatch of itr) { + if (prevBatch.length) { + const block = adjoinTokens( + [], + prevBatch, + thisBatch, + maxTokens, + overlapTokens, + ellipsis + ).join('') + + yield trimTextBlock(block) + } + + prevBatch = adjoinTokens( + prevBatch, + thisBatch, + [], + maxTokens, + overlapTokens, + ellipsis + ) + } + + if (prevBatch.length) { + const block = prevBatch.join('') + + yield trimTextBlock(block) + } +} + +// --- +// --- +// --- + +/** + * Slices the given string from start to stop token. This function has the same + * behavior as the Array.slice method but for text tokens. + */ +export function slice( + input: string, + startToken: number, + stopToken: number = Infinity, + options: { + toTextTokens: (input: string) => string[] + } = { toTextTokens: getTextTokens } +): string { + const { toTextTokens } = options + + const tokens = toTextTokens(input) + + const sliced = tokens.slice(startToken, stopToken) + + const result = sliced.join('') + + return result +} + +/** + * Splits the text of maxToken sizes with overlap tokens. + */ +export function split( + input: string, + maxTokens: number, + overlapTokens: number = 0, + options: { + toTextTokens: (input: string) => string[] + } = { toTextTokens: getTextTokens } +): string[] { + assert(overlapTokens >= 0, 'overlapTokens must be greater than or equal to 0') + assert(overlapTokens < maxTokens, 'overlapTokens must be less than maxTokens') + + const { toTextTokens } = options + + const chunks: string[] = [] + + const tokens = toTextTokens(input) + + const stepSize = maxTokens - overlapTokens + + for (let i = 0; i < tokens.length; i += stepSize) { + const chunk = tokens.slice(i, i + maxTokens).join('') + + if (chunks.length === 0 || !chunks[chunks.length - 1]?.endsWith(chunk)) { + chunks.push(chunk) + } + } + + return chunks +} diff --git a/packages/gpt/src/text-splitter.test.ts b/packages/gpt/src/text-splitter.test.ts new file mode 100644 index 0000000..cbe1f00 --- /dev/null +++ b/packages/gpt/src/text-splitter.test.ts @@ -0,0 +1,1195 @@ +import { getTextTokensLength } from './index' +import { + DEFAULT_SEPARATORS, + splitTextRecursive, + splitTextRecursiveByTokens, +} from './text-splitter' + +// --- +// --- +// --- + +// @note helper ported from LangChain.js test suite to generate lines of +// repeated characters for chunk boundary testing + +function textLineGenerator(char: string, length: number): string { + const line = new Array(length).join(char) + + return `${line}\n` +} + +// --- +// --- +// --- + +describe('splitTextRecursive', () => { + describe('basic splitting', () => { + it('returns single chunk for short text', () => { + const output = splitTextRecursive('Hello world', { + chunkSize: 100, + chunkOverlap: 0, + }) + + expect(output).toEqual(['Hello world']) + }) + + it('returns empty array for empty text', () => { + const output = splitTextRecursive('', { + chunkSize: 100, + chunkOverlap: 0, + }) + + expect(output).toEqual([]) + }) + + it('splits text on double newlines by default', () => { + const text = 'Hello\n\nWorld' + + const output = splitTextRecursive(text, { + chunkSize: 10, + chunkOverlap: 0, + }) + + expect(output).toEqual(['Hello', 'World']) + }) + + it('splits text on single newlines when no double newlines present', () => { + const text = 'Hello\nWorld' + + const output = splitTextRecursive(text, { + chunkSize: 6, + chunkOverlap: 0, + }) + + expect(output).toEqual(['Hello', 'World']) + }) + + it('splits text on spaces as last resort before characters', () => { + const text = 'foo bar baz' + + const output = splitTextRecursive(text, { + chunkSize: 5, + chunkOverlap: 0, + }) + + expect(output).toEqual(['foo', 'bar', 'baz']) + }) + + it('splits text into individual characters as final fallback', () => { + const text = 'abcde' + + const output = splitTextRecursive(text, { + chunkSize: 2, + chunkOverlap: 0, + }) + + expect(output).toEqual(['ab', 'cd', 'e']) + }) + }) + + // --- + // --- + // --- + + describe('overlap', () => { + it('creates overlapping chunks', () => { + const text = 'foo bar baz 123' + + const output = splitTextRecursive(text, { + chunkSize: 7, + chunkOverlap: 3, + keepSeparator: false, + }) + + expect(output).toEqual(['foo bar', 'bar baz', 'baz 123']) + }) + + it('handles overlap with single character separators', () => { + const text = 'aa ab ac ba bb' + + const output = splitTextRecursive(text, { + keepSeparator: false, + chunkSize: 7, + chunkOverlap: 3, + }) + + expect(output).toEqual(['aa ab', 'ab ac', 'ac ba', 'ba bb']) + }) + }) + + // --- + // --- + // --- + + describe('recursive behavior', () => { + it('uses double newline first then falls back to single newline', () => { + const text = + "Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.\nThis is a weird text to write, but gotta test the splittingggg some how.\n\nBye!\n\n-H." + + const output = splitTextRecursive(text, { + chunkSize: 10, + chunkOverlap: 1, + }) + + expect(output).toEqual([ + 'Hi.', + "I'm", + 'Harrison.', + 'How? Are?', + 'You?', + 'Okay then', + 'f f f f.', + 'This is a', + 'weird', + 'text to', + 'write,', + 'but gotta', + 'test the', + 'splitting', + 'gggg', + 'some how.', + 'Bye!', + '-H.', + ]) + }) + + it('handles multi-level separator fallback', () => { + const text = + 'Part A first paragraph.\n\nPart A second paragraph.\n\nPart B section.\nLine 1\nLine 2' + + const output = splitTextRecursive(text, { + chunkSize: 30, + chunkOverlap: 0, + }) + + expect(output.length).toBeGreaterThan(1) + + for (const chunk of output) { + expect(chunk.length).toBeLessThanOrEqual(30) + } + }) + }) + + // --- + // --- + // --- + + describe('keepSeparator', () => { + it('keeps separator when keepSeparator is true (default)', () => { + const text = 'Hello\n\nWorld' + + const output = splitTextRecursive(text, { + chunkSize: 8, + chunkOverlap: 0, + keepSeparator: true, + }) + + expect(output).toEqual(['Hello', 'World']) + }) + + it('removes separator when keepSeparator is false', () => { + const text = 'foo bar baz' + + const output = splitTextRecursive(text, { + chunkSize: 7, + chunkOverlap: 3, + keepSeparator: false, + }) + + expect(output).toEqual(['foo bar', 'bar baz']) + }) + }) + + // --- + // --- + // --- + + describe('custom separators', () => { + it('uses custom separators', () => { + const text = 'part1|part2|part3' + + const output = splitTextRecursive(text, { + chunkSize: 6, + chunkOverlap: 0, + separators: ['|', ''], + keepSeparator: false, + }) + + expect(output).toEqual(['part1', 'part2', 'part3']) + }) + + it('falls through custom separators in order', () => { + const text = 'section1##subsection1.1##subsection1.2\n\nsection2' + + const output = splitTextRecursive(text, { + chunkSize: 20, + chunkOverlap: 0, + separators: ['\n\n', '##', ' ', ''], + keepSeparator: false, + }) + + expect(output.length).toBeGreaterThan(1) + + for (const chunk of output) { + expect(chunk.length).toBeLessThanOrEqual(20) + } + }) + }) + + // --- + // --- + // --- + + describe('custom length function', () => { + it('uses a custom length function', () => { + // @note word count length function + const wordCount = (text: string) => + text.split(/\s+/).filter((w) => w.length > 0).length + + const text = 'one two three four five six seven eight' + + const output = splitTextRecursive(text, { + chunkSize: 3, + chunkOverlap: 0, + lengthFunction: wordCount, + }) + + expect(output.length).toBeGreaterThan(1) + }) + }) + + // --- + // --- + // --- + + describe('validation', () => { + it('throws when chunkOverlap >= chunkSize', () => { + expect(() => { + splitTextRecursive('test', { chunkSize: 2, chunkOverlap: 4 }) + }).toThrow('Cannot have chunkOverlap >= chunkSize') + + expect(() => { + splitTextRecursive('test', { chunkSize: 2, chunkOverlap: 2 }) + }).toThrow('Cannot have chunkOverlap >= chunkSize') + }) + }) + + // --- + // --- + // --- + + describe('DEFAULT_SEPARATORS', () => { + it('has the correct default separators', () => { + expect(DEFAULT_SEPARATORS).toEqual(['\n\n', '\n', ' ', '']) + }) + }) + + // --- + // --- + // --- + + describe('edge cases', () => { + it('handles text with only separators', () => { + const output = splitTextRecursive('\n\n\n\n', { + chunkSize: 5, + chunkOverlap: 0, + }) + + expect(output).toEqual([]) + }) + + it('handles single character text', () => { + const output = splitTextRecursive('a', { + chunkSize: 10, + chunkOverlap: 0, + }) + + expect(output).toEqual(['a']) + }) + + it('handles text with trailing whitespace', () => { + const output = splitTextRecursive('hello ', { + chunkSize: 10, + chunkOverlap: 0, + }) + + expect(output).toEqual(['hello']) + }) + + it('handles large chunk size', () => { + const text = 'This is a test string that should fit in one chunk.' + + const output = splitTextRecursive(text, { + chunkSize: 1000, + chunkOverlap: 0, + }) + + expect(output).toEqual([text]) + }) + + it('handles text with multiple consecutive separators', () => { + const text = 'Hello\n\n\n\nWorld' + + const output = splitTextRecursive(text, { + chunkSize: 10, + chunkOverlap: 0, + }) + + expect(output).toEqual(['Hello', 'World']) + }) + }) +}) + +// --- +// --- +// --- + +// @note the following tests are ported from langchain-ai/langchainjs +// libs/langchain-textsplitters/src/tests/text_splitter.test.ts to verify +// behavioral parity between our implementation and LangChain's +// RecursiveCharacterTextSplitter + +describe('langchain parity: CharacterTextSplitter equivalents', () => { + // @note LangChain's CharacterTextSplitter uses a single separator and then + // calls mergeSplits. We replicate this using separators: [sep] with + // keepSeparator: false. + + it('splits by character count with overlap', () => { + const output = splitTextRecursive('foo bar baz 123', { + separators: [' '], + chunkSize: 7, + chunkOverlap: 3, + keepSeparator: false, + }) + + expect(output).toEqual(['foo bar', 'bar baz', 'baz 123']) + }) + + it('does not create empty documents from double spaces', () => { + // @note LangChain's CharacterTextSplitter is non-recursive with a single + // separator. We replicate this with separators: [' ', ''] so the recursive + // splitter has a fallback. + const output = splitTextRecursive('foo bar', { + separators: [' ', ''], + chunkSize: 4, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual(['foo', 'bar']) + }) + + it('handles long words that exceed chunk size', () => { + const output = splitTextRecursive('foo bar baz a a', { + separators: [' ', ''], + chunkSize: 3, + chunkOverlap: 1, + keepSeparator: false, + }) + + expect(output).toEqual(['foo', 'bar', 'baz', 'a a']) + }) + + it('handles shorter words first then long words', () => { + const output = splitTextRecursive('a a foo bar baz', { + separators: [' ', ''], + chunkSize: 3, + chunkOverlap: 1, + keepSeparator: false, + }) + + expect(output).toEqual(['a a', 'foo', 'bar', 'baz']) + }) + + it('splits into characters with tiny chunk size', () => { + // @note unlike LangChain's CharacterTextSplitter (which leaves oversized + // splits intact), the recursive splitter falls through to '' separator + // and splits into individual characters when chunkSize < word length + const output = splitTextRecursive('foo bar baz', { + separators: [' ', ''], + chunkSize: 1, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual(['f', 'o', 'o', 'b', 'a', 'r', 'b', 'a', 'z']) + }) + + it('handles exhausted separators gracefully', () => { + // @note when separators list has no '' fallback, long words are emitted + // as-is rather than crashing + const output = splitTextRecursive('foo bar baz', { + separators: [' '], + chunkSize: 2, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual(['foo', 'bar', 'baz']) + }) +}) + +// --- +// --- +// --- + +// @note the following tests are ported from langchain-ai/langchain (Python) +// libs/text-splitters/tests/unit_tests/test_text_splitters.py to verify +// behavioral parity between our implementation and LangChain Python's +// RecursiveCharacterTextSplitter + +describe('langchain parity (python): CharacterTextSplitter equivalents', () => { + it('splits edge separator into small chunks', () => { + // @note Python: test_character_text_splitter_separtor_empty_doc + // "f b" with separator " ", chunk_size=2 → ["f", "b"] + const output = splitTextRecursive('f b', { + separators: [' ', ''], + chunkSize: 2, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual(['f', 'b']) + }) + + it('handles text with no matching separator', () => { + // @note Python: test_character_text_splitter_no_separator_in_text + // Single word with no separator present returns it as-is + const output = splitTextRecursive('singleword', { + separators: [' ', ''], + chunkSize: 10, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual(['singleword']) + }) + + it('returns empty array for whitespace-only input', () => { + // @note Python: test_character_text_splitter_whitespace_only + const output = splitTextRecursive(' ', { + separators: [' ', ''], + chunkSize: 5, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual([]) + }) + + it('merges splits respecting chunk size and overlap', () => { + // @note Python: test_merge_splits + // ["foo", "bar", "baz"] with separator " ", chunk_size=9, overlap=2 + // → "foo bar" (len 7 ≤ 9), then "baz" (can't merge further) + const output = splitTextRecursive('foo bar baz', { + separators: [' '], + chunkSize: 9, + chunkOverlap: 2, + keepSeparator: false, + }) + + expect(output).toEqual(['foo bar', 'baz']) + }) +}) + +// --- +// --- +// --- + +describe('langchain parity (python): RecursiveCharacterTextSplitter', () => { + it('splits with keepSeparator true using custom separators', () => { + // @note Python: test_iterative_text_splitter_keep_separator + // Text "....5X..3Y...4X....5Y..." with separators ["X", "Y"], + // keepSeparator=true, chunkSize=6 (5 + 1 for separator) + const output = splitTextRecursive('....5X..3Y...4X....5Y...', { + separators: ['X', 'Y'], + chunkSize: 6, + chunkOverlap: 0, + keepSeparator: true, + }) + + expect(output).toEqual(['....5', 'X..3', 'Y...4', 'X....5', 'Y...']) + }) + + it('splits with keepSeparator false using custom separators', () => { + // @note Python: test_iterative_text_splitter_discard_separator + // Same text but keepSeparator=false and chunkSize=5 + const output = splitTextRecursive('....5X..3Y...4X....5Y...', { + separators: ['X', 'Y'], + chunkSize: 5, + chunkOverlap: 0, + keepSeparator: false, + }) + + expect(output).toEqual(['....5', '..3', '...4', '....5', '...']) + }) + + it('validates that chunk overlap must be less than chunk size', () => { + // @note Python: test_character_text_splitting_args + // Python only rejects overlap > size, but JS LangChain and our + // implementation also reject overlap == size. This is a known difference. + expect(() => { + splitTextRecursive('test', { chunkSize: 2, chunkOverlap: 4 }) + }).toThrow('Cannot have chunkOverlap >= chunkSize') + }) + + it('rejects zero and negative chunk sizes', () => { + // @note Python: test_character_text_splitting_args validates + // chunk_size > 0 and chunk_overlap >= 0. Our implementation uses + // defaults when not provided, so this validates the throw path. + expect(() => { + splitTextRecursive('test', { chunkSize: 0, chunkOverlap: 0 }) + }).toThrow() + }) + + it('splits with keepSeparator=true (start) matching Python start behavior', () => { + // @note Python: test_recursive_character_text_splitter_keep_separators + // Python supports keep_separator="start" which prepends separator to + // the following chunk. Our keepSeparator=true uses regex lookahead + // which produces the same "start" behavior. + const output = splitTextRecursive('Apple,banana,orange and tomato.', { + separators: [',', '.'], + chunkSize: 10, + chunkOverlap: 0, + keepSeparator: true, + }) + + expect(output).toEqual(['Apple', ',banana', ',orange and tomato', '.']) + }) + + // @note Python also supports keep_separator="end" which appends the + // separator to the preceding chunk: ["Apple,", "banana,", "orange and tomato."] + // Our implementation only supports boolean keepSeparator (matching JS + // LangChain). "end" mode is a known Python-only feature difference. +}) + +// --- +// --- +// --- + +describe('langchain parity: RecursiveCharacterTextSplitter', () => { + it('produces one unique chunk for short content', () => { + const content = textLineGenerator('A', 70) + + const output = splitTextRecursive(content, { + chunkSize: 100, + chunkOverlap: 0, + }) + + expect(output).toEqual([content.trim()]) + }) + + it('splits two lines into separate chunks', () => { + const line1 = textLineGenerator('A', 70) + const line2 = textLineGenerator('B', 70) + const content = line1 + line2 + + const output = splitTextRecursive(content, { + chunkSize: 100, + chunkOverlap: 0, + }) + + expect(output).toEqual([line1.trim(), line2.trim()]) + }) + + it('splits identical lines into separate chunks', () => { + const line = textLineGenerator('A', 70) + const content = line + line + + const output = splitTextRecursive(content, { + chunkSize: 100, + chunkOverlap: 0, + }) + + expect(output).toEqual([line.trim(), line.trim()]) + }) + + it('handles content starting with newlines', () => { + const line1 = textLineGenerator('\n', 2) + const line2 = textLineGenerator('A', 70) + const line3 = textLineGenerator('\n', 4) + const line4 = textLineGenerator('B', 70) + + const content = line1 + line2 + line3 + line4 + + const output = splitTextRecursive(content, { + chunkSize: 100, + chunkOverlap: 0, + }) + + expect(output).toEqual([line2.trim(), line4.trim()]) + }) + + it('creates overlapping chunks from generated lines', () => { + const line1 = textLineGenerator('A', 70) + const line2 = textLineGenerator('B', 20) + const line3 = textLineGenerator('C', 70) + const content = line1 + line2 + line3 + + const output = splitTextRecursive(content, { + chunkSize: 100, + chunkOverlap: 30, + }) + + // @note first chunk contains line1 + line2, second contains line2 + line3 + expect(output).toEqual([(line1 + line2).trim(), (line2 + line3).trim()]) + }) + + it('handles overlap spanning multiple short lines', () => { + const line1 = textLineGenerator('A', 70) + const line2 = textLineGenerator('B', 10) + const line3 = textLineGenerator('C', 10) + const line4 = textLineGenerator('D', 70) + const content = line1 + line2 + line3 + line4 + + const output = splitTextRecursive(content, { + chunkSize: 100, + chunkOverlap: 30, + }) + + expect(output).toEqual([ + (line1 + line2 + line3).trim(), + (line2 + line3 + line4).trim(), + ]) + }) + + it('handles the iterative text splitter test case', () => { + const text = `Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.\nThis is a weird text to write, but gotta test the splittingggg some how.\n\nBye!\n\n-H.` + + const output = splitTextRecursive(text, { + chunkSize: 10, + chunkOverlap: 1, + }) + + expect(output).toEqual([ + 'Hi.', + "I'm", + 'Harrison.', + 'How? Are?', + 'You?', + 'Okay then', + 'f f f f.', + 'This is a', + 'weird', + 'text to', + 'write,', + 'but gotta', + 'test the', + 'splitting', + 'gggg', + 'some how.', + 'Bye!', + '-H.', + ]) + }) + + it('considers separator length correctly for chunk size', () => { + const output = splitTextRecursive('aa ab ac ba bb', { + keepSeparator: false, + chunkSize: 7, + chunkOverlap: 3, + }) + + expect(output).toEqual(['aa ab', 'ab ac', 'ac ba', 'ba bb']) + }) +}) + +// --- +// --- +// --- + +describe('langchain parity: language-specific separators', () => { + it('splits markdown content using markdown separators', () => { + const text = + '# 🦜️🔗 LangChain\n' + + '\n' + + '⚡ Building applications with LLMs through composability ⚡\n' + + '\n' + + '## Quick Install\n' + + '\n' + + '```bash\n' + + "# Hopefully this code block isn't split\n" + + 'pip install langchain\n' + + '```\n' + + '\n' + + 'As an open source project in a rapidly developing field, we are extremely open to contributions.' + + // @note these are the separators from + // RecursiveCharacterTextSplitter.getSeparatorsForLanguage('markdown') + const markdownSeparators = [ + '\n## ', + '\n### ', + '\n#### ', + '\n##### ', + '\n###### ', + '```\n\n', + '\n\n***\n\n', + '\n\n---\n\n', + '\n\n___\n\n', + '\n\n', + '\n', + ' ', + '', + ] + + const output = splitTextRecursive(text, { + separators: markdownSeparators, + chunkSize: 100, + chunkOverlap: 0, + keepSeparator: true, + }) + + expect(output).toEqual([ + '# 🦜️🔗 LangChain\n\n⚡ Building applications with LLMs through composability ⚡', + "## Quick Install\n\n```bash\n# Hopefully this code block isn't split\npip install langchain", + '```', + 'As an open source project in a rapidly developing field, we are extremely open to contributions.', + ]) + }) + + it('splits LaTeX content using LaTeX separators', () => { + const text = [ + '\\begin{document}', + '\\title{🦜️🔗 LangChain}', + '⚡ Building applications with LLMs through composability ⚡', + '', + '\\section{Quick Install}', + '', + '\\begin{verbatim}', + "Hopefully this code block isn't split", + 'pnpm install langchain', + '\\end{verbatim}', + '', + 'As an open source project in a rapidly developing field, we are extremely open to contributions.', + '', + '\\end{document}', + ].join('\n') + + // @note these are the separators from + // RecursiveCharacterTextSplitter.getSeparatorsForLanguage('latex') + const latexSeparators = [ + '\n\\chapter{', + '\n\\section{', + '\n\\subsection{', + '\n\\subsubsection{', + '\n\\begin{enumerate}', + '\n\\begin{itemize}', + '\n\\begin{description}', + '\n\\begin{list}', + '\n\\begin{quote}', + '\n\\begin{quotation}', + '\n\\begin{verse}', + '\n\\begin{verbatim}', + '\n\\begin{align}', + '$$', + '$', + '\n\n', + '\n', + ' ', + '', + ] + + const output = splitTextRecursive(text, { + separators: latexSeparators, + chunkSize: 100, + chunkOverlap: 0, + keepSeparator: true, + }) + + expect(output).toEqual([ + '\\begin{document}\n\\title{🦜️🔗 LangChain}\n⚡ Building applications with LLMs through composability ⚡', + '\\section{Quick Install}', + "\\begin{verbatim}\nHopefully this code block isn't split\npnpm install langchain\n\\end{verbatim}", + 'As an open source project in a rapidly developing field, we are extremely open to contributions.', + '\\end{document}', + ]) + }) + + it('splits HTML content using HTML separators', () => { + const text = [ + '', + '', + ' ', + ' 🦜️🔗 LangChain', + ' ', + ' ', + ' ', + '
', + '

🦜️🔗 LangChain

', + '

⚡ Building applications with LLMs through composability ⚡

', + '
', + '
', + ' As an open source project in a rapidly developing field, we are extremely open to contributions.', + '
', + ' ', + '', + ].join('\n') + + // @note these are the separators from + // RecursiveCharacterTextSplitter.getSeparatorsForLanguage('html') + const htmlSeparators = [ + '', + '
', + '

', + '
', + '

  • ', + '

    ', + '

    ', + '

    ', + '

    ', + '

    ', + '
    ', + '', + '', + '', + '
    ', + '', + '
      ', + '
        ', + '
        ', + '