diff --git a/.changeset/bounded-catalog-rebuilds.md b/.changeset/bounded-catalog-rebuilds.md new file mode 100644 index 0000000000..7b0630f161 --- /dev/null +++ b/.changeset/bounded-catalog-rebuilds.md @@ -0,0 +1,5 @@ +--- +"executor": patch +--- + +Background tool-catalog rebuilds share one executor-wide limit, set with the new `toolsSyncConcurrency` option, so reads that overlap cannot stack more rebuilds in one instance than the limit. A rebuild that is already queued is joined, not queued again, and a retry of a sync that never finished runs after the other stale catalogs. diff --git a/.changeset/oauth-refresh-lease.md b/.changeset/oauth-refresh-lease.md new file mode 100644 index 0000000000..5a8c374a29 --- /dev/null +++ b/.changeset/oauth-refresh-lease.md @@ -0,0 +1,5 @@ +--- +"executor": patch +--- + +Keep OAuth connections alive when two sessions or instances refresh the same connection at once: refreshes now coordinate through a lease in the shared database, so a rotating refresh token is only spent once. A tool catalog sync that started and never finished is no longer re-run on every read; it is retried after a 15 minute backoff. diff --git a/.changeset/oauth-refresh-stalled-holder.md b/.changeset/oauth-refresh-stalled-holder.md new file mode 100644 index 0000000000..e19ca1a9c9 --- /dev/null +++ b/.changeset/oauth-refresh-stalled-holder.md @@ -0,0 +1,5 @@ +--- +"executor": patch +--- + +OAuth connections no longer get revoked when a refresh stalls. A refresh that lost its lease, or has too little of it left, no longer sends the spent refresh token; the token request is bounded to end before the lease does; the lease is renewed before a slow save; and an invalid_grant that a peer's newer refresh already superseded returns the peer's token instead of marking the connection as needing a reconnect. diff --git a/apps/cloud/drizzle/0021_connection_refresh_lease.sql b/apps/cloud/drizzle/0021_connection_refresh_lease.sql new file mode 100644 index 0000000000..966308a2e8 --- /dev/null +++ b/apps/cloud/drizzle/0021_connection_refresh_lease.sql @@ -0,0 +1,4 @@ +ALTER TABLE "connection" ADD COLUMN "tools_sync_started_at" bigint;--> statement-breakpoint +ALTER TABLE "connection" ADD COLUMN "refresh_lease_until" bigint;--> statement-breakpoint +ALTER TABLE "connection" ADD COLUMN "refresh_lease_holder" text;--> statement-breakpoint +ALTER TABLE "connection" ADD COLUMN "oauth_refreshed_at" bigint; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0021_snapshot.json b/apps/cloud/drizzle/meta/0021_snapshot.json new file mode 100644 index 0000000000..bf4418e521 --- /dev/null +++ b/apps/cloud/drizzle/meta/0021_snapshot.json @@ -0,0 +1,1784 @@ +{ + "id": "003096c7-6702-4d05-905a-8fcc608faa3a", + "prevId": "88f2845b-be28-4ad3-92b2-2cac819478e5", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sign_in_at": { + "name": "last_sign_in_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_email_lower_idx": { + "name": "accounts_email_lower_idx", + "columns": [ + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.membership_tombstones": { + "name": "membership_tombstones", + "schema": "", + "columns": { + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "membership_tombstones_organization_id_idx": { + "name": "membership_tombstones_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "membership_tombstones_account_id_accounts_id_fk": { + "name": "membership_tombstones_account_id_accounts_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "membership_tombstones_organization_id_organizations_id_fk": { + "name": "membership_tombstones_organization_id_organizations_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_membership_id_unique": { + "name": "memberships_membership_id_unique", + "columns": [ + { + "expression": "membership_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_organization_id_idx": { + "name": "memberships_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backfilled_at": { + "name": "backfilled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workos_sync": { + "name": "workos_sync", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "range_start": { + "name": "range_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_completed_at": { + "name": "backfill_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "drained_at": { + "name": "drained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "tools_sync_started_at": { + "name": "tools_sync_started_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "refresh_lease_until": { + "name": "refresh_lease_until", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "refresh_lease_holder": { + "name": "refresh_lease_holder", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_refreshed_at": { + "name": "oauth_refreshed_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index 73842e4d59..d6fc154426 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -148,6 +148,13 @@ "when": 1789575639971, "tag": "0020_workos_sync_drained_at", "breakpoints": true + }, + { + "idx": 21, + "version": "7", + "when": 1790359874309, + "tag": "0021_connection_refresh_lease", + "breakpoints": true } ] } diff --git a/apps/cloud/src/db/executor-schema.ts b/apps/cloud/src/db/executor-schema.ts index 0db709b884..63ef229cf5 100644 --- a/apps/cloud/src/db/executor-schema.ts +++ b/apps/cloud/src/db/executor-schema.ts @@ -62,10 +62,14 @@ export const connection = pgTable( description: text("description"), last_health: json("last_health"), tools_synced_at: bigint("tools_synced_at", { mode: "bigint" }), + tools_sync_started_at: bigint("tools_sync_started_at", { mode: "bigint" }), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), expires_at: bigint("expires_at", { mode: "bigint" }), + refresh_lease_until: bigint("refresh_lease_until", { mode: "bigint" }), + refresh_lease_holder: text("refresh_lease_holder"), + oauth_refreshed_at: bigint("oauth_refreshed_at", { mode: "bigint" }), oauth_scope: text("oauth_scope"), oauth_token_url: text("oauth_token_url"), provider_state: json("provider_state"), diff --git a/apps/host-cloudflare/DELEGATION.md b/apps/host-cloudflare/DELEGATION.md new file mode 100644 index 0000000000..90a1279e60 --- /dev/null +++ b/apps/host-cloudflare/DELEGATION.md @@ -0,0 +1,100 @@ +# Delegated subjects on the Cloudflare host + +## The problem + +Executor scopes every row by `(tenant, owner, subject)`. A row is visible when +`owner = 'org'` **or** `subject` matches the caller's. Org-owned connections +therefore work for any caller, and personal connections work only for the person +who owns them. + +On this host, identity comes entirely from Cloudflare Access, which issues it two +ways: + +- a **human in a browser** carries `email` + `sub`, so the subject is their Access `sub`; +- a **service token** carries `common_name` (the token's client id) and no email, + so the subject is _the token_. + +A headless agent can only present a service token. Its subject is never any +human's, so it reaches `owner: 'org'` rows and nothing else. Access has no way to +express "this backend is acting for Alice": there is no delegation primitive in +it, and a backend cannot mint a user's Access JWT. + +## What this adds + +`applyDelegatedSubject` (in `src/auth/cloudflare-access.ts`) lets ONE configured +service token name the subject it is acting for, using two request headers: + +| Header | Meaning | +| -------------------------- | -------------------------------------------------------- | +| `X-Executor-Subject` | the subject to bind, i.e. that person's Access `sub` | +| `X-Executor-Subject-Email` | their email, so roles resolve as they would in a browser | + +Delegation runs **after** JWT verification, never instead of it. The caller is +always a fully verified Access principal first; delegation only re-binds who that +verified principal is acting for. + +## The gate + +In order: + +1. No delegation headers → the principal passes through untouched. +2. The caller is not the configured delegator → **reject the request** (`null` → + `Unauthorized`). Silently ignoring the header would let any Access-authenticated + human probe for delegation and learn whether it is enabled. +3. A delegator naming no subject → reject, for the same reason. + +Only a service token may delegate. A human principal always carries an `email`, so +requiring an empty one means a browser session can never delegate, even if it +somehow learned the delegator's id. + +## Configuration + +Set `ACCESS_DELEGATION_COMMON_NAME` to the `common_name` of the one service token +allowed to delegate. **Unset disables delegation entirely**, which is the correct +default: an instance with no headless agent in front of it should never accept a +delegated subject. + +```bash +bunx wrangler deploy --var ACCESS_DELEGATION_COMMON_NAME:.access +``` + +Keep that token the only service token in the Access application's policy, so the +set of principals that can delegate is exactly one. + +The calling agent then sends, per acting user: + +```http +CF-Access-Client-Id: .access +CF-Access-Client-Secret: +X-Executor-Subject: +X-Executor-Subject-Email: +``` + +The agent is responsible for mapping its own user identity (a Slack user id, say) +to that person's Access `sub`. A person must have signed in to the console once +before their subject exists. + +## Known limitation + +Access `groups` are not available to the delegator, so a delegated principal gets +admin from the `ADMIN_EMAILS` allowlist but never group-derived roles. If you rely +on Access groups for authorization, a delegated session will see less than the same +human sees in a browser. + +Delegation is also ignored under `ENABLE_DEV_AUTH`, which short-circuits to a fixed +admin before any of this runs. + +## Alternative, not implemented: a service binding + +If the calling agent is itself a Worker in the same Cloudflare account, a service +binding is a cleaner trust boundary than a service token plus a header: + +- no public hop, so nothing on the internet can reach the delegating path; +- no shared secret to rotate; +- the binding itself is the boundary, so the subject header needs no gate beyond it. + +It was not taken here because it needs a second auth path for `/mcp` and touches +worker routing, which is a materially larger delta to carry across upstream +releases than one pure function. Worth revisiting if this host ever grows a +first-class non-Access auth seam, or if upstream adds MCP OAuth over Access (see +the note at the end of `src/mcp/auth.ts`, which anticipates exactly that). diff --git a/apps/host-cloudflare/README.md b/apps/host-cloudflare/README.md index 9700570289..8bc2044e1f 100644 --- a/apps/host-cloudflare/README.md +++ b/apps/host-cloudflare/README.md @@ -64,6 +64,26 @@ The Access values are live Worker variables, not values in `wrangler.jsonc`. Wrangler's `keep_vars` option preserves them during later code deploys. Run the command above again whenever you need to change them. +### Redeploy after a merge + +There is no automatic deploy. To ship a merged change: + +```bash +git checkout skyward && git pull --ff-only +bun install +cd apps/host-cloudflare +bun run deploy # vite build -> assert-shell-asset -> wrangler deploy +``` + +Requires `bunx wrangler login` to the Skyward account. `keep_vars` preserves +the Access variables set above, so there is nothing else to pass. Do not +re-run `deploy:setup` for a routine deploy; that script is for first-time +provisioning. + +Verify with `bunx wrangler deployments list` (run from `apps/host-cloudflare`), +which lists the new version. Logs and traces are in Cloudflare Workers +Observability for `executor-cloudflare`. + ## Local development ```bash diff --git a/apps/host-cloudflare/package.json b/apps/host-cloudflare/package.json index b693834278..94ad2bca33 100644 --- a/apps/host-cloudflare/package.json +++ b/apps/host-cloudflare/package.json @@ -36,6 +36,7 @@ "@executor-js/sdk": "workspace:*", "@jitl/quickjs-wasmfile-release-sync": "catalog:", "@modelcontextprotocol/sdk": "^1.29.0", + "@sentry/cloudflare": "^10.48.0", "@tanstack/react-router": "catalog:", "drizzle-orm": "catalog:", "effect": "catalog:", diff --git a/apps/host-cloudflare/src/account/account-provider.test.ts b/apps/host-cloudflare/src/account/account-provider.test.ts new file mode 100644 index 0000000000..89d579536e --- /dev/null +++ b/apps/host-cloudflare/src/account/account-provider.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { principalFromAccessClaims } from "../auth/cloudflare-access"; +import type { CloudflareConfig } from "../config"; +import { memberRowFromPrincipal } from "./account-provider"; + +const config: CloudflareConfig = { + accessTeamDomain: "team.cloudflareaccess.com", + accessAud: "aud-tag", + accessNameClaim: "name", + accessGroupsClaim: "groups", + adminEmails: ["felipe@skyward.ai", "sergio@skyward.ai"], + organizationId: "default", + organizationName: "Default", + organizationSlug: "default", + secretKey: "x".repeat(32), + allowLocalNetwork: false, + webBaseUrl: "https://localhost", + enableDevAuth: false, +}; + +const rowFor = (claims: Record) => + memberRowFromPrincipal(principalFromAccessClaims(claims, config)); + +describe("memberRowFromPrincipal", () => { + it("reports an allowlisted human as an active admin", () => { + const row = rowFor({ sub: "felipe-sub", email: "felipe@skyward.ai" }); + expect(row.role).toBe("admin"); + expect(row.status).toBe("active"); + expect(row.isCurrentUser).toBe(true); + expect(row.userId).toBe("felipe@skyward.ai"); + // These three fields together are exactly what the shell's + // isTenantAdminMember() requires before it offers the Workspace connection + // owner (packages/react/src/lib/admin-access.ts). Not imported here: that is + // a browser package, and this host must not depend on it. + }); + + it("reports the second allowlisted admin too", () => { + expect(rowFor({ sub: "s", email: "sergio@skyward.ai" }).role).toBe("admin"); + }); + + it("is case-insensitive about the allowlist", () => { + expect(rowFor({ sub: "f", email: "FELIPE@skyward.ai" }).role).toBe("admin"); + }); + + it("reports a non-allowlisted teammate as a plain member", () => { + const row = rowFor({ sub: "o", email: "otro@skyward.ai" }); + expect(row.role).toBe("member"); + }); + + it("reports a service token as a plain member", () => { + expect(rowFor({ common_name: "agent-token.access" }).role).toBe("member"); + }); +}); diff --git a/apps/host-cloudflare/src/account/account-provider.ts b/apps/host-cloudflare/src/account/account-provider.ts index bbbbd3d522..8f379ef66a 100644 --- a/apps/host-cloudflare/src/account/account-provider.ts +++ b/apps/host-cloudflare/src/account/account-provider.ts @@ -4,6 +4,7 @@ import { AccountProvider, accountProviderMiddlewareLayer, type AccountHeaders, + type Principal, } from "@executor-js/api/server"; import { AccountError, AccountUnauthorized } from "@executor-js/api"; @@ -21,10 +22,44 @@ import type { CloudflareConfig } from "../config"; // shows no members page, so those methods are never reached from the UI; they // return empty (reads) or a clear "managed by Cloudflare Access" error (writes) // to satisfy the provider shape. +// +// `listMembers` is the ONE exception, and it is load-bearing. The console +// carries no role on the session (`/account/me` returns a user and an +// organization, never a role), so the shell derives "am I an admin" by finding +// the `isCurrentUser` row in this list and reading its role. Returning an empty +// list therefore made EVERY Access user a non-admin in the UI — which silently +// removed the Workspace (`owner: "org"`) choice from the connection-create flow, +// leaving personal connections as the only thing anyone could make, however +// `ADMIN_EMAILS` was set. So we project the Access principal into a single +// member row. There is no directory to enumerate here (Access owns membership), +// so the list is exactly one row: whoever is asking. // --------------------------------------------------------------------------- const NOT_IN_APP = "Managed by Cloudflare Access, not in the app."; +/** + * Project an Access principal into the ONE member row this host reports. + * + * `role` mirrors the Access-derived org role (`ADMIN_EMAILS` plus the groups + * claim), so "admin" in the console means admin at the server gate too, instead + * of the two disagreeing. `status` is always `"active"`: Access has no pending + * state — a principal either passed the Access policy or never reached us. + * + * Pure (no headers, no IO) so it is unit-testable, like + * `principalFromAccessClaims` in ../auth/cloudflare-access. + */ +export const memberRowFromPrincipal = (principal: Principal) => ({ + id: principal.accountId, + userId: principal.accountId, + email: principal.email, + name: principal.name, + avatarUrl: principal.avatarUrl, + role: principal.orgRole === "admin" ? "admin" : "member", + status: "active", + lastActiveAt: null, + isCurrentUser: true, +}); + export const cloudflareAccountProvider = ( config: CloudflareConfig, ): Layer.Layer => { @@ -66,7 +101,12 @@ export const cloudflareAccountProvider = ( listOrgApiKeys: () => Effect.succeed({ apiKeys: [] }), createOrgApiKey: () => forbiddenWrite, revokeOrgApiKey: () => forbiddenWrite, - listMembers: () => Effect.succeed({ members: [] }), + listMembers: (headers) => + principalFrom(headers).pipe( + Effect.map((principal) => + principal ? { members: [memberRowFromPrincipal(principal)] } : { members: [] }, + ), + ), listRoles: () => Effect.succeed({ roles: [] }), inviteMember: () => forbiddenWrite, removeMember: () => forbiddenWrite, diff --git a/apps/host-cloudflare/src/auth/cloudflare-access.test.ts b/apps/host-cloudflare/src/auth/cloudflare-access.test.ts index 1e1a7e00a7..f92d219a41 100644 --- a/apps/host-cloudflare/src/auth/cloudflare-access.test.ts +++ b/apps/host-cloudflare/src/auth/cloudflare-access.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "@effect/vitest"; -import { orgWriteAccessForPrincipal } from "@executor-js/host-mcp"; +import { orgWriteAccessForPrincipal, principalOwns } from "@executor-js/host-mcp"; +import type { Principal } from "@executor-js/host-mcp"; import type { CloudflareConfig } from "../config"; -import { principalFromAccessClaims } from "./cloudflare-access"; +import { applyDelegatedSubject, principalFromAccessClaims } from "./cloudflare-access"; const config: CloudflareConfig = { accessTeamDomain: "team.cloudflareaccess.com", @@ -21,12 +22,12 @@ const config: CloudflareConfig = { }; describe("principalFromAccessClaims", () => { - it("maps a human identity (email + sub + groups)", () => { + it("maps a human identity (email + sub + groups), keyed on the EMAIL not the sub", () => { const p = principalFromAccessClaims( { sub: "user-123", email: "person@example.com", name: "Person", groups: ["eng"] }, config, ); - expect(p.accountId).toBe("user-123"); + expect(p.accountId).toBe("person@example.com"); expect(p.email).toBe("person@example.com"); expect(p.name).toBe("Person"); expect(p.roles).toEqual(["eng"]); @@ -41,6 +42,26 @@ describe("principalFromAccessClaims", () => { expect(orgWriteAccessForPrincipal(p)).toBe("allowed"); }); + it("survives a seat being removed and re-added, which changes the Access sub", () => { + // Cloudflare documents `sub` as unique per email per account, but NOT durable: + // re-adding a user's seat mints a new one. Keying on it would orphan every + // connection that person owns. + const before = principalFromAccessClaims({ sub: "sub-1", email: "person@example.com" }, config); + const after = principalFromAccessClaims({ sub: "sub-2", email: "person@example.com" }, config); + expect(after.accountId).toBe(before.accountId); + }); + + it("is case-insensitive, so one person is never two accounts", () => { + const shouty = principalFromAccessClaims({ sub: "u", email: "Person@Example.com" }, config); + const quiet = principalFromAccessClaims({ sub: "u", email: "person@example.com" }, config); + expect(shouty.accountId).toBe(quiet.accountId); + }); + + it("falls back to the sub when a human identity carries no email", () => { + const p = principalFromAccessClaims({ sub: "user-123" }, config); + expect(p.accountId).toBe("user-123"); + }); + it("gives a SERVICE TOKEN (common_name, no email/sub) a stable identity", () => { // Cloudflare Access service-token JWT: common_name set, email/sub absent. const p = principalFromAccessClaims({ common_name: "df8a20db.access", type: "app" }, config); @@ -59,3 +80,79 @@ describe("principalFromAccessClaims", () => { expect(orgWriteAccessForPrincipal(p)).toBe("denied"); }); }); + +const DELEGATOR = "agent-token.access"; +const delegating: CloudflareConfig = { ...config, accessDelegationCommonName: DELEGATOR }; + +// The delegator itself: a service-token principal, exactly as Access presents one. +const delegatorPrincipal = () => + principalFromAccessClaims({ common_name: DELEGATOR, type: "app" }, delegating); + +describe("applyDelegatedSubject", () => { + it("passes an ordinary request through untouched", () => { + const p = principalFromAccessClaims({ sub: "user-123", email: "person@example.com" }, config); + expect(applyDelegatedSubject(p, config, { subject: null, email: null })).toBe(p); + }); + + it("re-binds the trusted delegator to the named subject", () => { + const p = applyDelegatedSubject(delegatorPrincipal(), delegating, { + subject: "alice@example.com", + email: "alice@example.com", + }); + expect(p?.accountId).toBe("alice@example.com"); + expect(p?.email).toBe("alice@example.com"); + expect(p?.roles).toEqual(["member"]); + }); + + it("mirrors the delegated human's admin standing", () => { + const p = applyDelegatedSubject(delegatorPrincipal(), delegating, { + subject: "ADMIN@example.com", + email: "ADMIN@example.com", + }); + expect(p?.roles).toContain("admin"); + expect(p?.orgRole).toBe("admin"); + }); + + it("REJECTS a human trying to delegate, rather than ignoring the header", () => { + const human = principalFromAccessClaims({ sub: "u", email: "person@example.com" }, delegating); + expect(applyDelegatedSubject(human, delegating, { subject: "alice", email: null })).toBeNull(); + }); + + it("REJECTS an admin human trying to delegate", () => { + const admin = principalFromAccessClaims({ sub: "u", email: "admin@example.com" }, delegating); + expect(applyDelegatedSubject(admin, delegating, { subject: "alice", email: null })).toBeNull(); + }); + + it("reaches the SAME rows the person reaches in a browser", () => { + // The whole point of delegation: an agent acting for Alice and Alice's own + // session must resolve to one identity, or the agent silently sees org rows. + const browser = principalFromAccessClaims( + { sub: "alice-sub", email: "alice@example.com" }, + delegating, + ); + const delegated = applyDelegatedSubject(delegatorPrincipal(), delegating, { + subject: "Alice@Example.com", + email: "Alice@Example.com", + }); + expect(delegated?.accountId).toBe(browser.accountId); + expect(principalOwns(browser, delegated as Principal)).toBe(true); + }); + + it("REJECTS a service token that is not the configured delegator", () => { + const other = principalFromAccessClaims({ common_name: "other.access" }, delegating); + expect(applyDelegatedSubject(other, delegating, { subject: "alice", email: null })).toBeNull(); + }); + + it("REJECTS delegation when no delegator is configured", () => { + const token = principalFromAccessClaims({ common_name: DELEGATOR }, config); + expect(applyDelegatedSubject(token, config, { subject: "alice", email: null })).toBeNull(); + }); + + it("REJECTS a delegator that names an email but no subject", () => { + const p = applyDelegatedSubject(delegatorPrincipal(), delegating, { + subject: " ", + email: "alice@example.com", + }); + expect(p).toBeNull(); + }); +}); diff --git a/apps/host-cloudflare/src/auth/cloudflare-access.ts b/apps/host-cloudflare/src/auth/cloudflare-access.ts index b311256c2e..a67a7471d6 100644 --- a/apps/host-cloudflare/src/auth/cloudflare-access.ts +++ b/apps/host-cloudflare/src/auth/cloudflare-access.ts @@ -17,6 +17,14 @@ import type { CloudflareConfig } from "../config"; // Roles come from the admin allowlist + the Access groups claim. // --------------------------------------------------------------------------- +/** + * Normalize an identity value into the key ownership is stored under. Lowercased + * and trimmed so the same person is one account however their IdP happens to + * capitalize them; every value that reaches here (email, Access UUID, service + * token common name) is case-insensitive in its own right. + */ +const identityKey = (value: string): string => value.trim().toLowerCase(); + /** * Map verified Access JWT claims onto the neutral `Principal`. Pure (no JWT * verification) so it is unit-testable. Handles both human identities (email + @@ -24,6 +32,14 @@ import type { CloudflareConfig } from "../config"; * `CF-Access-Client-Id`/`-Secret` headers — which carry `common_name` (the * token's client id) instead of email/sub. Single-tenant: every principal * belongs to the one configured org; admin comes from the email allowlist. + * + * A human is keyed by EMAIL, not by the Access `sub`. Cloudflare documents `sub` + * as unique to an email address per account but NOT durable: remove and re-add a + * user's seat and they come back with a different one, which would orphan every + * connection they own. The email is the identifier that survives that, and it is + * also the only one a delegating backend can know about a person (see + * `applyDelegatedSubject`), so keying on it is what lets a browser session and an + * agent acting for that same person reach the same rows. */ export const principalFromAccessClaims = ( claims: Record, @@ -39,7 +55,7 @@ export const principalFromAccessClaims = ( return { kind: "member", - accountId: sub || email || commonName, + accountId: identityKey(email || sub || commonName), organizationId: config.organizationId, organizationName: config.organizationName, organizationSlug: config.organizationSlug, @@ -52,6 +68,82 @@ export const principalFromAccessClaims = ( }; }; +/** Header naming the subject a trusted delegating caller is acting for. */ +export const DELEGATED_SUBJECT_HEADER = "X-Executor-Subject"; +/** Header carrying that subject's email, so roles resolve as they would in a browser. */ +export const DELEGATED_EMAIL_HEADER = "X-Executor-Subject-Email"; + +/** The delegation a request asserts, read straight off the headers (both null on + * an ordinary request). */ +export interface DelegatedIdentity { + readonly subject: string | null; + readonly email: string | null; +} + +export const readDelegatedIdentity = (request: Request): DelegatedIdentity => ({ + subject: request.headers.get(DELEGATED_SUBJECT_HEADER), + email: request.headers.get(DELEGATED_EMAIL_HEADER), +}); + +/** + * Re-bind a verified principal to the subject a TRUSTED service token says it is + * acting for. Cloudflare Access authenticates browsers and machines, but has no + * way to express "this backend is acting for Alice" — a service token's identity + * is the token. Without this, a headless agent can only ever reach `owner: "org"` + * rows, because its subject never matches any human's. + * + * The subject a delegator sends is that person's EMAIL, the same key a browser + * session resolves to in `principalFromAccessClaims`. The two must agree or the + * delegated run silently sees organization rows only. + * + * The gate, in order: + * - no delegation headers at all → the principal passes through untouched; + * - the caller is NOT the one configured delegator → `null`, i.e. REJECT the + * request. Silently ignoring the header would let any Access-authenticated + * human probe for delegation and learn whether it is enabled; + * - a delegator that names no subject → `null`, for the same reason. + * + * Only a service token may delegate: a human principal always carries an `email`, + * so requiring an empty one means a browser session can never delegate even if it + * somehow learned the delegator's id. + * + * Roles MIRROR the delegated human's real standing, so the agent reaches exactly + * what that person reaches in a browser. Note the one asymmetry: Access `groups` + * are not available to the delegator, so a delegated principal gets admin from the + * email allowlist but never group-derived roles. + * + * Pure (no request, no IO) so it is unit-testable, like `principalFromAccessClaims`. + */ +export const applyDelegatedSubject = ( + principal: Principal, + config: CloudflareConfig, + delegated: DelegatedIdentity, +): Principal | null => { + const subject = delegated.subject?.trim() ?? ""; + const email = delegated.email?.trim() ?? ""; + if (subject.length === 0 && email.length === 0) return principal; + + const delegator = identityKey(config.accessDelegationCommonName ?? ""); + const mayDelegate = + delegator.length > 0 && principal.email.length === 0 && principal.accountId === delegator; + if (!mayDelegate) return null; + if (subject.length === 0) return null; + + const isAdmin = email.length > 0 && config.adminEmails.includes(email.toLowerCase()); + return { + ...principal, + accountId: identityKey(subject), + email, + name: email.length > 0 ? email : subject, + roles: isAdmin ? ["admin"] : ["member"], + // Restated rather than left to the spread: `Principal` discriminates `orgRole` + // on `orgRoleModel`, so the spread alone leaves the union open and `orgRole` + // unassignable. This host only ever builds the "organization" arm. + orgRoleModel: "organization", + orgRole: isAdmin ? "admin" : "member", + }; +}; + /** * Resolve a request to its verified `Principal`, or `null` when the Access * assertion is missing/invalid. The single source of truth for "who is this @@ -97,7 +189,13 @@ export const makeAccessVerifier = (config: CloudflareConfig) => { }).pipe(Effect.orElseSucceed(() => null)); if (!verified) return null; - return principalFromAccessClaims(verified.payload as Record, config); + const principal = principalFromAccessClaims( + verified.payload as Record, + config, + ); + // Delegation runs AFTER verification, never instead of it: the caller is + // always a fully verified Access principal first. + return applyDelegatedSubject(principal, config, readDelegatedIdentity(request)); }); return { verify }; diff --git a/apps/host-cloudflare/src/config.test.ts b/apps/host-cloudflare/src/config.test.ts index 39aab1e7d3..f7a2c32b3d 100644 --- a/apps/host-cloudflare/src/config.test.ts +++ b/apps/host-cloudflare/src/config.test.ts @@ -72,3 +72,44 @@ describe("Cloudflare deployment configuration", () => { expect(config.vars).toHaveProperty("ENABLE_DEV_AUTH", "false"); }); }); + +describe("jevGateway", () => { + const access = { + ACCESS_TEAM_DOMAIN: "team.cloudflareaccess.com", + ACCESS_AUD: "aud-tag", + }; + + it("is configured when both ids are present", async () => { + const config = loadConfig( + makeEnv({ + ...access, + CLOUDFLARE_ACCOUNT_ID: "acct", + AI_GATEWAY_ID: "staging-gateway", + AI_GATEWAY_TOKEN: { get: () => Promise.resolve("tok") }, + }), + ); + expect(config.jevGateway?.accountId).toBe("acct"); + expect(config.jevGateway?.gatewayId).toBe("staging-gateway"); + // The binding is resolved per call, so config carries a resolver and never + // the secret itself. + await expect(config.jevGateway?.authToken?.()).resolves.toBe("tok"); + }); + + it("is OFF when an id is missing, so search degrades to lexical instead of 404ing", () => { + // A gateway URL built from a blank id resolves to a 404 that reads like + // "Jev found nothing" rather than "Jev was never configured". + expect( + loadConfig(makeEnv({ ...access, AI_GATEWAY_ID: "staging-gateway" })).jevGateway, + ).toBeUndefined(); + expect( + loadConfig(makeEnv({ ...access, CLOUDFLARE_ACCOUNT_ID: "acct" })).jevGateway, + ).toBeUndefined(); + }); + + it("keeps the token optional — an unauthenticated gateway needs none", () => { + const config = loadConfig( + makeEnv({ ...access, CLOUDFLARE_ACCOUNT_ID: "acct", AI_GATEWAY_ID: "g" }), + ); + expect(config.jevGateway?.authToken).toBeUndefined(); + }); +}); diff --git a/apps/host-cloudflare/src/config.ts b/apps/host-cloudflare/src/config.ts index c397c4ef87..376e9954c2 100644 --- a/apps/host-cloudflare/src/config.ts +++ b/apps/host-cloudflare/src/config.ts @@ -1,5 +1,6 @@ import type { D1Database, DurableObjectNamespace, R2Bucket } from "@cloudflare/workers-types"; +import type { JevGatewayConfig } from "@executor-js/execution"; import { isValidOrgSlug } from "@executor-js/api"; import { missingPublicOriginWarning, resolvePublicOrigin } from "@executor-js/sdk/public-origin"; @@ -13,6 +14,11 @@ let warnedNoCloudflareOrigin = false; // Worker; the only real secret is the at-rest secret-encryption key. // --------------------------------------------------------------------------- +/** A Secrets Store binding: the value is resolved per call, never held as a var. */ +export interface SecretsStoreBinding { + readonly get: () => Promise; +} + export const CLOUDFLARE_NAMESPACE = "executor_cloudflare"; export const CLOUDFLARE_SCHEMA_VERSION = "1.0.0"; @@ -40,6 +46,13 @@ export interface CloudflareEnv { readonly ACCESS_GROUPS_CLAIM?: string; /** Comma-separated emails granted the admin role. */ readonly ADMIN_EMAILS?: string; + /** + * The `common_name` of the ONE Access service token allowed to act on behalf + * of another subject (see `applyDelegatedSubject`). Unset disables delegation + * entirely, which is the correct default: an instance with no headless agent + * in front of it should never accept a delegated subject. + */ + readonly ACCESS_DELEGATION_COMMON_NAME?: string; /** The single organization id/name every authenticated user belongs to. */ readonly SELF_HOSTED_ORG_ID?: string; readonly SELF_HOSTED_ORG_NAME?: string; @@ -48,7 +61,25 @@ export interface CloudflareEnv { /** At-rest secret-encryption key (a `wrangler secret`, NOT a var). */ readonly EXECUTOR_SECRET_KEY?: string; readonly ALLOW_LOCAL_NETWORK?: string; + /** + * Cloudflare AI Gateway, for the Jev classifier (`custom-typesafe`). Every + * model call goes through a gateway rather than a provider directly, so the + * key stays in the gateway's BYOK store and never in a worker var — which is + * why there is no TYPESAFE_API_KEY here and must not be one. + */ + readonly CLOUDFLARE_ACCOUNT_ID?: string; + readonly AI_GATEWAY_ID?: string; + /** + * `cf-aig-authorization` bearer, bound from the Secrets Store — NOT a + * `wrangler secret` and never a var. A store binding hands back an object + * whose value is fetched on demand, so the token is never part of the + * worker's static configuration and is rotated in one place for every worker + * that binds it. + */ + readonly AI_GATEWAY_TOKEN?: SecretsStoreBinding; readonly VITE_PUBLIC_SITE_URL?: string; + /** Sentry DSN (a `wrangler secret`). Unset leaves Sentry disabled. */ + readonly SENTRY_DSN?: string; /** * Dev/single-user escape hatch: when "true", skip Cloudflare Access entirely * and treat every request as a fixed admin. For local `wrangler dev` and @@ -64,6 +95,9 @@ export interface CloudflareConfig { readonly accessNameClaim: string; readonly accessGroupsClaim: string; readonly adminEmails: readonly string[]; + /** See {@link CloudflareEnv.ACCESS_DELEGATION_COMMON_NAME}. Optional so an + * instance that never delegates carries no extra configuration. */ + readonly accessDelegationCommonName?: string; readonly organizationId: string; readonly organizationName: string; /** URL slug for org-prefixed console paths (`//policies`). */ @@ -74,6 +108,10 @@ export interface CloudflareConfig { * static URL — the per-request origin is used instead (see RequestWebOrigin). */ readonly webBaseUrl?: string; readonly enableDevAuth: boolean; + /** Present only when the gateway is fully configured; absent disables Jev + * rather than failing a search, so a missing var degrades ranking instead of + * breaking tool discovery. */ + readonly jevGateway?: JevGatewayConfig; } type CloudflareConfigEnv = Omit< @@ -86,6 +124,22 @@ type CloudflareAccessEnv = Pick< "ACCESS_TEAM_DOMAIN" | "ACCESS_AUD" | "ENABLE_DEV_AUTH" >; +// Both ids are required: a gateway URL missing either one resolves to a 404 that +// would look like "Jev found nothing" rather than "Jev was never configured". +const resolveJevGateway = (env: CloudflareConfigEnv): CloudflareConfig["jevGateway"] => { + const accountId = env.CLOUDFLARE_ACCOUNT_ID?.trim() ?? ""; + const gatewayId = env.AI_GATEWAY_ID?.trim() ?? ""; + if (accountId.length === 0 || gatewayId.length === 0) { + return undefined; + } + const token = env.AI_GATEWAY_TOKEN; + return { + accountId, + gatewayId, + authToken: token === undefined ? undefined : () => token.get(), + }; +}; + const splitLower = (value: string | undefined): readonly string[] => (value ?? "") .split(",") @@ -161,6 +215,7 @@ export const loadConfig = (env: CloudflareConfigEnv): CloudflareConfig => { accessNameClaim: env.ACCESS_NAME_CLAIM ?? "name", accessGroupsClaim: env.ACCESS_GROUPS_CLAIM ?? "groups", adminEmails: splitLower(env.ADMIN_EMAILS), + accessDelegationCommonName: env.ACCESS_DELEGATION_COMMON_NAME?.trim() || undefined, organizationId: env.SELF_HOSTED_ORG_ID ?? "default", organizationName: env.SELF_HOSTED_ORG_NAME ?? "Default", organizationSlug: resolveOrgSlug(env.SELF_HOSTED_ORG_SLUG), @@ -174,5 +229,6 @@ export const loadConfig = (env: CloudflareConfigEnv): CloudflareConfig => { // mirroring self-host (gated on enableDevAuth = local `wrangler dev`). webBaseUrl, enableDevAuth, + jevGateway: resolveJevGateway(env), }; }; diff --git a/apps/host-cloudflare/src/execution.ts b/apps/host-cloudflare/src/execution.ts index 1ee0e4b1e9..6e2174b7d8 100644 --- a/apps/host-cloudflare/src/execution.ts +++ b/apps/host-cloudflare/src/execution.ts @@ -56,6 +56,13 @@ export const makeCloudflareHostConfig = (config: CloudflareConfig): Layer.Layer< allowLocalNetwork: config.allowLocalNetwork, webBaseUrl: config.webBaseUrl, oauthCallbackPath: "/api/oauth/callback", + // Each MCP session's executor lives in a Durable Object with a small memory + // limit, and one rebuild holds a whole catalog (a large OpenAPI spec) in + // memory. Rebuild stale catalogs one at a time so many of them cannot take + // the session down together. + toolsSyncConcurrency: 1, + // Absent unless both gateway ids are set, which leaves search lexical. + ...(config.jevGateway === undefined ? {} : { jevGateway: config.jevGateway }), }); /** diff --git a/apps/host-cloudflare/src/mcp/session-durable-object.ts b/apps/host-cloudflare/src/mcp/session-durable-object.ts index 1ddd680093..92f8f263c6 100644 --- a/apps/host-cloudflare/src/mcp/session-durable-object.ts +++ b/apps/host-cloudflare/src/mcp/session-durable-object.ts @@ -24,6 +24,7 @@ import { type McpExecutionOwnerRoute, } from "@executor-js/cloudflare/mcp/execution-owner-directory"; import { mcpSessionStub } from "@executor-js/cloudflare/mcp/session-stub"; +import { makeJevToolDiscoveryProvider } from "@executor-js/execution"; import type { ResumeResponse } from "@executor-js/execution"; import { loadConfig, type CloudflareConfig, type CloudflareEnv } from "../config"; @@ -176,6 +177,15 @@ export class McpSessionDO extends McpAgentSessionDOBase = Layer.effect( + ErrorCapture, + Effect.gen(function* () { + const consoleCapture = yield* ErrorCapture; + return ErrorCapture.of({ + captureException: (cause) => + consoleCapture.captureException(cause).pipe( + Effect.tap((traceId) => + Effect.sync(() => { + const [error] = Cause.prettyErrors(cause); + Sentry.captureException(error ?? Cause.squash(cause), { tags: { traceId } }); + }), + ), + ), + }); + }), +).pipe(Layer.provide(consoleErrorCapture("cloudflare"))); + +/** + * A call the client was waiting on that will never get a real answer: its + * session Durable Object died (memory or CPU limit, deploy) or nothing came + * back before the deadline. The dying isolate cannot report itself, so the + * front worker, which answers the client for it, reports it instead. + */ +export const reportLostCall = (detail: Readonly>): void => { + Sentry.captureMessage(`MCP call lost: ${String(detail.reason ?? "unknown")}`, { + level: "error", + tags: { reason: String(detail.reason ?? "unknown") }, + extra: detail, + }); +}; diff --git a/apps/host-cloudflare/src/sentry.ts b/apps/host-cloudflare/src/sentry.ts new file mode 100644 index 0000000000..8350ab8aaf --- /dev/null +++ b/apps/host-cloudflare/src/sentry.ts @@ -0,0 +1,7 @@ +import type { CloudflareEnv } from "./config"; + +export const sentryOptions = (env: CloudflareEnv) => ({ + dsn: env.SENTRY_DSN, + tracesSampleRate: 0, + sendDefaultPii: false, +}); diff --git a/apps/host-cloudflare/src/worker.ts b/apps/host-cloudflare/src/worker.ts index b9964fac4a..7c6d6d8802 100644 --- a/apps/host-cloudflare/src/worker.ts +++ b/apps/host-cloudflare/src/worker.ts @@ -1,14 +1,27 @@ +import * as Sentry from "@sentry/cloudflare"; + import { makeCloudflareApp } from "./app"; import { cloudflareAccessConfigErrorMessage, missingCloudflareAccessVars, type CloudflareEnv, } from "./config"; +import { McpSessionDO as McpSessionDOBase } from "./mcp"; import { mcpResourceFromPath } from "./mcp/resource"; +import { reportLostCall } from "./observability"; +import { sentryOptions } from "./sentry"; + +(globalThis as { __executorReportLostCall?: typeof reportLostCall }).__executorReportLostCall = + reportLostCall; // The MCP Durable Object classes, bound in wrangler.jsonc. They must be exported -// at the Worker entry module scope for the runtime to find them. -export { McpExecutionOwnerDirectoryDO, McpSessionDO } from "./mcp"; +// at the Worker entry module scope for the runtime to find them. Wrapping the +// session DO initialises Sentry inside its isolate. +export { McpExecutionOwnerDirectoryDO } from "./mcp"; +export const McpSessionDO = Sentry.instrumentDurableObjectWithSentry( + sentryOptions, + McpSessionDOBase, +); // --------------------------------------------------------------------------- // The Worker fetch entry. Most requests go to `ExecutorApp.make`'s Effect web @@ -41,7 +54,7 @@ const accessConfigErrorResponse = (missingVars: readonly string[]): Response => }, }); -export default { +export default Sentry.withSentry(sentryOptions, { fetch: async (request: Request, env: CloudflareEnv, ctx: ExecutionContext): Promise => { const missingAccessVars = missingCloudflareAccessVars(env); if (missingAccessVars.length > 0) { @@ -55,4 +68,4 @@ export default { } return serve.app(request); }, -}; +} satisfies ExportedHandler); diff --git a/apps/host-cloudflare/wrangler.jsonc b/apps/host-cloudflare/wrangler.jsonc index f97e19507e..cb3838fa86 100644 --- a/apps/host-cloudflare/wrangler.jsonc +++ b/apps/host-cloudflare/wrangler.jsonc @@ -7,7 +7,10 @@ // Access configuration is set per installation after the first deploy. // Preserve those live bindings when this source config does not declare them. "keep_vars": true, - "observability": { "enabled": true }, + "observability": { + "enabled": true, + "traces": { "enabled": true, "head_sampling_rate": 1 }, + }, // The web UI (Workers Static Assets) — the shared multiplayer SPA built by // `vite build` into ./dist. `single-page-application` serves index.html for // client routes (e.g. /policies); `run_worker_first` forces the API, MCP, and @@ -29,7 +32,7 @@ { "binding": "DB", "database_name": "executor", - "database_id": "ae748ca1-032c-4427-a1a0-fe39db77d1a9", + "database_id": "833a3544-5c28-45bf-b964-a0846a783b73", }, ], // Plugin blob seam backend: multi-MB values (resolved OpenAPI specs, @@ -61,6 +64,16 @@ { "name": "MCP_EXECUTION_OWNER", "class_name": "McpExecutionOwnerDirectoryDO" }, ], }, + // Secrets Store, not `wrangler secret`: the value is fetched per call rather + // than baked into the worker's configuration, and one rotation in the store + // covers every worker that binds it. + "secrets_store_secrets": [ + { + "binding": "AI_GATEWAY_TOKEN", + "store_id": "18b901f1d8ca4cf4869408ac39159daa", + "secret_name": "AI_GATEWAY_TOKEN_STAGING", + }, + ], "migrations": [ { "tag": "v1", "new_sqlite_classes": ["McpSessionDO"] }, { "tag": "v2", "new_sqlite_classes": ["McpExecutionOwnerDirectoryDO"] }, @@ -71,6 +84,14 @@ // secret-encryption key) is a SECRET, set it with // `wrangler secret put EXECUTOR_SECRET_KEY`, never in vars. "vars": { + // Jev (the tool-search classifier) is reached through this AI Gateway as the + // account-level custom provider `custom-typesafe`. The provider key lives in + // the gateway's BYOK store, never here; AI_GATEWAY_TOKEN is the gateway's own + // bearer and is bound from the Secrets Store below, never a var or a + // `wrangler secret`. + // With either id missing, Jev is simply off and search falls back to lexical. + "CLOUDFLARE_ACCOUNT_ID": "b75ee9798c9c48a874af240b29724270", + "AI_GATEWAY_ID": "staging-gateway", "ACCESS_NAME_CLAIM": "name", "ACCESS_GROUPS_CLAIM": "groups", // Never preserve a production dev-auth override through keep_vars. diff --git a/apps/local/drizzle/0007_connection_refresh_lease.sql b/apps/local/drizzle/0007_connection_refresh_lease.sql new file mode 100644 index 0000000000..49e77cd7a4 --- /dev/null +++ b/apps/local/drizzle/0007_connection_refresh_lease.sql @@ -0,0 +1,3 @@ +ALTER TABLE `connection` ADD `refresh_lease_until` blob;--> statement-breakpoint +ALTER TABLE `connection` ADD `refresh_lease_holder` text;--> statement-breakpoint +ALTER TABLE `connection` ADD `oauth_refreshed_at` blob; \ No newline at end of file diff --git a/apps/local/drizzle/meta/0007_snapshot.json b/apps/local/drizzle/meta/0007_snapshot.json new file mode 100644 index 0000000000..fe149a1a96 --- /dev/null +++ b/apps/local/drizzle/meta/0007_snapshot.json @@ -0,0 +1,969 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "f764126c-fb31-4e16-b72e-b84a67aad09e", + "prevId": "89a4fd1b-f0f6-4482-a991-0db78c859f76", + "tables": { + "blob": { + "name": "blob", + "columns": { + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": ["id"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "connection": { + "name": "connection", + "columns": { + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_ids": { + "name": "item_ids", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_write": { + "name": "credential_write", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_lease_until": { + "name": "refresh_lease_until", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_lease_holder": { + "name": "refresh_lease_holder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_refreshed_at": { + "name": "oauth_refreshed_at", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_state": { + "name": "provider_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": ["tenant", "owner", "subject", "integration", "name"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "definition": { + "name": "definition", + "columns": { + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection": { + "name": "connection", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "schema": { + "name": "schema", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": ["tenant", "owner", "subject", "integration", "connection", "name"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integration": { + "name": "integration", + "columns": { + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "can_remove": { + "name": "can_remove", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "can_refresh": { + "name": "can_refresh", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": ["tenant", "slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_client": { + "name": "oauth_client", + "columns": { + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "credential_write": { + "name": "credential_write", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": ["tenant", "owner", "subject", "slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_session": { + "name": "oauth_session", + "columns": { + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": ["tenant", "state"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_storage": { + "name": "plugin_storage", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection": { + "name": "collection", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": ["tenant", "owner", "subject", "plugin_id", "collection", "key"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tool": { + "name": "tool", + "columns": { + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection": { + "name": "connection", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_schema": { + "name": "input_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_schema": { + "name": "output_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "annotations": { + "name": "annotations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": ["tenant", "owner", "subject", "integration", "connection", "name"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tool_policy": { + "name": "tool_policy", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tenant": { + "name": "tenant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": ["tenant", "owner", "subject", "id"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/apps/local/drizzle/meta/_journal.json b/apps/local/drizzle/meta/_journal.json index dbe786204c..7866505a1b 100644 --- a/apps/local/drizzle/meta/_journal.json +++ b/apps/local/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1788255902609, "tag": "0006_bored_landau", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1790359874595, + "tag": "0007_connection_refresh_lease", + "breakpoints": true } ] } diff --git a/apps/local/src/db/executor-schema.ts b/apps/local/src/db/executor-schema.ts index a7f6ceb48e..3688db0456 100644 --- a/apps/local/src/db/executor-schema.ts +++ b/apps/local/src/db/executor-schema.ts @@ -43,6 +43,9 @@ export const connection = sqliteTable( oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), expires_at: blob("expires_at"), + refresh_lease_until: blob("refresh_lease_until"), + refresh_lease_holder: text("refresh_lease_holder"), + oauth_refreshed_at: blob("oauth_refreshed_at"), oauth_scope: text("oauth_scope"), oauth_token_url: text("oauth_token_url"), provider_state: text("provider_state"), diff --git a/bun.lock b/bun.lock index 78ee6bcded..dc5a396dad 100644 --- a/bun.lock +++ b/bun.lock @@ -192,6 +192,7 @@ "@executor-js/sdk": "workspace:*", "@jitl/quickjs-wasmfile-release-sync": "catalog:", "@modelcontextprotocol/sdk": "^1.29.0", + "@sentry/cloudflare": "^10.48.0", "@tanstack/react-router": "catalog:", "drizzle-orm": "catalog:", "effect": "catalog:", diff --git a/packages/core/api/src/oauth/api.ts b/packages/core/api/src/oauth/api.ts index 5d0eee3d41..8a79d7cca7 100644 --- a/packages/core/api/src/oauth/api.ts +++ b/packages/core/api/src/oauth/api.ts @@ -60,12 +60,22 @@ const ConnectionResponse = Schema.Struct({ // createClient — register an owner-scoped OAuth app. // --------------------------------------------------------------------------- +/** The grant set, spelled ONCE. It was previously repeated per struct, which is + * how `github_app` reached `createClient` but not the summary response — a + * client you could create and then never list. Keep new grants here only. */ +const OAuthGrantSchema = Schema.Literals([ + "authorization_code", + "client_credentials", + "id_jag", + "github_app", +]); + const CreateClientPayload = Schema.Struct({ owner: Owner, slug: OAuthClientSlug, authorizationUrl: Schema.String, tokenUrl: Schema.String, - grant: Schema.Literals(["authorization_code", "client_credentials", "id_jag"]), + grant: OAuthGrantSchema, clientId: Schema.String, clientSecret: Schema.String, tokenEndpointAuthMethod: Schema.optional(TokenEndpointAuthMethodSchema), @@ -114,7 +124,7 @@ const RegisterDynamicResponse = Schema.Struct({ const OAuthClientSummaryResponse = Schema.Struct({ owner: Owner, slug: OAuthClientSlug, - grant: Schema.Literals(["authorization_code", "client_credentials", "id_jag"]), + grant: OAuthGrantSchema, authorizationUrl: Schema.String, tokenUrl: Schema.String, resource: Schema.optional(Schema.NullOr(Schema.String)), diff --git a/packages/core/api/src/server/execution-stack.ts b/packages/core/api/src/server/execution-stack.ts index 3aa0dd6b8d..72837e83f2 100644 --- a/packages/core/api/src/server/execution-stack.ts +++ b/packages/core/api/src/server/execution-stack.ts @@ -30,6 +30,7 @@ import type { McpResource } from "@executor-js/host-mcp"; import type { AnyPlugin, Executor, ExecutorConfig, StorageFailure } from "@executor-js/sdk"; import { createExecutionEngine, + makeJevToolDiscoveryProvider, type ExecutionEngine, type ExecutionEngineConfig, } from "@executor-js/execution"; @@ -139,9 +140,25 @@ export const makeExecutionStack = < const { decorate } = yield* EngineDecorator.asEffect().pipe( Effect.withSpan("executor.stack.decorator"), ); + // Tool search stays lexical unless this host configured a gateway, which is + // the right default for a host without one. Where a gateway IS configured Jev + // does the ranking, because the lexical ranker only matches English and the + // queries this fork serves are Spanish as often as not. + const hostConfig = yield* HostConfig.asEffect(); + const jevGateway = hostConfig.jevGateway; const engine = yield* Effect.sync(() => decorate( - createExecutionEngine({ executor, codeExecutor }), + createExecutionEngine({ + executor, + codeExecutor, + ...(jevGateway === undefined + ? {} + : { + toolDiscoveryProvider: makeJevToolDiscoveryProvider({ + gateway: jevGateway, + }), + }), + }), { accountId, organizationId, diff --git a/packages/core/api/src/server/scoped-executor.ts b/packages/core/api/src/server/scoped-executor.ts index 749839be3e..28dfc4cda7 100644 --- a/packages/core/api/src/server/scoped-executor.ts +++ b/packages/core/api/src/server/scoped-executor.ts @@ -34,6 +34,7 @@ import { Context, Effect, Option } from "effect"; import type { McpResource } from "@executor-js/host-mcp"; +import type { JevGatewayConfig } from "@executor-js/execution"; import { createExecutor, Subject, @@ -64,6 +65,12 @@ export interface HostConfigShape { * production hosts leave it off. Drives `makeHostedHttpClientLayer`. */ readonly allowLocalNetwork: boolean; + /** + * AI Gateway for the Jev classifier. Present only on a host that configured + * one; absent leaves tool search purely lexical, which is the correct default + * for a host with no gateway rather than a failure. + */ + readonly jevGateway?: JevGatewayConfig; /** Require TLS for public outbound requests from both execution and admin views. */ readonly requireTls?: boolean; /** @@ -126,6 +133,12 @@ export interface HostConfigShape { * operator knob. */ readonly toolsSyncTtlMs?: number | null; + /** + * Forwarded verbatim to `ExecutorConfig.toolsSyncConcurrency`: how many + * background catalog rebuilds one executor runs at the same time. Hosts whose + * instances have a small memory limit set it low. Omit for the SDK default. + */ + readonly toolsSyncConcurrency?: number; /** * Forwarded to `ExecutorConfig.waitUntil`: the host's keep-alive * for background work that outlives a request (stale tool-catalog rebuilds @@ -334,6 +347,9 @@ export const makeScopedExecutor = < fetch: hostedFetch, onIntegrationChange: config.onIntegrationChange, ...(config.toolsSyncTtlMs !== undefined ? { toolsSyncTtlMs: config.toolsSyncTtlMs } : {}), + ...(config.toolsSyncConcurrency !== undefined + ? { toolsSyncConcurrency: config.toolsSyncConcurrency } + : {}), ...(waitUntil !== undefined ? { waitUntil } : {}), onElicitation: "accept-all", ...(options?.orgWrites === undefined ? {} : { orgWrites: options.orgWrites }), diff --git a/packages/core/execution/src/index.ts b/packages/core/execution/src/index.ts index 23175a01b3..ba22385a6a 100644 --- a/packages/core/execution/src/index.ts +++ b/packages/core/execution/src/index.ts @@ -38,3 +38,9 @@ export { type PagedResult, type ToolDiscoveryResult, } from "./tool-invoker"; + +export { + makeJevToolDiscoveryProvider, + type JevToolDiscoveryOptions, +} from "./jev-discovery"; +export type { JevGatewayConfig } from "./jev"; diff --git a/packages/core/execution/src/jev-discovery.test.ts b/packages/core/execution/src/jev-discovery.test.ts new file mode 100644 index 0000000000..466ca22619 --- /dev/null +++ b/packages/core/execution/src/jev-discovery.test.ts @@ -0,0 +1,288 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Layer, Ref } from "effect"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import { makeJevToolDiscoveryProvider } from "./jev-discovery"; +import type { + PagedResult, + ToolDiscoveryInput, + ToolDiscoveryProvider, + ToolDiscoveryResult, +} from "./tool-invoker"; + +// The shape the catalog really yields: a prefixed `address` and its own +// `integration`. Fixtures that invented a `path` field hid a production crash. +const TOOLS = [ + { + address: "tools.linear.issues.list", + integration: "linear", + name: "list issues", + description: "List Linear issues", + }, + { + address: "tools.github.pulls.list", + integration: "github", + name: "list pull requests", + description: "List GitHub pull requests", + }, +]; + +// Only the slice of Executor this provider touches: the tool list it scores. +const executorWith = (tools: readonly Record[]) => + // A test double for the ONE Executor method this provider calls; building a + // whole Executor would test the fixture rather than the rescue. + // oxlint-disable-next-line executor/no-double-cast -- boundary: test double + ({ + tools: { list: () => Effect.succeed(tools) }, + }) as unknown as ToolDiscoveryInput["executor"]; + +const input = ( + overrides: Partial = {}, +): ToolDiscoveryInput => ({ + executor: executorWith(TOOLS), + query: "muestrame mis tareas pendientes", + limit: 10, + offset: 0, + ...overrides, +}); + +const emptyPage: PagedResult = { + items: [], + total: 0, + hasMore: false, + nextOffset: null, +}; + +const lexical = ( + result: PagedResult, + calls?: { count: number }, +): ToolDiscoveryProvider => ({ + searchTools: () => + Effect.sync(() => { + if (calls !== undefined) { + calls.count += 1; + } + return result; + }), +}); + +const jevReturning = ( + body: string, +): Effect.Effect<{ + readonly layer: Layer.Layer; + readonly calls: Ref.Ref; +}> => + Effect.gen(function* () { + const calls = yield* Ref.make(0); + const layer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + yield* Ref.update(calls, (n) => n + 1); + return HttpClientResponse.fromWeb( + request, + new Response(body, { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + }), + ), + ); + return { layer, calls }; + }); + +const GATEWAY = { accountId: "acct", gatewayId: "staging-gateway" }; + +describe("makeJevToolDiscoveryProvider", () => { + it.effect("scores with Jev even when lexical already matched", () => + Effect.gen(function* () { + const { layer, calls } = yield* jevReturning(`{"answers":{"1":{"noul":0.95}}}`); + const hit: PagedResult = { + items: [ + { + path: "linear.issues.list", + name: "list issues", + integration: "linear", + score: 90, + }, + ], + total: 1, + hasMore: false, + nextOffset: null, + }; + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(hit), + }); + + const result = yield* provider + .searchTools(input()) + .pipe(Effect.provide(layer)); + + expect(yield* Ref.get(calls)).toBe(1); + // Jev's ranking wins over lexical's: it scored the GitHub tool, which the + // lexical hit never contained, and the lexical score of 90 is not returned. + expect(result.items.map((item) => item.path)).toEqual([ + "github.pulls.list", + ]); + expect(result.items[0]?.score).toBe(95); + })); + + it.effect("rescues a cross-language query the lexical ranker drops entirely", () => + Effect.gen(function* () { + // "tareas pendientes" shares no token with `linear.issues.list`, so the + // lexical ranker returns nothing — the cliff this exists for. + const { layer } = yield* jevReturning( + `{"answers":{"0":{"noul":0.68},"1":{"noul":0.1}}}`, + ); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + const result = yield* provider + .searchTools(input()) + .pipe(Effect.provide(layer)); + + expect(result.items.map((item) => item.path)).toEqual([ + "linear.issues.list", + ]); + expect(result.items[0]?.score).toBe(68); + expect(result.total).toBe(1); + })); + + it.effect("keeps the lexical answer when Jev fails, so search never gets worse", () => + Effect.gen(function* () { + const layer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response("boom", { status: 500 }), + ), + ), + ), + ); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + const result = yield* provider + .searchTools(input()) + .pipe(Effect.provide(layer)); + + expect(result).toEqual(emptyPage); + })); + + it.effect("drops a weak opinion instead of offering it as a match", () => + Effect.gen(function* () { + const { layer } = yield* jevReturning( + `{"answers":{"0":{"noul":0.05},"1":{"noul":0.02}}}`, + ); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + const result = yield* provider + .searchTools(input()) + .pipe(Effect.provide(layer)); + + expect(result).toEqual(emptyPage); + })); + + it.effect("skips the rescue for an empty query, which is enumeration not search", () => + Effect.gen(function* () { + const { layer, calls } = yield* jevReturning(`{"answers":{}}`); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + yield* provider + .searchTools(input({ query: " " })) + .pipe(Effect.provide(layer)); + + expect(yield* Ref.get(calls)).toBe(0); + })); + + it.effect("shards a catalog too large for one call rather than truncating it", () => + Effect.gen(function* () { + // Descriptions are capped before scoring, so what fills a shard is the + // number of tools. 400 of them does not fit one state budget. + const many = Array.from({ length: 400 }, (_, index) => ({ + address: `tools.integration${index}.do.thing`, + integration: `integration${index}`, + name: `tool ${index}`, + description: "d".repeat(400), + })); + const { layer, calls } = yield* jevReturning(`{"answers":{"0":{"noul":0.9}}}`); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + const result = yield* provider + .searchTools(input({ executor: executorWith(many) })) + .pipe(Effect.provide(layer)); + + // Growth costs more calls, never a silently truncated catalog. + expect(yield* Ref.get(calls)).toBeGreaterThan(1); + // Each shard answers about ITS OWN tools, so id "0" resolves per shard and + // every shard contributes its own top match. + expect(result.items.length).toBe(yield* Ref.get(calls)); + })); + + it.effect("reads tools that carry an address rather than a path", () => + Effect.gen(function* () { + // The catalog yields `address` (prefixed) and its own `integration`; reading + // `path` off it directly was undefined and took the whole search down with + // a TypeError in production. + const addressed = [ + { + address: "tools.linear.issues.list", + name: "list issues", + integration: "linear", + description: "List Linear issues", + }, + ]; + const { layer } = yield* jevReturning(`{"answers":{"0":{"noul":0.9}}}`); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + const result = yield* provider + .searchTools(input({ executor: executorWith(addressed) })) + .pipe(Effect.provide(layer)); + + expect(result.items[0]?.path).toBe("linear.issues.list"); + expect(result.items[0]?.integration).toBe("linear"); + })); + + it.effect("scopes the rescue to the requested namespace", () => + Effect.gen(function* () { + const { layer } = yield* jevReturning(`{"answers":{"0":{"noul":0.9}}}`); + const provider = makeJevToolDiscoveryProvider({ + gateway: { ...GATEWAY }, + httpClientLayer: layer, + delegate: lexical(emptyPage), + }); + + const result = yield* provider + .searchTools(input({ namespace: "linear" })) + .pipe(Effect.provide(layer)); + + expect(result.items.map((item) => item.path)).toEqual([ + "linear.issues.list", + ]); + })); +}); diff --git a/packages/core/execution/src/jev-discovery.ts b/packages/core/execution/src/jev-discovery.ts new file mode 100644 index 0000000000..3c300e1b42 --- /dev/null +++ b/packages/core/execution/src/jev-discovery.ts @@ -0,0 +1,231 @@ +import { Effect } from "effect"; + +import { askJev, JEV_STATE_CHAR_BUDGET, type JevGatewayConfig, type JevQuestion } from "./jev"; +import { + defaultToolDiscoveryProvider, + matchesNamespace, + paginate, + toSearchableTool, + type SearchableTool, + type ToolDiscoveryInput, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "./tool-invoker"; + +// --------------------------------------------------------------------------- +// Semantic ranking for tool search. +// +// Jev IS the ranker here; lexical is the fallback, not the first pass. The +// repo's users write Spanglish, and a token-overlap scorer is close to noise for +// that: it drops a tool entirely when nothing overlaps, and it "matches" on an +// incidental word like a namespace when something does. Ranking by meaning is +// the job, so the semantic scorer does it and the lexical one stands by for when +// Jev is unavailable. +// +// The catalog is scored in SHARDS sized by Jev's state budget rather than a +// guessed tool count, because what fills that budget is description length, not +// the number of tools: a handful of verbose MCP tools can outweigh a hundred +// terse ones. Shards run in parallel and their scores merge into one ranking, so +// growth costs more calls rather than a silently truncated catalog. Each shard is +// its own pass and passes are the part that costs, so the chunks are made as +// large as the budget allows rather than small and many. +// --------------------------------------------------------------------------- + +/** Below this, Jev's opinion is too weak to be worth showing as a match. */ +const MIN_PROBABILITY = 0.3; + +/** + * How much of Jev's state budget one shard may use. Headroom is left for the + * questions, which are billed against a larger total but still share the state + * ceiling with the longest one. + */ +const SHARD_CHAR_BUDGET = Math.floor(JEV_STATE_CHAR_BUDGET * 0.6); + +/** A single tool's description is truncated to this before scoring: a verbose + * one would otherwise consume a shard on its own, and the opening sentence is + * what carries the meaning. */ +const MAX_DESCRIPTION_CHARS = 300; + +/** Shards scored at once. Enough to cover a large catalog in one round trip + * without opening an unbounded number of connections from a Worker. */ +const SHARD_CONCURRENCY = 4; + +export interface JevToolDiscoveryOptions { + readonly gateway: JevGatewayConfig; + /** The lexical provider to try first. Defaults to the built-in one. */ + readonly delegate?: ToolDiscoveryProvider; + /** Injected for tests; `askJev` defaults to the platform fetch client. */ + readonly httpClientLayer?: JevToolDiscoveryHttpLayer; +} + +/** The HTTP layer `askJev` accepts, re-exported so a caller can inject one. */ +export type JevToolDiscoveryHttpLayer = NonNullable< + Parameters[0]["httpClientLayer"] +>; + +const describeTool = (tool: SearchableTool): string => + tool.description === undefined + ? `${tool.path} (${tool.name})` + : `${tool.path} (${tool.name}): ${tool.description.slice(0, MAX_DESCRIPTION_CHARS)}`; + +export const makeJevToolDiscoveryProvider = ( + options: JevToolDiscoveryOptions, +): ToolDiscoveryProvider => { + const delegate = options.delegate ?? defaultToolDiscoveryProvider; + + return { + searchTools: (input: ToolDiscoveryInput) => + Effect.gen(function* () { + const lexical = yield* delegate.searchTools(input); + const query = input.query.trim(); + // Whether the rescue ran, and whether it helped, is the whole question + // this feature exists to answer. Unannotated it is invisible: a search + // that Jev saved looks exactly like one lexical got right. + yield* Effect.annotateCurrentSpan({ + "executor.search.jev.lexical_total": lexical.total, + }); + // An empty query is enumeration, not search: there is nothing to be + // semantically closer to. Every other query gets scored. + if (query.length === 0) { + yield* Effect.annotateCurrentSpan({ + "executor.search.jev.attempted": false, + "executor.search.jev.skipped_reason": "empty_query", + }); + return lexical; + } + + const listStartedAt = Date.now(); + const all = yield* input.executor.tools + .list({ includeAnnotations: false }) + .pipe(Effect.orElseSucceed(() => [])); + yield* Effect.logInfo("jev search listed tools", { + tools: all.length, + durationMs: Date.now() - listStartedAt, + namespace: input.namespace ?? null, + }); + const namespace = input.namespace?.trim(); + // Normalised and scoped exactly as the lexical ranker does, so turning Jev + // on cannot change WHICH tools a namespace search considers. + const candidates = all + .map(toSearchableTool) + .filter((tool) => matchesNamespace(tool, namespace)); + if (candidates.length === 0) { + yield* Effect.annotateCurrentSpan({ + "executor.search.jev.attempted": false, + "executor.search.jev.skipped_reason": "no_candidates", + }); + return lexical; + } + + // Shard by CHARACTERS, not by tool count: what fills Jev's state budget + // is description length, and a few verbose tools outweigh many terse ones. + const shards: SearchableTool[][] = []; + let shard: SearchableTool[] = []; + let shardChars = 0; + for (const tool of candidates) { + const cost = describeTool(tool).length + 1; + if (shard.length > 0 && shardChars + cost > SHARD_CHAR_BUDGET) { + shards.push(shard); + shard = []; + shardChars = 0; + } + // push, not spread: rebuilding the array per tool is O(n^2) in exactly + // the path that exists for large catalogs. + shard.push(tool); + shardChars += cost; + } + if (shard.length > 0) { + shards.push(shard); + } + + // Each shard carries its own question ids, so an answer is resolved + // against the tools of the shard that asked — a global index would + // misattribute every score past the first shard. + yield* Effect.logInfo("jev search sharded", { + candidates: candidates.length, + shards: shards.length, + toolsPerShard: shards.map((entry) => entry.length), + charsPerShard: shards.map((entry) => + entry.reduce((sum, tool) => sum + describeTool(tool).length + 1, 0), + ), + }); + const scoreShard = (tools: readonly SearchableTool[], shardIndex: number) => + askJev({ + config: options.gateway, + state: [ + "Each question asks whether one tool answers the user's request.", + "The tools available:", + ...tools.map(describeTool), + ].join("\n"), + questions: tools.map((tool, index) => ({ + id: String(index), + instructions: `Would ${tool.path} answer this request: ${query}`, + })) satisfies JevQuestion[], + ...(options.httpClientLayer === undefined + ? {} + : { httpClientLayer: options.httpClientLayer }), + }).pipe( + // One failed shard must not lose the others: it contributes nothing + // and the ranking is built from whatever came back. + Effect.tapError((error) => + Effect.logWarning("jev shard failed", { + shard: shardIndex, + tools: tools.length, + message: error.message, + status: error.status ?? null, + }), + ), + Effect.orElseSucceed(() => []), + Effect.map((answers) => + answers.flatMap((answer) => { + const tool = tools[Number(answer.id)]; + return tool === undefined ? [] : [{ tool, probability: answer.probability }]; + }), + ), + ); + + const scored = (yield* Effect.all( + shards.map((entry, index) => scoreShard(entry, index)), + { + concurrency: SHARD_CONCURRENCY, + }, + )).flat(); + + // Jev IS the ranking. Lexical only stands in when Jev produced nothing, + // which means the gateway failed or every shard came back empty. + const ranked: ToolDiscoveryResult[] = scored + .filter((entry) => entry.probability >= MIN_PROBABILITY) + .map((entry) => ({ + path: entry.tool.path, + name: entry.tool.name, + integration: entry.tool.integration, + score: Math.round(entry.probability * 100), + ...(entry.tool.description === undefined + ? {} + : { description: entry.tool.description }), + })); + + yield* Effect.logInfo("jev search ranked", { + candidates: candidates.length, + scored: scored.length, + ranked: ranked.length, + lexicalFallback: ranked.length === 0, + }); + yield* Effect.annotateCurrentSpan({ + "executor.search.jev.attempted": true, + "executor.search.jev.candidate_count": candidates.length, + "executor.search.jev.shard_count": shards.length, + "executor.search.jev.scored_count": scored.length, + "executor.search.jev.ranked_count": ranked.length, + "executor.search.jev.lexical_fallback": ranked.length === 0, + }); + if (ranked.length === 0) { + return lexical; + } + ranked.sort( + (left, right) => right.score - left.score || left.path.localeCompare(right.path), + ); + return paginate(ranked, input.offset, input.limit); + }), + }; +}; diff --git a/packages/core/execution/src/jev.test.ts b/packages/core/execution/src/jev.test.ts new file mode 100644 index 0000000000..0e1566ac82 --- /dev/null +++ b/packages/core/execution/src/jev.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Layer, Predicate, Ref } from "effect"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import { JEV_MODEL, JEV_STATE_CHAR_BUDGET, JevError, askJev } from "./jev"; + +interface RecordedRequest { + readonly url: string; + readonly headers: Readonly>; + readonly body: string; +} + +// Records the outgoing request so the wire contract can be asserted: the gateway +// path, the auth header, and the body shape are all things a silent change to +// would leave the ranker quietly returning nothing. +const recordingClient = ( + respond: () => Response = () => new Response(`{"answers":{}}`, { status: 200 }), +): Effect.Effect<{ + readonly layer: Layer.Layer; + readonly requests: Ref.Ref>; +}> => + Effect.gen(function* () { + const requests = yield* Ref.make>([]); + const layer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + // `bodyJsonUnsafe` encodes to bytes, so decode rather than expecting a + // string — reading the wrong field silently captured "" and made every + // body assertion vacuous. + const raw = + typeof request.body === "object" && + request.body !== null && + "body" in request.body + ? request.body.body + : undefined; + const body = + typeof raw === "string" + ? raw + : raw instanceof Uint8Array + ? new TextDecoder().decode(raw) + : ""; + yield* Ref.update(requests, (xs) => [ + ...xs, + { url: request.url, headers: request.headers, body }, + ]); + return HttpClientResponse.fromWeb(request, respond()); + }), + ), + ); + return { layer, requests }; + }); + +const CONFIG = { + accountId: "acct", + gatewayId: "staging-gateway", + authToken: () => Promise.resolve("gw-token"), +}; + +describe("askJev", () => { + it.effect("asks every question in ONE call, against one shared state", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient( + () => + new Response(`{"answers":{"a":{"noul":0.9},"b":{"noul":0.1}}}`, { + status: 200, + }), + ); + const answers = yield* askJev({ + config: CONFIG, + state: "the catalog", + questions: [ + { id: "a", instructions: "is this about linear?" }, + { id: "b", instructions: "is this about billing?" }, + ], + httpClientLayer: layer, + }); + + // One call, N questions — the whole reason this is cheap enough to rank a catalog. + expect(yield* Ref.get(requests)).toHaveLength(1); + expect(answers).toEqual([ + { id: "a", probability: 0.9 }, + { id: "b", probability: 0.1 }, + ]); + })); + + it.effect("posts to the gateway's custom-provider path, re-supplying the version segment", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + yield* askJev({ + config: CONFIG, + state: "s", + questions: [{ id: "a", instructions: "q" }], + httpClientLayer: layer, + }); + const [request] = yield* Ref.get(requests); + expect(request?.url).toBe( + "https://gateway.ai.cloudflare.com/v1/acct/staging-gateway/custom-typesafe/v1/systemone", + ); + })); + + it.effect("authenticates with cf-aig-authorization and NEVER an Authorization header", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + yield* askJev({ + config: CONFIG, + state: "s", + questions: [{ id: "a", instructions: "q" }], + httpClientLayer: layer, + }); + const [request] = yield* Ref.get(requests); + expect(request?.headers["cf-aig-authorization"]).toBe("Bearer gw-token"); + // An Authorization header would be forwarded to the provider and override + // the gateway's stored BYOK key, so the request must never carry one. + expect(request?.headers["authorization"]).toBeUndefined(); + })); + + it.effect("resolves the token per call, so a rotated secret is picked up without redeploying", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + let issued = 0; + const rotating = { + accountId: "acct", + gatewayId: "staging-gateway", + authToken: () => Promise.resolve(`token-${(issued += 1)}`), + }; + const ask = () => + askJev({ + config: rotating, + state: "s", + questions: [{ id: "a", instructions: "q" }], + httpClientLayer: layer, + }); + yield* ask(); + yield* ask(); + + const sent = yield* Ref.get(requests); + expect(sent[0]?.headers["cf-aig-authorization"]).toBe("Bearer token-1"); + expect(sent[1]?.headers["cf-aig-authorization"]).toBe("Bearer token-2"); + })); + + it.effect("fails clearly when the secret cannot be resolved, instead of calling unauthenticated", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + const error = yield* Effect.flip( + askJev({ + config: { + accountId: "acct", + gatewayId: "g", + // A store read that fails, produced by Effect rather than a hand-rolled + // rejection — the binding's `get()` returns a promise, so this is the + // honest shape of the Secrets Store being unavailable. + authToken: () => + Effect.runPromise( + Effect.fail(new JevError({ message: "store unavailable" })), + ), + }, + state: "s", + questions: [{ id: "a", instructions: "q" }], + httpClientLayer: layer, + }), + ); + expect(Predicate.isTagged(error, "JevError")).toBe(true); + // An unauthenticated call would 401 and read as "Jev is broken" rather + // than "the secret could not be read". + expect(yield* Ref.get(requests)).toHaveLength(0); + })); + + it.effect("sends the model and keys questions by id", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + yield* askJev({ + config: CONFIG, + state: "the state", + questions: [{ id: "tool-1", instructions: "matches?" }], + httpClientLayer: layer, + }); + const [request] = yield* Ref.get(requests); + expect(request?.body).toContain(`"model":"${JEV_MODEL}"`); + expect(request?.body).toContain(`"state":"the state"`); + expect(request?.body).toContain(`"tool-1":{"type":"noul","instructions":"matches?"}`); + })); + + it.effect("refuses a duplicate question id instead of silently dropping one", () => + Effect.gen(function* () { + const { layer } = yield* recordingClient(); + const error = yield* Effect.flip( + askJev({ + config: CONFIG, + state: "s", + questions: [ + { id: "same", instructions: "first" }, + { id: "same", instructions: "second" }, + ], + httpClientLayer: layer, + }), + ); + expect(Predicate.isTagged(error, "JevError")).toBe(true); + })); + + it.effect("refuses a state over budget rather than paying for a rejected request", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + const error = yield* Effect.flip( + askJev({ + config: CONFIG, + state: "x".repeat(JEV_STATE_CHAR_BUDGET), + questions: [{ id: "a", instructions: "q" }], + httpClientLayer: layer, + }), + ); + expect(Predicate.isTagged(error, "JevError")).toBe(true); + expect(yield* Ref.get(requests)).toHaveLength(0); + })); + + it.effect("makes no call at all when there is nothing to ask", () => + Effect.gen(function* () { + const { layer, requests } = yield* recordingClient(); + const answers = yield* askJev({ + config: CONFIG, + state: "s", + questions: [], + httpClientLayer: layer, + }); + expect(answers).toEqual([]); + expect(yield* Ref.get(requests)).toHaveLength(0); + })); + + it.effect("fails on a non-2xx instead of reporting no matches", () => + Effect.gen(function* () { + const { layer } = yield* recordingClient( + () => new Response("nope", { status: 500 }), + ); + const error = yield* Effect.flip( + askJev({ + config: CONFIG, + state: "s", + questions: [{ id: "a", instructions: "q" }], + httpClientLayer: layer, + }), + ); + expect(error.status).toBe(500); + })); + + it.effect("drops an answer Jev declined to score rather than reading it as zero", () => + Effect.gen(function* () { + const { layer } = yield* recordingClient( + () => + new Response(`{"answers":{"a":{"noul":0.5},"b":{}}}`, { status: 200 }), + ); + const answers = yield* askJev({ + config: CONFIG, + state: "s", + questions: [ + { id: "a", instructions: "q" }, + { id: "b", instructions: "q" }, + ], + httpClientLayer: layer, + }); + // "no opinion" must not become "score 0" — that would rank a tool as a + // confident non-match instead of an unknown. + expect(answers).toEqual([{ id: "a", probability: 0.5 }]); + })); +}); diff --git a/packages/core/execution/src/jev.ts b/packages/core/execution/src/jev.ts new file mode 100644 index 0000000000..0f4d522b31 --- /dev/null +++ b/packages/core/execution/src/jev.ts @@ -0,0 +1,199 @@ +import * as Data from "effect/Data"; +import { Effect } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; +import type { Layer } from "effect"; + +// --------------------------------------------------------------------------- +// Jev (TypeSafe System One) — a classifier, not a chat model. +// +// It answers many small questions about one shared `state` in a single call, and +// returns a probability per question. That shape is what makes it cheap enough to +// score a whole tool catalog: the questions are nearly free, it is the extra +// PASSES that cost, so one call with N questions beats N calls. +// +// Reached through the Cloudflare AI Gateway like every other provider, as an +// account-level CUSTOM provider (`custom-typesafe`). Two consequences of that +// which are easy to get wrong: +// - the gateway does not carry through the configured base_url's trailing +// version segment, so the request path re-supplies `/v1/systemone`; +// - auth is `cf-aig-authorization` ONLY. An `Authorization` header would be +// forwarded to the provider and override the gateway's stored BYOK key. +// --------------------------------------------------------------------------- + +export const JEV_MODEL = "jev-1.13.0"; + +// Jev caps the state it reasons over; questions are billed against a larger total. +// Both are in CHARACTERS here (~3 per token) because that is what a caller can +// cheaply measure before paying for a request that would be rejected. +export const JEV_STATE_CHAR_BUDGET = 32_000 * 3; +export const JEV_TOTAL_CHAR_BUDGET = 64_000 * 3; + +export class JevError extends Data.TaggedError("JevError")<{ + readonly message: string; + readonly status?: number; + readonly cause?: unknown; +}> {} + +export interface JevGatewayConfig { + /** Cloudflare account that owns the gateway. */ + readonly accountId: string; + /** Gateway id, e.g. `staging-gateway`. */ + readonly gatewayId: string; + /** + * Resolves the `cf-aig-authorization` bearer, per call. A function rather than + * a string because the token comes from the Secrets Store: it is fetched on + * demand and never held in the worker's configuration. + */ + readonly authToken?: () => Promise; +} + +export interface JevQuestion { + readonly id: string; + readonly instructions: string; +} + +export interface JevAnswer { + readonly id: string; + readonly probability: number; +} + +export interface AskJevOptions { + readonly config: JevGatewayConfig; + /** The shared context every question is asked against. */ + readonly state: string; + readonly questions: readonly JevQuestion[]; + /** Injected for tests. Defaults to the platform fetch-backed client. */ + readonly httpClientLayer?: Layer.Layer; +} + +export const jevGatewayUrl = (config: JevGatewayConfig): string => + `https://gateway.ai.cloudflare.com/v1/${config.accountId}/${config.gatewayId}/custom-typesafe/v1/systemone`; + +const overBudget = (state: string, questions: readonly JevQuestion[]): string | null => { + const longest = questions.reduce( + (max, question) => Math.max(max, question.instructions.length), + 0, + ); + if (state.length + longest > JEV_STATE_CHAR_BUDGET) { + return `state plus the longest question exceeds ${JEV_STATE_CHAR_BUDGET} characters`; + } + const total = questions.reduce( + (sum, question) => sum + question.instructions.length, + state.length, + ); + return total > JEV_TOTAL_CHAR_BUDGET + ? `state plus all questions exceeds ${JEV_TOTAL_CHAR_BUDGET} characters` + : null; +}; + +// The body keys questions by id, so a duplicate id would silently drop one +// question and misattribute its answer to the other. +const duplicateId = (questions: readonly JevQuestion[]): string | null => { + const seen = new Set(); + for (const question of questions) { + if (seen.has(question.id)) { + return question.id; + } + seen.add(question.id); + } + return null; +}; + +const readAnswers = (body: unknown): readonly JevAnswer[] => { + if (typeof body !== "object" || body === null || !("answers" in body)) { + return []; + } + const answers = (body as { readonly answers: unknown }).answers; + if (typeof answers !== "object" || answers === null) { + return []; + } + return Object.entries(answers).flatMap(([id, answer]) => { + const noul = + typeof answer === "object" && answer !== null && "noul" in answer + ? (answer as { readonly noul: unknown }).noul + : undefined; + return typeof noul === "number" ? [{ id, probability: noul }] : []; + }); +}; + +/** + * Score every question against one state. Returns an answer per question Jev + * scored; a question it declines to score is simply absent rather than zero, so + * a caller can tell "not relevant" from "no opinion". + */ +export const askJev = (options: AskJevOptions): Effect.Effect => + Effect.gen(function* () { + if (options.questions.length === 0) { + return []; + } + const duplicate = duplicateId(options.questions); + if (duplicate !== null) { + return yield* new JevError({ + message: `Duplicate Jev question id "${duplicate}"`, + }); + } + const tooBig = overBudget(options.state, options.questions); + if (tooBig !== null) { + return yield* new JevError({ message: `Jev request rejected: ${tooBig}` }); + } + + const questions: Record = {}; + for (const question of options.questions) { + questions[question.id] = { + type: "noul", + instructions: question.instructions, + }; + } + + const resolveToken = options.config.authToken; + const token = + resolveToken === undefined + ? undefined + : yield* Effect.tryPromise({ + try: () => resolveToken(), + catch: (cause) => + new JevError({ + message: "Could not resolve the AI Gateway token", + cause, + }), + }); + + const client = yield* HttpClient.HttpClient; + const request = HttpClientRequest.post(jevGatewayUrl(options.config)).pipe( + HttpClientRequest.setHeaders({ + "Content-Type": "application/json", + ...(token === undefined ? {} : { "cf-aig-authorization": `Bearer ${token}` }), + }), + HttpClientRequest.bodyJsonUnsafe({ + model: JEV_MODEL, + state: options.state, + questions, + }), + ); + + const requestStartedAt = Date.now(); + const response = yield* client + .execute(request) + .pipe(Effect.mapError((cause) => new JevError({ message: "Jev request failed", cause }))); + yield* Effect.logInfo("jev call answered", { + status: response.status, + durationMs: Date.now() - requestStartedAt, + stateChars: options.state.length, + questions: options.questions.length, + }); + if (response.status < 200 || response.status >= 300) { + return yield* new JevError({ + message: `Jev returned ${response.status}`, + status: response.status, + }); + } + const body = yield* response.json.pipe( + Effect.mapError((cause) => new JevError({ message: "Jev response was not JSON", cause })), + ); + const answers = readAnswers(body); + yield* Effect.logInfo("jev call parsed", { + answers: answers.length, + questions: options.questions.length, + }); + return answers; + }).pipe(Effect.provide(options.httpClientLayer ?? FetchHttpClient.layer)); diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index a0c59a2cb8..0881d5ac3f 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -429,7 +429,10 @@ export type ExecutorIntegrationListItem = { }; export type ToolDiscoveryInput = { - readonly executor: Executor; + /** Only the tool catalog, matching what `searchTools` itself declares: a + * passthrough caller builds a filtered `tools.list` and has no whole + * `Executor` to hand over. */ + readonly executor: { readonly tools: Pick }; readonly query: string; readonly namespace?: string; readonly limit: number; @@ -462,7 +465,9 @@ export type PagedResult = { readonly nextOffset: number | null; }; -const paginate = (all: readonly T[], offset: number, limit: number): PagedResult => { +/** Slice a ranked list into a page. Exported so another discovery provider pages + * identically — `hasMore`/`nextOffset` are easy to get subtly wrong twice. */ +export const paginate = (all: readonly T[], offset: number, limit: number): PagedResult => { const total = all.length; const start = Math.min(Math.max(offset, 0), total); const items = all.slice(start, start + limit); @@ -478,14 +483,14 @@ const paginate = (all: readonly T[], offset: number, limit: number): PagedRes /** What `searchTools` ranks over — the sandbox-callable path plus the v2 * identity fields a query can match against. */ -type SearchableTool = { +export type SearchableTool = { readonly path: string; readonly integration: string; readonly name: string; readonly description?: string; }; -const toSearchableTool = (tool: Tool): SearchableTool => ({ +export const toSearchableTool = (tool: Tool): SearchableTool => ({ path: addressToPath(String(tool.address)), integration: String(tool.integration), name: String(tool.name), @@ -582,7 +587,7 @@ const scorePreparedField = ( }; }; -const matchesNamespace = (tool: SearchableTool, namespace?: string): boolean => { +export const matchesNamespace = (tool: SearchableTool, namespace?: string): boolean => { if (!namespace || normalizeSearchText(namespace).length === 0) { return true; } diff --git a/packages/core/sdk/src/connections.test.ts b/packages/core/sdk/src/connections.test.ts index 1fd653c093..b014813e38 100644 --- a/packages/core/sdk/src/connections.test.ts +++ b/packages/core/sdk/src/connections.test.ts @@ -2042,6 +2042,217 @@ describe("tool catalog sync safety", () => { ), ); + it.effect( + "a sync that started and never finished is not retried on every read until its backoff passes", + () => + Effect.scoped( + Effect.gen(function* () { + let resolutions = 0; + const guardedPlugin = definePlugin(() => ({ + id: "guarded" as const, + credentialProviders: [memoryProvider()], + storage: () => ({}), + resolveTools: () => + Effect.sync(() => { + resolutions += 1; + return { + tools: [{ name: ToolName.make("deploy"), description: "deploy" }], + }; + }), + invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), + extension: (ctx) => ({ + seed: () => + ctx.core.integrations.register({ + slug: INTEG, + description: "Vercel", + config: {}, + }), + }), + }))(); + const config = makeTestConfig({ plugins: [guardedPlugin] as const }); + const executor = yield* createExecutor({ ...config, toolsSyncGraceMs: null }); + yield* executor.guarded.seed(); + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration: INTEG, + template: TEMPLATE, + value: "secret-token", + }); + const resolutionsAfterCreate = resolutions; + + // A sync began and the isolate running it died before it finished: + // the start is stamped, the finish never was. + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b.and(b("integration", "=", String(INTEG)), b("name", "=", "main")), + set: { tools_synced_at: null, tools_sync_started_at: Date.now() }, + }), + ); + + const firstRead = yield* executor.tools.list({ integration: INTEG }); + yield* executor.tools.list({ integration: INTEG }); + expect(resolutions, "reads inside the backoff do not re-run the crashed sync").toBe( + resolutionsAfterCreate, + ); + expect( + firstRead.map((tool) => String(tool.name)), + "reads inside the backoff serve the existing catalog", + ).toEqual(["deploy"]); + + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b.and(b("integration", "=", String(INTEG)), b("name", "=", "main")), + set: { tools_sync_started_at: Date.now() - 16 * 60_000 }, + }), + ); + yield* executor.tools.list({ integration: INTEG }); + expect(resolutions, "a read after the backoff retries the sync once").toBe( + resolutionsAfterCreate + 1, + ); + }), + ), + ); + + it.live( + "a read over many stale catalogs rebuilds them one at a time and answers without waiting for all of them", + () => + Effect.scoped( + Effect.gen(function* () { + const names = ["a", "b", "c", "d", "e"]; + let parked = false; + const release = yield* Deferred.make(); + let inFlight = 0; + let maxInFlight = 0; + const rebuilt: string[] = []; + const heavyPlugin = definePlugin(() => ({ + id: "heavy" as const, + credentialProviders: [memoryProvider()], + storage: () => ({}), + resolveTools: ({ connection }) => + Effect.gen(function* () { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + if (parked) yield* Deferred.await(release); + inFlight -= 1; + if (parked) rebuilt.push(String(connection.name)); + return { tools: [{ name: ToolName.make("deploy"), description: "deploy" }] }; + }), + invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), + extension: (ctx) => ({ + seed: () => + ctx.core.integrations.register({ slug: INTEG, description: "Vercel", config: {} }), + }), + }))(); + const config = makeTestConfig({ plugins: [heavyPlugin] as const }); + const executor = yield* createExecutor({ + ...config, + toolsSyncConcurrency: 1, + toolsSyncGraceMs: 50, + }); + yield* executor.heavy.seed(); + for (const name of names) { + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make(name), + integration: INTEG, + template: TEMPLATE, + value: "secret-token", + }); + } + parked = true; + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + set: { tools_synced_at: null }, + }), + ); + + const firstRead = yield* executor.tools + .list({ integration: INTEG }) + .pipe(Effect.timeoutOption("2 seconds")); + const secondRead = yield* executor.tools + .list({ integration: INTEG }) + .pipe(Effect.timeoutOption("2 seconds")); + expect( + Option.map(firstRead, (tools) => tools.length), + "the read answers from the persisted rows while the rebuilds wait", + ).toEqual(Option.some(5)); + expect(Option.isSome(secondRead)).toBe(true); + expect(maxInFlight, "only one rebuild runs at a time, across both reads").toBe(1); + + yield* Deferred.succeed(release, undefined); + yield* Effect.sync(() => rebuilt.length).pipe( + Effect.repeat({ until: (count) => count >= 5, schedule: Schedule.spaced("10 millis") }), + Effect.timeout("5 seconds"), + ); + yield* Effect.sleep("100 millis"); + expect( + [...rebuilt].sort(), + "every catalog rebuilds exactly once; the second read joined the queued rebuilds", + ).toEqual(names); + expect(maxInFlight).toBe(1); + }), + ), + ); + + it.effect("a retry of a sync that never finished rebuilds after the other stale catalogs", () => + Effect.scoped( + Effect.gen(function* () { + let recording = false; + const order: string[] = []; + const orderedPlugin = definePlugin(() => ({ + id: "ordered" as const, + credentialProviders: [memoryProvider()], + storage: () => ({}), + resolveTools: ({ connection }) => + Effect.sync(() => { + if (recording) order.push(String(connection.name)); + return { tools: [{ name: ToolName.make("deploy"), description: "deploy" }] }; + }), + invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), + extension: (ctx) => ({ + seed: () => + ctx.core.integrations.register({ slug: INTEG, description: "Vercel", config: {} }), + }), + }))(); + const config = makeTestConfig({ plugins: [orderedPlugin] as const }); + const executor = yield* createExecutor({ + ...config, + toolsSyncConcurrency: 1, + toolsSyncGraceMs: null, + }); + yield* executor.ordered.seed(); + for (const name of ["crashed", "healthy"]) { + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make(name), + integration: INTEG, + template: TEMPLATE, + value: "secret-token", + }); + } + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + set: { tools_synced_at: null }, + }), + ); + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b.and(b("integration", "=", String(INTEG)), b("name", "=", "crashed")), + set: { tools_sync_started_at: Date.now() - 16 * 60_000 }, + }), + ); + recording = true; + + yield* executor.tools.list({ integration: INTEG }); + yield* executor.tools.list({ integration: INTEG }); + expect(order).toEqual(["healthy", "crashed"]); + }), + ), + ); + it.effect( "background sync preserves a nonzero remote catalog when a plugin returns authoritative empty", () => diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 8014584695..1abb610912 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -227,6 +227,12 @@ export const coreTables = defineTables({ // Epoch ms of the last tool (re)production for this connection. Stale // vs the integration's `config_revised_at` → re-produced on next read. tools_synced_at: nullableBigintColumn("tools_synced_at"), + // Epoch ms the current tool (re)production STARTED; cleared when it + // finishes. Still set means an attempt has not finished: running + // elsewhere, or its isolate died mid-rebuild. The stale scan backs off + // such a connection instead of re-running the same rebuild on every + // read. + tools_sync_started_at: nullableBigintColumn("tools_sync_started_at"), oauth_client: nullableTextColumn("oauth_client"), // The OWNER of `oauth_client` (a Personal connection may be minted through // a shared Workspace app), set together with `oauth_client`; null for @@ -235,6 +241,16 @@ export const coreTables = defineTables({ oauth_client_owner: nullableTextColumn("oauth_client_owner"), refresh_item_id: nullableTextColumn("refresh_item_id"), expires_at: nullableBigintColumn("expires_at"), + // Cross-instance refresh lease. A refresher claims it with a + // conditional UPDATE before spending the rotating refresh token, so two + // isolates never send the same one. Epoch ms the claim lapses (covering + // a holder that dies mid-refresh) and the claimant's opaque id. + refresh_lease_until: nullableBigintColumn("refresh_lease_until"), + refresh_lease_holder: nullableTextColumn("refresh_lease_holder"), + // Epoch ms of the last persisted token refresh. A refresher that waited + // on a peer's lease compares it with the row it started from to tell + // whether the peer already minted a token it can use. + oauth_refreshed_at: nullableBigintColumn("oauth_refreshed_at"), oauth_scope: nullableTextColumn("oauth_scope"), // Per-connection token endpoint override. Set only when the code was // redeemed at a region other than the oauth_client's configured token host diff --git a/packages/core/sdk/src/core-tools.ts b/packages/core/sdk/src/core-tools.ts index 461e04d8a9..2a483ae5a5 100644 --- a/packages/core/sdk/src/core-tools.ts +++ b/packages/core/sdk/src/core-tools.ts @@ -33,7 +33,12 @@ const schemaToStandard = (schema: Schema.Decoder): StaticToolSchema< >; const OwnerSchema = Schema.Literals(["org", "user"]); -const OAuthGrantSchema = Schema.Literals(["authorization_code", "client_credentials"]); +const OAuthGrantSchema = Schema.Literals([ + "authorization_code", + "client_credentials", + "id_jag", + "github_app", +]); // --------------------------------------------------------------------------- // Schemas diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index fdbc9b7671..eb27d6e5d3 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -209,12 +209,14 @@ import { refreshAccessToken, exchangeClientCredentials, isPermanentTokenRejection, + OAUTH2_DEFAULT_TIMEOUT_MS, isUnusableSuccessTokenResponse, optionalScopesFromAuthorizationUrl, shouldRefreshToken, type OAuth2TokenResponse, type OAuthEndpointUrlPolicy, } from "./oauth-helpers"; +import { mintGithubAppToken } from "./github-app"; import { ENTERPRISE_MANAGED_PROVIDER_STATE_KEY, enterpriseManagedStateFrom, @@ -795,6 +797,25 @@ export interface ExecutorConfig { const os = ownerSubject(owner); return (b: AnyCb) => @@ -1752,7 +1822,7 @@ const makePluginStorageFacade = (input: { if (validationError) return yield* validationError; const rows = yield* input.core.findMany("plugin_storage", { - where: whereFor(definition.name), + where: whereForPrefix(definition.name, queryInput?.keyPrefix), }); const filtered = sortByOwnerPrecedence(rows) .filter((row) => @@ -1828,7 +1898,7 @@ const makePluginStorageFacade = (input: { list: (storageInput) => Effect.gen(function* () { const rows = yield* input.core.findMany("plugin_storage", { - where: whereFor(storageInput.collection), + where: whereForPrefix(storageInput.collection, storageInput.keyPrefix), }); return sortByOwnerPrecedence(rows) .filter((row) => @@ -2300,6 +2370,10 @@ export const createExecutor = => { const existingState = decodeJsonColumn(row.provider_state); const mergedState = @@ -2319,6 +2393,11 @@ export const createExecutor = = { expires_at: nextExpiresAt, + oauth_refreshed_at: Date.now(), updated_at: new Date(), }; if (token.scope !== undefined) set.oauth_scope = token.scope; @@ -2574,12 +2656,126 @@ export const createExecutor = ({ + owner: row.owner as Owner, + integration: IntegrationSlug.make(row.integration), + name: ConnectionName.make(row.name), + }); + + const connectionRowWhere = (row: ConnectionRow) => (b: AnyCb) => + b.and( + byOwner(row.owner as Owner)(b), + b("integration", "=", String(row.integration)), + b("name", "=", String(row.name)), + ); + + const refreshedAtOf = (row: ConnectionRow): number | null => + row.oauth_refreshed_at == null ? null : Number(row.oauth_refreshed_at); + + const refreshLeaseMarginMs = Math.min( + OAUTH_REFRESH_LEASE_MARGIN_MS, + Math.floor(refreshLeaseMs / 6), + ); + + const pollDelay = Effect.promise( + () => new Promise((resolve) => setTimeout(resolve, OAUTH_REFRESH_LEASE_POLL_MS)), + ); + + /** How long the grant may take, or null when this holder must not send it: + * the lease is no longer ours, a peer already committed a refresh, or too + * little of the lease is left to finish before a waiter may take over. */ + const remainingGrantBudget = ( + row: ConnectionRow, + lease: RefreshLease, + ): Effect.Effect => + Effect.gen(function* () { + const current = yield* findConnectionRow(connectionRefOf(row)); + if (current === null || current.refresh_lease_holder !== lease.holder) return null; + if (refreshedAtOf(current) !== lease.startStamp) return null; + const budget = lease.leaseUntil - refreshLeaseMarginMs - Date.now(); + return budget >= refreshLeaseMarginMs ? budget : null; + }); + + /** After a successful grant and before persisting it: renew the lease so a + * slow persist cannot let a waiter take over while the spent refresh token + * is still the stored one, then look for a peer that committed first. A + * peer can only have done so against a provider that tolerates reuse; its + * token is then the stored one and ours is dropped. Otherwise our token is + * the grant's valid successor and is persisted even if the renewal did not + * land. Returns the peer's access token, or null to persist ours. */ + const settleLeasedGrant = ( + row: ConnectionRow, + provider: CredentialProvider, + lease: RefreshLease, + ): Effect.Effect => + Effect.gen(function* () { + const renewedUntil = Date.now() + refreshLeaseMs; + yield* core.updateMany("connection", { + where: (b: AnyCb) => + b.and(connectionRowWhere(row)(b), b("refresh_lease_holder", "=", lease.holder)), + set: { refresh_lease_until: renewedUntil }, + }); + const current = yield* findConnectionRow(connectionRefOf(row)); + if (current?.refresh_lease_holder === lease.holder) lease.leaseUntil = renewedUntil; + if (current === null || refreshedAtOf(current) === lease.startStamp) return null; + const access = yield* provider.get(ProviderItemId.make(accessTokenItemIdOf(current))); + if (access !== null) { + yield* Effect.annotateCurrentSpan({ + "executor.oauth.refresh.peer_committed_first": true, + }); + } + return access; + }); + + /** Decide what an invalid_grant on our refresh grant means before calling + * the connection dead. A peer that spent the same refresh token first left + * a newer `oauth_refreshed_at` (use its access token) or at least a + * different stored refresh token (its persist is still landing: wait for + * it). A holder that took over a lapsed lease also waits, for a stalled + * holder's late commit. Returns the peer's access token, `"pending"` when a + * newer refresh token is stored but never got its access token in time, + * or null when nothing supersedes the rejection. */ + const findSupersedingRefresh = ( + row: ConnectionRow, + provider: CredentialProvider, + sentRefreshToken: string, + startStamp: number | null, + waitForLateCommit: boolean, + ): Effect.Effect => { + const deadline = Date.now() + refreshLeaseMs; + const look = (): Effect.Effect => + Effect.gen(function* () { + const current = yield* findConnectionRow(connectionRefOf(row)); + if (current === null) return null; + if (refreshedAtOf(current) !== startStamp) { + const access = yield* provider.get(ProviderItemId.make(accessTokenItemIdOf(current))); + if (access !== null) return access; + } + const storedRefreshToken = current.refresh_item_id + ? yield* provider.get(ProviderItemId.make(current.refresh_item_id)) + : null; + const peerRotated = + storedRefreshToken !== null && storedRefreshToken !== sentRefreshToken; + if (!peerRotated && !waitForLateCommit) return null; + if (Date.now() >= deadline) return peerRotated ? "pending" : null; + yield* pollDelay; + return yield* look(); + }); + return look(); + }; + // Perform the actual refresh-token grant and persist the rotated material. const performTokenRefresh = ( row: ConnectionRow, provider: CredentialProvider, trigger: RefreshTrigger, - ): Effect.Effect => + lease?: RefreshLease, + ): Effect.Effect< + string | null | RefreshLeaseLost, + StorageFailure | CredentialResolutionError + > => Effect.gen(function* () { const owner = row.owner as Owner; const reauth = ( @@ -2730,6 +2926,41 @@ export const createExecutor = + new StorageError({ + // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: OAuth2Error carries a typed `message` + message: `GitHub App token request failed: ${cause.message}`, + cause, + }), + ), + ); + yield* persistRefreshedToken(row, provider, token); + return token.access_token; + } + // client_credentials (machine-to-machine) has NO refresh token — the // token is RE-MINTED from the client id/secret. The authorization_code // path below needs a stored refresh token. Branching on grant here is @@ -2808,6 +3039,17 @@ export const createExecutor = { // An RFC 6749 §5.2 error code is the AS's definitive @@ -2876,19 +3123,57 @@ export const createExecutor = + // A rejection is only the grant's death when nothing newer + // superseded it: a peer that spent the same token first + // left its own token behind, and that is what the caller + // gets. Otherwise persist the definitive verdict so the NEXT + // refresh skips the doomed grant (see the known-dead gate + // above) and the connection shows `expired` without waiting + // for a probe. + Effect.catch((error) => Predicate.isTagged(error, "CredentialResolutionError") && error.reauthRequired === true - ? // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: CredentialResolutionError carries a typed `message` field - markRefreshGrantDead(row, error.message, credentialFailureReason(error)) - : Effect.void, + ? Effect.gen(function* () { + const startStamp = lease?.startStamp ?? refreshedAtOf(row); + const superseding = yield* findSupersedingRefresh( + row, + provider, + refreshToken, + startStamp, + lease?.tookOverLapsedLease === true, + ); + if (superseding !== null && superseding !== "pending") { + yield* Effect.annotateCurrentSpan({ + "executor.oauth.refresh.superseded_by_peer": true, + }); + return { supersededBy: superseding }; + } + if (superseding === "pending") { + return yield* new StorageError({ + message: `A newer OAuth refresh for ${row.owner}/${row.integration}/${row.name} is still being saved; retry the call.`, + cause: undefined, + }); + } + yield* markRefreshGrantDead( + row, + // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: CredentialResolutionError carries a typed `message` field + error.message, + credentialFailureReason(error), + startStamp, + ); + return yield* error; + }) + : Effect.fail(error), ), ); }); + if (token === REFRESH_LEASE_LOST) return REFRESH_LEASE_LOST; + if ("supersededBy" in token) return token.supersededBy; + if (lease !== undefined && storedRefreshToken !== undefined) { + const peerAccess = yield* settleLeasedGrant(row, provider, lease); + if (peerAccess !== null) return peerAccess; + } yield* persistRefreshedToken(row, provider, token, storedRefreshToken); return token.access_token; }).pipe( @@ -2945,6 +3230,100 @@ export const createExecutor = + core + .updateMany("connection", { + where: (b: AnyCb) => + b.and(connectionRowWhere(row)(b), b("refresh_lease_holder", "=", holder)), + set: { refresh_lease_until: null, refresh_lease_holder: null }, + }) + .pipe(Effect.ignore); + + /** The cross-instance layer of refresh single-flight. The in-isolate gate + * (`refreshInFlight`) only dedupes callers sharing one root DB handle; on + * Cloudflare every MCP session is its own Durable Object, so two sessions + * would otherwise spend the same rotating refresh token and the + * authorization server would revoke the grant ("refresh token reuse"). + * + * Claim: a conditional UPDATE that only lands on a free or lapsed lease, + * stamping an id unique to this attempt; reading the row back says who + * won (the facade's `updateMany` reports no row count). A loser never + * sends a grant: it polls until the holder's persist shows up as a newer + * `oauth_refreshed_at` than the row it started from and returns that + * stored access token, or until the lease lapses (a holder that died) and + * it can claim. The same comparison runs right after a win, so a caller + * holding a row read before a peer's refresh finished uses the peer's + * token rather than spending the successor refresh token for nothing. */ + const leasedTokenRefresh = ( + seen: ConnectionRow, + provider: CredentialProvider, + trigger: RefreshTrigger, + ): Effect.Effect => { + const holder = crypto.randomUUID(); + const giveUpAt = Date.now() + refreshLeaseMs + refreshLeaseMs; + const timedOut = () => + new StorageError({ + message: `Timed out waiting for another instance to refresh the OAuth token for ${seen.owner}/${seen.integration}/${seen.name}; retry the call.`, + cause: undefined, + }); + // `observed` is the last row this loop read before claiming: whether it + // named another holder is what says a won claim took over a lapsed lease. + const attempt = ( + observed: ConnectionRow, + ): Effect.Effect => + Effect.gen(function* () { + const now = Date.now(); + yield* core.updateMany("connection", { + where: (b: AnyCb) => + b.and( + connectionRowWhere(seen)(b), + b.or( + b.isNull("refresh_lease_until"), + b("refresh_lease_until", "<=", now - refreshLeaseMarginMs), + ), + ), + set: { refresh_lease_until: now + refreshLeaseMs, refresh_lease_holder: holder }, + }); + const fresh = yield* findConnectionRow(connectionRefOf(seen)); + if (fresh === null) { + const access = yield* performTokenRefresh(seen, provider, trigger); + return access === REFRESH_LEASE_LOST ? null : access; + } + const won = fresh.refresh_lease_holder === holder; + + const peerStamp = refreshedAtOf(fresh); + if (peerStamp !== null && peerStamp !== refreshedAtOf(seen)) { + const access = yield* provider.get(ProviderItemId.make(accessTokenItemIdOf(fresh))); + if (access !== null) { + if (won) yield* releaseRefreshLease(seen, holder); + yield* Effect.annotateCurrentSpan({ "executor.oauth.refresh.joined_peer": true }); + return access; + } + } + + if (won) { + const previousHolder = observed.refresh_lease_holder; + const lease: RefreshLease = { + holder, + leaseUntil: now + refreshLeaseMs, + startStamp: peerStamp, + tookOverLapsedLease: previousHolder != null && previousHolder !== holder, + }; + const outcome = yield* performTokenRefresh(fresh, provider, trigger, lease).pipe( + Effect.ensuring(releaseRefreshLease(seen, holder)), + ); + if (outcome !== REFRESH_LEASE_LOST) return outcome; + if (Date.now() >= giveUpAt) return yield* timedOut(); + return yield* attempt(fresh); + } + + if (Date.now() >= giveUpAt) return yield* timedOut(); + yield* pollDelay; + return yield* attempt(fresh); + }); + return attempt(seen); + }; + const refreshConnectionToken = ( row: ConnectionRow, provider: CredentialProvider, @@ -2985,7 +3364,7 @@ export const createExecutor = Deferred.done(deferred, exit)), Effect.ensuring(Effect.sync(() => void refreshInFlight.delete(key))), @@ -3521,6 +3900,14 @@ export const createExecutor = (effect: Effect.Effect) => catalogPersistLock.withPermits(1)(transaction(effect)); @@ -3553,10 +3940,11 @@ export const createExecutor = + values.reduce( + (max, value) => Math.max(max, JSON.stringify(value ?? null).length), + 0, + ); + yield* Effect.logInfo("executor catalog rebuild persisting", { + integration: String(ref.integration), + connection: String(ref.name), + toolRows: toolRows.length, + definitionRows: definitionRows.length, + largestToolSchemaChars: largestJsonChars( + toolRows.flatMap((row) => [row.input_schema, row.output_schema]), + ), + largestDefinitionChars: largestJsonChars(definitionRows.map((row) => row.schema)), + }); yield* persistCatalog( Effect.gen(function* () { yield* core.deleteMany("tool", { where }); @@ -3756,6 +4176,10 @@ export const createExecutor = rowToTool( @@ -3803,7 +4227,36 @@ export const createExecutor = entry.mode).pipe( + const rebuildFields = { + integration: String(ref.integration), + connection: String(ref.name), + mode: requestedMode, + }; + const produce = Effect.suspend(() => { + const startedAt = Date.now(); + return Effect.logInfo("executor catalog rebuild started", rebuildFields).pipe( + Effect.andThen(produceConnectionToolsUnshared(integrationRow, ref, () => entry.mode)), + Effect.tap((tools) => + Effect.logInfo("executor catalog rebuild finished", { + ...rebuildFields, + tools: tools.length, + durationMs: Date.now() - startedAt, + }), + ), + Effect.tapCause((cause) => + Effect.logWarning("executor catalog rebuild failed", { + ...rebuildFields, + durationMs: Date.now() - startedAt, + cause: String(cause).slice(0, 500), + }), + ), + ); + }); + const run = ( + requestedMode === "background" + ? backgroundRebuildPermits.withPermits(1)(produce) + : produce + ).pipe( Effect.exit, Effect.flatMap((exit) => Deferred.done(entry.deferred, exit)), Effect.ensuring(Effect.sync(() => void toolProductionInFlight.delete(key))), @@ -5602,7 +6055,8 @@ export const createExecutor = [] = []; const deferred: Effect.Effect[] = []; + const urgentRetries: Effect.Effect[] = []; + const deferredRetries: Effect.Effect[] = []; + const retries = (queue: Effect.Effect[]) => + queue === urgent ? urgentRetries : deferredRetries; for (const connection of connections) { const integrationRow = integrationBySlug.get(connection.integration); if (!integrationRow) continue; @@ -5639,7 +6097,39 @@ export const createExecutor = 0) { const background = yield* Effect.forkDetach( Effect.all(deferred, { concurrency: STALE_TOOLS_SYNC_CONCURRENCY }), @@ -5678,9 +6175,7 @@ export const createExecutor = ((resolve) => background.addObserver(() => resolve(undefined))), ); } - yield* Effect.all(urgent, { - concurrency: STALE_TOOLS_SYNC_CONCURRENCY, - }); + yield* Fiber.join(urgentFiber); }); // How long a tools read waits for the stale sync before answering from diff --git a/packages/core/sdk/src/github-app.test.ts b/packages/core/sdk/src/github-app.test.ts new file mode 100644 index 0000000000..183020f621 --- /dev/null +++ b/packages/core/sdk/src/github-app.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { createPrivateKey } from "node:crypto"; + +import { mintGithubAppToken, signGithubAppJwt } from "./github-app"; + +// A throwaway PKCS#1 key, generated for this test only. +const { privateKey, publicKey } = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], +); +const pkcs8 = new Uint8Array(await crypto.subtle.exportKey("pkcs8", privateKey)); +const pem = (label: string, bytes: Uint8Array) => + `-----BEGIN ${label}-----\n${btoa(String.fromCharCode(...bytes)).replace(/(.{64})/g, "$1\n")}\n-----END ${label}-----\n`; +const PKCS8_PEM = pem("PRIVATE KEY", pkcs8); + +// The SAME key re-encoded as PKCS#1 — the form GitHub actually hands out +// (`BEGIN RSA PRIVATE KEY`). Produced by node:crypto rather than by our own +// `pkcs1ToPkcs8`, so the round-trip test below is independent of the code it +// exercises. Without this fixture the DER-wrapping path is never executed. +const PKCS1_PEM = createPrivateKey({ key: Buffer.from(pkcs8), format: "der", type: "pkcs8" }) + .export({ type: "pkcs1", format: "pem" }) + .toString(); + +const decode = (segment: string) => + // oxlint-disable-next-line executor/no-json-parse -- boundary: decoding a JWT segment this test just produced, to assert its claims + JSON.parse( + new TextDecoder().decode( + Uint8Array.from(atob(segment.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0)), + ), + ); + +describe("signGithubAppJwt", () => { + it("accepts a PKCS#1 key, the form GitHub hands out", async () => { + expect(PKCS1_PEM).toContain("BEGIN RSA PRIVATE KEY"); + const jwt = await signGithubAppJwt({ appId: "3602580", privateKeyPem: PKCS1_PEM }); + const [head, payload, signature] = jwt.split("."); + // Verifying against the public key proves the DER re-wrap produced a key + // WebCrypto could import and sign with — not merely that it did not throw. + const ok = await crypto.subtle.verify( + "RSASSA-PKCS1-v1_5", + publicKey, + Uint8Array.from(atob(signature!.replace(/-/g, "+").replace(/_/g, "/")), (c) => + c.charCodeAt(0), + ), + new TextEncoder().encode(`${head}.${payload}`), + ); + expect(ok).toBe(true); + }); + + it("signs a verifiable RS256 JWT with the app id as issuer", async () => { + const now = 1_700_000_000_000; + const jwt = await signGithubAppJwt({ appId: "3602580", privateKeyPem: PKCS8_PEM, now }); + const [head, payload, signature] = jwt.split("."); + + expect(decode(head!)).toEqual({ alg: "RS256", typ: "JWT" }); + const claims = decode(payload!); + expect(claims.iss).toBe("3602580"); + // Backdated against clock skew, and inside GitHub's 10-minute ceiling. + expect(claims.iat).toBeLessThan(now / 1000); + expect(claims.exp - claims.iat).toBeLessThanOrEqual(600); + + const ok = await crypto.subtle.verify( + "RSASSA-PKCS1-v1_5", + publicKey, + Uint8Array.from(atob(signature!.replace(/-/g, "+").replace(/_/g, "/")), (c) => + c.charCodeAt(0), + ), + new TextEncoder().encode(`${head}.${payload}`), + ); + expect(ok).toBe(true); + }); +}); + +const mint = (fetchImpl: typeof globalThis.fetch, now = 1_700_000_000_000) => + mintGithubAppToken({ + appId: "3602580", + privateKeyPem: PKCS8_PEM, + tokenUrl: "https://api.github.com/app/installations/130473318/access_tokens", + fetch: fetchImpl, + now, + }); + +const jsonResponse = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + +describe("mintGithubAppToken", () => { + it("normalizes GitHub's { token, expires_at } into an OAuth token response", async () => { + const now = Date.parse("2026-09-18T08:22:49Z"); + const token = await Effect.runPromise( + mint(async () => jsonResponse({ token: "ghs_abc", expires_at: "2026-09-18T09:22:49Z" }), now), + ); + expect(token.access_token).toBe("ghs_abc"); + expect(token.token_type).toBe("bearer"); + expect(token.expires_in).toBe(3600); + }); + + it("accepts a standard { access_token, expires_in } response too", async () => { + const token = await Effect.runPromise( + mint(async () => jsonResponse({ access_token: "tok", expires_in: 900 })), + ); + expect(token.access_token).toBe("tok"); + expect(token.expires_in).toBe(900); + }); + + it("sends the signed JWT as a bearer to the configured token url", async () => { + let seenUrl: string | undefined; + let seenAuth: string | null = null; + await Effect.runPromise( + mint(async (url, init) => { + seenUrl = String(url); + seenAuth = new Headers(init?.headers).get("authorization"); + return jsonResponse({ token: "t" }); + }), + ); + expect(seenUrl).toBe("https://api.github.com/app/installations/130473318/access_tokens"); + expect(seenAuth).toMatch(/^Bearer [\w-]+\.[\w-]+\.[\w-]+$/); + }); + + it("surfaces the endpoint's message on failure", async () => { + await expect( + Effect.runPromise(mint(async () => jsonResponse({ message: "Bad credentials" }, 401))), + ).rejects.toThrow(/Bad credentials/); + }); + + it("fails when the endpoint returns no token", async () => { + await expect(Effect.runPromise(mint(async () => jsonResponse({})))).rejects.toThrow(/no token/); + }); +}); diff --git a/packages/core/sdk/src/github-app.ts b/packages/core/sdk/src/github-app.ts new file mode 100644 index 0000000000..20c9ed0784 --- /dev/null +++ b/packages/core/sdk/src/github-app.ts @@ -0,0 +1,200 @@ +import { Effect, Encoding } from "effect"; + +import { OAUTH2_DEFAULT_TIMEOUT_MS, OAuth2Error, type OAuth2TokenResponse } from "./oauth-helpers"; + +// --------------------------------------------------------------------------- +// GitHub App installation tokens — the `github_app` grant. +// +// A GitHub App has no static credential. You sign a short-lived JWT with the +// App's private key, POST it to the installation's token endpoint, and get back +// a token that expires in ONE HOUR (fixed by GitHub, not configurable). That is +// why this is a grant rather than a pasted API key: the executor's existing +// credential lifecycle already re-mints on `expires_at` and retries once on a +// 401, so wiring the mint in here makes the expiry invisible. +// +// It reuses the `oauth_client` row as-is — no new columns, no migration: +// client_id → the App ID, signed as the JWT `iss` +// secret → the App's PEM private key (resolved from the credential store) +// token_url → the full mint endpoint, which is what carries the installation +// id (`/app/installations//access_tokens`) +// +// The endpoint is just `token_url`, so the broad shape — sign a JWT with a +// private key, POST it, get a bearer — would suit other providers. It is NOT +// generic as written, though: the vendor `Accept` header, the PKCS#1 unwrap and +// GitHub's `{token, expires_at}` envelope are all specific. Hence the specific +// name. The neutral RFC 7523 grant already exists separately as `id_jag`. +// +// One cost of piggybacking on `oauth_client`: the row IS the installation, so an +// app installed on N organizations needs N rows, each holding a copy of the same +// RSA private key. +// --------------------------------------------------------------------------- + +/** How far ahead of NOW the JWT expires. With the 60s backdate below this puts + * `exp - iat` at exactly 600s, GitHub's documented ceiling and its own example. */ +const JWT_LIFETIME_SECONDS = 540; +/** Backdate `iat` against clock skew between us and GitHub, per their guidance. */ +const JWT_BACKDATE_SECONDS = 60; + +const encodeJson = (value: unknown): string => Encoding.encodeBase64Url(JSON.stringify(value)); + +/** DER length bytes for a definite-length item. */ +const derLength = (length: number): readonly number[] => { + if (length < 0x80) return [length]; + const bytes: number[] = []; + for (let n = length; n > 0; n >>= 8) bytes.unshift(n & 0xff); + return [0x80 | bytes.length, ...bytes]; +}; + +/** ASN.1 AlgorithmIdentifier for rsaEncryption, plus the version INTEGER that + * precedes it inside a PKCS#8 PrivateKeyInfo. */ +const PKCS8_RSA_PREAMBLE = [ + 0x02, 0x01, 0x00, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, + 0x05, 0x00, +] as const; + +/** + * Wrap a PKCS#1 RSAPrivateKey in a PKCS#8 PrivateKeyInfo. + * + * Needed because GitHub hands out `BEGIN RSA PRIVATE KEY` (PKCS#1) while + * WebCrypto's `importKey` only accepts `pkcs8`. The wrap is a fixed ASN.1 + * envelope around the original bytes, so it is a re-encoding, not a conversion. + */ +const pkcs1ToPkcs8 = (pkcs1: Uint8Array): Uint8Array => { + const body = [...PKCS8_RSA_PREAMBLE, 0x04, ...derLength(pkcs1.length), ...pkcs1]; + return new Uint8Array([0x30, ...derLength(body.length), ...body]); +}; + +/** Decode a PEM body to DER, wrapping PKCS#1 keys so WebCrypto accepts them. */ +const derFromPem = (pem: string): Uint8Array => { + const body = pem.replace(/-----[^-]+-----/g, "").replace(/\s+/g, ""); + const der = Uint8Array.from(atob(body), (c) => c.charCodeAt(0)); + return /BEGIN RSA PRIVATE KEY/.test(pem) ? pkcs1ToPkcs8(der) : der; +}; + +/** Sign the app JWT GitHub expects: RS256, `iss` = app id, ≤10 minute lifetime. */ +export const signGithubAppJwt = async (input: { + readonly appId: string; + readonly privateKeyPem: string; + readonly now?: number; +}): Promise => { + const nowSeconds = Math.floor((input.now ?? Date.now()) / 1000); + const header = encodeJson({ alg: "RS256", typ: "JWT" }); + const payload = encodeJson({ + iat: nowSeconds - JWT_BACKDATE_SECONDS, + exp: nowSeconds + JWT_LIFETIME_SECONDS, + iss: input.appId, + }); + const key = await crypto.subtle.importKey( + "pkcs8", + derFromPem(input.privateKeyPem), + { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + false, + ["sign"], + ); + const signature = await crypto.subtle.sign( + "RSASSA-PKCS1-v1_5", + key, + new TextEncoder().encode(`${header}.${payload}`), + ); + return `${header}.${payload}.${Encoding.encodeBase64Url(new Uint8Array(signature))}`; +}; + +/** Seconds until `expires_at`, floored at 0 — GitHub reports an absolute + * timestamp where the executor's lifecycle wants a relative `expires_in`. */ +const expiresInFrom = (expiresAt: unknown, now: number): number | undefined => { + if (typeof expiresAt !== "string") return undefined; + const at = Date.parse(expiresAt); + return Number.isNaN(at) ? undefined : Math.max(0, Math.floor((at - now) / 1000)); +}; + +/** Probe an untrusted token-endpoint body. Mirrors the same helpers in + * ./oauth-helpers: the value is only read for a token and an expiry, never + * decoded into a domain type, so a parse failure just means "no JSON body". */ +const jsonBody = (text: string): Record | null => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: probing an untrusted token-endpoint body; unparseable means "no envelope" + try { + // oxlint-disable-next-line executor/no-json-parse -- boundary: same untrusted-body probe; only a token string and an expiry are read back out + const parsed: unknown = JSON.parse(text); + return typeof parsed === "object" && parsed !== null + ? (parsed as Record) + : null; + } catch { + return null; + } +}; + +const endpointMessage = (body: Record | null, status: number): string => + typeof body?.message === "string" ? body.message : `HTTP ${status}`; + +/** + * Mint an installation access token. + * + * Returns the standard `OAuth2TokenResponse` so every caller downstream — the + * connect path, the re-mint path, `persistRefreshedToken` — treats it exactly + * like any other grant's token. GitHub answers `{ token, expires_at }` rather + * than `{ access_token, expires_in }`, so both shapes are accepted: the standard + * one first, GitHub's as the fallback. + */ +export const mintGithubAppToken = (input: { + readonly appId: string; + readonly privateKeyPem: string; + readonly tokenUrl: string; + readonly fetch?: typeof globalThis.fetch; + readonly now?: number; +}): Effect.Effect => + Effect.gen(function* () { + const now = input.now ?? Date.now(); + + const jwt = yield* Effect.tryPromise({ + try: () => signGithubAppJwt({ appId: input.appId, privateKeyPem: input.privateKeyPem, now }), + catch: () => + new OAuth2Error({ + message: + "GitHub App token mint failed: could not sign the app JWT. Check that the credential is the app's RSA private key in PEM form.", + }), + }); + + const result = yield* Effect.tryPromise({ + try: async () => { + // oxlint-disable-next-line executor/no-raw-fetch -- boundary: provider token exchange is the SDK's HTTP boundary and preserves its injected fetch seam + const response = await (input.fetch ?? globalThis.fetch)(input.tokenUrl, { + method: "POST", + // Every other grant reaches the wire with a timeout; without one a hung + // endpoint hangs the invoke that triggered the re-mint. + signal: AbortSignal.timeout(OAUTH2_DEFAULT_TIMEOUT_MS), + headers: { + authorization: `Bearer ${jwt}`, + accept: "application/vnd.github+json", + "user-agent": "executor", + }, + }); + return { ok: response.ok, status: response.status, text: await response.text() }; + }, + catch: () => + new OAuth2Error({ + message: `GitHub App token mint failed: could not reach ${input.tokenUrl}`, + }), + }); + + const body = jsonBody(result.text); + if (!result.ok) { + return yield* new OAuth2Error({ + message: `GitHub App token mint failed: ${endpointMessage(body, result.status)}`, + }); + } + + const token = body?.access_token ?? body?.token; + if (typeof token !== "string" || token.length === 0) { + return yield* new OAuth2Error({ + message: "GitHub App token mint failed: endpoint returned no token", + }); + } + + const expiresIn = + typeof body?.expires_in === "number" ? body.expires_in : expiresInFrom(body?.expires_at, now); + return { + access_token: token, + token_type: "bearer", + ...(expiresIn === undefined ? {} : { expires_in: expiresIn }), + } satisfies OAuth2TokenResponse; + }); diff --git a/packages/core/sdk/src/oauth-client.ts b/packages/core/sdk/src/oauth-client.ts index 8778205014..0293f09d7d 100644 --- a/packages/core/sdk/src/oauth-client.ts +++ b/packages/core/sdk/src/oauth-client.ts @@ -94,7 +94,10 @@ export type EnterpriseIdentityProviderDescriptor = * Such a client's `clientId`/`clientSecret`/`tokenUrl` are its registration at * the MCP server's Resource Authorization Server — the IdP registration is a * second client, named on the connect request. */ -export type OAuthGrant = "authorization_code" | "client_credentials" | "id_jag"; +/** `github_app` mints a GitHub App installation token by signing a JWT with the + * app's private key (see ./github-app). Like `client_credentials` it has no + * human step and no refresh token — the token is re-minted on expiry. */ +export type OAuthGrant = "authorization_code" | "client_credentials" | "id_jag" | "github_app"; /** Provider OAuth config an integration declares as one of its auth templates — * what to request. (The flow itself runs off the self-contained OAuthClient.) diff --git a/packages/core/sdk/src/oauth-flow.test.ts b/packages/core/sdk/src/oauth-flow.test.ts index b98d500716..688c2f6ae0 100644 --- a/packages/core/sdk/src/oauth-flow.test.ts +++ b/packages/core/sdk/src/oauth-flow.test.ts @@ -1596,6 +1596,10 @@ describe("oauth token refresh in resolveConnectionValue", () => { // second would simply join the first's grant — closing the very // window this test exists to open. A second replica holds a second // handle, which is what the extra `withQueryContext` wrapper is. + // + // The cross-instance refresh lease would make B wait for A; a short + // lease is A stalling past it, so B takes over and the window stays + // open. const config = { ...makeTestConfig({ plugins: [oauthPlugin] as const, @@ -1603,6 +1607,7 @@ describe("oauth token refresh in resolveConnectionValue", () => { subject: SHARED_STORE_SUBJECT, }), providers: [sharedStore], + oauthRefreshLeaseMs: 200, }; const instanceA = yield* createExecutor(config); const instanceB = yield* createExecutor({ @@ -1691,8 +1696,9 @@ describe("oauth token refresh in resolveConnectionValue", () => { ).toEqual([]); // The gate did still run for A — on an item of its own, holding no - // credential. Its grant then failed on the spent token, so it - // persisted nothing: this one write is everything A wrote. + // credential. It then found its lease gone and never sent the spent + // token, so it persisted nothing: this one write is everything A + // wrote. const writtenByA = writes.slice(writesBeforeAResumes); expect(writtenByA, "the resumed refresher wrote exactly one item").toHaveLength(1); expect(writtenByA[0], "and it was not the refresh token's own item").not.toBe( @@ -1703,15 +1709,15 @@ describe("oauth token refresh in resolveConnectionValue", () => { "the item it wrote carries no credential", ).not.toContain(store.get(writtenByA[0]!)); - // Both instances really did reach the authorization server, so the - // interleaving under test happened rather than being short-circuited. + // Only B reached the authorization server: A stalled past its lease, + // so it gave up the spent token instead of sending it. expect( (yield* server.requests).filter( (request) => request.path === "/token" && request.body.includes("grant_type=refresh_token"), ), - "both instances sent a refresh grant", - ).toHaveLength(2); + "only the peer that held the lease sent a refresh grant", + ).toHaveLength(1); }), ), ); @@ -2899,3 +2905,371 @@ describe("resource-less client sends no resource parameter (#1789)", () => { ), ); }); + +// Two instances (two isolates, two replicas) over ONE database and ONE +// credential store, which is what a Cloudflare deployment is: every MCP +// session is its own Durable Object. The in-isolate refresh gate cannot see +// across them, so only a lease in the shared database keeps both from spending +// the same rotating refresh token. +describe("oauth refresh across instances", () => { + const twoInstancesOverOneExpiredConnection = (options: { + readonly oauthRefreshLeaseMs?: number; + readonly revokeGrantOnReuse?: boolean; + }) => + Effect.gen(function* () { + const server = yield* serveOAuthTestServer({ + scopes: ["read"], + ...(options.revokeGrantOnReuse === undefined + ? {} + : { revokeGrantOnReuse: options.revokeGrantOnReuse }), + }); + const store = new Map(); + const pause = { + next: false, + reached: yield* Deferred.make(), + resume: yield* Deferred.make(), + }; + // One-shot stop on the next write of a refresh token: the persist after + // a grant succeeded. + const pauseWrite = { + next: false, + reached: yield* Deferred.make(), + resume: yield* Deferred.make(), + }; + // One-shot stop on instance A's next refresh grant, before it leaves for + // the authorization server. Releasing forwards it; an abort of the + // request while it waits is recorded and rejects it the way a real + // fetch would. + const grantA = { + next: false, + reached: yield* Deferred.make(), + release: yield* Deferred.make(), + abortedAt: null as number | null, + }; + const grantSentAt = { a: [] as number[], b: [] as number[] }; + const isRefreshGrant = (init: RequestInit | undefined) => + String(init?.body ?? "").includes("refresh_token"); + const fetchFor = + (instance: "a" | "b"): typeof globalThis.fetch => + async (input, init) => { + if (instance === "a" && grantA.next && isRefreshGrant(init)) { + grantA.next = false; + await Effect.runPromise(Deferred.succeed(grantA.reached, undefined)); + const signal = init?.signal; + await new Promise((resolve, reject) => { + const onAbort = () => { + grantA.abortedAt = Date.now(); + // oxlint-disable-next-line executor/no-promise-reject -- test boundary: an aborted fetch must reject with the signal's reason, as the platform fetch does + reject(signal?.reason); + }; + if (signal?.aborted) return onAbort(); + signal?.addEventListener("abort", onAbort, { once: true }); + void Effect.runPromise(Deferred.await(grantA.release)).then(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }); + }); + } + if (isRefreshGrant(init)) grantSentAt[instance].push(Date.now()); + // oxlint-disable-next-line executor/no-raw-fetch -- test boundary: the latch wraps the platform fetch and must delegate back to it to hold a refresh grant before it leaves + return globalThis.fetch(input, init); + }; + const sharedStore: CredentialProvider = { + key: ProviderKey.make("shared-memory"), + writable: true, + get: (id) => + Effect.gen(function* () { + const value = store.get(String(id)) ?? null; + if (pause.next && String(id).endsWith(":refresh")) { + pause.next = false; + yield* Deferred.succeed(pause.reached, undefined); + yield* Deferred.await(pause.resume); + } + return value; + }), + set: (id, value) => + Effect.gen(function* () { + if (pauseWrite.next && String(id).endsWith(":refresh")) { + pauseWrite.next = false; + yield* Deferred.succeed(pauseWrite.reached, undefined); + yield* Deferred.await(pauseWrite.resume); + } + store.set(String(id), value); + }), + delete: (id) => Effect.sync(() => void store.delete(String(id))), + }; + const config = { + ...makeTestConfig({ + plugins: [oauthPlugin] as const, + tenant: SHARED_STORE_TENANT, + subject: SHARED_STORE_SUBJECT, + }), + providers: [sharedStore], + ...(options.oauthRefreshLeaseMs === undefined + ? {} + : { oauthRefreshLeaseMs: options.oauthRefreshLeaseMs }), + }; + const instanceA = yield* createExecutor({ ...config, fetch: fetchFor("a") }); + const instanceB = yield* createExecutor({ + ...config, + fetch: fetchFor("b"), + db: withQueryContext(config.testDb.db, { + tenant: SHARED_STORE_TENANT, + subject: SHARED_STORE_SUBJECT, + }), + }); + yield* Effect.addFinalizer(() => + Effect.promise(() => config.testDb.close()).pipe(Effect.ignore), + ); + yield* Effect.addFinalizer(() => instanceA.close().pipe(Effect.ignore)); + yield* Effect.addFinalizer(() => instanceB.close().pipe(Effect.ignore)); + yield* Effect.addFinalizer(() => Deferred.succeed(grantA.release, undefined)); + + yield* instanceA.acme.seed(); + yield* instanceA.oauth.createClient({ + owner: "org", + slug: CLIENT, + authorizationUrl: server.authorizationEndpoint, + tokenUrl: server.tokenEndpoint, + grant: "authorization_code", + clientId: "test-client", + clientSecret: "test-secret", + }); + const started = yield* instanceA.oauth.start({ + owner: "org", + client: CLIENT, + clientOwner: "org", + name: ConnectionName.make("main"), + integration: INTEG, + template: TEMPLATE, + }); + if (started.status !== "redirect") { + return yield* Effect.die("expected an authorization redirect"); + } + const callback = yield* server.completeAuthorizationCodeFlow({ + authorizationUrl: started.authorizationUrl, + }); + yield* instanceA.oauth.complete({ state: started.state, code: callback.code }); + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("name", "=", "main"), + set: { expires_at: Date.now() - 60_000 }, + }), + ); + + const refreshGrants = server.requests.pipe( + Effect.map( + (requests) => + requests.filter( + (request) => + request.path === "/token" && request.body.includes("grant_type=refresh_token"), + ).length, + ), + ); + const connectionRow = Effect.promise(() => + config.db.findFirst("connection", { where: (b) => b("name", "=", "main") }), + ); + const markedDead = connectionRow.pipe( + Effect.map( + (row) => + (row?.provider_state as { oauthReauthRequiredAt?: number } | null) + ?.oauthReauthRequiredAt !== undefined, + ), + ); + return { + server, + config, + instanceA, + instanceB, + pause, + pauseWrite, + grantA, + grantSentAt, + refreshGrants, + connectionRow, + markedDead, + }; + }); + + const whoami = ToolAddress.make("tools.acme.org.main.whoami"); + + const realSleep = (ms: number) => + Effect.promise(() => new Promise((resolve) => setTimeout(resolve, ms))); + + it.effect( + "two instances refreshing one connection at once send one refresh grant and both get a working token", + () => + Effect.scoped( + Effect.gen(function* () { + const { server, instanceA, instanceB, pause, refreshGrants } = + yield* twoInstancesOverOneExpiredConnection({}); + + // A is mid-refresh: it has the stored refresh token in hand and has + // not yet spent it. + pause.next = true; + const refresherA = yield* Effect.forkChild(instanceA.execute(whoami, {})); + yield* Deferred.await(pause.reached); + + // B needs a token too, right now. Given the time, it would spend the + // same refresh token A is holding. + const refresherB = yield* Effect.forkChild(instanceB.execute(whoami, {})); + yield* realSleep(300); + + yield* Deferred.succeed(pause.resume, undefined); + const resultA = (yield* Fiber.join(refresherA)) as { token: string }; + const resultB = (yield* Fiber.join(refresherB)) as { token: string }; + + expect(yield* refreshGrants, "only one instance spent the refresh token").toBe(1); + expect(yield* server.acceptsAccessToken(resultA.token), "A's token works").toBe(true); + expect(yield* server.acceptsAccessToken(resultB.token), "B's token works").toBe(true); + }), + ), + ); + + it.effect("a lease left by an instance that died mid-refresh is taken over once it lapses", () => + Effect.scoped( + Effect.gen(function* () { + const { server, config, instanceA, refreshGrants } = + yield* twoInstancesOverOneExpiredConnection({ oauthRefreshLeaseMs: 1_200 }); + + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("name", "=", "main"), + set: { + refresh_lease_until: Date.now() + 400, + refresh_lease_holder: "an-isolate-that-died", + }, + }), + ); + + const result = (yield* instanceA.execute(whoami, {})) as { token: string }; + + expect(yield* refreshGrants, "the survivor refreshed once the lease lapsed").toBe(1); + expect(yield* server.acceptsAccessToken(result.token)).toBe(true); + const row = yield* Effect.promise(() => + config.db.findFirst("connection", { where: (b) => b("name", "=", "main") }), + ); + expect(row?.refresh_lease_holder, "the survivor released its own lease").toBeNull(); + }), + ), + ); + + it.effect("a holder that stalls past its lease before sending never spends the token", () => + Effect.scoped( + Effect.gen(function* () { + const { server, instanceA, instanceB, pause, refreshGrants, markedDead } = + yield* twoInstancesOverOneExpiredConnection({ + oauthRefreshLeaseMs: 600, + revokeGrantOnReuse: true, + }); + + // A has the stored refresh token in hand and stalls there, past its + // lease. B takes the lapsed lease over and refreshes to completion. + pause.next = true; + const refresherA = yield* Effect.forkChild(instanceA.execute(whoami, {})); + yield* Deferred.await(pause.reached); + const resultB = (yield* instanceB.execute(whoami, {})) as { token: string }; + + // A wakes holding a token B already spent. Sending it would be a reuse, + // and this provider revokes the whole grant on reuse. + yield* Deferred.succeed(pause.resume, undefined); + const resultA = (yield* Fiber.join(refresherA)) as { token: string }; + + expect(yield* refreshGrants, "only the lease's live holder sent a grant").toBe(1); + expect(yield* server.acceptsAccessToken(resultA.token), "A's token works").toBe(true); + expect(yield* server.acceptsAccessToken(resultB.token), "B's token works").toBe(true); + expect(yield* markedDead, "the connection is still healthy").toBe(false); + }), + ), + ); + + it.effect("a grant still waiting on the provider is abandoned before its lease lapses", () => + Effect.scoped( + Effect.gen(function* () { + const { server, instanceA, instanceB, grantA, grantSentAt, refreshGrants } = + yield* twoInstancesOverOneExpiredConnection({ oauthRefreshLeaseMs: 1_200 }); + + // A's grant leaves and the provider never answers it. + grantA.next = true; + const refresherA = yield* Effect.forkChild(Effect.exit(instanceA.execute(whoami, {}))); + yield* Deferred.await(grantA.reached); + const resultB = (yield* instanceB.execute(whoami, {})) as { token: string }; + yield* Fiber.join(refresherA); + + expect(grantA.abortedAt, "A gave up on its own grant").not.toBeNull(); + expect(grantSentAt.b, "B sent one grant").toHaveLength(1); + expect( + grantSentAt.b[0]! >= grantA.abortedAt!, + "B only sent once A's request was abandoned", + ).toBe(true); + expect(yield* refreshGrants, "the provider saw only B's grant").toBe(1); + expect(yield* server.acceptsAccessToken(resultB.token)).toBe(true); + }), + ), + ); + + it.effect( + "an invalid_grant after a peer rotated the token returns the peer's token and keeps the connection", + () => + Effect.scoped( + Effect.gen(function* () { + const { server, config, instanceA, instanceB, grantA, markedDead } = + yield* twoInstancesOverOneExpiredConnection({ oauthRefreshLeaseMs: 3_000 }); + + // A's grant is on its way when its lease is lost (the isolate was + // paused long enough for it to lapse), and B refreshes meanwhile. + grantA.next = true; + const refresherA = yield* Effect.forkChild(instanceA.execute(whoami, {})); + yield* Deferred.await(grantA.reached); + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("name", "=", "main"), + set: { refresh_lease_until: Date.now() - 60_000 }, + }), + ); + const resultB = (yield* instanceB.execute(whoami, {})) as { token: string }; + + // A's spent refresh token now reaches the provider and is refused. + yield* Deferred.succeed(grantA.release, undefined); + const resultA = (yield* Fiber.join(refresherA)) as { token: string }; + + expect(resultA.token, "A uses the token B minted").toBe(resultB.token); + expect(yield* server.acceptsAccessToken(resultA.token)).toBe(true); + expect(yield* markedDead, "the connection is not marked dead").toBe(false); + }), + ), + ); + + it.effect("a slow persist keeps the lease, so a waiter never spends the token again", () => + Effect.scoped( + Effect.gen(function* () { + const { server, instanceA, instanceB, grantA, pauseWrite, refreshGrants, markedDead } = + yield* twoInstancesOverOneExpiredConnection({ + oauthRefreshLeaseMs: 1_000, + revokeGrantOnReuse: true, + }); + + // A's grant takes a while and then succeeds; saving its rotated token + // then takes long enough to outlast the lease A first claimed. + grantA.next = true; + pauseWrite.next = true; + const refresherA = yield* Effect.forkChild(instanceA.execute(whoami, {})); + yield* Deferred.await(grantA.reached); + yield* realSleep(400); + yield* Deferred.succeed(grantA.release, undefined); + yield* Deferred.await(pauseWrite.reached); + const refresherB = yield* Effect.forkChild(instanceB.execute(whoami, {})); + yield* realSleep(800); + yield* Deferred.succeed(pauseWrite.resume, undefined); + + const resultA = (yield* Fiber.join(refresherA)) as { token: string }; + const resultB = (yield* Fiber.join(refresherB)) as { token: string }; + + expect(yield* refreshGrants, "the spent token was never sent again").toBe(1); + expect(yield* server.acceptsAccessToken(resultA.token), "A's token works").toBe(true); + expect(yield* server.acceptsAccessToken(resultB.token), "B's token works").toBe(true); + expect(yield* markedDead, "the connection is still healthy").toBe(false); + }), + ), + ); +}); diff --git a/packages/core/sdk/src/oauth-service.ts b/packages/core/sdk/src/oauth-service.ts index 7d8d7d46ea..2b799bd944 100644 --- a/packages/core/sdk/src/oauth-service.ts +++ b/packages/core/sdk/src/oauth-service.ts @@ -109,6 +109,7 @@ import { type OAuth2TokenResponse, type OAuthEndpointUrlPolicy, } from "./oauth-helpers"; +import { mintGithubAppToken } from "./github-app"; import { OAUTH2_SESSION_TTL_MS, encodeOAuthCallbackState } from "./oauth"; import { canonicalIssuerUrl, hostOfUrl, isDcrClassifiedRow, parseUrl } from "./oauth-gc"; @@ -461,7 +462,10 @@ const clientOwnerFromPayload = (payload: unknown): Owner | null => { * `authorization_code`; an unknown grant means a corrupt row and callers that * drive token exchange (`loadClient`) must fail loudly rather than guessing. */ const parseGrant = (grant: unknown): OAuthGrant | null => - grant === "client_credentials" || grant === "authorization_code" || grant === "id_jag" + grant === "client_credentials" || + grant === "authorization_code" || + grant === "id_jag" || + grant === "github_app" ? grant : null; @@ -1829,6 +1833,42 @@ export const makeOAuthService = (deps: OAuthServiceDeps): OAuthService => { } } + // github_app: no browser and no exchange — sign a JWT with the app's + // private key (held as this client's secret) and mint straight from the + // installation endpoint in `tokenUrl`. + if (client.grant === "github_app") { + const token = yield* mintGithubAppToken({ + appId: client.clientId, + privateKeyPem: client.clientSecret ?? "", + tokenUrl: client.tokenUrl, + fetch, + }).pipe( + Effect.mapError( + (cause) => + // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: OAuth2Error carries a typed `message` field + new OAuthStartError({ message: cause.message }), + ), + ); + const connection = yield* mintFromToken( + { ...input, name }, + client, + token, + requestedScopes, + input.clientOwner, + null, + ).pipe( + Effect.mapError((cause) => + Predicate.isTagged(cause, "OrgWriteDeniedError") + ? cause + : new OAuthStartError({ + // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: StorageFailure carries a typed `message` field + message: `Failed to mint OAuth connection: ${cause.message}`, + }), + ), + ); + return { status: "connected", connection } as const; + } + // client_credentials: exchange immediately and mint the connection. if (client.grant === "client_credentials") { const token = yield* exchangeClientCredentials({ diff --git a/packages/core/sdk/src/testing/oauth-test-server.ts b/packages/core/sdk/src/testing/oauth-test-server.ts index 0380e389f0..bedabd44cb 100644 --- a/packages/core/sdk/src/testing/oauth-test-server.ts +++ b/packages/core/sdk/src/testing/oauth-test-server.ts @@ -65,6 +65,12 @@ export interface OAuthTestServerOptions { readonly scopes?: readonly string[]; readonly omitTokenResponseScopes?: readonly string[]; readonly supportRefresh?: boolean; + /** Treat a spent refresh token presented again as theft, as providers that + * enforce strict rotation do: the reuse answers `invalid_grant` and revokes + * every live refresh and access token of that grant, so the connection is + * dead until the user signs in again. Without it a spent token is merely + * unknown and its successor stays valid. */ + readonly revokeGrantOnReuse?: boolean; readonly tokenExpiresInSeconds?: number; /** Refuse a refresh-token grant whose `scope` parameter names anything outside * this list, answering the RFC 6749 §5.2 envelope Railway returns: @@ -211,6 +217,9 @@ interface RefreshTokenRecord { readonly username: string; readonly scope: string | null; readonly resource: string | null; + /** Every token minted from one authorization shares it, so a detected reuse + * can revoke the whole grant. */ + readonly grantId: string; } const JsonObject = Schema.Record(Schema.String, Schema.Unknown); @@ -570,6 +579,13 @@ export const serveOAuthTestServer = ( const transactions = new Map(); const authorizationCodes = new Map(); const refreshTokens = new Map(); + const spentRefreshTokens = new Map(); + const accessTokensByGrant = new Map>(); + const recordGrantAccessToken = (grantId: string, accessToken: string) => { + const tokens = accessTokensByGrant.get(grantId) ?? new Set(); + tokens.add(accessToken); + accessTokensByGrant.set(grantId, tokens); + }; const defaultClientId = options.defaultClientId ?? "test-client"; const defaultClientSecret = options.defaultClientSecret ?? "test-secret"; @@ -827,11 +843,14 @@ export const serveOAuthTestServer = ( const accessToken = `at_${randomUUID()}`; const refreshToken = `rt_${randomUUID()}`; yield* Ref.update(issuedAccessTokens, (tokens) => new Set([...tokens, accessToken])); + const grantId = randomUUID(); + recordGrantAccessToken(grantId, accessToken); refreshTokens.set(refreshToken, { clientId, username: record.username, scope: record.scope, resource: record.resource, + grantId, }); const scope = tokenResponseScope(record.scope); return jsonResponse( @@ -851,6 +870,21 @@ export const serveOAuthTestServer = ( if (grantType === "refresh_token") { const refreshToken = params.get("refresh_token"); const record = refreshToken ? refreshTokens.get(refreshToken) : undefined; + const reusedGrantId = + options.revokeGrantOnReuse === true && refreshToken + ? spentRefreshTokens.get(refreshToken) + : undefined; + if (reusedGrantId !== undefined) { + for (const [token, live] of [...refreshTokens]) { + if (live.grantId === reusedGrantId) refreshTokens.delete(token); + } + const revoked = accessTokensByGrant.get(reusedGrantId) ?? new Set(); + yield* Ref.update( + issuedAccessTokens, + (tokens) => new Set([...tokens].filter((token) => !revoked.has(token))), + ); + return oauthError(400, "invalid_grant", "Refresh token reuse detected"); + } if (!supportRefresh || !refreshToken || !record || record.clientId !== clientId) { const rejection = options.refreshRejection; return rejection @@ -874,7 +908,9 @@ export const serveOAuthTestServer = ( const nextAccessToken = `at_${randomUUID()}`; const nextRefreshToken = `rt_${randomUUID()}`; refreshTokens.delete(refreshToken); + spentRefreshTokens.set(refreshToken, record.grantId); refreshTokens.set(nextRefreshToken, record); + recordGrantAccessToken(record.grantId, nextAccessToken); yield* Ref.update( issuedAccessTokens, (tokens) => new Set([...tokens, nextAccessToken]), diff --git a/packages/hosts/cloudflare/src/mcp/agents-post-stream-loss.test.ts b/packages/hosts/cloudflare/src/mcp/agents-post-stream-loss.test.ts index 1681709ca6..503d300d49 100644 --- a/packages/hosts/cloudflare/src/mcp/agents-post-stream-loss.test.ts +++ b/packages/hosts/cloudflare/src/mcp/agents-post-stream-loss.test.ts @@ -208,6 +208,31 @@ describe("POST bridge: lost-execution visibility", () => { } }); + it("reports each lost call to the host's lost-call hook, and nothing on the happy path", async () => { + const reported: unknown[] = []; + const holder = globalThis as { __executorReportLostCall?: (detail: unknown) => void }; + holder.__executorReportLostCall = (detail) => reported.push(detail); + try { + const delivered = await postToBridge(toolCall(1)); + const deliveredBody = drainResponse(delivered.response); + emitResponse(delivered.ws, { id: 1, jsonrpc: "2.0", result: { ok: true } }); + await flushMicrotasks(); + emitAbnormalClose(delivered.ws); + await deliveredBody; + expect(reported).toEqual([]); + + const lost = await postToBridge(toolCall(2)); + const lostBody = drainResponse(lost.response); + emitAbnormalClose(lost.ws, 1006, "WebSocket disconnected without sending Close frame."); + await lostBody; + expect(reported).toEqual([ + expect.objectContaining({ outstandingCount: 1, reason: "session_reset" }), + ]); + } finally { + delete holder.__executorReportLostCall; + } + }); + it("writes nothing extra when the response was delivered before the close", async () => { const { response, ws } = await postToBridge(toolCall(1)); const drained = drainResponse(response); diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index f7c1a349f7..daa843cd4e 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -52,7 +52,7 @@ import type { import type * as Tracer from "effect/Tracer"; import { createExecutionEngine, - searchTools, + defaultToolDiscoveryProvider, formatExecuteResult, formatPausedExecution, formatTtlDuration, @@ -65,6 +65,7 @@ import { type Skill, type ExecutionEngine, type ExecutionEngineConfig, + type ToolDiscoveryProvider, type ResumeResponse, type ExecutionResult, type PausedExecution, @@ -211,6 +212,12 @@ type SharedMcpServerConfig = { * Invoke is marked destructive for client approval. Requires `tools`. */ readonly mode?: McpToolMode; + /** + * Ranks `search`. Passed explicitly because a caller that supplies a prebuilt + * `engine` gives this server no way to reach the engine's own provider, and + * passthrough search must rank the same way codemode's `tools.search` does. + */ + readonly toolDiscoveryProvider?: ToolDiscoveryProvider; /** * The scoped executor's tool catalog, for passthrough mode. Structurally * satisfied by `executor.tools`. Hosts that never serve passthrough may @@ -1278,6 +1285,10 @@ const registerPassthroughTools = ( args: unknown, extra: McpRequestJoinKeys, ) => Effect.Effect, + /** Ranks `search`. Passed in because this function receives ports, not the + * server config — and passthrough search must rank the same way codemode's + * `tools.search` does. */ + toolDiscoveryProvider: ToolDiscoveryProvider, ): Effect.Effect => Effect.gen(function* () { const context = yield* Effect.context(); @@ -1419,7 +1430,15 @@ const registerPassthroughTools = ( .pipe(Effect.map((items) => items.filter((tool) => tool.static !== true))), }, }; - const page = yield* searchTools(discovery, query, limit, { offset }); + // Through the configured provider, not the raw ranker: passthrough + // search must rank the same way codemode's `tools.search` does, or + // semantic ranking silently applies to one surface and not the other. + const page = yield* toolDiscoveryProvider.searchTools({ + executor: discovery, + query, + limit, + offset, + }); const candidates = yield* Effect.forEach( page.items, (match) => @@ -2049,6 +2068,7 @@ export const createExecutorMcpServer = ( config.connections, config.integrations, executePassthroughCall, + config.toolDiscoveryProvider ?? defaultToolDiscoveryProvider, ); } diff --git a/packages/plugins/mcp/src/sdk/connection.test.ts b/packages/plugins/mcp/src/sdk/connection.test.ts index 2c20c4ee22..9eb8606ebf 100644 --- a/packages/plugins/mcp/src/sdk/connection.test.ts +++ b/packages/plugins/mcp/src/sdk/connection.test.ts @@ -7,7 +7,7 @@ import { HttpClientResponse, } from "effect/unstable/http"; -import { createMcpConnector } from "./connection"; +import { createMcpConnector, withDefaultUserAgent } from "./connection"; const endpoint = "https://internal.example/mcp"; @@ -83,3 +83,69 @@ describe("MCP remote transport failures", () => { }), ); }); + +describe("MCP remote transport User-Agent", () => { + // The unit tests below cover the merge rule, but only this one proves the + // header survives the SDK's requestInit and reaches the wire — it is the + // test that fails if the default is computed and then dropped. + it.effect("sends the default User-Agent on the wire", () => + Effect.gen(function* () { + const seen: Array> = []; + const httpClientLayer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => { + seen.push(request.headers); + return Effect.fail( + new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request, + cause: { code: "ECONNREFUSED", message: "stop here" }, + }), + }), + ); + }), + ); + + yield* createMcpConnector({ + transport: "remote", + endpoint, + remoteTransport: "streamable-http", + httpClientLayer, + }).pipe(Effect.flip); + + expect(seen[0]?.["user-agent"]).toBe("skywardai-ua"); + }), + ); +}); + +describe("withDefaultUserAgent", () => { + it("adds a User-Agent when none is configured", () => { + // Workers send none, and a Vercel-fronted MCP edge answers a UA-less + // request with 403 before it ever evaluates the credential. + expect(withDefaultUserAgent({})).toEqual({ "User-Agent": "skywardai-ua" }); + }); + + it("keeps existing headers alongside it", () => { + expect(withDefaultUserAgent({ Authorization: "api-key abc" })).toEqual({ + Authorization: "api-key abc", + "User-Agent": "skywardai-ua", + }); + }); + + it("never overrides an explicitly configured User-Agent", () => { + expect(withDefaultUserAgent({ "User-Agent": "mine/2.0" })).toEqual({ + "User-Agent": "mine/2.0", + }); + }); + + it("matches the configured header case-insensitively, since HTTP names are", () => { + expect(withDefaultUserAgent({ "user-agent": "mine/2.0" })).toEqual({ + "user-agent": "mine/2.0", + }); + }); + + it("does not mutate the caller's headers", () => { + const original = { Authorization: "api-key abc" }; + withDefaultUserAgent(original); + expect(original).toEqual({ Authorization: "api-key abc" }); + }); +}); diff --git a/packages/plugins/mcp/src/sdk/connection.ts b/packages/plugins/mcp/src/sdk/connection.ts index 344de1f640..21ecf9d872 100644 --- a/packages/plugins/mcp/src/sdk/connection.ts +++ b/packages/plugins/mcp/src/sdk/connection.ts @@ -547,6 +547,28 @@ const connectClient = (input: { // Public factory // --------------------------------------------------------------------------- +/** The OpenAPI plugin defaults its own User-Agent for the same reason and sends + * `"executor"` (see its `DEFAULT_USER_AGENT`). The two values deliberately + * differ so MCP traffic is distinguishable in an upstream's logs; converge them + * if that ever stops being worth the second string to allowlist. */ +const DEFAULT_USER_AGENT = "skywardai-ua"; + +/** Default a User-Agent into a remote MCP server's headers. + * + * Workers' `fetch` sends NO User-Agent, and some providers' edge/WAF answer a + * UA-less request with 403 BEFORE auth is evaluated (Vercel-fronted MCP servers + * do this), which reads downstream as a credential failure. Only the header's + * PRESENCE matters. An explicitly configured User-Agent always wins, matched + * case-insensitively because HTTP header names are — a plain spread default + * would leave both keys and the wire value would become "default, configured". + */ +export const withDefaultUserAgent = ( + headers: Readonly>, +): Record => + Object.keys(headers).some((name) => name.toLowerCase() === "user-agent") + ? { ...headers } + : { ...headers, "User-Agent": DEFAULT_USER_AGENT }; + export const createMcpConnector = (input: ConnectorInput): McpConnector => { if (input.transport === "stdio") { const command = input.command.trim(); @@ -626,9 +648,9 @@ export const createMcpConnector = (input: ConnectorInput): McpConnector => { } // Remote transport - const headers = input.headers ?? {}; + const headers = withDefaultUserAgent(input.headers ?? {}); const remoteTransport = input.remoteTransport ?? "auto"; - const requestInit = Object.keys(headers).length > 0 ? { headers } : undefined; + const requestInit = { headers }; const fetch = input.httpClientLayer ? fetchFromHttpClientLayer(input.httpClientLayer, input.staticOAuthBearer === true) : undefined; diff --git a/packages/plugins/mcp/src/sdk/plugin.test.ts b/packages/plugins/mcp/src/sdk/plugin.test.ts index 487874677b..e9fd318a59 100644 --- a/packages/plugins/mcp/src/sdk/plugin.test.ts +++ b/packages/plugins/mcp/src/sdk/plugin.test.ts @@ -1,5 +1,16 @@ import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option, Predicate, Schema, Tracer } from "effect"; +import { + Cause, + Effect, + Exit, + Layer, + Logger, + Option, + Predicate, + References, + Schema, + Tracer, +} from "effect"; import { fileURLToPath } from "node:url"; import { HttpClient, @@ -1390,6 +1401,52 @@ describe("mcpPlugin", () => { }), ), ); + + it.effect("probeEndpoint logs why it rejected an endpoint, without the credential", () => + Effect.scoped( + Effect.gen(function* () { + const SECRET = "synthetic-probe-secret"; + const server = yield* serveTestHttpApp(() => + Effect.succeed(HttpServerResponse.html("not an MCP server")), + ); + const config = makeTestConfig({ plugins: [mcpPlugin()] as const }); + const executor = yield* createExecutor(config); + + const warnings: Array<{ + readonly message: string; + readonly annotations: Record; + }> = []; + const capture = Logger.make((options) => { + if (options.logLevel !== "Warn") return; + warnings.push({ + message: String(options.message), + annotations: { ...options.fiber.getRef(References.CurrentLogAnnotations) }, + }); + }); + + const exit = yield* executor.mcp + .probeEndpoint({ endpoint: server.url("/mcp"), headers: { "X-Api-Key": SECRET } }) + .pipe(Effect.exit, Effect.provide(Logger.layer([capture]))); + + expect(Exit.isFailure(exit)).toBe(true); + expect( + warnings.find((w) => w.message.includes("discovery failed"))?.annotations, + ).toMatchObject({ + "mcp.error.name": "McpToolDiscoveryError", + "mcp.endpoint": server.url("/mcp"), + }); + expect(warnings.find((w) => w.message.includes("rejected"))?.annotations).toMatchObject({ + "mcp.probe.kind": "not-mcp", + "mcp.probe.category": "wrong-shape", + "mcp.probe.reason": "2xx POST body is not a JSON-RPC envelope", + }); + expect(JSON.stringify(warnings)).not.toContain(SECRET); + + yield* executor.close(); + yield* Effect.promise(() => config.testDb.close()); + }), + ), + ); }); // --------------------------------------------------------------------------- diff --git a/packages/plugins/mcp/src/sdk/plugin.ts b/packages/plugins/mcp/src/sdk/plugin.ts index 079b89dce5..fe057d5ec9 100644 --- a/packages/plugins/mcp/src/sdk/plugin.ts +++ b/packages/plugins/mcp/src/sdk/plugin.ts @@ -610,6 +610,59 @@ export const userFacingProbeMessage = ( ? "Couldn't reach this URL. Check the address, your network, and that the server is running." : "This URL doesn't appear to host an MCP server. Double-check the address, including the path."; +/** Structural fields of the errors the plugin folds into fallbacks. Typed, so + * renaming a field on McpConnectionError or McpToolDiscoveryError breaks the + * build instead of silently dropping it from the logs. */ +interface SwallowedMcpError { + readonly name?: string; + readonly message: string; + readonly httpStatus?: number; + readonly failureKind?: string; + readonly stage?: string; + readonly transport?: string; +} + +const logMcpWarning = (message: string, annotations: Record) => + Effect.logWarning(message).pipe( + Effect.annotateLogs(annotations), + Effect.andThen(Effect.annotateCurrentSpan(annotations)), + ); + +/** Log a failure the plugin is about to fold into a fallback (probe, detect, + * discovery), so the reason survives in the host's logs. Only structural + * fields are logged and the endpoint is sanitized: a cause or request can + * carry the credential headers the caller supplied. */ +const logSwallowedMcpError = (message: string, failure: SwallowedMcpError, endpoint?: string) => + logMcpWarning(message, { + ...(endpoint === undefined ? {} : endpointTelemetryAttributes("mcp.endpoint", endpoint)), + "mcp.error.name": failure.name ?? "unknown", + "mcp.error.message": failure.message, + ...(failure.httpStatus === undefined ? {} : { "mcp.error.http_status": failure.httpStatus }), + ...(failure.failureKind === undefined ? {} : { "mcp.error.failure_kind": failure.failureKind }), + ...(failure.stage === undefined ? {} : { "mcp.error.stage": failure.stage }), + ...(failure.transport === undefined ? {} : { "mcp.error.transport": failure.transport }), + }); + +/** Marking the catalog stale is best-effort: the next tools read re-lists + * either way, so a failure here is logged, not surfaced. */ +const markToolsStaleOrLog = ( + connections: PluginCtx["connections"], + ref: Parameters[0], +) => + connections + .markToolsStale(ref) + .pipe(Effect.catch((error) => logSwallowedMcpError("mcp: failed to mark tools stale", error))); + +const logRejectedProbeShape = (endpoint: string, shape: McpShapeProbeResult) => + shape.kind === "mcp" + ? Effect.void + : logMcpWarning("mcp: probe rejected endpoint", { + ...endpointTelemetryAttributes("mcp.endpoint", endpoint), + "mcp.probe.kind": shape.kind, + "mcp.probe.reason": shape.reason, + ...(shape.kind === "not-mcp" ? { "mcp.probe.category": shape.category } : {}), + }); + // --------------------------------------------------------------------------- // MCP-SDK OAuth provider adapter — wraps a pre-resolved access token so the // transport sends it as a Bearer header. Refresh is core's responsibility @@ -1011,8 +1064,10 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { httpClientLayer, }).pipe( Effect.map((d) => ({ ok: true as const, ...d })), - Effect.catch(() => - Effect.succeed({ ok: false as const, manifest: null, versionNegotiation: null }), + Effect.catch((error) => + logSwallowedMcpError("mcp: probe discovery failed", error, trimmed).pipe( + Effect.as({ ok: false as const, manifest: null, versionNegotiation: null }), + ), ), Effect.withSpan("mcp.plugin.discover_tools"), ); @@ -1042,6 +1097,7 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { headers: probeHeaders, queryParams: probeQueryParams, }); + yield* logRejectedProbeShape(trimmed, shape); // A `not-mcp`/auth-required shape only proves the endpoint returned // 401, but the add-flow recovery is the same as a spec-compliant MCP @@ -1078,7 +1134,13 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { const probeResult = yield* ctx.oauth.probe({ url: trimmed }).pipe( Effect.map((oauth) => ({ ok: true as const, oauth })), - Effect.catch(() => Effect.succeed({ ok: false as const, oauth: null })), + Effect.catch((error) => + logSwallowedMcpError( + "mcp: probe found no usable OAuth metadata", + error, + trimmed, + ).pipe(Effect.as({ ok: false as const, oauth: null })), + ), Effect.withSpan("mcp.plugin.probe_oauth"), ); @@ -1515,6 +1577,9 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { // Distinguish the two ways this can fail by TAG, not by reading a // message off an unknown: the plugin refused, or we never reached // it at all. + Effect.tapError((error) => + logSwallowedMcpError("mcp: codex plugin access check failed", error), + ), Effect.catchTags({ McpConnectionError: () => Effect.succeed({ @@ -1568,11 +1633,18 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { Effect.gen(function* () { const parsed = parseMcpIntegrationConfig(config); if (!parsed) return { tools: [] as readonly ToolDef[], incomplete: true }; + const remoteEndpoint = parsed.transport === "remote" ? parsed.endpoint : undefined; // Discovery tolerates unresolved credentials (an open server lists // tools unauthenticated; a bad value just yields zero tools). const values = yield* getValues().pipe( - Effect.orElseSucceed(() => ({}) as Record), + Effect.catch((error) => + logSwallowedMcpError( + "mcp: discovery continues without unresolved credential values", + error, + remoteEndpoint, + ).pipe(Effect.as({} as Record)), + ), ); const built = yield* buildConnectorInput( @@ -1584,6 +1656,11 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { ).pipe(Effect.result); if (Result.isFailure(built)) { + yield* logSwallowedMcpError( + "mcp: connector could not be built; tools left incomplete", + built.failure, + remoteEndpoint, + ); return { tools: [] as readonly ToolDef[], incomplete: true, @@ -1598,6 +1675,11 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { }), ); if (Result.isFailure(discovered)) { + yield* logSwallowedMcpError( + "mcp: tool discovery failed; tools left incomplete", + discovered.failure, + remoteEndpoint, + ); const reauthorizationRequired = discovered.failure.reauthorizationRequired === true; return { tools: [] as readonly ToolDef[], @@ -1716,9 +1798,7 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { }, }).pipe( Effect.onExit(() => - toolListChanged - ? ctx.connections.markToolsStale(connectionRef).pipe(Effect.ignore) - : Effect.void, + toolListChanged ? markToolsStaleOrLog(ctx.connections, connectionRef) : Effect.void, ), ); @@ -1730,9 +1810,9 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { // spec's protocol error. Same meaning: the persisted catalog // drifted. Mark it stale and answer with the typed drift failure. if (isUnknownToolMessage(errorMessage, stamp.toolName)) { - return yield* ctx.connections - .markToolsStale(connectionRef) - .pipe(Effect.ignore, Effect.as(unknownToolFailure(String(toolRow.name), credential))); + return yield* markToolsStaleOrLog(ctx.connections, connectionRef).pipe( + Effect.as(unknownToolFailure(String(toolRow.name), credential)), + ); } if (parsed.transport === "remote") { const recoveredSlackConnectFile = yield* recoverSlackConnectFile({ @@ -1797,13 +1877,11 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { ); } if (error.unknownTool === true) { - return ctx.connections - .markToolsStale({ - owner: credential.owner, - integration: credential.integration, - name: credential.connection, - }) - .pipe(Effect.ignore, Effect.as(unknownToolFailure(String(toolRow.name), credential))); + return markToolsStaleOrLog(ctx.connections, { + owner: credential.owner, + integration: credential.integration, + name: credential.connection, + }).pipe(Effect.as(unknownToolFailure(String(toolRow.name), credential))); } // The server refused the call itself (typically -32602 invalid // params: an argument outside the schema's enum, a missing required @@ -1865,7 +1943,11 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { httpClientLayer, }).pipe( Effect.map(() => true), - Effect.catch(() => Effect.succeed(false)), + Effect.catch((error) => + logSwallowedMcpError("mcp: detect discovery failed", error, trimmed).pipe( + Effect.as(false), + ), + ), Effect.withSpan("mcp.plugin.discover_tools"), ); @@ -1880,6 +1962,7 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { } const shape = yield* probeMcpEndpointShape(trimmed, { httpClientLayer }); + yield* logRejectedProbeShape(trimmed, shape); if (shape.kind === "mcp") { return { kind: MCP_PLUGIN_ID, diff --git a/packages/plugins/openapi/src/sdk/store.test.ts b/packages/plugins/openapi/src/sdk/store.test.ts index ef4a4ed2cf..a9177fa78c 100644 --- a/packages/plugins/openapi/src/sdk/store.test.ts +++ b/packages/plugins/openapi/src/sdk/store.test.ts @@ -13,6 +13,103 @@ import { import { makeDefaultOpenapiStore } from "./store"; import { OperationBinding } from "./types"; +function makeRecordingStore() { + const rows = new Map(); + const listPrefixes: Array = []; + const now = new Date(); + const storageKey = (collection: string, key: string) => `${collection}\0${key}`; + const seed = (owner: "org" | "user", collection: string, key: string, data: unknown) => { + const entry: PluginStorageEntry = { + id: storageKey(collection, key), + owner, + pluginId: "openapi", + collection, + key, + data, + createdAt: now, + updatedAt: now, + }; + rows.set(entry.id, entry); + return entry; + }; + const unusedCollection: ReturnType = { + get: () => Effect.succeed(null), + getForOwner: () => Effect.succeed(null), + list: () => Effect.succeed([]), + put: (input) => Effect.succeed(seed(input.owner, "unused", input.key, input.data)), + query: () => Effect.succeed([]), + count: () => Effect.succeed(0), + remove: () => Effect.void, + }; + const pluginStorage: PluginStorageFacade = { + collection: () => unusedCollection, + get: (input: { readonly collection: string; readonly key: string }) => + Effect.succeed( + (rows.get(storageKey(input.collection, input.key)) as PluginStorageEntry | undefined) ?? + null, + ), + getForOwner: (input: { readonly collection: string; readonly key: string }) => + Effect.succeed( + (rows.get(storageKey(input.collection, input.key)) as PluginStorageEntry | undefined) ?? + null, + ), + list: (input: { readonly collection: string; readonly keyPrefix?: string }) => + Effect.sync(() => { + listPrefixes.push(input.keyPrefix); + return [...rows.values()].filter( + (row) => + row.collection === input.collection && + (input.keyPrefix === undefined || row.key.startsWith(input.keyPrefix)), + ) as PluginStorageEntry[]; + }), + put: (input: { + readonly owner: "org" | "user"; + readonly collection: string; + readonly key: string; + readonly data: unknown; + }) => + Effect.succeed( + seed(input.owner, input.collection, input.key, input.data) as PluginStorageEntry, + ), + putMany: (input) => + Effect.sync(() => { + for (const entry of input.entries) + seed(input.owner, entry.collection, entry.key, entry.data); + }), + remove: (input) => + Effect.sync(() => { + rows.delete(storageKey(input.collection, input.key)); + }), + removeMany: (input) => + Effect.sync(() => { + for (const entry of input.entries) rows.delete(storageKey(entry.collection, entry.key)); + }), + }; + const blobs: PluginBlobStore = { + get: () => Effect.succeed(null), + put: () => Effect.void, + delete: () => Effect.void, + has: () => Effect.succeed(false), + }; + const store = makeDefaultOpenapiStore({ + owner: { tenant: Tenant.make("tenant"), subject: Subject.make("subject") }, + blobs, + pluginStorage, + } satisfies StorageDeps); + return { store, seed, listPrefixes }; +} + +function getBinding(pathTemplate: string) { + return OperationBinding.make({ + method: "get", + servers: [], + pathTemplate, + parameters: [], + requestBody: Option.none(), + responseBody: Option.none(), + }); +} + describe("OpenAPI operation store", () => { it.effect("bounds operation storage keys while preserving tool-name lookup", () => Effect.gen(function* () { @@ -146,4 +243,31 @@ describe("OpenAPI operation store", () => { expect(operation?.binding.pathTemplate).toBe("/users/{userId}/messages"); }), ); + + it.effect("lists one integration's operations without reading other integrations' rows", () => + Effect.gen(function* () { + const { store, seed, listPrefixes } = makeRecordingStore(); + yield* store.putOperations("google_sheets", [ + { integration: "google_sheets", toolName: "values.get", binding: getBinding("/values") }, + ]); + seed("org", "operation", "google_sheets.sheets.copyTo", { + integration: "google_sheets", + toolName: "sheets.copyTo", + binding: { method: "post", servers: [], pathTemplate: "/copyTo", parameters: [] }, + }); + seed("org", "operation", "op.someotherhash.tool", { + integration: "cloudflare_api", + toolName: "zones.list", + binding: { notABinding: true }, + }); + + const operations = yield* store.listOperations("google_sheets"); + + expect(operations.map((operation) => operation.toolName).sort()).toEqual([ + "sheets.copyTo", + "values.get", + ]); + expect(listPrefixes).not.toContain(undefined); + }), + ); }); diff --git a/packages/plugins/openapi/src/sdk/store.ts b/packages/plugins/openapi/src/sdk/store.ts index 548a521031..8028c997e4 100644 --- a/packages/plugins/openapi/src/sdk/store.ts +++ b/packages/plugins/openapi/src/sdk/store.ts @@ -147,15 +147,31 @@ export const makeDefaultOpenapiStore = ({ pluginStorage, blobs }: StorageDeps): ...(operation.description !== undefined ? { description: operation.description } : {}), }); - const listRows = (integration: string) => + const belongsTo = (row: PluginStorageEntry, integration: string): boolean => { + const decoded = decodeOperationStorage(row.data); + return Option.isSome(decoded) && decoded.value.integration === integration; + }; + + const listRowsWithPrefix = (integration: string, keyPrefix: string) => pluginStorage - .list({ collection: OPERATION_COLLECTION }) + .list({ collection: OPERATION_COLLECTION, keyPrefix }) .pipe( Effect.map((rows: readonly PluginStorageEntry[]) => - rows.filter((row) => rowToOperation(row)?.integration === integration), + rows.filter((row) => belongsTo(row, integration)), ), ); + const listRows = (integration: string) => + Effect.all([ + listRowsWithPrefix(integration, `${OPERATION_KEY_VERSION}.${stableKeyHash(integration)}.`), + listRowsWithPrefix(integration, `${integration}.`), + ]).pipe( + Effect.map(([current, legacy]) => { + const seen = new Set(current.map((row) => row.key)); + return [...current, ...legacy.filter((row) => !seen.has(row.key))]; + }), + ); + const removeOperations = (integration: string) => Effect.gen(function* () { const rows = yield* listRows(integration); diff --git a/patches/agents@0.17.3.patch b/patches/agents@0.17.3.patch index 7eec44acf0..c195a9cd83 100644 --- a/patches/agents@0.17.3.patch +++ b/patches/agents@0.17.3.patch @@ -1,18 +1,3 @@ -diff --git a/node_modules/agents/.bun-tag-37e67b70862be2b2 b/.bun-tag-37e67b70862be2b2 -new file mode 100644 -index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 -diff --git a/node_modules/agents/.bun-tag-61b2f1517ab5ced4 b/.bun-tag-61b2f1517ab5ced4 -new file mode 100644 -index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 -diff --git a/node_modules/agents/.bun-tag-61d29b56a6079f1d b/.bun-tag-61d29b56a6079f1d -new file mode 100644 -index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 -diff --git a/node_modules/agents/.bun-tag-a6a47855632a2623 b/.bun-tag-a6a47855632a2623 -new file mode 100644 -index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 -diff --git a/node_modules/agents/.bun-tag-c0c639aa2299e502 b/.bun-tag-c0c639aa2299e502 -new file mode 100644 -index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/dist/agent-tool-types-CNyE1iz_.d.ts b/dist/agent-tool-types-CNyE1iz_.d.ts index 571eececebd5a1eaf7f2fbf5278801c4d34728ba..010811ae74ef2d27f12f7fb2506848f1939d91d6 100644 --- a/dist/agent-tool-types-CNyE1iz_.d.ts @@ -95,7 +80,7 @@ index c8fad448e8797b89690a99d93490d1363851b225..d80f66f1532c29dda0a0477dc1ec9ef5 McpAgent, type McpAuthContext, diff --git a/dist/mcp/index.js b/dist/mcp/index.js -index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c28786cca392 100644 +index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..30dcc9bbb148db7bfe9f649de332289c89a06c12 100644 --- a/dist/mcp/index.js +++ b/dist/mcp/index.js @@ -28,13 +28,60 @@ import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/ @@ -196,7 +181,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 request.headers.forEach((value, key) => { existingHeaders[key] = value; }); -@@ -206,47 +273,159 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { +@@ -206,47 +273,166 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { jsonrpc: "2.0" }); return new Response(body, { status: 500 }); @@ -276,7 +261,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 + * The happy path is byte-identical: every id has been reported + * responded by then, so the set is empty and nothing is written. + */ -+ const __finishAbnormally = async (reason) => { ++ const __finishAbnormally = async (reason, detail = {}) => { + if (__finished) return; + __finished = true; + clearTimeout(__responseDeadline); @@ -286,8 +271,15 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 + event: "mcp_post_stream_lost", + outstandingCount: __outstanding.size, + reason, -+ sessionId ++ sessionId, ++ ...detail + })); ++ globalThis.__executorReportLostCall?.({ ++ outstandingCount: __outstanding.size, ++ reason, ++ sessionId, ++ ...detail ++ }); + for (const id of __outstanding.values()) __forwardSse(encoder.encode(`event: message\ndata: ${JSON.stringify(sessionResetErrorResponse(id, reason))}\n\n`)); + __outstanding.clear(); + } @@ -351,7 +343,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 + async function onError(_error) { + // The DO end of the bridge failed. Nothing still + // outstanding will ever be answered. -+ await __finishAbnormally("session_reset"); ++ await __finishAbnormally("session_reset", { bridge: "error", error: String(_error?.message ?? _error ?? "") }); } onError(error).catch(console.error); }); @@ -376,14 +368,14 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 + await writer.close().catch(() => {}); + return; + } -+ await __finishAbnormally("session_reset"); ++ await __finishAbnormally("session_reset", { bridge: "close", closeCode: closeEvent?.code ?? null, closeReason: closeEvent?.reason ?? null, wasClean: closeEvent?.wasClean ?? null }); } - onClose().catch(console.error); + onClose(event).catch(console.error); }); return new Response(readable, { headers: { -@@ -279,10 +458,16 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { +@@ -279,10 +465,16 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { id: null, jsonrpc: "2.0" }), { status: 400 }); @@ -404,7 +396,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 props: ctx.props, jurisdiction: options.jurisdiction }); -@@ -306,27 +491,116 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { +@@ -306,27 +498,116 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { if (!ws) { await writer.close(); return new Response("Failed to establish WS to DO", { status: 500 }); @@ -536,7 +528,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 return new Response(readable, { headers: { "Cache-Control": "no-cache", -@@ -389,10 +663,16 @@ const createLegacySseHandler = (basePath, namespace, options = {}) => { +@@ -389,10 +670,16 @@ const createLegacySseHandler = (basePath, namespace, options = {}) => { const url = new URL(request.url); if (request.method === "GET" && basePattern.test(url)) { const sessionId = url.searchParams.get("sessionId") || namespace.newUniqueId().toString(); @@ -557,7 +549,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 endpointUrl.pathname = encodeURI(`${basePath}/message`); endpointUrl.searchParams.set("sessionId", sessionId); const endpointMessage = `event: endpoint\ndata: ${endpointUrl.pathname + endpointUrl.search + endpointUrl.hash}\n\n`; -@@ -414,35 +694,94 @@ const createLegacySseHandler = (basePath, namespace, options = {}) => { +@@ -414,35 +701,94 @@ const createLegacySseHandler = (basePath, namespace, options = {}) => { console.error("Failed to establish WebSocket connection"); await writer.close(); return new Response("Failed to establish WebSocket connection", { status: 500 }); @@ -670,7 +662,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 console.error("Error closing SSE connection:", error); } } -@@ -586,6 +925,8 @@ var StreamableHTTPServerTransport = class { +@@ -586,6 +932,8 @@ var StreamableHTTPServerTransport = class { constructor(options) { this._started = false; this._streamResponseIds = /* @__PURE__ */ new Map(); @@ -679,7 +671,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 const { agent } = getCurrentAgent(); if (!agent) throw new Error("McpAgent was not found in Transport constructor"); this._agent = agent; -@@ -627,23 +968,146 @@ var StreamableHTTPServerTransport = class { +@@ -627,23 +975,146 @@ var StreamableHTTPServerTransport = class { const resumedStreamId = await this._eventStore.getStreamIdForEventId?.(lastEventId); if (resumedStreamId) { const resumeState = { streamId: resumedStreamId }; @@ -832,7 +824,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 } /** * Close any connection (other than `selfId`) currently bound to -@@ -651,6 +1115,9 @@ var StreamableHTTPServerTransport = class { +@@ -651,6 +1122,9 @@ var StreamableHTTPServerTransport = class { * Closing rather than mutating sibling state mirrors how the SDK's * single `_streamMapping` entry gives last-writer-wins for free, and * keeps `send()` from routing to a stale bridge. @@ -842,7 +834,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 */ supersedePriorStreamConnections(agent, selfId, streamId) { for (const other of agent.getConnections()) { -@@ -664,12 +1131,14 @@ var StreamableHTTPServerTransport = class { +@@ -664,12 +1138,14 @@ var StreamableHTTPServerTransport = class { * Only used when resumability is enabled */ async replayEvents(lastEventId) { @@ -858,7 +850,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 this.writeSSEEvent(connection, message, eventId); } catch (error) { this.onerror?.(error); -@@ -678,6 +1147,45 @@ var StreamableHTTPServerTransport = class { +@@ -678,6 +1154,45 @@ var StreamableHTTPServerTransport = class { } catch (error) { this.onerror?.(error); } @@ -904,7 +896,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 } /** * Writes an event to the SSE stream with proper formatting -@@ -689,10 +1197,69 @@ var StreamableHTTPServerTransport = class { +@@ -689,10 +1204,69 @@ var StreamableHTTPServerTransport = class { return connection.send(JSON.stringify({ type: "cf_mcp_agent_event", event: eventData, @@ -974,7 +966,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 * Handles POST requests containing JSON-RPC messages */ async handlePostRequest(req, parsedBody) { -@@ -733,6 +1300,22 @@ var StreamableHTTPServerTransport = class { +@@ -733,6 +1307,22 @@ var StreamableHTTPServerTransport = class { }; connection.setState(postState); if (this._eventStore) await agent.setStreamRequestIds(streamId, requestIds); @@ -997,7 +989,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 for (const message of messages) { if (this.messageInterceptor) { if (await this.messageInterceptor(message, { -@@ -760,7 +1343,22 @@ var StreamableHTTPServerTransport = class { +@@ -760,7 +1350,22 @@ var StreamableHTTPServerTransport = class { * when the originating WS has dropped. */ async sendOnStream(agent, streamId, relatedIds, liveConnection, message, requestId) { @@ -1021,7 +1013,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 let shouldClose = false; if (isJSONRPCResultResponse(message) || isJSONRPCErrorResponse(message)) { let responseIds = this._streamResponseIds.get(streamId); -@@ -777,9 +1375,11 @@ var StreamableHTTPServerTransport = class { +@@ -777,9 +1382,11 @@ var StreamableHTTPServerTransport = class { } catch (error) { this.onerror?.(error); } @@ -1035,7 +1027,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 } } async send(message, options) { -@@ -798,14 +1398,19 @@ var StreamableHTTPServerTransport = class { +@@ -798,14 +1405,19 @@ var StreamableHTTPServerTransport = class { * * Sent on exactly one stream, per MCP: "the server MUST send each of * its JSON-RPC messages on only one of the connected streams; it MUST @@ -1059,7 +1051,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 if (standalone) this.writeSSEEvent(standalone, message, eventId); } /** -@@ -861,12 +1466,10 @@ var StreamableHTTPServerTransport = class { +@@ -861,12 +1473,10 @@ var StreamableHTTPServerTransport = class { * * ## Lifecycle * @@ -1076,7 +1068,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 * * Standalone GET stream events (`_GET_stream`) are *not* cleared * automatically; they accumulate for the lifetime of the DO. Bounded -@@ -893,12 +1496,34 @@ var DurableObjectEventStore = class DurableObjectEventStore { +@@ -893,12 +1503,34 @@ var DurableObjectEventStore = class DurableObjectEventStore { } async storeEvent(streamId, message) { if (streamId.includes(":")) throw new Error(`DurableObjectEventStore: streamId must not contain ':' (got ${JSON.stringify(streamId)})`); @@ -1111,7 +1103,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 return eventId; } async getStreamIdForEventId(eventId) { -@@ -915,9 +1540,59 @@ var DurableObjectEventStore = class DurableObjectEventStore { +@@ -915,9 +1547,59 @@ var DurableObjectEventStore = class DurableObjectEventStore { start: startKey, limit: DurableObjectEventStore.REPLAY_LIMIT }); @@ -1172,7 +1164,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 /** * Drop the event log for a single stream. Called by the transport * immediately after a POST's final response has been written to the -@@ -973,6 +1648,13 @@ DurableObjectEventStore.EVENT_KEY_PREFIX = "__mcp_event__:"; +@@ -973,6 +1655,13 @@ DurableObjectEventStore.EVENT_KEY_PREFIX = "__mcp_event__:"; DurableObjectEventStore.SEQ_PAD = 16; DurableObjectEventStore.DELETE_CHUNK = 128; DurableObjectEventStore.REPLAY_LIMIT = 1e3; @@ -1186,7 +1178,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 //#endregion //#region src/mcp/client-transports.ts /** -@@ -1355,6 +2037,26 @@ function experimental_createMcpHandler(server, options = {}) { +@@ -1355,6 +2044,26 @@ function experimental_createMcpHandler(server, options = {}) { } //#endregion //#region src/mcp/index.ts @@ -1213,7 +1205,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 var McpAgent = class McpAgent extends Agent { constructor(..._args) { super(..._args); -@@ -1369,18 +2071,121 @@ var McpAgent = class McpAgent extends Agent { +@@ -1369,18 +2078,121 @@ var McpAgent = class McpAgent extends Agent { async getInitializeRequest() { return this.ctx.storage.get("initializeRequest"); } @@ -1338,7 +1330,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 /** * Reverse lookup: find which POST stream a given `requestId` belongs * to, and return the stream's full `requestIds` list in the same -@@ -1407,10 +2212,14 @@ var McpAgent = class McpAgent extends Agent { +@@ -1407,10 +2219,14 @@ var McpAgent = class McpAgent extends Agent { limit: STREAM_REQS_SCAN_LIMIT }); if (rows.size === STREAM_REQS_SCAN_LIMIT) console.warn(`McpAgent: getStreamForRequestId hit the ${STREAM_REQS_SCAN_LIMIT}-key scan cap; stale __mcp_stream_reqs__ entries may be accumulating from abandoned POSTs`); @@ -1357,7 +1349,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 } /** Read the transport type for this agent. * This relies on the naming scheme being `sse:${sessionId}`, -@@ -1498,6 +2307,12 @@ var McpAgent = class McpAgent extends Agent { +@@ -1498,6 +2314,12 @@ var McpAgent = class McpAgent extends Agent { } /** Sets up the MCP transport and server every time the Agent is started.*/ async onStart(props) { @@ -1370,7 +1362,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 if (props) await this.updateProps(props); else this.props = await this.ctx.storage.get("props"); await this.init(); -@@ -1516,23 +2331,36 @@ var McpAgent = class McpAgent extends Agent { +@@ -1516,23 +2338,36 @@ var McpAgent = class McpAgent extends Agent { return; } break; @@ -1422,7 +1414,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c287 } } } -@@ -1697,7 +2525,9 @@ var McpAgent = class McpAgent extends Agent { +@@ -1697,7 +2532,9 @@ var McpAgent = class McpAgent extends Agent { } }; McpAgent.STREAM_REQS_KEY_PREFIX = "__mcp_stream_reqs__:";