From 5b46b9d70176be4d08743afcc35a436b0cc8971b Mon Sep 17 00:00:00 2001 From: ssdeanx Date: Thu, 8 Jan 2026 09:55:30 -0500 Subject: [PATCH] feat: refactor vector storage configuration and implement LanceDB - Removed the Qdrant vector configuration file (`qdrant.ts`) and its associated logic. - Introduced a new LanceDB vector configuration file (`lance.ts`) to support vector storage and similarity search using LanceDB. - Implemented comprehensive settings for LanceDB, including memory management, embedding generation, and advanced filtering capabilities. - Added functions for generating embeddings, validating filters, and performing storage operations with detailed logging. - Created new JSON files for crawler statistics and session pool state to track request metrics and session management. - Added a request queue JSON file to manage requests with unique identifiers and metadata. chore: update storage and request queue structures - Created `SDK_CRAWLER_STATISTICS_0.json` to log crawler performance metrics. - Created `SDK_SESSION_POOL_STATE.json` to manage session states and usability. - Created a request queue JSON file to handle requests with relevant metadata and error tracking. --- .github/agents/gpt-5-beast-mode.agent.md | 2 +- package-lock.json | 72 +- package.json | 5 +- src/mastra/agents/codingAgents.ts | 3 +- src/mastra/agents/dane.ts | 60 +- src/mastra/config/lance.ts | 325 ++++++++ src/mastra/config/pg-storage.ts | 4 +- src/mastra/config/{vector => }/qdrant.ts | 2 +- src/mastra/config/upstash.ts | 2 +- src/mastra/config/vector/lance.ts | 697 ------------------ .../default/SDK_CRAWLER_STATISTICS_0.json | 27 + .../default/SDK_SESSION_POOL_STATE.json | 26 + .../default/jBMmaaywUPlwSK3.json | 9 + 13 files changed, 498 insertions(+), 736 deletions(-) create mode 100644 src/mastra/config/lance.ts rename src/mastra/config/{vector => }/qdrant.ts (99%) delete mode 100644 src/mastra/config/vector/lance.ts create mode 100644 src/mastra/public/storage/key_value_stores/default/SDK_CRAWLER_STATISTICS_0.json create mode 100644 src/mastra/public/storage/key_value_stores/default/SDK_SESSION_POOL_STATE.json create mode 100644 src/mastra/public/storage/request_queues/default/jBMmaaywUPlwSK3.json diff --git a/.github/agents/gpt-5-beast-mode.agent.md b/.github/agents/gpt-5-beast-mode.agent.md index aab5c393..a9f18425 100644 --- a/.github/agents/gpt-5-beast-mode.agent.md +++ b/.github/agents/gpt-5-beast-mode.agent.md @@ -1,6 +1,6 @@ --- description: 'Beast Mode 2.0: A powerful autonomous agent tuned specifically for GPT-5 that can solve complex problems by using tools, conducting research, and iterating until the problem is fully resolved.' -tools: ['vscode', 'execute', 'read', 'edit', 'search', 'web', 'ast-grep-mcp/*', 'mastrabeta/mastraBlog', 'mastrabeta/mastraChanges', 'mastrabeta/mastraDocs', 'mastrabeta/mastraMigration', 'mastrabeta/mastraExamples', 'next-devtools/*', 's-ai/*', 'agent', 'vscode.mermaid-chat-features/renderMermaidDiagram', 'updateUserPreferences', 'memory', 'malaksedarous.copilot-context-optimizer/askAboutFile', 'malaksedarous.copilot-context-optimizer/runAndExtract', 'malaksedarous.copilot-context-optimizer/askFollowUp', 'malaksedarous.copilot-context-optimizer/researchTopic', 'malaksedarous.copilot-context-optimizer/deepResearch', 'ms-python.python/getPythonEnvironmentInfo', 'ms-python.python/getPythonExecutableCommand', 'ms-python.python/installPythonPackage', 'ms-python.python/configurePythonEnvironment', 'ms-vscode.vscode-websearchforcopilot/websearch', 'the0807.uv-toolkit/uv-init', 'the0807.uv-toolkit/uv-sync', 'the0807.uv-toolkit/uv-add', 'the0807.uv-toolkit/uv-add-dev', 'the0807.uv-toolkit/uv-upgrade', 'the0807.uv-toolkit/uv-clean', 'the0807.uv-toolkit/uv-lock', 'the0807.uv-toolkit/uv-venv', 'the0807.uv-toolkit/uv-run', 'the0807.uv-toolkit/uv-script-dep', 'the0807.uv-toolkit/uv-python-install', 'the0807.uv-toolkit/uv-python-pin', 'the0807.uv-toolkit/uv-tool-install', 'the0807.uv-toolkit/uvx-run', 'the0807.uv-toolkit/uv-activate-venv', 'todo'] +tools: ['vscode', 'execute', 'read', 'edit', 'search', 'web', 'ast-grep-mcp/*', 'mastrabeta/mastraBlog', 'mastrabeta/mastraChanges', 'mastrabeta/mastraDocs', 'mastrabeta/mastraExamples', 'mastrabeta/mastraMigration', 'next-devtools/*', 's-ai/*', 'agent', 'vscode.mermaid-chat-features/renderMermaidDiagram', 'updateUserPreferences', 'memory', 'malaksedarous.copilot-context-optimizer/askAboutFile', 'malaksedarous.copilot-context-optimizer/runAndExtract', 'malaksedarous.copilot-context-optimizer/askFollowUp', 'malaksedarous.copilot-context-optimizer/researchTopic', 'malaksedarous.copilot-context-optimizer/deepResearch', 'ms-python.python/getPythonEnvironmentInfo', 'ms-python.python/getPythonExecutableCommand', 'ms-python.python/installPythonPackage', 'ms-python.python/configurePythonEnvironment', 'ms-vscode.vscode-websearchforcopilot/websearch', 'the0807.uv-toolkit/uv-init', 'the0807.uv-toolkit/uv-sync', 'the0807.uv-toolkit/uv-add', 'the0807.uv-toolkit/uv-add-dev', 'the0807.uv-toolkit/uv-upgrade', 'the0807.uv-toolkit/uv-clean', 'the0807.uv-toolkit/uv-lock', 'the0807.uv-toolkit/uv-venv', 'the0807.uv-toolkit/uv-run', 'the0807.uv-toolkit/uv-script-dep', 'the0807.uv-toolkit/uv-python-install', 'the0807.uv-toolkit/uv-python-pin', 'the0807.uv-toolkit/uv-tool-install', 'the0807.uv-toolkit/uvx-run', 'the0807.uv-toolkit/uv-activate-venv', 'todo'] infer: true name: 'GPT 5 Beast Mode' --- diff --git a/package-lock.json b/package-lock.json index c68eb4f3..6b785629 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "@mastra/observability": "^1.0.0-beta.10", "@mastra/otel-exporter": "^1.0.0-beta.11", "@mastra/pg": "^1.0.0-beta.12", + "@mastra/qdrant": "^1.0.0-beta.3", "@mastra/rag": "^2.0.0-beta.5", "@mastra/react": "^0.1.0-beta.20", "@mastra/upstash": "^1.0.0-beta.10", @@ -94,7 +95,7 @@ "excalidraw-to-svg": "^3.1.0", "fast-glob": "^3.3.3", "fast-xml-parser": "^5.3.3", - "framer-motion": "^12.24.11", + "framer-motion": "^12.24.12", "gray-matter": "^4.0.3", "husky": "^9.1.7", "input-otp": "^1.4.2", @@ -108,7 +109,7 @@ "monaco-editor-webpack-plugin": "^7.1.1", "monaco-languages": "^2.11.1", "monaco-themes": "^0.4.8", - "motion": "^12.24.11", + "motion": "^12.24.12", "nanoid": "^5.1.6", "next": "^16.1.1", "next-themes": "^0.4.6", @@ -6218,6 +6219,21 @@ "@mastra/core": ">=1.0.0-0 <2.0.0-0" } }, + "node_modules/@mastra/qdrant": { + "version": "1.0.0-beta.3", + "resolved": "https://registry.npmjs.org/@mastra/qdrant/-/qdrant-1.0.0-beta.3.tgz", + "integrity": "sha512-ICBcB1ZTrLgs4zf5xV2FfH9T6yzoxNHA5KqS32KXKfq38eR4qL7ImHpkTAzvROuqiGi2SLLEUZUVveIavUnNMw==", + "license": "Apache-2.0", + "dependencies": { + "@qdrant/js-client-rest": "^1.15.1" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "@mastra/core": ">=1.0.0-0 <2.0.0-0" + } + }, "node_modules/@mastra/rag": { "version": "2.0.0-beta.5", "resolved": "https://registry.npmjs.org/@mastra/rag/-/rag-2.0.0-beta.5.tgz", @@ -9857,6 +9873,42 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@qdrant/js-client-rest": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/@qdrant/js-client-rest/-/js-client-rest-1.16.2.tgz", + "integrity": "sha512-Zm4wEZURrZ24a+Hmm4l1QQYjiz975Ep3vF0yzWR7ICGcxittNz47YK2iBOk8kb8qseCu8pg7WmO1HOIsO8alvw==", + "license": "Apache-2.0", + "dependencies": { + "@qdrant/openapi-typescript-fetch": "1.2.6", + "undici": "^6.0.0" + }, + "engines": { + "node": ">=18.17.0", + "pnpm": ">=8" + }, + "peerDependencies": { + "typescript": ">=4.7" + } + }, + "node_modules/@qdrant/js-client-rest/node_modules/undici": { + "version": "6.23.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz", + "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/@qdrant/openapi-typescript-fetch": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@qdrant/openapi-typescript-fetch/-/openapi-typescript-fetch-1.2.6.tgz", + "integrity": "sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0", + "pnpm": ">=8" + } + }, "node_modules/@radix-ui/number": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", @@ -14495,6 +14547,7 @@ "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-18.1.0.tgz", "integrity": "sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@swc/helpers": "^0.5.11", "@types/command-line-args": "^5.2.3", @@ -19524,9 +19577,9 @@ "license": "MIT" }, "node_modules/framer-motion": { - "version": "12.24.11", - "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.24.11.tgz", - "integrity": "sha512-cNTxTGvFKcY/eQcuGMpG2rb7DOxEFW+A8h5MFhwPqQrn4Jyz9hFhJAlHpngxZIkS0Kk+rZiYmuxNMgO5dGxeYQ==", + "version": "12.24.12", + "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.24.12.tgz", + "integrity": "sha512-W+tBOI1SDGNMH4D4mADY95qYd16Drke2Tj9zlGlwTGSCi6yy8wbMmPY1mvirfcTK8HBeuuCd2PflHdN/zbL4ew==", "license": "MIT", "dependencies": { "motion-dom": "^12.24.11", @@ -25235,12 +25288,12 @@ } }, "node_modules/motion": { - "version": "12.24.11", - "resolved": "https://registry.npmjs.org/motion/-/motion-12.24.11.tgz", - "integrity": "sha512-nfon8E7NDNh42QTJHwB9gjnfdGLFLf0eCu0izAJ7qmZ1NFODuVp+n20kfe5r5PEQlJyONjI0znSKm7QN8szQAQ==", + "version": "12.24.12", + "resolved": "https://registry.npmjs.org/motion/-/motion-12.24.12.tgz", + "integrity": "sha512-usaP62NpHmM8++QrEnNoCco6qrtK1AtzkeHfgW+4qICE0k7ykK+dPJGaRjEzo7sF1GcrYskrGBB/r5RtqnminQ==", "license": "MIT", "dependencies": { - "framer-motion": "^12.24.11", + "framer-motion": "^12.24.12", "tslib": "^2.4.0" }, "peerDependencies": { @@ -34112,7 +34165,6 @@ "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", "license": "ISC", - "peer": true, "peerDependencies": { "zod": "^3.25 || ^4" } diff --git a/package.json b/package.json index e4f3f851..5a8b582f 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@mastra/observability": "^1.0.0-beta.10", "@mastra/otel-exporter": "^1.0.0-beta.11", "@mastra/pg": "^1.0.0-beta.12", + "@mastra/qdrant": "^1.0.0-beta.3", "@mastra/rag": "^2.0.0-beta.5", "@mastra/react": "^0.1.0-beta.20", "@mastra/upstash": "^1.0.0-beta.10", @@ -127,7 +128,7 @@ "excalidraw-to-svg": "^3.1.0", "fast-glob": "^3.3.3", "fast-xml-parser": "^5.3.3", - "framer-motion": "^12.24.11", + "framer-motion": "^12.24.12", "gray-matter": "^4.0.3", "husky": "^9.1.7", "input-otp": "^1.4.2", @@ -141,7 +142,7 @@ "monaco-editor-webpack-plugin": "^7.1.1", "monaco-languages": "^2.11.1", "monaco-themes": "^0.4.8", - "motion": "^12.24.11", + "motion": "^12.24.12", "nanoid": "^5.1.6", "next": "^16.1.1", "next-themes": "^0.4.6", diff --git a/src/mastra/agents/codingAgents.ts b/src/mastra/agents/codingAgents.ts index be03dfcf..078f8a1b 100644 --- a/src/mastra/agents/codingAgents.ts +++ b/src/mastra/agents/codingAgents.ts @@ -46,7 +46,7 @@ export const codeArchitectAgent = new Agent({ instructions: ({ requestContext }: { requestContext: RequestContext }) => { const userTier = requestContext.get('user-tier') ?? 'free' const language = requestContext.get('language') ?? 'en' - const projectRoot = requestContext.get('projectRoot') ?? process.cwd() + // const projectRoot = requestContext.get('projectRoot') ?? process.cwd() return { role: 'system', @@ -55,7 +55,6 @@ export const codeArchitectAgent = new Agent({ **Context:** - User Tier: ${userTier} - Language: ${language} -- Project Root: ${projectRoot} **Core Capabilities:** 1. **Architecture Analysis**: Evaluate existing code structure, identify patterns and anti-patterns diff --git a/src/mastra/agents/dane.ts b/src/mastra/agents/dane.ts index 56fe9f3c..97b9b788 100644 --- a/src/mastra/agents/dane.ts +++ b/src/mastra/agents/dane.ts @@ -5,7 +5,7 @@ import { listEvents } from '../tools/calendar-tool' import { execaTool } from '../tools/execa-tool' import type { GoogleGenerativeAIProviderOptions } from '@ai-sdk/google' import type { RequestContext } from '@mastra/core/request-context' -import { TokenLimiterProcessor } from '@mastra/core/processors' +import { BatchPartsProcessor, TokenLimiterProcessor } from '@mastra/core/processors' import { webScraperTool } from '../tools' export interface DaneContext { @@ -41,7 +41,13 @@ export const daneCommitMessage = new Agent({ }, model: googleAIFlashLite, memory: pgMemory, - outputProcessors: [new TokenLimiterProcessor(1048576)] + outputProcessors: [new TokenLimiterProcessor(128576), + new BatchPartsProcessor({ + batchSize: 10, + maxWaitTime: 75, + emitOnNonText: true + }), + ] }) export const daneIssueLabeler = new Agent({ @@ -70,7 +76,13 @@ export const daneIssueLabeler = new Agent({ }, model: googleAIFlashLite, memory: pgMemory, - outputProcessors: [new TokenLimiterProcessor(1048576)] + outputProcessors: [new TokenLimiterProcessor(128576), + new BatchPartsProcessor({ + batchSize: 10, + maxWaitTime: 75, + emitOnNonText: true + }), + ] }) export const daneLinkChecker = new Agent({ @@ -104,7 +116,13 @@ export const daneLinkChecker = new Agent({ }, model: googleAIFlashLite, memory: pgMemory, - outputProcessors: [new TokenLimiterProcessor(1048576)] + outputProcessors: [new TokenLimiterProcessor(128576), + new BatchPartsProcessor({ + batchSize: 10, + maxWaitTime: 75, + emitOnNonText: true + }), + ] }) export const daneChangeLog = new Agent({ @@ -140,7 +158,13 @@ export const daneChangeLog = new Agent({ }, model: googleAIFlashLite, memory: pgMemory, - outputProcessors: [new TokenLimiterProcessor(1048576)] + outputProcessors: [new TokenLimiterProcessor(128576), + new BatchPartsProcessor({ + batchSize: 10, + maxWaitTime: 75, + emitOnNonText: true + }), + ] }) export const dane = new Agent({ @@ -160,19 +184,9 @@ export const dane = new Agent({ # Our tools: - ## readPDF - Makes you a powerful agent capable of reading PDF files. - - ## fsTool - Makes you a powerful agent capable of reading and writing files to the local filesystem. - ## execaTool Makes you a powerful agent capabale of executing files on the local system. - ## googleSearch - Makes you a powerful agent capabale answering all questions by finding the answer on Google search. - Pass the query as a JS object. If you have links, ALWAYS CITE YOUR SOURCES. - ## browserTool Makes you a powerful agent capable of scraping the web. Pass the url as a JS object. @@ -180,12 +194,12 @@ export const dane = new Agent({ Makes you a powerful agent capable of listing events on a calendar. When using this tool ONLY RETURN RELEVANT EVENTS. DO NOT ATTEMPT TO DO ANYTHING MORE. - ## imageTool - Makes you a powerful agent capable of generating images and saving them to disk. Pass the directory and an image prompt. + ## webScraperTool + Makes you a powerful agent capable of scraping web pages. Use this tool to get information from web pages. # Rules * **Tool Efficiency:** Do NOT use the same tool repetitively or back-to-back for the same query. - * DO NOT ATTEMPT TO USE GENERAL KNOWLEDGE. Use the 'googleSearch' tool to find the answer. + * DO NOT ATTEMPT TO USE GENERAL KNOWLEDGE. Use the 'webScraperTool' tool to find the answer. * Don't reference tools when you communicate with the user. Do not mention what tools you are using. * Tell the user what you are doing. `, @@ -193,7 +207,7 @@ export const dane = new Agent({ google: { thinkingConfig: { includeThoughts: true, - thinkingLevel: 'high', + thinkingLevel: 'low', }, responseModalities: ['TEXT'], } satisfies GoogleGenerativeAIProviderOptions, @@ -208,5 +222,11 @@ export const dane = new Agent({ webScraperTool, listEvents, }, - outputProcessors: [new TokenLimiterProcessor(1048576)] + outputProcessors: [new TokenLimiterProcessor(128576), + new BatchPartsProcessor({ + batchSize: 10, + maxWaitTime: 75, + emitOnNonText: true + }), + ] }) diff --git a/src/mastra/config/lance.ts b/src/mastra/config/lance.ts new file mode 100644 index 00000000..24af6e14 --- /dev/null +++ b/src/mastra/config/lance.ts @@ -0,0 +1,325 @@ +import { LanceVectorStore, LanceStorage } from '@mastra/lance' +import { createVectorQueryTool, createGraphRAGTool } from '@mastra/rag' +import { google } from '@ai-sdk/google' +import { embedMany } from 'ai' +import { log } from './logger +import { Memory } from '@mastra/memory' + +/** + * LanceDB Vector configuration for the Governed RAG system + * Uses LanceDB for vector storage and similarity search + */ + +// Configuration constants +const LANCE_CONFIG = { + dbPath: process.env.LANCE_DB_PATH ?? '/tmp/lance_db', + tableName: process.env.LANCE_TABLE_NAME ?? 'governed_rag', + // Google Gemini gemini-embedding-001 supports flexible dimensions: 128-3072 + // Recommended: 768, 1536, 3072 + embeddingDimension: parseInt( + process.env.LANCE_EMBEDDING_DIMENSION ?? '1536' + ), + embeddingModel: google.textEmbedding('gemini-embedding-001'), +} as const + +// LanceDB Storage configuration constants +const LANCE_STORAGE_CONFIG = { + // Storage database configuration + storageName: process.env.LANCE_STORAGE_NAME ?? 'governed-rag-storage', + dbUri: process.env.LANCE_DB_URI ?? '/tmp/lance_storage_db', + // Storage options + storageOptions: { + timeout: process.env.LANCE_STORAGE_TIMEOUT ?? '60s', + }, + // Table prefix for environment isolation + tablePrefix: process.env.LANCE_STORAGE_TABLE_PREFIX ?? 'dev_', +} as const + + +// Using LanceStorage.create with uri, name, and prefix as per Mastra v1 storage specs +export const lanceStorage = await LanceStorage.create( + LANCE_STORAGE_CONFIG.dbUri, + LANCE_STORAGE_CONFIG.storageName, + LANCE_STORAGE_CONFIG.tablePrefix +) + +const vectorStore = await LanceVectorStore.create("/path/to/db"); + + +/** + * LanceDB Memory instance configured like pgMemory (pg-storage.ts) + */ +export const lanceMemory = new Memory({ + storage: lanceStorage, + vector: vectorStore, + embedder: google.textEmbedding('gemini-embedding-001'), + options: { + generateTitle: process.env.LANCE_THREAD_GENERATE_TITLE !== 'false', + // Message management + lastMessages: parseInt(process.env.LANCE_MEMORY_LAST_MESSAGES ?? '500'), + // Advanced semantic recall with LanceDB configuration + semanticRecall: { + topK: parseInt(process.env.LANCE_SEMANTIC_TOP_K ?? '5'), + messageRange: { + before: parseInt( + process.env.LANCE_SEMANTIC_RANGE_BEFORE ?? '3' + ), + after: parseInt(process.env.LANCE_SEMANTIC_RANGE_AFTER ?? '2'), + }, + scope: 'resource', // 'resource' | 'thread' + // LanceDB-specific index configuration + indexConfig: {}, + }, + // Enhanced working memory with supported template + workingMemory: { + enabled: true, + scope: 'resource', // 'resource' | 'thread' + version: 'vnext', // Enable the improved/experimental tool + template: `# User Profile & Context + ## Personal Information + - **Name**: [To be learned] + - **Role/Title**: [To be learned] + - **Organization**: [To be learned] + - **Location**: [To be learned] + - **Time Zone**: [To be learned] + + ## Communication Preferences + - **Preferred Communication Style**: [To be learned] + - **Response Length Preference**: [To be learned] + - **Technical Level**: [To be learned] + + ## Current Context + - **Active Projects**: [To be learned] + - **Current Goals**: [To be learned] + - **Recent Activities**: [To be learned] + - **Pain Points**: [To be learned] + + ## Long-term Memory + - **Key Achievements**: [To be learned] + - **Important Relationships**: [To be learned] + - **Recurring Patterns**: [To be learned] + - **Preferences & Habits**: [To be learned] + + ## Session Notes + - **Today's Focus**: [To be learned] + - **Outstanding Questions**: [To be learned] + - **Action Items**: [To be learned] + - **Follow-ups Needed**: [To be learned] + `, + }, + }, +}) + +log.info('LanceDB Memory configured with comprehensive settings', { + storageName: LANCE_STORAGE_CONFIG.storageName, + dbUri: LANCE_STORAGE_CONFIG.dbUri, + memoryOptions: { + lastMessages: parseInt(process.env.LANCE_MEMORY_LAST_MESSAGES ?? '500'), + semanticRecall: { + topK: parseInt(process.env.LANCE_SEMANTIC_TOP_K ?? '5'), + }, + workingMemory: { + enabled: true, + version: 'vnext', + }, + }, +}) + +/** + * LanceDB-compatible filter format for vector queries + * Supports Sift query syntax for metadata filtering + * + * LanceDB-specific features: + * - Supports advanced filtering with nested conditions + * - Payload (metadata) fields must be explicitly indexed for filtering + * - Efficient handling of geo-spatial queries + * - Special handling for null and empty values + * - Vector-specific filtering capabilities + * - Datetime values must be in RFC 3339 format + */ +export interface LanceMetadataFilter { + [key: string]: + | string + | number + | boolean + | LanceMetadataFilter + | LanceMetadataFilter[] + | Array + | undefined +} + +/** + * Raw LanceDB filter format expected by the library + * Using Record for compatibility with @mastra/lance types + */ +export type LanceRawFilter = Record + +// Graph-based RAG tool using LanceDB +export const lanceGraphTool = createGraphRAGTool({ + id: 'lance-graph-rag', + description: + 'Graph-based retrieval augmented generation using LanceDB for advanced semantic search and context retrieval.', + // Supported vector store and index options + vectorStoreName: 'lanceStore', + indexName: LANCE_CONFIG.tableName, + model: LANCE_CONFIG.embeddingModel, + // Supported graph options for LanceDB + graphOptions: { + dimension: LANCE_CONFIG.embeddingDimension, + threshold: parseFloat(process.env.LANCE_GRAPH_THRESHOLD ?? '0.7'), + randomWalkSteps: parseInt( + process.env.LANCE_GRAPH_RANDOM_WALK_STEPS ?? '10' + ), + restartProb: parseFloat(process.env.LANCE_GRAPH_RESTART_PROB ?? '0.15'), + }, + includeSources: true, + // Filtering and ranking + enableFilter: true, +}) + +// LanceDB query tool for semantic search +export const lanceQueryTool = createVectorQueryTool({ + id: 'lance-vector-query', + description: + 'LanceDB similarity search for semantic content retrieval and question answering.', + // Supported vector store and index options + vectorStoreName: 'lanceStore', + indexName: LANCE_CONFIG.tableName, + model: LANCE_CONFIG.embeddingModel, + // Supported database configuration for LanceDB + databaseConfig: { + lanceDb: { + minScore: parseFloat(process.env.LANCE_MIN_SCORE ?? '0.7'), + // LanceDB specific parameters + maxResults: parseInt(process.env.LANCE_MAX_RESULTS ?? '100'), + }, + }, + includeVectors: true, + // Advanced filtering + enableFilter: true, + includeSources: true, +}) + +/** + * Transform LanceDB metadata filter to raw filter format + * Handles any necessary conversions between our interface and library expectations + */ +export function transformToLanceFilter( + filter: LanceMetadataFilter +): LanceRawFilter { + // For LanceDB, the filter format is already compatible + // This function can be expanded if any transformations are needed + return filter as LanceRawFilter +} + +/** + * Validate LanceDB metadata filter according to documented constraints + */ +export function validateLanceFilter( + filter: LanceMetadataFilter +): LanceMetadataFilter { + // Basic validation - LanceDB has specific requirements for indexed fields + if (filter === null || typeof filter !== 'object') { + throw new Error('Filter must be a valid object') + } + + return filter +} + +/** + * Generate embeddings for text content using LanceDB configuration + */ +export async function generateEmbeddings( + texts: string[], + options: { + model?: string + dimensions?: number + } = {} +): Promise { + try { + const model = options.model ?? LANCE_CONFIG.embeddingModel + const { embeddings } = await embedMany({ + values: texts, + model, + }) + + log.info('Embeddings generated successfully', { + textCount: texts.length, + embeddingDimension: embeddings[0]?.length, + model: options.model ?? 'gemini-embedding-001', + }) + + return embeddings + } catch (error) { + log.error('Failed to generate embeddings', { error: String(error) }) + throw error + } +} + +/** + * Format storage messages for LanceDB-specific requirements + */ +export function formatStorageMessages( + messages: Array<{ + id: string + content: string + role: string + createdAt?: Date + metadata?: Record + }> +): Array<{ + id: string + content: string + role: string + createdAt: Date + metadata: Record +}> { + return messages.map((message) => ({ + id: message.id, + content: message.content, + role: message.role, + createdAt: message.createdAt ?? new Date(), + metadata: message.metadata ?? {}, + })) +} + +/** + * Perform storage operations with LanceDB-specific error handling and tracing + */ +export async function performStorageOperation( + operation: () => Promise, + operationName: string, + metadata: Record = {} +): Promise { + const startTime = Date.now() + + try { + log.info( + `Starting LanceDB storage operation: ${operationName}`, + metadata + ) + + const result = await operation() + + const duration = Date.now() - startTime + log.info(`LanceDB storage operation completed: ${operationName}`, { + ...metadata, + duration, + success: true, + }) + + return result + } catch (error) { + const duration = Date.now() - startTime + log.error(`LanceDB storage operation failed: ${operationName}`, { + ...metadata, + duration, + error: String(error), + success: false, + }) + throw error + } +} + +// Export configuration for external use +export { LANCE_CONFIG, LANCE_STORAGE_CONFIG } diff --git a/src/mastra/config/pg-storage.ts b/src/mastra/config/pg-storage.ts index 54ff0eab..00b4b626 100644 --- a/src/mastra/config/pg-storage.ts +++ b/src/mastra/config/pg-storage.ts @@ -96,9 +96,9 @@ export const pgMemory = new Memory({ scope: 'resource', // 'resource' | 'thread' // HNSW index configuration to support high-dimensional embeddings (>2000 dimensions) indexConfig: { - type: 'hnsw', // flat index type (supports dimensions > 4000, unlike HNSW limit of 2000) + type: 'ivfflat', // flat index type (supports dimensions > 4000, unlike HNSW limit of 2000) metric: 'cosine', // Distance metric for normalized embeddings - hnsw: { m: parseFloat(process.env.PG_M ?? '32'), efConstruction: parseInt(process.env.PG_EF ?? '96') }, // hnsw configuration + ivf: { lists: 1080 }, // IVFFlat configuration }, threshold: 0.75, // Similarity threshold for semantic recall indexName: 'memory_messages_3072', // Index name for semantic recall diff --git a/src/mastra/config/vector/qdrant.ts b/src/mastra/config/qdrant.ts similarity index 99% rename from src/mastra/config/vector/qdrant.ts rename to src/mastra/config/qdrant.ts index c1a2e498..203d024d 100644 --- a/src/mastra/config/vector/qdrant.ts +++ b/src/mastra/config/qdrant.ts @@ -2,7 +2,7 @@ import { QdrantVector } from '@mastra/qdrant'; import { createVectorQueryTool, createGraphRAGTool } from '@mastra/rag'; import { google } from '@ai-sdk/google'; import { embedMany } from 'ai'; -import { log } from '../logger'; +import { log } from './logger'; /** * Qdrant-compatible filter format for vector queries diff --git a/src/mastra/config/upstash.ts b/src/mastra/config/upstash.ts index 8e66997c..27835c5e 100644 --- a/src/mastra/config/upstash.ts +++ b/src/mastra/config/upstash.ts @@ -144,7 +144,7 @@ export const upstashQueryTool = createVectorQueryTool({ // Supported database configuration for PgVector providerOptions: { google: { retrievalConfig: { dimensions: 1536 } }, - } satisfies GoogleGenerativeAIProviderOptions, + }, databaseConfig: { diff --git a/src/mastra/config/vector/lance.ts b/src/mastra/config/vector/lance.ts deleted file mode 100644 index 75d4a096..00000000 --- a/src/mastra/config/vector/lance.ts +++ /dev/null @@ -1,697 +0,0 @@ -import { LanceVectorStore, LanceStorage } from "@mastra/lance"; -import { createVectorQueryTool, createGraphRAGTool } from "@mastra/rag"; -import { google } from '@ai-sdk/google'; -import { embedMany } from 'ai'; -import { log } from '../logger'; -import { Memory } from '@mastra/memory'; -import { TokenLimiter } from '@mastra/memory/processors'; -import { maskStreamTags } from '@mastra/core'; - -/** - * LanceDB Vector configuration for the Governed RAG system - * Uses LanceDB for vector storage and similarity search - */ - -// Configuration constants -const LANCE_CONFIG = { - dbPath: process.env.LANCE_DB_PATH ?? '/tmp/lance_db', - tableName: process.env.LANCE_TABLE_NAME ?? 'governed_rag', - // Google Gemini gemini-embedding-001 supports flexible dimensions: 128-3072 - // Recommended: 768, 1536, 3072 - embeddingDimension: parseInt(process.env.LANCE_EMBEDDING_DIMENSION ?? "1536"), - embeddingModel: google.textEmbedding("gemini-embedding-001"), -} as const; - -/** - * Initialize LanceDB Vector store with proper configuration - */ -let lanceStore: LanceVectorStore; - -/** - * Initialize the LanceDB store - */ -export async function initializeLanceStore(): Promise { - lanceStore ??= await LanceVectorStore.create(LANCE_CONFIG.dbPath); -} - -/** - * LanceDB-compatible filter format for vector queries - * Supports Sift query syntax for metadata filtering - * - * LanceDB-specific features: - * - Supports advanced filtering with nested conditions - * - Payload (metadata) fields must be explicitly indexed for filtering - * - Efficient handling of geo-spatial queries - * - Special handling for null and empty values - * - Vector-specific filtering capabilities - * - Datetime values must be in RFC 3339 format - */ -export interface LanceMetadataFilter { - [key: string]: string | number | boolean | LanceMetadataFilter | LanceMetadataFilter[] | Array | undefined; -} - -/** - * Raw LanceDB filter format expected by the library - * Using Record for compatibility with @mastra/lance types - */ -export type LanceRawFilter = Record; - -/** - * Create and configure the vector index - */ -export async function initializeVectorIndex(): Promise { - try { - await initializeLanceStore(); - - await lanceStore.createIndex({ - tableName: LANCE_CONFIG.tableName, - indexName: LANCE_CONFIG.tableName, - dimension: LANCE_CONFIG.embeddingDimension, - metric: "cosine", // LanceDB supports cosine, euclidean, and dotproduct - }); - - log.info("Vector index created", { - tableName: LANCE_CONFIG.tableName, - dimension: LANCE_CONFIG.embeddingDimension - }); - } catch (error: unknown) { - // Index might already exist, which is fine - const errorObj = error as { message?: string }; - const isAlreadyExists = errorObj.message?.includes('already exists') ?? false; - if (isAlreadyExists) { - log.info("Vector index already exists", { tableName: LANCE_CONFIG.tableName }); - } else { - log.error("Failed to create vector index", { error: String(error) }); - throw error; - } - } -} - -/** - * Process document content and generate embeddings - * Simplified chunking for basic vector setup without AI extraction - */ -export async function processDocument( - content: string, - options: { - chunkSize?: number; - chunkOverlap?: number; - } = {} -): Promise<{ - chunks: Array<{ - text: string; - metadata?: Record; - }>; - embeddings: number[][]; -}> { - try { - // Simple text-based chunking without AI extraction - const chunkSize = options.chunkSize ?? 1000; - const chunkOverlap = options.chunkOverlap ?? 200; - - const chunks: Array<{ text: string; metadata?: Record }> = []; - - // Split content into overlapping chunks - for (let i = 0; i < content.length; i += chunkSize - chunkOverlap) { - const chunkText = content.slice(i, i + chunkSize); - if (chunkText.trim()) { - chunks.push({ - text: chunkText, - metadata: { - chunkIndex: chunks.length, - startPosition: i, - endPosition: i + chunkText.length, - totalLength: content.length, - }, - }); - } - } - - // Generate embeddings for all chunks - const { embeddings } = await embedMany({ - values: chunks.map((chunk) => chunk.text), - model: LANCE_CONFIG.embeddingModel, - }); - - log.info("Document processed successfully", { - chunksCount: chunks.length, - chunkSize, - chunkOverlap, - embeddingDimension: embeddings[0]?.length - }); - - return { chunks, embeddings }; - } catch (error) { - log.error("Failed to process document", { error: String(error) }); - throw error; - } -} - -/** - * Store document chunks and their embeddings - */ -export async function storeDocumentEmbeddings( - chunks: Array<{ - text: string; - metadata?: Record; - }>, - embeddings: number[][], - baseMetadata: Record = {} -): Promise { - try { - await initializeLanceStore(); - - // Prepare metadata for each chunk - const metadata = chunks.map((chunk, index) => ({ - ...baseMetadata, - text: chunk.text, - chunkIndex: index, - createdAt: new Date(), - ...chunk.metadata, - })); - - // Upsert vectors with metadata - const ids = await lanceStore.upsert({ - tableName: LANCE_CONFIG.tableName, - indexName: LANCE_CONFIG.tableName, - vectors: embeddings, - metadata, - }); - - log.info("Document embeddings stored", { - tableName: LANCE_CONFIG.tableName, - vectorsCount: embeddings.length - }); - - return ids; - } catch (error) { - log.error("Failed to store document embeddings", { error: String(error) }); - throw error; - } -} - -/** - * Query similar documents with metadata filtering - */ -export async function querySimilarDocuments( - queryText: string, - options: { - topK?: number; - filter?: LanceMetadataFilter; - includeVector?: boolean; - } = {} -): Promise; - vector?: number[]; -}>> { - try { - await initializeLanceStore(); - - // Generate embedding for the query - const { embeddings: [queryEmbedding] } = await embedMany({ - values: [queryText], - model: LANCE_CONFIG.embeddingModel, - }); - - // Query the vector store - const results = await lanceStore.query({ - tableName: LANCE_CONFIG.tableName, - indexName: LANCE_CONFIG.tableName, - queryVector: queryEmbedding, - topK: options.topK ?? 10, - filter: options.filter, - includeVector: options.includeVector ?? false, - }); - - log.info("Vector query completed", { - tableName: LANCE_CONFIG.tableName, - resultsCount: results.length, - topK: options.topK ?? 10 - }); - - // Ensure metadata is always defined - return results.map(result => ({ - ...result, - metadata: result.metadata ?? {} - })); - } catch (error) { - log.error("Failed to query similar documents", { error: String(error) }); - throw error; - } -} - -/** - * Clean up resources - */ -export async function disconnectVectorStore(): Promise { - try { - // LanceDB doesn't have a disconnect method, but we can clean up the reference - lanceStore = undefined!; - log.info("LanceDB Vector store disconnected"); - } catch (error) { - log.error("Failed to disconnect vector store", { error: String(error) }); - throw error; - } -} - -// Graph-based RAG tool using LanceDB -export const lanceGraphTool = createGraphRAGTool({ - id: 'lance-graph-rag', - description: - 'Graph-based retrieval augmented generation using LanceDB for advanced semantic search and context retrieval.', - // Supported vector store and index options - vectorStoreName: 'lanceStore', - indexName: LANCE_CONFIG.tableName, - model: LANCE_CONFIG.embeddingModel, - // Supported graph options for LanceDB - graphOptions: { - dimension: LANCE_CONFIG.embeddingDimension, - threshold: parseFloat(process.env.LANCE_GRAPH_THRESHOLD ?? '0.7'), - randomWalkSteps: parseInt(process.env.LANCE_GRAPH_RANDOM_WALK_STEPS ?? '10'), - restartProb: parseFloat(process.env.LANCE_GRAPH_RESTART_PROB ?? '0.15'), - }, - includeSources: true, - // Filtering and ranking - enableFilter: true, -}); - -// LanceDB query tool for semantic search -export const lanceQueryTool = createVectorQueryTool({ - id: 'lance-vector-query', - description: - 'LanceDB similarity search for semantic content retrieval and question answering.', - // Supported vector store and index options - vectorStoreName: 'lanceStore', - indexName: LANCE_CONFIG.tableName, - model: LANCE_CONFIG.embeddingModel, - // Supported database configuration for LanceDB - databaseConfig: { - lanceDb: { - minScore: parseFloat(process.env.LANCE_MIN_SCORE ?? '0.7'), - // LanceDB specific parameters - maxResults: parseInt(process.env.LANCE_MAX_RESULTS ?? '100'), - }, - }, - includeVectors: true, - // Advanced filtering - enableFilter: true, - includeSources: true, -}); - -/** - * Transform LanceDB metadata filter to raw filter format - * Handles any necessary conversions between our interface and library expectations - */ -export function transformToLanceFilter(filter: LanceMetadataFilter): LanceRawFilter { - // For LanceDB, the filter format is already compatible - // This function can be expanded if any transformations are needed - return filter as LanceRawFilter; -} - -/** - * Validate LanceDB metadata filter according to documented constraints - */ -export function validateLanceFilter(filter: LanceMetadataFilter): LanceMetadataFilter { - // Basic validation - LanceDB has specific requirements for indexed fields - if (filter === null || typeof filter !== 'object') { - throw new Error('Filter must be a valid object'); - } - - return filter; -} - -/** - * LanceDB Storage configuration for the Governed RAG system - * Uses LanceDB for both vector operations and persistent storage - */ - -// LanceDB Storage configuration constants -const LANCE_STORAGE_CONFIG = { - // Storage database configuration - storageName: process.env.LANCE_STORAGE_NAME ?? 'governed-rag-storage', - dbUri: process.env.LANCE_DB_URI ?? '/tmp/lance_storage_db', - // Storage options - storageOptions: { - timeout: process.env.LANCE_STORAGE_TIMEOUT ?? '60s', - }, - // Table prefix for environment isolation - tablePrefix: process.env.LANCE_STORAGE_TABLE_PREFIX ?? 'dev_', -} as const; - -/** - * Initialize LanceDB storage with proper configuration - */ -let lanceStorage: LanceStorage | null = null; - -export async function initializeLanceStorage(): Promise { - try { - // Initialize LanceDB storage - lanceStorage = await LanceStorage.create( - LANCE_STORAGE_CONFIG.storageName, - LANCE_STORAGE_CONFIG.dbUri, - { - storageOptions: LANCE_STORAGE_CONFIG.storageOptions, - } - ); - - log.info('LanceDB storage initialized successfully', { - storageName: LANCE_STORAGE_CONFIG.storageName, - dbUri: LANCE_STORAGE_CONFIG.dbUri, - tablePrefix: LANCE_STORAGE_CONFIG.tablePrefix - }); - } catch (error) { - log.error('Failed to initialize LanceDB storage', { error: String(error) }); - throw error; - } -} - -/** - * Get or initialize the LanceDB storage instance - */ -export async function getLanceStorage(): Promise { - if (lanceStorage === null) { - await initializeLanceStorage(); - } - return lanceStorage!; -} - -/** - * LanceDB Memory instance configured with comprehensive settings - * Note: Ensure initializeLanceStorage() and initializeLanceStore() are called before using - */ -export const lanceMemory = new Memory({ - storage: lanceStorage!, - vector: lanceStore!, - embedder: google.textEmbedding('gemini-embedding-001'), - options: { - // Message management - lastMessages: parseInt(process.env.LANCE_MEMORY_LAST_MESSAGES ?? '500'), - // Advanced semantic recall with LanceDB configuration - semanticRecall: { - topK: parseInt(process.env.LANCE_SEMANTIC_TOP_K ?? '5'), - messageRange: { - before: parseInt(process.env.LANCE_SEMANTIC_RANGE_BEFORE ?? '3'), - after: parseInt(process.env.LANCE_SEMANTIC_RANGE_AFTER ?? '2'), - }, - scope: 'resource', // 'resource' | 'thread' - // LanceDB-specific index configuration - indexConfig: {} - }, - // Enhanced working memory with supported template - workingMemory: { - enabled: true, - scope: 'resource', // 'resource' | 'thread' - version: 'vnext', // Enable the improved/experimental tool - template: `# User Profile & Context - ## Personal Information - - **Name**: [To be learned] - - **Role/Title**: [To be learned] - - **Organization**: [To be learned] - - **Location**: [To be learned] - - **Time Zone**: [To be learned] - - ## Communication Preferences - - **Preferred Communication Style**: [To be learned] - - **Response Length Preference**: [To be learned] - - **Technical Level**: [To be learned] - - ## Current Context - - **Active Projects**: [To be learned] - - **Current Goals**: [To be learned] - - **Recent Activities**: [To be learned] - - **Pain Points**: [To be learned] - - ## Long-term Memory - - **Key Achievements**: [To be learned] - - **Important Relationships**: [To be learned] - - **Recurring Patterns**: [To be learned] - - **Preferences & Habits**: [To be learned] - - ## Session Notes - - **Today's Focus**: [To be learned] - - **Outstanding Questions**: [To be learned] - - **Action Items**: [To be learned] - - **Follow-ups Needed**: [To be learned] - `, - }, - // Thread management with supported options - threads: { - generateTitle: process.env.LANCE_THREAD_GENERATE_TITLE !== 'false', - }, - }, - processors: [new TokenLimiter(1048576)], -}); - -log.info('LanceDB Memory configured with comprehensive settings', { - storageName: LANCE_STORAGE_CONFIG.storageName, - dbUri: LANCE_STORAGE_CONFIG.dbUri, - memoryOptions: { - lastMessages: parseInt(process.env.LANCE_MEMORY_LAST_MESSAGES ?? '500'), - semanticRecall: { - topK: parseInt(process.env.LANCE_SEMANTIC_TOP_K ?? '5'), - }, - workingMemory: { - enabled: true, - version: 'vnext', - }, - }, -}); - -/** - * Utility function to create a masked stream for sensitive data - * This properly uses maskStreamTags to mask content between XML tags in streams - */ -export function createMaskedStream( - inputStream: AsyncIterable, - sensitiveTags: string[] = ['password', 'secret', 'token', 'key'] -): AsyncIterable { - // Chain multiple maskStreamTags calls for different sensitive tags - let maskedStream = inputStream; - for (const tag of sensitiveTags) { - maskedStream = maskStreamTags(maskedStream, tag); - } - return maskedStream; -} - -/** - * Utility function to mask sensitive data in message content for logging - */ -export function maskSensitiveMessageData( - content: string, - sensitiveFields: string[] = ['password', 'secret', 'token', 'key', 'apiKey'] -): string { - let maskedContent = content; - - // Mask sensitive fields in JSON-like structures - for (const field of sensitiveFields) { - // Match field:"value" or field: "value" or "field": "value" patterns - const regex = new RegExp(`("${field}"\\s*:\\s*)"[^"]*"`, 'gi'); - maskedContent = maskedContent.replace(regex, `$1"[MASKED]"`); - } - - return maskedContent; -} - -/** - * Generate embeddings for text content using LanceDB configuration - */ -export async function generateEmbeddings( - texts: string[], - options: { - model?: string; - dimensions?: number; - } = {} -): Promise { - try { - const model = options.model ?? LANCE_CONFIG.embeddingModel; - const { embeddings } = await embedMany({ - values: texts, - model, - }); - - log.info('Embeddings generated successfully', { - textCount: texts.length, - embeddingDimension: embeddings[0]?.length, - model: options.model ?? 'gemini-embedding-001' - }); - - return embeddings; - } catch (error) { - log.error('Failed to generate embeddings', { error: String(error) }); - throw error; - } -} - -/** - * Check LanceDB database health and connectivity - */ -export async function checkDatabaseHealth(): Promise<{ - status: 'healthy' | 'unhealthy'; - details: Record; - timestamp: string; -}> { - try { - await initializeLanceStore(); - - // Basic connectivity check - try to access the store - const isConnected = lanceStore !== null; - - // Check if we can perform basic operations - let canQuery = false; - try { - // Try a simple query to test functionality - await lanceStore.query({ - tableName: LANCE_CONFIG.tableName, - indexName: LANCE_CONFIG.tableName, - queryVector: new Array(LANCE_CONFIG.embeddingDimension).fill(0), - topK: 1, - }); - canQuery = true; - } catch (queryError) { - // Table might not exist yet, which is fine - if (String(queryError).includes('not found') || String(queryError).includes('does not exist')) { - canQuery = true; // Expected for new databases - } - } - - const healthStatus = isConnected && canQuery ? 'healthy' : 'unhealthy'; - - const details = { - dbPath: LANCE_CONFIG.dbPath, - tableName: LANCE_CONFIG.tableName, - embeddingDimension: LANCE_CONFIG.embeddingDimension, - isConnected, - canQuery, - storageInitialized: lanceStorage !== null, - }; - - log.info('LanceDB health check completed', { - status: healthStatus, - ...details - }); - - return { - status: healthStatus, - details, - timestamp: new Date().toISOString(), - }; - } catch (error) { - log.error('LanceDB health check failed', { error: String(error) }); - return { - status: 'unhealthy', - details: { error: String(error) }, - timestamp: new Date().toISOString(), - }; - } -} - -/** - * Initialize LanceDB database schema and tables - */ -export async function initializeDatabase(): Promise { - try { - log.info('Initializing LanceDB database', { - dbPath: LANCE_CONFIG.dbPath, - tableName: LANCE_CONFIG.tableName - }); - - // Initialize both vector and storage - await initializeLanceStore(); - await initializeLanceStorage(); - - // Create vector index if it doesn't exist - await initializeVectorIndex(); - - log.info('LanceDB database initialized successfully'); - } catch (error) { - log.error('Failed to initialize LanceDB database', { error: String(error) }); - throw error; - } -} - -/** - * Shutdown LanceDB database connections - */ -export async function shutdownDatabase(): Promise { - try { - log.info('Shutting down LanceDB database connections'); - - // Disconnect vector store - await disconnectVectorStore(); - - // Clear storage reference - lanceStorage = null; - - log.info('LanceDB database connections shut down successfully'); - } catch (error) { - log.error('Failed to shutdown LanceDB database', { error: String(error) }); - throw error; - } -} - -/** - * Format storage messages for LanceDB-specific requirements - */ -export function formatStorageMessages( - messages: Array<{ - id: string; - content: string; - role: string; - createdAt?: Date; - metadata?: Record; - }> -): Array<{ - id: string; - content: string; - role: string; - createdAt: Date; - metadata: Record; -}> { - return messages.map(message => ({ - id: message.id, - content: maskSensitiveMessageData(message.content), - role: message.role, - createdAt: message.createdAt ?? new Date(), - metadata: message.metadata ?? {}, - })); -} - -/** - * Perform storage operations with LanceDB-specific error handling and tracing - */ -export async function performStorageOperation( - operation: () => Promise, - operationName: string, - metadata: Record = {} -): Promise { - const startTime = Date.now(); - - try { - log.info(`Starting LanceDB storage operation: ${operationName}`, metadata); - - const result = await operation(); - - const duration = Date.now() - startTime; - log.info(`LanceDB storage operation completed: ${operationName}`, { - ...metadata, - duration, - success: true - }); - - return result; - } catch (error) { - const duration = Date.now() - startTime; - log.error(`LanceDB storage operation failed: ${operationName}`, { - ...metadata, - duration, - error: String(error), - success: false - }); - throw error; - } -} - -// Export configuration for external use -export { LANCE_CONFIG, LANCE_STORAGE_CONFIG }; diff --git a/src/mastra/public/storage/key_value_stores/default/SDK_CRAWLER_STATISTICS_0.json b/src/mastra/public/storage/key_value_stores/default/SDK_CRAWLER_STATISTICS_0.json new file mode 100644 index 00000000..6dc63e2e --- /dev/null +++ b/src/mastra/public/storage/key_value_stores/default/SDK_CRAWLER_STATISTICS_0.json @@ -0,0 +1,27 @@ +{ + "requestsFinished": 1, + "requestsFailed": 0, + "requestsRetries": 0, + "requestsFailedPerMinute": 0, + "requestsFinishedPerMinute": 16, + "requestMinDurationMillis": 2907, + "requestMaxDurationMillis": 2907, + "requestTotalFailedDurationMillis": 0, + "requestTotalFinishedDurationMillis": 2907, + "crawlerStartedAt": "2026-01-08T14:09:03.077Z", + "crawlerFinishedAt": "2026-01-08T14:09:06.880Z", + "statsPersistedAt": "2026-01-08T14:09:06.880Z", + "crawlerRuntimeMillis": 3850, + "crawlerLastStartTimestamp": 1767881343030, + "requestRetryHistogram": [ + 1 + ], + "statsId": 0, + "requestAvgFailedDurationMillis": null, + "requestAvgFinishedDurationMillis": 2907, + "requestTotalDurationMillis": 2907, + "requestsTotal": 1, + "requestsWithStatusCode": {}, + "errors": {}, + "retryErrors": {} +} \ No newline at end of file diff --git a/src/mastra/public/storage/key_value_stores/default/SDK_SESSION_POOL_STATE.json b/src/mastra/public/storage/key_value_stores/default/SDK_SESSION_POOL_STATE.json new file mode 100644 index 00000000..82a7e838 --- /dev/null +++ b/src/mastra/public/storage/key_value_stores/default/SDK_SESSION_POOL_STATE.json @@ -0,0 +1,26 @@ +{ + "usableSessionsCount": 1, + "retiredSessionsCount": 0, + "sessions": [ + { + "id": "session_TgweIdckIO", + "cookieJar": { + "version": "tough-cookie@6.0.0", + "storeType": "MemoryCookieStore", + "rejectPublicSuffixes": true, + "enableLooseMode": false, + "allowSpecialUseDomain": true, + "prefixSecurity": "silent", + "cookies": [] + }, + "userData": {}, + "maxErrorScore": 3, + "errorScoreDecrement": 0.5, + "expiresAt": "2026-01-08T14:59:03.949Z", + "createdAt": "2026-01-08T14:09:03.949Z", + "usageCount": 1, + "maxUsageCount": 50, + "errorScore": 0 + } + ] +} \ No newline at end of file diff --git a/src/mastra/public/storage/request_queues/default/jBMmaaywUPlwSK3.json b/src/mastra/public/storage/request_queues/default/jBMmaaywUPlwSK3.json new file mode 100644 index 00000000..48c55edd --- /dev/null +++ b/src/mastra/public/storage/request_queues/default/jBMmaaywUPlwSK3.json @@ -0,0 +1,9 @@ +{ + "id": "jBMmaaywUPlwSK3", + "json": "{\"id\":\"jBMmaaywUPlwSK3\",\"url\":\"https://news.ycombinator.com/\",\"loadedUrl\":\"https://news.ycombinator.com/\",\"uniqueKey\":\"https://news.ycombinator.com\",\"method\":\"GET\",\"noRetry\":false,\"retryCount\":0,\"errorMessages\":[],\"headers\":{},\"userData\":{\"depth\":0,\"__crawlee\":{\"state\":4}},\"handledAt\":\"2026-01-08T14:09:06.855Z\"}", + "method": "GET", + "orderNo": null, + "retryCount": 0, + "uniqueKey": "https://news.ycombinator.com", + "url": "https://news.ycombinator.com/" +} \ No newline at end of file