From 266171b4a8d5f701bf636aae41cf1aec3f4a1991 Mon Sep 17 00:00:00 2001 From: toyes3429 Date: Wed, 21 Oct 2026 19:57:11 +0000 Subject: [PATCH] feat: add inverted index search --- src/lib/search/inverted-index.ts | 253 +++++++++++++++++++++++ tests/inverted-index.test.ts | 338 +++++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+) create mode 100644 src/lib/search/inverted-index.ts create mode 100644 tests/inverted-index.test.ts diff --git a/src/lib/search/inverted-index.ts b/src/lib/search/inverted-index.ts new file mode 100644 index 0000000..76a4a56 --- /dev/null +++ b/src/lib/search/inverted-index.ts @@ -0,0 +1,253 @@ + +/** + * Lightweight in-memory inverted index for offline text search. + * + * Maps normalized terms to documents and their normalized term frequencies. + * Supports multiple text fields, AND/OR queries, stop words, and ranked + * search without external dependencies. + */ + +export interface IndexOptions { + /** Words excluded from indexing and searching. */ + stopWords?: Set; + /** Preserve case when tokenizing. Defaults to false. */ + caseSensitive?: boolean; +} + +export interface SearchOptions { + /** AND requires every query term; OR requires at least one. Defaults to AND. */ + operator?: 'AND' | 'OR'; + /** Minimum relevance score required for a result. */ + minScore?: number; + /** Maximum number of results to return. */ + limit?: number; +} + +export interface SearchResult { + document: T; + score: number; + matchedTokens: string[]; +} + +export class InvertedIndex { + private readonly documents = new Map< + string, + { document: TDocument; tokenCount: number } + >(); + + private readonly postings = new Map>(); + private readonly stopWords: Set; + private readonly caseSensitive: boolean; + + constructor(options: IndexOptions = {}) { + this.caseSensitive = options.caseSensitive ?? false; + this.stopWords = new Set( + [...(options.stopWords ?? [])].map((word) => this.normalize(word)), + ); + } + + private normalize(value: string): string { + return this.caseSensitive ? value : value.toLowerCase(); + } + + private tokenize(text: string): string[] { + return (text.match(/[\p{L}\p{N}]+/gu) ?? []) + .map((token) => this.normalize(token)) + .filter((token) => !this.stopWords.has(token)); + } + + /** + * Adds or replaces a document, indexing only the specified string fields. + */ + addDocument(doc: TDocument, fields: (keyof TDocument)[]): void { + if (typeof doc.id !== 'string' || doc.id.length === 0) { + throw new TypeError('Document id must be a non-empty string.'); + } + + // Replacing an ID must not leave stale terms in the index. + this.removeDocument(doc.id); + + const frequencies = new Map(); + let tokenCount = 0; + + for (const field of fields) { + const value = doc[field]; + if (typeof value !== 'string') continue; + + for (const token of this.tokenize(value)) { + frequencies.set(token, (frequencies.get(token) ?? 0) + 1); + tokenCount++; + } + } + + this.documents.set(doc.id, { document: doc, tokenCount }); + + if (tokenCount === 0) return; + + for (const [term, frequency] of frequencies) { + let termPostings = this.postings.get(term); + + if (!termPostings) { + termPostings = new Map(); + this.postings.set(term, termPostings); + } + + // Normalized TF: occurrences of the term / indexed document tokens. + termPostings.set(doc.id, frequency / tokenCount); + } + } + + /** Removes a document and all of its term postings. */ + removeDocument(id: string): void { + if (!this.documents.has(id)) return; + + this.documents.delete(id); + + for (const [term, termPostings] of this.postings) { + termPostings.delete(id); + + if (termPostings.size === 0) { + this.postings.delete(term); + } + } + } + + /** + * Searches exact terms and term prefixes, ranking matches by cumulative TF. + * Prefix matching allows a query such as "pro" to match "project". + */ + search( + query: string, + options: SearchOptions = {}, + ): SearchResult[] { + const queryTerms = [...new Set(this.tokenize(query))]; + + if (queryTerms.length === 0) return []; + + const operator = options.operator ?? 'AND'; + const minScore = options.minScore ?? 0; + const limit = options.limit ?? Infinity; + + if ( + !['AND', 'OR'].includes(operator) || + !Number.isFinite(minScore) || + minScore < 0 || + !(limit === Infinity || (Number.isInteger(limit) && limit >= 0)) + ) { + throw new RangeError('Invalid search options.'); + } + + if (limit === 0) return []; + + + + const vocabulary = [...this.postings.keys()]; + const scores = new Map(); + const matchedByDocument = new Map>(); + + // Track whether this is the first term in an AND query. + let firstTerm = true; + + for (const queryTerm of queryTerms) { + // Aggregate scores across exact and prefix matches for this query term. + const termScores = new Map(); + + const matchingTerms = vocabulary.filter( + (term) => term === queryTerm || term.startsWith(queryTerm), + ); + + for (const term of matchingTerms) { + const termPostings = this.postings.get(term); + if (!termPostings) continue; + + for (const [documentId, frequency] of termPostings) { + termScores.set( + documentId, + (termScores.get(documentId) ?? 0) + frequency, + ); + } + } + + if (operator === 'AND') { + if (firstTerm) { + // Initialize candidates with documents matching the first query term. + for (const [documentId, score] of termScores) { + scores.set(documentId, score); + matchedByDocument.set(documentId, new Set([queryTerm])); + } + } else { + // Intersect candidates with each subsequent term's matching documents. + for (const documentId of scores.keys()) { + const termScore = termScores.get(documentId); + + if (termScore === undefined) { + // A document missing any required term cannot satisfy AND. + scores.delete(documentId); + matchedByDocument.delete(documentId); + } else { + // Accumulate relevance scores for documents matching every term. + scores.set(documentId, scores.get(documentId)! + termScore); + matchedByDocument.get(documentId)!.add(queryTerm); + } + } + } + } else { + // OR keeps documents matching any query term and accumulates their scores. + for (const [documentId, score] of termScores) { + scores.set(documentId, (scores.get(documentId) ?? 0) + score); + + let matched = matchedByDocument.get(documentId); + if (!matched) { + matched = new Set(); + matchedByDocument.set(documentId, matched); + } + + matched.add(queryTerm); + } + } + + firstTerm = false; + } + + + return [...scores.entries()] + .filter(([id, score]) => { + const matched = matchedByDocument.get(id); + const matchesQuery = + operator === 'OR' || matched?.size === queryTerms.length; + + return matchesQuery && score >= minScore; + }) + .sort(([idA, scoreA], [idB, scoreB]) => { + return scoreB - scoreA || idA.localeCompare(idB); + }) + .slice(0, limit) + .map(([id, score]) => ({ + document: this.documents.get(id)!.document, + score, + matchedTokens: [...(matchedByDocument.get(id) ?? [])].sort(), + })); + } + + /** Returns whether a document ID is indexed. */ + hasDocument(id: string): boolean { + return this.documents.has(id); + } + + /** Removes all documents and postings. */ + clear(): void { + this.documents.clear(); + this.postings.clear(); + } + + /** Number of indexed documents, including documents with no searchable terms. */ + get documentCount(): number { + return this.documents.size; + } + + /** Number of unique indexed terms. */ + get termCount(): number { + return this.postings.size; + } +} + diff --git a/tests/inverted-index.test.ts b/tests/inverted-index.test.ts new file mode 100644 index 0000000..6454c7a --- /dev/null +++ b/tests/inverted-index.test.ts @@ -0,0 +1,338 @@ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { InvertedIndex } from '../src/lib/search/inverted-index'; + +interface TestDocument { + id: string; + title: string; + content: string; + category?: string; +} + +function createIndex( + options: ConstructorParameters>[0] = {}, +) { + return new InvertedIndex(options); +} + +describe('InvertedIndex', () => { + it('indexes selected string fields and reports document and term counts', () => { + const index = createIndex(); + + index.addDocument( + { + id: '1', + title: 'Project Alpha', + content: 'Build software', + category: 'work', + }, + ['title', 'content'], + ); + + assert.equal(index.documentCount, 1); + assert.equal(index.termCount, 4); + assert.equal(index.hasDocument('1'), true); + assert.equal(index.hasDocument('missing'), false); + }); + + it('matches terms case-insensitively and ignores punctuation', () => { + const index = createIndex(); + + index.addDocument( + { + id: '1', + title: 'Project Alpha!', + content: 'Build software.', + }, + ['title', 'content'], + ); + + assert.equal(index.search('PROJECT').length, 1); + assert.equal(index.search('alpha!!!').length, 1); + assert.deepEqual(index.search('!!!'), []); + }); + + it('normalizes term frequency and ranks more relevant documents first', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha', content: 'alpha alpha beta' }, + ['title', 'content'], + ); + index.addDocument( + { id: '2', title: 'alpha alpha', content: 'alpha alpha alpha' }, + ['title', 'content'], + ); + + const results = index.search('alpha'); + + assert.equal(results.length, 2); + assert.equal(results[0].document.id, '2'); + assert.ok(results[0].score > results[1].score); + }); + + it('uses AND to require every query term', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha beta', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '2', title: 'alpha', content: '' }, + ['title', 'content'], + ); + + const results = index.search('alpha beta', { operator: 'AND' }); + + assert.deepEqual( + results.map((result) => result.document.id), + ['1'], + ); + }); + + it('uses OR to match any query term', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '2', title: 'beta', content: '' }, + ['title', 'content'], + ); + + const results = index.search('alpha beta', { operator: 'OR' }); + + assert.equal(results.length, 2); + assert.deepEqual( + new Set(results.map((result) => result.document.id)), + new Set(['1', '2']), + ); + }); + + it('accumulates OR scores across query terms and initializes missing scores', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha beta', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '2', title: 'alpha gamma delta epsilon', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '3', title: 'beta', content: '' }, + ['title', 'content'], + ); + + const results = index.search('alpha beta', { operator: 'OR' }); + const resultById = new Map( + results.map((result) => [result.document.id, result]), + ); + + assert.equal(results.length, 3); + assert.deepEqual(resultById.get('1')?.matchedTokens, ['alpha', 'beta']); + assert.deepEqual(resultById.get('2')?.matchedTokens, ['alpha']); + assert.deepEqual(resultById.get('3')?.matchedTokens, ['beta']); + + assert.equal(resultById.get('1')?.score, 1); + assert.equal(resultById.get('2')?.score, 0.25); + assert.equal(resultById.get('3')?.score, 1); + }); + + it('supports prefix matching', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'application apply', content: 'banana' }, + ['title', 'content'], + ); + + assert.equal(index.search('app').length, 1); + assert.deepEqual(index.search('bananx'), []); + }); + + it('ignores configured stop words', () => { + const index = createIndex({ stopWords: new Set(['the', 'and']) }); + + index.addDocument( + { id: '1', title: 'the project', content: 'and software' }, + ['title', 'content'], + ); + + assert.equal(index.search('the').length, 0); + assert.equal(index.search('project').length, 1); + }); + + it('replaces documents without retaining stale terms', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '1', title: 'beta', content: '' }, + ['title', 'content'], + ); + + assert.deepEqual(index.search('alpha'), []); + assert.equal(index.search('beta').length, 1); + assert.equal(index.documentCount, 1); + }); + + it('removes documents and their unused postings', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'unique alpha', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '2', title: 'beta alpha', content: '' }, + ['title', 'content'], + ); + + index.removeDocument('1'); + + assert.equal(index.hasDocument('1'), false); + assert.equal(index.documentCount, 1); + assert.deepEqual(index.search('unique'), []); + assert.equal(index.termCount, 2); + + index.removeDocument('2'); + + assert.equal(index.documentCount, 0); + assert.equal(index.termCount, 0); + assert.deepEqual(index.search('alpha'), []); + }); + + it('clears all documents and terms', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha beta', content: '' }, + ['title', 'content'], + ); + index.clear(); + + assert.equal(index.documentCount, 0); + assert.equal(index.termCount, 0); + assert.deepEqual(index.search('alpha'), []); + }); + + it('returns no results for empty or punctuation-only queries', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha', content: '' }, + ['title', 'content'], + ); + + assert.deepEqual(index.search(''), []); + assert.deepEqual(index.search(' '), []); + assert.deepEqual(index.search('!!!'), []); + }); + + it('handles documents containing no tokenizable characters', () => { + const index = createIndex(); + + index.addDocument( + { id: 'empty-text', title: '!!! ...', content: '---' }, + ['title', 'content'], + ); + + assert.equal(index.documentCount, 1); + assert.equal(index.termCount, 0); + assert.deepEqual(index.search('!!!'), []); + }); + + it('ignores selected fields that are not strings', () => { + const index = new InvertedIndex<{ + id: string; + title: string; + content: string; + count: number; + }>(); + + index.addDocument( + { id: '1', title: 'alpha', content: '', count: 123 }, + ['title', 'content', 'count'], + ); + + assert.deepEqual(index.search('123'), []); + assert.equal(index.search('alpha').length, 1); + }); + + it('supports case-sensitive indexing', () => { + const index = createIndex({ caseSensitive: true }); + + index.addDocument( + { id: '1', title: 'Alpha alpha', content: '' }, + ['title', 'content'], + ); + + assert.equal(index.search('Alpha').length, 1); + assert.equal(index.search('ALPHA').length, 0); + }); + + it('applies result limits and minimum scores', () => { + const index = createIndex(); + + index.addDocument( + { id: '1', title: 'alpha', content: '' }, + ['title', 'content'], + ); + index.addDocument( + { id: '2', title: 'alpha beta', content: '' }, + ['title', 'content'], + ); + + assert.equal(index.search('alpha', { limit: 1 }).length, 1); + assert.equal(index.search('alpha', { limit: 0 }).length, 0); + assert.equal(index.search('alpha', { minScore: 0 }).length, 2); + assert.deepEqual(index.search('alpha', { minScore: 100 }), []); + }); + + it('rejects invalid search options', () => { + const index = createIndex(); + + assert.throws( + () => index.search('alpha', { operator: 'XOR' as 'AND' }), + RangeError, + ); + assert.throws( + () => index.search('alpha', { minScore: -1 }), + RangeError, + ); + assert.throws( + () => index.search('alpha', { minScore: Infinity }), + RangeError, + ); + assert.throws( + () => index.search('alpha', { limit: -1 }), + RangeError, + ); + assert.throws( + () => index.search('alpha', { limit: 1.5 }), + RangeError, + ); + }); + + it('rejects empty document IDs', () => { + const index = createIndex(); + + assert.throws( + () => + index.addDocument( + { id: '', title: 'alpha', content: '' }, + ['title', 'content'], + ), + TypeError, + ); + }); +});