diff --git a/docs/api-overview.md b/docs/api-overview.md index e5bffd1ac..287ce8d34 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -177,6 +177,34 @@ Errors extending `PropagatingToolError` (for example the delegation depth guard) are the exception: they are rethrown and abort the run instead of being shown to the model. +## Models, tokens and cost + +Dependency-free helpers for budgeting and context decisions. + +```ts +import { estimateTokens, estimateCost, getModelInfo, registerModel } from '@loushy/build-ai-agent'; + +// A custom or self-hosted model: add it (or override a built-in) before use. +registerModel({ + id: 'my-llama', + provider: 'ollama', + contextWindow: 32768, + inputCostPerMTok: 0.2, // optional; omit for unknown or free + outputCostPerMTok: 0.6, +}); + +const used = estimateTokens([{ role: 'user', content: 'Summarise this report' }], { model: 'my-llama' }); +const info = getModelInfo('openai/gpt-4o-mini'); // exact id, `provider/id`, or a dated snapshot +const usd = estimateCost({ inputTokens: used, outputTokens: 500 }, 'my-llama'); // undefined if unpriced +``` + +- `estimateTokens(input, { model?, estimator? })` is a heuristic (about 4 characters per token for English, more for CJK and other scripts, plus per-message overhead and tool-call JSON). Expect roughly 15-20% error on English: fine for compaction and budgets, not for billing. Plug in a real tokenizer with `setTokenEstimator(fn)` or `options.estimator`. +- `getModelInfo(id)` matches the exact id, then `provider/id`, then dated snapshots (`gpt-4o-mini-2024-07-18` resolves to `gpt-4o-mini`). Unknown models return `undefined`. +- `registerModel(info)` adds or overrides an entry; the latest registration wins. +- `estimateCost(usage, model)` returns USD, or `undefined` (not `0`) when the model or its prices are unknown. + +The built-in context windows and prices are a dated snapshot (see the retrieval date and sources at the top of `src/models/modelData.ts`). Providers change prices and models, so override entries with `registerModel` when you need billing-grade numbers. + ## Flows, evals, observability and security - `FlowBuilder` / `FlowExecutor` - multi-step workflow graphs. diff --git a/src/index.ts b/src/index.ts index 18e5470a1..927a82d95 100644 --- a/src/index.ts +++ b/src/index.ts @@ -44,6 +44,9 @@ export * from './storage'; // Templates export * from './templates'; +// Token estimation and model registry (LOU-W1) +export * from './models'; + // Utils export * from './utils'; diff --git a/src/models/estimateTokens.ts b/src/models/estimateTokens.ts new file mode 100644 index 000000000..f59b25372 --- /dev/null +++ b/src/models/estimateTokens.ts @@ -0,0 +1,145 @@ +/** + * Dependency-free token estimation (LOU-W1). + * + * A heuristic, not a tokenizer: it exists so budgeting and context-compaction + * code can answer "roughly how big is this prompt?" without shipping a + * tokenizer for every provider. + */ + +import type { Message } from '../providers/llm'; + +/** Anything `estimateTokens` accepts. */ +export type TokenEstimateInput = string | Message | Message[]; + +/** A replacement estimator, e.g. one backed by `tiktoken` or a provider's count endpoint. */ +export type TokenEstimator = (input: TokenEstimateInput, options: { model?: string }) => number; + +/** Options for {@link estimateTokens}. */ +export interface EstimateTokensOptions { + /** + * Model id the estimate is for. The built-in heuristic ignores it; it is + * forwarded to custom estimators so they can pick a model-specific tokenizer. + */ + model?: string; + /** Use this estimator for this call instead of the built-in heuristic (or the one set with `setTokenEstimator`). */ + estimator?: TokenEstimator; +} + +/** Tokens added per message for role markers and framing. */ +const MESSAGE_OVERHEAD_TOKENS = 4; + +/** Tokens per code point for Latin/ASCII text (~4 chars per token). */ +const LATIN_WEIGHT = 0.25; +/** CJK ideographs, kana and Hangul: about one token per character. */ +const CJK_WEIGHT = 1; +/** Other non-Latin scripts (Cyrillic, Arabic, Hebrew, Thai, Indic, ...). */ +const OTHER_SCRIPT_WEIGHT = 0.5; +/** Emoji and pictographs usually cost several byte-level tokens. */ +const EMOJI_WEIGHT = 2; + +type Range = readonly [number, number]; + +/** Code point ranges for CJK ideographs, kana, Hangul and CJK punctuation. */ +const CJK_RANGES: readonly Range[] = [ + [0x2e80, 0x2fdf], + [0x3000, 0x303f], + [0x3040, 0x30ff], + [0x3100, 0x312f], + [0x3400, 0x4dbf], + [0x4e00, 0x9fff], + [0xac00, 0xd7af], + [0xf900, 0xfaff], + [0xff00, 0xffef], + [0x20000, 0x2fa1f], +]; + +/** Code point ranges for emoji and pictographs. */ +const EMOJI_RANGES: readonly Range[] = [ + [0x2600, 0x27bf], + [0x1f000, 0x1faff], +]; + +/** Zero-width joiner, variation selectors and combining marks glue characters together and cost nothing extra. */ +const ZERO_WEIGHT_RANGES: readonly Range[] = [ + [0x0300, 0x036f], + [0x200d, 0x200d], + [0xfe00, 0xfe0f], +]; + +function inRanges(cp: number, ranges: readonly Range[]): boolean { + return ranges.some(([lo, hi]) => cp >= lo && cp <= hi); +} + +function codePointWeight(cp: number): number { + if (cp < 0x0300) return LATIN_WEIGHT; + if (inRanges(cp, ZERO_WEIGHT_RANGES)) return 0; + if (inRanges(cp, CJK_RANGES)) return CJK_WEIGHT; + if (inRanges(cp, EMOJI_RANGES)) return EMOJI_WEIGHT; + return OTHER_SCRIPT_WEIGHT; +} + +function estimateText(text: string): number { + let tokens = 0; + for (const ch of text) { + tokens += codePointWeight(ch.codePointAt(0) ?? 0); + } + return Math.ceil(tokens); +} + +function toText(value: unknown): string { + if (typeof value === 'string') return value; + if (value === undefined || value === null) return ''; + return JSON.stringify(value) ?? ''; +} + +function estimateMessage(message: Message): number { + let tokens = MESSAGE_OVERHEAD_TOKENS + estimateText(toText(message.content)); + if (message.name) tokens += estimateText(message.name); + if (message.toolName) tokens += estimateText(message.toolName); + if (message.toolCalls?.length) tokens += estimateText(JSON.stringify(message.toolCalls)); + return tokens; +} + +function defaultEstimator(input: TokenEstimateInput): number { + if (typeof input === 'string') return estimateText(input); + if (Array.isArray(input)) return input.reduce((sum, m) => sum + estimateMessage(m), 0); + return estimateMessage(input); +} + +let globalEstimator: TokenEstimator = defaultEstimator; + +/** + * Replace the estimator used by every `estimateTokens` call, e.g. with a real + * tokenizer. Call with no argument to restore the built-in heuristic. + * + * @example + * setTokenEstimator((input) => countWithMyTokenizer(input)); + */ +export function setTokenEstimator(estimator?: TokenEstimator): void { + globalEstimator = estimator ?? defaultEstimator; +} + +/** + * Estimate how many tokens a string, message or conversation will use. + * + * Heuristic, no tokenizer: about 4 characters per token for Latin text, + * about 1 token per CJK character, about 0.5 per character for other + * non-Latin scripts and 2 per emoji (code points are counted, not UTF-16 + * units), plus 4 tokens of overhead per message and the JSON length of tool + * calls. Tool results are the message `content` (typically JSON) and are + * counted as text. + * + * Accuracy: an estimate for budgeting and compaction decisions. It is + * typically within ~15-20% for English prose and can be further off for + * code, minified JSON or rare scripts. It is NOT suitable for billing; use + * the usage a provider reports for that. + * + * @example + * estimateTokens('Hello, world!'); // 4 + * estimateTokens([{ role: 'user', content: 'Hi there' }]); // 6 + * estimateTokens(messages, { estimator: myTokenizer }); + */ +export function estimateTokens(input: TokenEstimateInput, options: EstimateTokensOptions = {}): number { + const estimator = options.estimator ?? globalEstimator; + return estimator(input, { model: options.model }); +} diff --git a/src/models/index.ts b/src/models/index.ts new file mode 100644 index 000000000..d58a59a91 --- /dev/null +++ b/src/models/index.ts @@ -0,0 +1,7 @@ +/** + * Models, tokens and cost (LOU-W1): token estimation and a model registry + * with context windows and prices. + */ + +export * from './estimateTokens'; +export * from './registry'; diff --git a/src/models/modelData.ts b/src/models/modelData.ts new file mode 100644 index 000000000..283d49fd1 --- /dev/null +++ b/src/models/modelData.ts @@ -0,0 +1,35 @@ +import type { ModelInfo } from './registry'; + +/** + * Built-in model table. A dated snapshot, not a live price list. + * + * Retrieved 2026-10-01. Sources (provider-published): + * - OpenAI: https://developers.openai.com/api/docs/pricing and + * https://developers.openai.com/api/docs/models/ (standard tier, + * short-context prices; context window and max output from each model page) + * - Anthropic: https://platform.claude.com/docs/en/docs/about-claude/models/overview + * ("128K" max output read as 128000, "1M" context as 1000000) + * - Ollama: https://ollama.com/library/ (context window only; local + * models have no per-token price, so prices are left undefined) + * + * Only models whose numbers were read from those pages are listed; anything + * unverified is left undefined. Override with `registerModel` for + * billing-grade numbers. + */ +export const BUILT_IN_MODELS: readonly ModelInfo[] = [ + { id: 'gpt-4o', provider: 'openai', contextWindow: 128000, maxOutputTokens: 16384, inputCostPerMTok: 2.5, outputCostPerMTok: 10 }, + { id: 'gpt-4o-mini', provider: 'openai', contextWindow: 128000, maxOutputTokens: 16384, inputCostPerMTok: 0.15, outputCostPerMTok: 0.6 }, + { id: 'gpt-4.1', provider: 'openai', contextWindow: 1047576, maxOutputTokens: 32768, inputCostPerMTok: 2, outputCostPerMTok: 8 }, + { id: 'gpt-4.1-mini', provider: 'openai', contextWindow: 1000000, maxOutputTokens: 32768, inputCostPerMTok: 0.4, outputCostPerMTok: 1.6 }, + { id: 'gpt-4.1-nano', provider: 'openai', contextWindow: 1000000, maxOutputTokens: 32768, inputCostPerMTok: 0.1, outputCostPerMTok: 0.4 }, + { id: 'gpt-5', provider: 'openai', contextWindow: 400000, maxOutputTokens: 128000, inputCostPerMTok: 1.25, outputCostPerMTok: 10 }, + { id: 'gpt-5-mini', provider: 'openai', contextWindow: 400000, maxOutputTokens: 128000, inputCostPerMTok: 0.25, outputCostPerMTok: 2 }, + { id: 'o3', provider: 'openai', contextWindow: 200000, maxOutputTokens: 100000, inputCostPerMTok: 2, outputCostPerMTok: 8 }, + { id: 'o4-mini', provider: 'openai', contextWindow: 200000, maxOutputTokens: 100000, inputCostPerMTok: 1.1, outputCostPerMTok: 4.4 }, + { id: 'claude-fable-5-1', provider: 'anthropic', contextWindow: 1000000, maxOutputTokens: 128000, inputCostPerMTok: 10, outputCostPerMTok: 50 }, + { id: 'claude-opus-5-5', provider: 'anthropic', contextWindow: 1000000, maxOutputTokens: 128000, inputCostPerMTok: 4, outputCostPerMTok: 20 }, + { id: 'claude-sonnet-5-5', provider: 'anthropic', contextWindow: 1000000, maxOutputTokens: 128000, inputCostPerMTok: 2, outputCostPerMTok: 10 }, + { id: 'claude-haiku-4-5', provider: 'anthropic', contextWindow: 200000, maxOutputTokens: 64000, inputCostPerMTok: 1, outputCostPerMTok: 5 }, + { id: 'llama3.1', provider: 'ollama', contextWindow: 131072 }, + { id: 'mistral', provider: 'ollama', contextWindow: 32768 }, +]; diff --git a/src/models/models.test.ts b/src/models/models.test.ts new file mode 100644 index 000000000..864204eda --- /dev/null +++ b/src/models/models.test.ts @@ -0,0 +1,136 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import type { Message } from '../providers/llm'; +import { estimateCost, estimateTokens, getModelInfo, registerModel, setTokenEstimator } from './index'; + +describe('estimateTokens', () => { + afterEach(() => setTokenEstimator()); + + it('is roughly chars/4 for English', () => { + const text = 'The quick brown fox jumps over the lazy dog. '.repeat(20); + const tokens = estimateTokens(text); + expect(tokens).toBeGreaterThan(text.length / 4 - 1); + expect(tokens).toBeLessThanOrEqual(Math.ceil(text.length / 4)); + }); + + it('is monotonic in length', () => { + let previous = -1; + for (const n of [0, 1, 10, 100, 1000]) { + const tokens = estimateTokens('a'.repeat(n)); + expect(tokens).toBeGreaterThanOrEqual(previous); + previous = tokens; + } + expect(estimateTokens('')).toBe(0); + }); + + it('counts CJK at a much higher ratio than English, by code point', () => { + const cjk = '你好世界'.repeat(25); // 100 characters + expect(estimateTokens(cjk)).toBeGreaterThanOrEqual(80); + expect(estimateTokens(cjk)).toBeGreaterThan(estimateTokens('a'.repeat(100)) * 3); + // astral CJK extension characters are 2 UTF-16 units but one code point + expect(estimateTokens('\u{20000}'.repeat(10))).toBe(10); + }); + + it('counts other scripts between Latin and CJK', () => { + const cyrillic = 'привет'.repeat(10); + const tokens = estimateTokens(cyrillic); + expect(tokens).toBeGreaterThan(estimateTokens('a'.repeat(60))); + expect(tokens).toBeLessThan(estimateTokens('你'.repeat(60))); + }); + + it('counts emoji per code point and ignores joiners', () => { + expect(estimateTokens('😀'.repeat(5))).toBe(10); + expect(estimateTokens('👨‍👩')).toBe(estimateTokens('👨👩')); + }); + + it('adds per-message overhead and counts tool calls and results', () => { + const user: Message = { role: 'user', content: 'Hi there' }; + const withCall: Message = { + role: 'assistant', + content: '', + toolCalls: [{ id: 'call_1', type: 'function', function: { name: 'search', arguments: '{"q":"weather in Paris"}' } }], + }; + const result: Message = { role: 'tool', toolCallId: 'call_1', toolName: 'search', content: '{"temp":21,"unit":"C"}' }; + + expect(estimateTokens(user)).toBeGreaterThan(estimateTokens('Hi there')); + expect(estimateTokens(withCall)).toBeGreaterThan(estimateTokens({ role: 'assistant', content: '' })); + expect(estimateTokens([user, withCall, result])).toBe( + estimateTokens(user) + estimateTokens(withCall) + estimateTokens(result) + ); + expect(estimateTokens([user, user])).toBe(estimateTokens(user) * 2); + }); + + it('tolerates non-string content', () => { + const odd = { role: 'user', content: { parts: ['a', 'b'] } } as unknown as Message; + expect(estimateTokens(odd)).toBeGreaterThan(4); + const missing = { role: 'user' } as unknown as Message; + expect(estimateTokens(missing)).toBe(4); + }); + + it('supports a per-call and a global custom estimator', () => { + const calls: Array = []; + const fixed = (_input: unknown, opts: { model?: string }) => { + calls.push(opts.model); + return 42; + }; + expect(estimateTokens('anything', { estimator: fixed, model: 'gpt-4o' })).toBe(42); + expect(calls).toEqual(['gpt-4o']); + + setTokenEstimator(() => 7); + expect(estimateTokens('anything')).toBe(7); + setTokenEstimator(); + expect(estimateTokens('abcd')).toBe(1); + }); +}); + +describe('model registry', () => { + it('finds exact ids', () => { + expect(getModelInfo('gpt-4o-mini')).toMatchObject({ provider: 'openai', contextWindow: 128000 }); + expect(getModelInfo('claude-sonnet-5-5')?.provider).toBe('anthropic'); + }); + + it('resolves provider-prefixed ids', () => { + expect(getModelInfo('openai/gpt-4o-mini')?.id).toBe('gpt-4o-mini'); + expect(getModelInfo('anthropic/claude-haiku-4-5')?.id).toBe('claude-haiku-4-5'); + }); + + it('resolves dated snapshots and tags to the longest base id', () => { + expect(getModelInfo('gpt-4o-mini-2024-07-18')?.id).toBe('gpt-4o-mini'); + expect(getModelInfo('gpt-4o-2024-08-06')?.id).toBe('gpt-4o'); + expect(getModelInfo('claude-haiku-4-5-20251001')?.id).toBe('claude-haiku-4-5'); + expect(getModelInfo('ollama/llama3.1:8b')?.id).toBe('llama3.1'); + }); + + it('does not match a different model that merely shares a prefix', () => { + expect(getModelInfo('o3-mini')).toBeUndefined(); + expect(getModelInfo('gpt-5.4')).toBeUndefined(); + expect(getModelInfo('does-not-exist')).toBeUndefined(); + }); + + it('lets later registrations add and override', () => { + registerModel({ id: 'my-llama', provider: 'ollama', contextWindow: 8192 }); + expect(getModelInfo('my-llama')?.contextWindow).toBe(8192); + expect(getModelInfo('my-llama-2025-01-01')?.id).toBe('my-llama'); + + const original = getModelInfo('o4-mini')!; + registerModel({ ...original, inputCostPerMTok: 1 }); + try { + expect(getModelInfo('o4-mini')?.inputCostPerMTok).toBe(1); + } finally { + registerModel(original); + } + expect(getModelInfo('o4-mini')?.inputCostPerMTok).toBe(original.inputCostPerMTok); + }); +}); + +describe('estimateCost', () => { + it('computes USD from per-million-token prices', () => { + expect(estimateCost({ inputTokens: 1_000_000, outputTokens: 1_000_000 }, 'gpt-4o')).toBeCloseTo(12.5); + expect(estimateCost({ inputTokens: 2000, outputTokens: 500 }, 'openai/gpt-4o-mini')).toBeCloseTo(0.0006); + expect(estimateCost({ inputTokens: 0, outputTokens: 0 }, 'gpt-4o')).toBe(0); + }); + + it('returns undefined, not 0, for unknown models or missing prices', () => { + expect(estimateCost({ inputTokens: 10, outputTokens: 10 }, 'nope')).toBeUndefined(); + expect(estimateCost({ inputTokens: 10, outputTokens: 10 }, 'llama3.1')).toBeUndefined(); + }); +}); diff --git a/src/models/registry.ts b/src/models/registry.ts new file mode 100644 index 000000000..7b2d187e7 --- /dev/null +++ b/src/models/registry.ts @@ -0,0 +1,85 @@ +/** + * Model registry: context windows and prices (LOU-W1). + */ + +import { BUILT_IN_MODELS } from './modelData'; + +/** Static facts about a model. */ +export interface ModelInfo { + /** Model id as sent to the provider, e.g. `gpt-4o-mini`. */ + id: string; + /** Provider name, e.g. `openai`, `anthropic`, `ollama`. */ + provider: string; + /** Maximum tokens of context. */ + contextWindow: number; + /** Maximum tokens the model can generate in one response. */ + maxOutputTokens?: number; + /** USD per million input tokens. */ + inputCostPerMTok?: number; + /** USD per million output tokens. */ + outputCostPerMTok?: number; +} + +const registry = new Map(); +for (const info of BUILT_IN_MODELS) registry.set(info.id, info); + +/** + * Add a model or override a built-in one (later registration wins). Use this + * for self-hosted or fine-tuned models, and to override the built-in price + * snapshot with your own prices. + * + * @example + * registerModel({ id: 'my-llama', provider: 'ollama', contextWindow: 32768 }); + */ +export function registerModel(info: ModelInfo): void { + registry.set(info.id, { ...info }); +} + +/** A snapshot/tag suffix: `-2024-07-18`, `-20251001`, `-latest`, `:8b`, `@20251001`. */ +const SNAPSHOT_SUFFIX = /^([:@]|-(\d|latest))/; + +function lookupExactOrPrefix(id: string): ModelInfo | undefined { + const exact = registry.get(id); + if (exact) return exact; + let best: ModelInfo | undefined; + for (const info of registry.values()) { + const isSnapshot = id.startsWith(info.id) && SNAPSHOT_SUFFIX.test(id.slice(info.id.length)); + if (isSnapshot && (!best || info.id.length > best.id.length)) best = info; + } + return best; +} + +/** + * Look up a model. Matching order: exact id, then the longest registered id + * the given id is a snapshot of (so `gpt-4o-mini-2024-07-18` resolves to + * `gpt-4o-mini`, but `o3-mini` does not resolve to `o3`), each also tried + * without a `provider/` prefix (as in `resolveProvider('openai/gpt-4o-mini')`). + * Returns `undefined` for unknown models. + * + * @example + * getModelInfo('openai/gpt-4o-mini')?.contextWindow; // 128000 + */ +export function getModelInfo(model: string): ModelInfo | undefined { + const direct = lookupExactOrPrefix(model); + if (direct) return direct; + const slash = model.indexOf('/'); + return slash > 0 ? lookupExactOrPrefix(model.slice(slash + 1)) : undefined; +} + +/** + * Estimate the USD cost of a request from its token usage. Returns + * `undefined` (never 0) when the model, or either of its prices, is unknown. + * Built-in prices are a dated snapshot: register your own with + * {@link registerModel} for billing-grade numbers. + * + * @example + * estimateCost({ inputTokens: 1_000_000, outputTokens: 0 }, 'gpt-4o'); // 2.5 + */ +export function estimateCost( + usage: { inputTokens: number; outputTokens: number }, + model: string +): number | undefined { + const info = getModelInfo(model); + if (info?.inputCostPerMTok === undefined || info.outputCostPerMTok === undefined) return undefined; + return (usage.inputTokens * info.inputCostPerMTok + usage.outputTokens * info.outputCostPerMTok) / 1_000_000; +}