LOU-W1: token estimation and a model registry (context window + pricing)

docs/api-overview.md

@@ -177,6 +177,34 @@ Errors extending `PropagatingToolError` (for example the delegation depth
 guard) are the exception: they are rethrown and abort the run instead of being
 shown to the model.
 
+## Models, tokens and cost
+
+Dependency-free helpers for budgeting and context decisions.
+
+```ts
+import { estimateTokens, estimateCost, getModelInfo, registerModel } from '@loushy/build-ai-agent';
+
+// A custom or self-hosted model: add it (or override a built-in) before use.
+registerModel({
+  id: 'my-llama',
+  provider: 'ollama',
+  contextWindow: 32768,
+  inputCostPerMTok: 0.2, // optional; omit for unknown or free
+  outputCostPerMTok: 0.6,
+});
+
+const used = estimateTokens([{ role: 'user', content: 'Summarise this report' }], { model: 'my-llama' });
+const info = getModelInfo('openai/gpt-4o-mini'); // exact id, `provider/id`, or a dated snapshot
+const usd = estimateCost({ inputTokens: used, outputTokens: 500 }, 'my-llama'); // undefined if unpriced
+```
+
+- `estimateTokens(input, { model?, estimator? })` is a heuristic (about 4 characters per token for English, more for CJK and other scripts, plus per-message overhead and tool-call JSON). Expect roughly 15-20% error on English: fine for compaction and budgets, not for billing. Plug in a real tokenizer with `setTokenEstimator(fn)` or `options.estimator`.
+- `getModelInfo(id)` matches the exact id, then `provider/id`, then dated snapshots (`gpt-4o-mini-2024-07-18` resolves to `gpt-4o-mini`). Unknown models return `undefined`.
+- `registerModel(info)` adds or overrides an entry; the latest registration wins.
+- `estimateCost(usage, model)` returns USD, or `undefined` (not `0`) when the model or its prices are unknown.
+
+The built-in context windows and prices are a dated snapshot (see the retrieval date and sources at the top of `src/models/modelData.ts`). Providers change prices and models, so override entries with `registerModel` when you need billing-grade numbers.
+
 ## Flows, evals, observability and security
 
 - `FlowBuilder` / `FlowExecutor` - multi-step workflow graphs.

src/index.ts

@@ -44,6 +44,9 @@ export * from './storage';
 // Templates
 export * from './templates';
 
+// Token estimation and model registry (LOU-W1)
+export * from './models';
+
 // Utils
 export * from './utils';
 

src/models/estimateTokens.ts

@@ -0,0 +1,145 @@
+/**
+ * Dependency-free token estimation (LOU-W1).
+ *
+ * A heuristic, not a tokenizer: it exists so budgeting and context-compaction
+ * code can answer "roughly how big is this prompt?" without shipping a
+ * tokenizer for every provider.
+ */
+
+import type { Message } from '../providers/llm';
+
+/** Anything `estimateTokens` accepts. */
+export type TokenEstimateInput = string | Message | Message[];
+
+/** A replacement estimator, e.g. one backed by `tiktoken` or a provider's count endpoint. */
+export type TokenEstimator = (input: TokenEstimateInput, options: { model?: string }) => number;
+
+/** Options for {@link estimateTokens}. */
+export interface EstimateTokensOptions {
+  /**
+   * Model id the estimate is for. The built-in heuristic ignores it; it is
+   * forwarded to custom estimators so they can pick a model-specific tokenizer.
+   */
+  model?: string;
+  /** Use this estimator for this call instead of the built-in heuristic (or the one set with `setTokenEstimator`). */
+  estimator?: TokenEstimator;
+}
+
+/** Tokens added per message for role markers and framing. */
+const MESSAGE_OVERHEAD_TOKENS = 4;
+
+/** Tokens per code point for Latin/ASCII text (~4 chars per token). */
+const LATIN_WEIGHT = 0.25;
+/** CJK ideographs, kana and Hangul: about one token per character. */
+const CJK_WEIGHT = 1;
+/** Other non-Latin scripts (Cyrillic, Arabic, Hebrew, Thai, Indic, ...). */
+const OTHER_SCRIPT_WEIGHT = 0.5;
+/** Emoji and pictographs usually cost several byte-level tokens. */
+const EMOJI_WEIGHT = 2;
+
+type Range = readonly [number, number];
+
+/** Code point ranges for CJK ideographs, kana, Hangul and CJK punctuation. */
+const CJK_RANGES: readonly Range[] = [
+  [0x2e80, 0x2fdf],
+  [0x3000, 0x303f],
+  [0x3040, 0x30ff],
+  [0x3100, 0x312f],
+  [0x3400, 0x4dbf],
+  [0x4e00, 0x9fff],
+  [0xac00, 0xd7af],
+  [0xf900, 0xfaff],
+  [0xff00, 0xffef],
+  [0x20000, 0x2fa1f],
+];
+
+/** Code point ranges for emoji and pictographs. */
+const EMOJI_RANGES: readonly Range[] = [
+  [0x2600, 0x27bf],
+  [0x1f000, 0x1faff],
+];
+
+/** Zero-width joiner, variation selectors and combining marks glue characters together and cost nothing extra. */
+const ZERO_WEIGHT_RANGES: readonly Range[] = [
+  [0x0300, 0x036f],
+  [0x200d, 0x200d],
+  [0xfe00, 0xfe0f],
+];
+
+function inRanges(cp: number, ranges: readonly Range[]): boolean {
+  return ranges.some(([lo, hi]) => cp >= lo && cp <= hi);
+}
+
+function codePointWeight(cp: number): number {
+  if (cp < 0x0300) return LATIN_WEIGHT;
+  if (inRanges(cp, ZERO_WEIGHT_RANGES)) return 0;
+  if (inRanges(cp, CJK_RANGES)) return CJK_WEIGHT;
+  if (inRanges(cp, EMOJI_RANGES)) return EMOJI_WEIGHT;
+  return OTHER_SCRIPT_WEIGHT;
+}
+
+function estimateText(text: string): number {
+  let tokens = 0;
+  for (const ch of text) {
+    tokens += codePointWeight(ch.codePointAt(0) ?? 0);
+  }
+  return Math.ceil(tokens);
+}
+
+function toText(value: unknown): string {
+  if (typeof value === 'string') return value;
+  if (value === undefined || value === null) return '';
+  return JSON.stringify(value) ?? '';
+}
+
+function estimateMessage(message: Message): number {
+  let tokens = MESSAGE_OVERHEAD_TOKENS + estimateText(toText(message.content));
+  if (message.name) tokens += estimateText(message.name);
+  if (message.toolName) tokens += estimateText(message.toolName);
+  if (message.toolCalls?.length) tokens += estimateText(JSON.stringify(message.toolCalls));
+  return tokens;
+}
+
+function defaultEstimator(input: TokenEstimateInput): number {
+  if (typeof input === 'string') return estimateText(input);
+  if (Array.isArray(input)) return input.reduce((sum, m) => sum + estimateMessage(m), 0);
+  return estimateMessage(input);
+}
+
+let globalEstimator: TokenEstimator = defaultEstimator;
+
+/**
+ * Replace the estimator used by every `estimateTokens` call, e.g. with a real
+ * tokenizer. Call with no argument to restore the built-in heuristic.
+ *
+ * @example
+ * setTokenEstimator((input) => countWithMyTokenizer(input));
+ */
+export function setTokenEstimator(estimator?: TokenEstimator): void {
+  globalEstimator = estimator ?? defaultEstimator;
+}
+
+/**
+ * Estimate how many tokens a string, message or conversation will use.
+ *
+ * Heuristic, no tokenizer: about 4 characters per token for Latin text,
+ * about 1 token per CJK character, about 0.5 per character for other
+ * non-Latin scripts and 2 per emoji (code points are counted, not UTF-16
+ * units), plus 4 tokens of overhead per message and the JSON length of tool
+ * calls. Tool results are the message `content` (typically JSON) and are
+ * counted as text.
+ *
+ * Accuracy: an estimate for budgeting and compaction decisions. It is
+ * typically within ~15-20% for English prose and can be further off for
+ * code, minified JSON or rare scripts. It is NOT suitable for billing; use
+ * the usage a provider reports for that.
+ *
+ * @example
+ * estimateTokens('Hello, world!');                         // 4
+ * estimateTokens([{ role: 'user', content: 'Hi there' }]); // 6
+ * estimateTokens(messages, { estimator: myTokenizer });
+ */
+export function estimateTokens(input: TokenEstimateInput, options: EstimateTokensOptions = {}): number {
+  const estimator = options.estimator ?? globalEstimator;
+  return estimator(input, { model: options.model });
+}

src/models/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Models, tokens and cost (LOU-W1): token estimation and a model registry
+ * with context windows and prices.
+ */
+
+export * from './estimateTokens';
+export * from './registry';

src/models/modelData.ts

@@ -0,0 +1,35 @@
+import type { ModelInfo } from './registry';
+
+/**
+ * Built-in model table. A dated snapshot, not a live price list.
+ *
+ * Retrieved 2026-10-01. Sources (provider-published):
+ *  - OpenAI: https://developers.openai.com/api/docs/pricing and
+ *    https://developers.openai.com/api/docs/models/<model> (standard tier,
+ *    short-context prices; context window and max output from each model page)
+ *  - Anthropic: https://platform.claude.com/docs/en/docs/about-claude/models/overview
+ *    ("128K" max output read as 128000, "1M" context as 1000000)
+ *  - Ollama: https://ollama.com/library/<model> (context window only; local
+ *    models have no per-token price, so prices are left undefined)
+ *
+ * Only models whose numbers were read from those pages are listed; anything
+ * unverified is left undefined. Override with `registerModel` for
+ * billing-grade numbers.
+ */
+export const BUILT_IN_MODELS: readonly ModelInfo[] = [
+  { id: 'gpt-4o', provider: 'openai', contextWindow: 128000, maxOutputTokens: 16384, inputCostPerMTok: 2.5, outputCostPerMTok: 10 },
+  { id: 'gpt-4o-mini', provider: 'openai', contextWindow: 128000, maxOutputTokens: 16384, inputCostPerMTok: 0.15, outputCostPerMTok: 0.6 },
+  { id: 'gpt-4.1', provider: 'openai', contextWindow: 1047576, maxOutputTokens: 32768, inputCostPerMTok: 2, outputCostPerMTok: 8 },
+  { id: 'gpt-4.1-mini', provider: 'openai', contextWindow: 1000000, maxOutputTokens: 32768, inputCostPerMTok: 0.4, outputCostPerMTok: 1.6 },
+  { id: 'gpt-4.1-nano', provider: 'openai', contextWindow: 1000000, maxOutputTokens: 32768, inputCostPerMTok: 0.1, outputCostPerMTok: 0.4 },
+  { id: 'gpt-5', provider: 'openai', contextWindow: 400000, maxOutputTokens: 128000, inputCostPerMTok: 1.25, outputCostPerMTok: 10 },
+  { id: 'gpt-5-mini', provider: 'openai', contextWindow: 400000, maxOutputTokens: 128000, inputCostPerMTok: 0.25, outputCostPerMTok: 2 },
+  { id: 'o3', provider: 'openai', contextWindow: 200000, maxOutputTokens: 100000, inputCostPerMTok: 2, outputCostPerMTok: 8 },
+  { id: 'o4-mini', provider: 'openai', contextWindow: 200000, maxOutputTokens: 100000, inputCostPerMTok: 1.1, outputCostPerMTok: 4.4 },
+  { id: 'claude-fable-5-1', provider: 'anthropic', contextWindow: 1000000, maxOutputTokens: 128000, inputCostPerMTok: 10, outputCostPerMTok: 50 },
+  { id: 'claude-opus-5-5', provider: 'anthropic', contextWindow: 1000000, maxOutputTokens: 128000, inputCostPerMTok: 4, outputCostPerMTok: 20 },
+  { id: 'claude-sonnet-5-5', provider: 'anthropic', contextWindow: 1000000, maxOutputTokens: 128000, inputCostPerMTok: 2, outputCostPerMTok: 10 },
+  { id: 'claude-haiku-4-5', provider: 'anthropic', contextWindow: 200000, maxOutputTokens: 64000, inputCostPerMTok: 1, outputCostPerMTok: 5 },
+  { id: 'llama3.1', provider: 'ollama', contextWindow: 131072 },
+  { id: 'mistral', provider: 'ollama', contextWindow: 32768 },
+];

src/models/models.test.ts

@@ -0,0 +1,136 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import type { Message } from '../providers/llm';
+import { estimateCost, estimateTokens, getModelInfo, registerModel, setTokenEstimator } from './index';
+
+describe('estimateTokens', () => {
+  afterEach(() => setTokenEstimator());
+
+  it('is roughly chars/4 for English', () => {
+    const text = 'The quick brown fox jumps over the lazy dog. '.repeat(20);
+    const tokens = estimateTokens(text);
+    expect(tokens).toBeGreaterThan(text.length / 4 - 1);
+    expect(tokens).toBeLessThanOrEqual(Math.ceil(text.length / 4));
+  });
+
+  it('is monotonic in length', () => {
+    let previous = -1;
+    for (const n of [0, 1, 10, 100, 1000]) {
+      const tokens = estimateTokens('a'.repeat(n));
+      expect(tokens).toBeGreaterThanOrEqual(previous);
+      previous = tokens;
+    }
+    expect(estimateTokens('')).toBe(0);
+  });
+
+  it('counts CJK at a much higher ratio than English, by code point', () => {
+    const cjk = '你好世界'.repeat(25); // 100 characters
+    expect(estimateTokens(cjk)).toBeGreaterThanOrEqual(80);
+    expect(estimateTokens(cjk)).toBeGreaterThan(estimateTokens('a'.repeat(100)) * 3);
+    // astral CJK extension characters are 2 UTF-16 units but one code point
+    expect(estimateTokens('\u{20000}'.repeat(10))).toBe(10);
+  });
+
+  it('counts other scripts between Latin and CJK', () => {
+    const cyrillic = 'привет'.repeat(10);
+    const tokens = estimateTokens(cyrillic);
+    expect(tokens).toBeGreaterThan(estimateTokens('a'.repeat(60)));
+    expect(tokens).toBeLessThan(estimateTokens('你'.repeat(60)));
+  });
+
+  it('counts emoji per code point and ignores joiners', () => {
+    expect(estimateTokens('😀'.repeat(5))).toBe(10);
+    expect(estimateTokens('👨‍👩')).toBe(estimateTokens('👨👩'));
+  });
+
+  it('adds per-message overhead and counts tool calls and results', () => {
+    const user: Message = { role: 'user', content: 'Hi there' };
+    const withCall: Message = {
+      role: 'assistant',
+      content: '',
+      toolCalls: [{ id: 'call_1', type: 'function', function: { name: 'search', arguments: '{"q":"weather in Paris"}' } }],
+    };
+    const result: Message = { role: 'tool', toolCallId: 'call_1', toolName: 'search', content: '{"temp":21,"unit":"C"}' };
+
+    expect(estimateTokens(user)).toBeGreaterThan(estimateTokens('Hi there'));
+    expect(estimateTokens(withCall)).toBeGreaterThan(estimateTokens({ role: 'assistant', content: '' }));
+    expect(estimateTokens([user, withCall, result])).toBe(
+      estimateTokens(user) + estimateTokens(withCall) + estimateTokens(result)
+    );
+    expect(estimateTokens([user, user])).toBe(estimateTokens(user) * 2);
+  });
+
+  it('tolerates non-string content', () => {
+    const odd = { role: 'user', content: { parts: ['a', 'b'] } } as unknown as Message;
+    expect(estimateTokens(odd)).toBeGreaterThan(4);
+    const missing = { role: 'user' } as unknown as Message;
+    expect(estimateTokens(missing)).toBe(4);
+  });
+
+  it('supports a per-call and a global custom estimator', () => {
+    const calls: Array<string | undefined> = [];
+    const fixed = (_input: unknown, opts: { model?: string }) => {
+      calls.push(opts.model);
+      return 42;
+    };
+    expect(estimateTokens('anything', { estimator: fixed, model: 'gpt-4o' })).toBe(42);
+    expect(calls).toEqual(['gpt-4o']);
+
+    setTokenEstimator(() => 7);
+    expect(estimateTokens('anything')).toBe(7);
+    setTokenEstimator();
+    expect(estimateTokens('abcd')).toBe(1);
+  });
+});
+
+describe('model registry', () => {
+  it('finds exact ids', () => {
+    expect(getModelInfo('gpt-4o-mini')).toMatchObject({ provider: 'openai', contextWindow: 128000 });
+    expect(getModelInfo('claude-sonnet-5-5')?.provider).toBe('anthropic');
+  });
+
+  it('resolves provider-prefixed ids', () => {
+    expect(getModelInfo('openai/gpt-4o-mini')?.id).toBe('gpt-4o-mini');
+    expect(getModelInfo('anthropic/claude-haiku-4-5')?.id).toBe('claude-haiku-4-5');
+  });
+
+  it('resolves dated snapshots and tags to the longest base id', () => {
+    expect(getModelInfo('gpt-4o-mini-2024-07-18')?.id).toBe('gpt-4o-mini');
+    expect(getModelInfo('gpt-4o-2024-08-06')?.id).toBe('gpt-4o');
+    expect(getModelInfo('claude-haiku-4-5-20251001')?.id).toBe('claude-haiku-4-5');
+    expect(getModelInfo('ollama/llama3.1:8b')?.id).toBe('llama3.1');
+  });
+
+  it('does not match a different model that merely shares a prefix', () => {
+    expect(getModelInfo('o3-mini')).toBeUndefined();
+    expect(getModelInfo('gpt-5.4')).toBeUndefined();
+    expect(getModelInfo('does-not-exist')).toBeUndefined();
+  });
+
+  it('lets later registrations add and override', () => {
+    registerModel({ id: 'my-llama', provider: 'ollama', contextWindow: 8192 });
+    expect(getModelInfo('my-llama')?.contextWindow).toBe(8192);
+    expect(getModelInfo('my-llama-2025-01-01')?.id).toBe('my-llama');
+
+    const original = getModelInfo('o4-mini')!;
+    registerModel({ ...original, inputCostPerMTok: 1 });
+    try {
+      expect(getModelInfo('o4-mini')?.inputCostPerMTok).toBe(1);
+    } finally {
+      registerModel(original);
+    }
+    expect(getModelInfo('o4-mini')?.inputCostPerMTok).toBe(original.inputCostPerMTok);
+  });
+});
+
+describe('estimateCost', () => {
+  it('computes USD from per-million-token prices', () => {
+    expect(estimateCost({ inputTokens: 1_000_000, outputTokens: 1_000_000 }, 'gpt-4o')).toBeCloseTo(12.5);
+    expect(estimateCost({ inputTokens: 2000, outputTokens: 500 }, 'openai/gpt-4o-mini')).toBeCloseTo(0.0006);
+    expect(estimateCost({ inputTokens: 0, outputTokens: 0 }, 'gpt-4o')).toBe(0);
+  });
+
+  it('returns undefined, not 0, for unknown models or missing prices', () => {
+    expect(estimateCost({ inputTokens: 10, outputTokens: 10 }, 'nope')).toBeUndefined();
+    expect(estimateCost({ inputTokens: 10, outputTokens: 10 }, 'llama3.1')).toBeUndefined();
+  });
+});