From 006fef2f97caef4ef474b838bd8710bbc5cebd29 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Tue, 20 Oct 2026 05:39:48 +0300 Subject: [PATCH] V41: Say how to price a model when a cost limit can't be enforced Co-Authored-By: Claude Opus 5.5 --- src/execution/budget.test.ts | 17 +++++++++++++++++ src/execution/budget.ts | 31 +++++++++++++++++++------------ src/providers/ollamaCost.test.ts | 2 +- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/execution/budget.test.ts b/src/execution/budget.test.ts index efbb4b2c..70ff2a0d 100644 --- a/src/execution/budget.test.ts +++ b/src/execution/budget.test.ts @@ -217,6 +217,23 @@ describe('unknown cost under maxCostUsd', () => { warn.mockRestore(); }); + it('names the model, the option and the docs link, once per model across repeated steps', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + vi.resetModules(); + const { startBudget: fresh } = await import('./budget'); + const byModel = { 'lmstudio/qwen': { inputTokens: 1, outputTokens: 1, calls: 1 } }; + const usage: RunUsage = { inputTokens: 1, outputTokens: 1, totalTokens: 2, costUsd: undefined, modelCalls: 1, estimated: false, promptTokens: 1, completionTokens: 1, byModel }; + const budget = fresh({ maxCostUsd: 1 })!; + for (let step = 1; step <= 3; step++) budget.check(usage, step); + expect(warn).toHaveBeenCalledTimes(1); + const text = String(warn.mock.calls[0][0]); + expect(text).toContain('lmstudio/qwen'); + expect(text).toContain('NOT enforced'); + expect(text).toContain('registerModel('); + expect(text).toContain('https://lousho.com/models-and-cost#models-and-the-price-table'); + warn.mockRestore(); + }); + it('enforces the cap on the partial total when only some steps have a known price', () => { const budget = startBudget({ maxCostUsd: 1 })!; const usage: RunUsage = { inputTokens: 1, outputTokens: 1, totalTokens: 2, costUsd: 1.5, costPartial: true, modelCalls: 2, estimated: false, promptTokens: 1, completionTokens: 1, byModel: {} }; diff --git a/src/execution/budget.ts b/src/execution/budget.ts index 3c3bb84c..d93174ed 100644 --- a/src/execution/budget.ts +++ b/src/execution/budget.ts @@ -16,7 +16,7 @@ export interface RunLimits { maxTokens?: number; maxInputTokens?: number; maxOutputTokens?: number; - /** USD (`usage.costUsd`): the provider-reported cost, else the registry's estimate; not checked (a warning is logged once) while a model used has unknown pricing. */ + /** USD (`usage.costUsd`): the provider-reported cost, else the registry's estimate; not checked (a warning naming the model is logged once) while a model used has unknown pricing. */ maxCostUsd?: number; /** Wall-clock time; an in-flight model or tool call is aborted when it runs out. */ maxDurationMs?: number; @@ -78,18 +78,25 @@ const SPENT_KEYS: Record, keyof BudgetSpen maxSteps: 'steps', }; -let warnedUnknownCost = false; +const warnedUnknownCost = new Set(); -/** Warns (once per process) that `maxCostUsd` cannot be enforced because a model used has no known price. */ -function warnUnknownCost(): void { - if (warnedUnknownCost) return; - warnedUnknownCost = true; +/** Warns (once per process and model) that `maxCostUsd` cannot be enforced because `model` has no known price. */ +function warnUnknownCost(model: string): void { + if (warnedUnknownCost.has(model)) return; + warnedUnknownCost.add(model); console.warn( - '[lousho] limits.maxCostUsd is set, but the cost of a model call is unknown (the model is not in the price registry and the provider reported no cost), ' + - 'so the cost limit is not enforced. Register a price with registerModel({ id, provider, inputCostPerMTok, outputCostPerMTok }), or use a provider that reports cost (OpenRouter does).' + `[lousho] limits.maxCostUsd is set, but the price of model '${model}' is unknown (it is not in the price registry and the provider reported no cost), ` + + `so the cost limit is NOT enforced for it. Register a price: registerModel({ id: '${model}', provider: '', contextWindow: 8192, inputCostPerMTok: 0, outputCostPerMTok: 0 }) ` + + '(0 counts the model as free), or use a provider that reports cost (OpenRouter does). See https://lousho.com/models-and-cost#models-and-the-price-table' ); } +/** The models of `byModel` with no known cost, else `'unknown'` when none is recorded. */ +function unpricedModels(byModel: RunUsage['byModel'] | undefined): string[] { + const ids = Object.entries(byModel ?? {}).filter(([, m]) => m.costUsd === undefined).map(([id]) => id); + return ids.length > 0 ? ids : ['unknown']; +} + const NOTHING_SPENT: BudgetSpent = { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0, steps: 0, durationMs: 0 }; /** `a + b`, field by field (`costUsd` is unknown when either is). */ @@ -125,11 +132,11 @@ export interface RunBudget { } /** The first spend limit of `max` that `total` reaches; `afterModelCall` leaves out `maxSteps`, `quietUnknownCost` the unknown-cost warning. */ -function spendLimitReached(max: RunLimits, total: BudgetSpent, afterModelCall: boolean, quietUnknownCost: boolean): Omit | undefined { +function spendLimitReached(max: RunLimits, total: BudgetSpent, afterModelCall: boolean, quietUnknownCost: boolean, byModel?: RunUsage['byModel']): Omit | undefined { for (const [limit, key] of Object.entries(SPENT_KEYS) as Array<[Exclude, keyof BudgetSpent]>) { const value = total[key]; const cap = max[limit]; - if (cap !== undefined && value === undefined && limit === 'maxCostUsd' && !quietUnknownCost) warnUnknownCost(); + if (cap !== undefined && value === undefined && limit === 'maxCostUsd' && !quietUnknownCost) unpricedModels(byModel).forEach(warnUnknownCost); if (cap === undefined || value === undefined || (afterModelCall && limit === 'maxSteps')) continue; if (value >= cap) return { limit, value, max: cap }; } @@ -145,12 +152,12 @@ export function startBudget(limits?: RunLimits, session?: SessionBudget, signal? ]; if (scopes.length === 0) return undefined; const startedAt = Date.now(); - const check = (usage: Pick, steps: number, afterModelCall = false, toolErrors = 0) => { + const check = (usage: Pick & Partial>, steps: number, afterModelCall = false, toolErrors = 0) => { const now = runSpent(usage, steps, Date.now() - startedAt); for (const { scope, limits: max, spent } of scopes) { // Failed tool calls count per run, also under a session's limits. if (max.maxToolErrors !== undefined && toolErrors >= max.maxToolErrors) return { limit: 'maxToolErrors' as const, value: toolErrors, max: max.maxToolErrors, scope }; - const reached = spendLimitReached(max, addSpent(spent, now), afterModelCall, quietUnknownCost); + const reached = spendLimitReached(max, addSpent(spent, now), afterModelCall, quietUnknownCost, usage.byModel); if (reached) return { ...reached, scope }; } return undefined; diff --git a/src/providers/ollamaCost.test.ts b/src/providers/ollamaCost.test.ts index 35d37da5..64c40f68 100644 --- a/src/providers/ollamaCost.test.ts +++ b/src/providers/ollamaCost.test.ts @@ -87,7 +87,7 @@ describe.skipIf(installedAiMajor !== 4 && !ollamaV2Installed)('Ollama llama3 cos expect(toolRuns).toBe(1); expect(result.usage).toMatchObject({ inputTokens: 2000, outputTokens: 1000, modelCalls: 2, estimated: false }); expect(result.usage.costUsd).toBeUndefined(); - expect(warn).toHaveBeenCalledWith(expect.stringMatching(/maxCostUsd is set, but the cost of a model call is unknown/)); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/maxCostUsd is set, but the price of model 'llama3' is unknown.*NOT enforced/)); }); it('with a registered price, maxCostUsd is enforced on an Ollama run', async () => {