Repository navigation
Expand file tree
/
Copy pathstrategy.ts
More file actions
258 lines (230 loc) · 9.2 KB
/
Copy pathstrategy.ts
File metadata and controls
258 lines (230 loc) · 9.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
/**
* Router Strategy Registry
*
* Pluggable strategy system for request routing.
* Default: RulesStrategy — identical to the original inline route() logic, <1ms.
*/
import type {
Tier,
TierConfig,
Promotion,
RoutingDecision,
RouterStrategy,
RouterOptions,
} from "./types.js";
import { classifyByRules } from "./rules.js";
import { selectModel } from "./selector.js";
import { estimateTokens, sampleForScan, scanLimit } from "./tokens.js";
/**
* Apply active time-windowed promotions to tier configs.
* Returns a new tierConfigs object with promotion overrides merged in.
* Expired or not-yet-active promotions are ignored.
*/
function applyPromotions(
tierConfigs: Record<Tier, TierConfig>,
promotions: Promotion[] | undefined,
profile: "auto" | "eco" | "premium" | "agentic",
now: Date = new Date(),
): Record<Tier, TierConfig> {
if (!promotions || promotions.length === 0) return tierConfigs;
let result = tierConfigs;
for (const promo of promotions) {
// Check time window
const start = new Date(promo.startDate);
const end = new Date(promo.endDate);
if (now < start || now >= end) continue;
// Check profile filter
if (promo.profiles && !promo.profiles.includes(profile)) continue;
// Shallow-clone on first mutation
if (result === tierConfigs) {
result = { ...tierConfigs };
for (const t of Object.keys(result) as Tier[]) {
result[t] = { ...result[t] };
}
}
// Merge overrides
for (const [tier, override] of Object.entries(promo.tierOverrides) as [
Tier,
Partial<TierConfig>,
][]) {
if (!result[tier]) continue;
if (override.primary) result[tier].primary = override.primary;
if (override.fallback) result[tier].fallback = override.fallback;
}
}
return result;
}
/**
* Remove host-declared-dead models from every tier chain, promoting the first
* surviving rung to primary. A tier whose chain is entirely dead keeps its
* original config — the router has nothing live to offer there, and inventing
* a model would hide the outage from the host that reported it.
*/
export function applyUnavailableModels(
tierConfigs: Record<Tier, TierConfig>,
unavailableModels: readonly string[] | undefined,
): Record<Tier, TierConfig> {
if (!unavailableModels || unavailableModels.length === 0) return tierConfigs;
const dead = new Set(unavailableModels);
let result = tierConfigs;
for (const tier of Object.keys(tierConfigs) as Tier[]) {
const config = tierConfigs[tier];
const alive = [config.primary, ...config.fallback].filter((model) => !dead.has(model));
if (alive.length === 0 || (alive[0] === config.primary && alive.length === config.fallback.length + 1)) {
continue;
}
if (result === tierConfigs) result = { ...tierConfigs };
result[tier] = { primary: alive[0], fallback: alive.slice(1) };
}
return result;
}
/**
* Rules-based routing strategy.
* Extracted from the original route() in index.ts — logic is identical.
* Attaches tierConfigs and profile to the decision for downstream use.
*/
export class RulesStrategy implements RouterStrategy {
readonly name = "rules";
route(
prompt: string,
systemPrompt: string | undefined,
maxOutputTokens: number,
options: RouterOptions,
): RoutingDecision {
const { config, modelPricing } = options;
const fullText = `${systemPrompt ?? ""} ${prompt}`;
const estimatedTokens = estimateTokens(fullText);
const limit = scanLimit(config.classifier.promptTruncationChars);
const scannedPrompt = sampleForScan(prompt, limit);
const scannedSystemPrompt = systemPrompt ? sampleForScan(systemPrompt, limit) : undefined;
// --- Rule-based classification (runs first to get agenticScore) ---
const ruleResult = classifyByRules(
scannedPrompt,
scannedSystemPrompt,
estimatedTokens,
config.scoring,
);
// --- Select tier configs based on routing profile ---
const { routingProfile } = options;
let tierConfigs: Record<Tier, { primary: string; fallback: string[] }>;
let profileSuffix: string;
let profile: RoutingDecision["profile"];
if (routingProfile === "eco") {
// `ecoTiers: null` explicitly disables the special eco tier set while
// keeping eco routing semantics. Fall back to regular tiers instead of
// dropping into auto routing (which could select agentic tiers).
tierConfigs = config.ecoTiers ?? config.tiers;
profileSuffix = config.ecoTiers ? " | eco" : " | eco (default tiers)";
profile = "eco";
} else if (routingProfile === "premium") {
// `premiumTiers: null` disables the premium-specific tier set but the
// request is still a premium-profile request, so use regular tiers while
// preserving premium metadata/cost semantics.
tierConfigs = config.premiumTiers ?? config.tiers;
profileSuffix = config.premiumTiers ? " | premium" : " | premium (default tiers)";
profile = "premium";
} else {
// Auto profile (or undefined): intelligent routing with agentic detection.
//
// `agenticMode` semantics:
// - `true` → force agentic tiers (ignore heuristics)
// - `false` → disable agentic tiers entirely (even if tools are present)
// - `undefined` → auto-detect via heuristics (tools present OR high agenticScore)
const agenticScore = ruleResult.agenticScore ?? 0;
const isAutoAgentic = agenticScore >= 0.5;
const agenticModeSetting = config.overrides.agenticMode;
const hasToolsInRequest = options.requiresTools ?? options.hasTools ?? false;
let useAgenticTiers: boolean;
if (agenticModeSetting === false) {
// Explicitly disabled — never use agentic tiers
useAgenticTiers = false;
} else if (agenticModeSetting === true) {
// Explicitly enabled — use agentic tiers if available
useAgenticTiers = config.agenticTiers != null;
} else {
// Auto-detect
useAgenticTiers = (hasToolsInRequest || isAutoAgentic) && config.agenticTiers != null;
}
tierConfigs = useAgenticTiers ? config.agenticTiers! : config.tiers;
profileSuffix = useAgenticTiers ? ` | agentic${hasToolsInRequest ? " (tools)" : ""}` : "";
profile = useAgenticTiers ? "agentic" : "auto";
}
// Apply time-windowed promotions
tierConfigs = applyPromotions(tierConfigs, config.promotions, profile!, options.now);
// Hard-remove models the host has observed dead at the gateway. After
// promotions, so a promo cannot resurrect a rung the host just killed.
tierConfigs = applyUnavailableModels(tierConfigs, options.unavailableModels);
const agenticScoreValue = ruleResult.agenticScore;
// --- Override: large context → force COMPLEX ---
if (estimatedTokens > config.overrides.maxTokensForceComplex) {
const decision = selectModel(
"COMPLEX",
0.95,
"rules",
`Input exceeds ${config.overrides.maxTokensForceComplex} tokens${profileSuffix}`,
tierConfigs,
modelPricing,
estimatedTokens,
maxOutputTokens,
routingProfile,
agenticScoreValue,
);
return { ...decision, tierConfigs, profile };
}
// Structured output is a protocol fact the host states (`response_format`
// → requiresStructuredOutput). It used to be inferred from the system
// prompt as well, but agent hosts describe their tool schemas there on
// every turn, which upgraded every one of their requests off SIMPLE.
const hasStructuredOutput = options.requiresStructuredOutput === true;
let tier: Tier;
let confidence: number;
const method: "rules" | "llm" = "rules";
let reasoning = `score=${ruleResult.score.toFixed(2)} | ${ruleResult.signals.join(", ")}`;
if (ruleResult.tier !== null) {
tier = ruleResult.tier;
confidence = ruleResult.confidence;
} else {
// Ambiguous — default to configurable tier (no external API call)
tier = config.overrides.ambiguousDefaultTier;
confidence = 0.5;
reasoning += ` | ambiguous -> default: ${tier}`;
}
// Apply structured output minimum tier
if (hasStructuredOutput) {
const tierRank: Record<Tier, number> = { SIMPLE: 0, MEDIUM: 1, COMPLEX: 2, REASONING: 3 };
const minTier = config.overrides.structuredOutputMinTier;
if (tierRank[tier] < tierRank[minTier]) {
reasoning += ` | upgraded to ${minTier} (structured output)`;
tier = minTier;
}
}
// Add routing profile suffix to reasoning
reasoning += profileSuffix;
const decision = selectModel(
tier,
confidence,
method,
reasoning,
tierConfigs,
modelPricing,
estimatedTokens,
maxOutputTokens,
routingProfile,
agenticScoreValue,
);
return { ...decision, tierConfigs, profile };
}
}
// --- Strategy Registry ---
const registry = new Map<string, RouterStrategy>();
registry.set("rules", new RulesStrategy());
export function getStrategy(name: string): RouterStrategy {
const strategy = registry.get(name);
if (!strategy) {
throw new Error(`Unknown routing strategy: ${name}`);
}
return strategy;
}
export function registerStrategy(strategy: RouterStrategy): void {
registry.set(strategy.name, strategy);
}