From 43372edd668ec0884b96a766b594997247bb77db Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Wed, 28 Oct 2026 23:53:42 +0300 Subject: [PATCH 1/2] feat(observe): evaluator rules API Add the evaluator_rules table (migration 5) and CRUD under /api/v1/evaluators: code rules ({ source }) and judge rules ({ model, rubric }) with a trace filter and sample rate. Rules are stored only; nothing compiles source or calls a model yet. Co-Authored-By: Claude Opus 5.5 --- docs/observability.md | 33 ++++ llms-full.txt | 33 ++++ src/observe/evaluators.ts | 152 ++++++++++++++++++ src/observe/routes/evaluators.test.ts | 217 ++++++++++++++++++++++++++ src/observe/routes/evaluators.ts | 164 +++++++++++++++++++ src/observe/routes/index.ts | 4 +- src/observe/schema.ts | 17 ++ 7 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 src/observe/evaluators.ts create mode 100644 src/observe/routes/evaluators.test.ts create mode 100644 src/observe/routes/evaluators.ts diff --git a/docs/observability.md b/docs/observability.md index db215cbcb..8e0b34573 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -584,6 +584,39 @@ example body may be up to 1 MiB. Retention never deletes datasets or examples. An example keeps its `source_trace_id` after that trace is pruned. +### Online evaluator rules + +An evaluator rule says which new traces to score and how. A `code` rule holds +the source of a function `(trace) => number | { value, comment }`, and a +`judge` rule holds a model and a rubric to grade by. The daemon only stores +rules for now: it does not compile a `source` or call a model yet. Running +rules on new traces comes separately. The routes take the same token as +ingest: + +| Route | Answer | +| --- | --- | +| `POST /api/v1/evaluators` `{ "name", "kind", "config", "filter"?, "sample_rate"?, "enabled"? }` | `201 { evaluator }`. A taken name is a `409`. | +| `GET /api/v1/evaluators?limit&offset&kind&enabled` | `{ evaluators, total, limit, offset }`, oldest first. `limit` defaults to 100 (max 1000). `kind` is `code` or `judge` and `enabled` is `true` or `false`; any other value is a `400`. | +| `GET /api/v1/evaluators/:id` | `{ evaluator }`. | +| `PUT /api/v1/evaluators/:id` | `{ evaluator }`. The body is any subset of the create fields. The merged rule is checked as a whole, so changing `kind` needs a matching `config` in the same request. Renaming onto a taken name is a `409`. | +| `DELETE /api/v1/evaluators/:id` | `204`. | + +A rule row is `{ rule_id, name, enabled, filter, sample_rate, kind, config, +created_at, updated_at }`. The fields are: + +- `name`: 1-200 characters after trimming, unique. +- `enabled`: a boolean, `true` by default. +- `filter`: `{ "agent_id"?, "status"?, "name_contains"? }`, `{}` by default. + `status` is `ok` or `error`, and each value is a non-empty string. +- `sample_rate`: a number from 0 to 1, `1` by default. +- `kind` and `config`: `code` takes `{ "source" }` (up to 64 KiB of text), and + `judge` takes `{ "model", "rubric" }` (up to 200 and 20 000 characters). + +An unknown field, an unknown `kind` or a `config` that does not match the +`kind` is a `400` with code `LOUSHO_REQUEST_INVALID`. An unknown rule is a +`404` with the same code. `updated_at` moves on every `PUT`. The body may be up +to 128 KiB. Retention never deletes rules. + ### Retention The daemon keeps 30 days of runs by default. On its first sweep after start, diff --git a/llms-full.txt b/llms-full.txt index d275bfe2f..75e7422ee 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -14388,6 +14388,39 @@ example body may be up to 1 MiB. Retention never deletes datasets or examples. An example keeps its `source_trace_id` after that trace is pruned. +### Online evaluator rules + +An evaluator rule says which new traces to score and how. A `code` rule holds +the source of a function `(trace) => number | { value, comment }`, and a +`judge` rule holds a model and a rubric to grade by. The daemon only stores +rules for now: it does not compile a `source` or call a model yet. Running +rules on new traces comes separately. The routes take the same token as +ingest: + +| Route | Answer | +| --- | --- | +| `POST /api/v1/evaluators` `{ "name", "kind", "config", "filter"?, "sample_rate"?, "enabled"? }` | `201 { evaluator }`. A taken name is a `409`. | +| `GET /api/v1/evaluators?limit&offset&kind&enabled` | `{ evaluators, total, limit, offset }`, oldest first. `limit` defaults to 100 (max 1000). `kind` is `code` or `judge` and `enabled` is `true` or `false`; any other value is a `400`. | +| `GET /api/v1/evaluators/:id` | `{ evaluator }`. | +| `PUT /api/v1/evaluators/:id` | `{ evaluator }`. The body is any subset of the create fields. The merged rule is checked as a whole, so changing `kind` needs a matching `config` in the same request. Renaming onto a taken name is a `409`. | +| `DELETE /api/v1/evaluators/:id` | `204`. | + +A rule row is `{ rule_id, name, enabled, filter, sample_rate, kind, config, +created_at, updated_at }`. The fields are: + +- `name`: 1-200 characters after trimming, unique. +- `enabled`: a boolean, `true` by default. +- `filter`: `{ "agent_id"?, "status"?, "name_contains"? }`, `{}` by default. + `status` is `ok` or `error`, and each value is a non-empty string. +- `sample_rate`: a number from 0 to 1, `1` by default. +- `kind` and `config`: `code` takes `{ "source" }` (up to 64 KiB of text), and + `judge` takes `{ "model", "rubric" }` (up to 200 and 20 000 characters). + +An unknown field, an unknown `kind` or a `config` that does not match the +`kind` is a `400` with code `LOUSHO_REQUEST_INVALID`. An unknown rule is a +`404` with the same code. `updated_at` moves on every `PUT`. The body may be up +to 128 KiB. Retention never deletes rules. + ### Retention The daemon keeps 30 days of runs by default. On its first sweep after start, diff --git a/src/observe/evaluators.ts b/src/observe/evaluators.ts new file mode 100644 index 000000000..313118709 --- /dev/null +++ b/src/observe/evaluators.ts @@ -0,0 +1,152 @@ +/** + * Online evaluator rules data access: which traces to score (`filter`, + * `sample_rate`) and how (`kind` 'code' with `{ source }`, or 'judge' with + * `{ model, rubric }`). Rules are only stored here; nothing compiles a + * `source` or calls a model. Kept out of store.ts so retention never + * learns about them. + */ + +import { randomUUID } from 'node:crypto'; + +import type { SqlDatabase } from '../storage/sqlite/driver'; + +/** Which traces a rule applies to; every key set must match. */ +export interface EvaluatorRuleFilter { + agent_id?: string; + status?: 'ok' | 'error'; + name_contains?: string; +} + +/** A `code` rule's config: the source of `(trace) => number | { value, comment }`. */ +export interface CodeEvaluatorConfig { + source: string; +} + +/** A `judge` rule's config: the model that grades and the rubric it grades by. */ +export interface JudgeEvaluatorConfig { + model: string; + rubric: string; +} + +/** The editable fields of a rule; `config` matches `kind`. */ +export type EvaluatorRuleFields = { + name: string; + enabled: boolean; + filter: EvaluatorRuleFilter; + sample_rate: number; +} & ({ kind: 'code'; config: CodeEvaluatorConfig } | { kind: 'judge'; config: JudgeEvaluatorConfig }); + +/** One stored rule. */ +export type EvaluatorRuleRow = EvaluatorRuleFields & { + rule_id: string; + created_at: number; + updated_at: number; +}; + +function toRule(row: Record): EvaluatorRuleRow { + return { + rule_id: String(row.rule_id), + name: String(row.name), + enabled: Number(row.enabled) === 1, + filter: JSON.parse(String(row.filter)) as EvaluatorRuleFilter, + sample_rate: Number(row.sample_rate), + kind: String(row.kind), + config: JSON.parse(String(row.config)) as unknown, + created_at: Number(row.created_at), + updated_at: Number(row.updated_at), + } as EvaluatorRuleRow; +} + +/** The rule `ruleId`, or `undefined`. */ +export function getEvaluatorRule(db: SqlDatabase, ruleId: string): EvaluatorRuleRow | undefined { + const row = db.prepare('SELECT * FROM evaluator_rules WHERE rule_id = ?').get(ruleId); + return row === undefined ? undefined : toRule(row); +} + +/** Whether a rule other than `exceptId` is called `name`. */ +function nameTaken(db: SqlDatabase, name: string, exceptId: string | null): boolean { + const row = db.prepare('SELECT rule_id FROM evaluator_rules WHERE name = ?').get(name); + return row !== undefined && String(row.rule_id) !== exceptId; +} + +/** Create a rule; `undefined` when the name is taken. */ +export function createEvaluatorRule( + db: SqlDatabase, + fields: EvaluatorRuleFields, + now: number = Date.now() +): EvaluatorRuleRow | undefined { + if (nameTaken(db, fields.name, null)) return undefined; + const ruleId = randomUUID(); + db.prepare( + `INSERT INTO evaluator_rules (rule_id, name, enabled, filter, sample_rate, kind, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + ruleId, + fields.name, + fields.enabled ? 1 : 0, + JSON.stringify(fields.filter), + fields.sample_rate, + fields.kind, + JSON.stringify(fields.config), + now, + now + ); + return getEvaluatorRule(db, ruleId); +} + +/** + * Replace a rule's fields and bump `updated_at`: `'missing'` for an unknown + * id, `'name-taken'` when another rule has the name. + */ +export function updateEvaluatorRule( + db: SqlDatabase, + ruleId: string, + fields: EvaluatorRuleFields, + now: number = Date.now() +): EvaluatorRuleRow | 'missing' | 'name-taken' { + const current = getEvaluatorRule(db, ruleId); + if (current === undefined) return 'missing'; + if (nameTaken(db, fields.name, ruleId)) return 'name-taken'; + db.prepare( + `UPDATE evaluator_rules SET name = ?, enabled = ?, filter = ?, sample_rate = ?, kind = ?, config = ?, updated_at = ? + WHERE rule_id = ?` + ).run( + fields.name, + fields.enabled ? 1 : 0, + JSON.stringify(fields.filter), + fields.sample_rate, + fields.kind, + JSON.stringify(fields.config), + Math.max(now, current.updated_at + 1), + ruleId + ); + return getEvaluatorRule(db, ruleId) as EvaluatorRuleRow; +} + +/** Delete a rule; `false` when it is unknown. */ +export function deleteEvaluatorRule(db: SqlDatabase, ruleId: string): boolean { + return Number(db.prepare('DELETE FROM evaluator_rules WHERE rule_id = ?').run(ruleId).changes) > 0; +} + +/** A page of rules, oldest first (insertion order breaks ties), plus the total matching. */ +export function listEvaluatorRules( + db: SqlDatabase, + options: { limit: number; offset: number; kind?: 'code' | 'judge'; enabled?: boolean } +): { evaluators: EvaluatorRuleRow[]; total: number } { + const where: string[] = []; + const args: (string | number)[] = []; + if (options.kind !== undefined) { + where.push('kind = ?'); + args.push(options.kind); + } + if (options.enabled !== undefined) { + where.push('enabled = ?'); + args.push(options.enabled ? 1 : 0); + } + const clause = where.length > 0 ? ` WHERE ${where.join(' AND ')}` : ''; + const rows = db + .prepare(`SELECT * FROM evaluator_rules${clause} ORDER BY created_at ASC, rowid ASC LIMIT ? OFFSET ?`) + .all(...args, options.limit, options.offset); + const total = db.prepare(`SELECT COUNT(*) AS n FROM evaluator_rules${clause}`).get(...args); + return { evaluators: rows.map(toRule), total: Number(total?.n ?? 0) }; +} diff --git a/src/observe/routes/evaluators.test.ts b/src/observe/routes/evaluators.test.ts new file mode 100644 index 000000000..d26d5ea46 --- /dev/null +++ b/src/observe/routes/evaluators.test.ts @@ -0,0 +1,217 @@ +/** + * Online evaluator rules API: create/list/get/update/delete rules, with the + * kind/config pairing, filter shape and sample rate validated. + */ +import { describe, expect, it } from 'vitest'; +import type * as http from 'node:http'; +import { openDb } from '../store'; +import type { EvaluatorRuleRow } from '../evaluators'; +import type { SqlDatabase } from '../../storage/sqlite/driver'; +// Through the registry, production's load order (see scores.test.ts). +import { matchRoute } from './index'; + +interface Captured { + status: number; + body: unknown; +} + +/** Calls the matched route with a recording ServerResponse stand-in. */ +async function call( + database: SqlDatabase, + method: 'GET' | 'POST' | 'PUT' | 'DELETE', + path: string, + body?: unknown +): Promise { + const captured: Captured = { status: 0, body: undefined }; + const res = { + writeHead(status: number) { + captured.status = status; + return res; + }, + end(chunk?: string) { + captured.body = chunk === undefined || chunk === '' ? undefined : JSON.parse(chunk); + }, + } as unknown as http.ServerResponse; + const [pathname, search = ''] = path.split('?'); + const matched = matchRoute(method, pathname); + if (!matched) throw new Error(`no route for ${method} ${pathname}`); + const text = typeof body === 'string' ? body : JSON.stringify(body ?? {}); + await matched.route.handle({ + db: database, + dbPath: ':memory:', + req: {} as http.IncomingMessage, + res, + params: matched.params, + query: new URLSearchParams(search), + readBody: () => Promise.resolve(text), + }); + return captured; +} + +async function withDb(work: (database: SqlDatabase) => Promise): Promise { + const database = openDb(':memory:'); + try { + await work(database); + } finally { + database.close(); + } +} + +const codeRule = { name: 'length', kind: 'code', config: { source: '(trace) => 1' } }; +const judgeRule = { name: 'helpful', kind: 'judge', config: { model: 'gpt-4o-mini', rubric: 'Is it helpful?' } }; + +const create = (database: SqlDatabase, body: unknown) => call(database, 'POST', '/api/v1/evaluators', body); +const rule = (captured: Captured) => (captured.body as { evaluator: EvaluatorRuleRow }).evaluator; +const invalid = { code: 'LOUSHO_REQUEST_INVALID' }; + +describe('evaluator rules API', () => { + it('creates code and judge rules with defaults applied', async () => { + await withDb(async (database) => { + const code = await create(database, codeRule); + expect(code.status).toBe(201); + expect(rule(code)).toMatchObject({ + name: 'length', + enabled: true, + filter: {}, + sample_rate: 1, + kind: 'code', + config: { source: '(trace) => 1' }, + }); + expect(typeof rule(code).rule_id).toBe('string'); + expect(rule(code).updated_at).toBe(rule(code).created_at); + + const judge = await create(database, { + ...judgeRule, + name: ' helpful ', + enabled: false, + sample_rate: 0.25, + filter: { agent_id: 'support', status: 'error', name_contains: 'refund' }, + }); + expect(judge.status).toBe(201); + expect(rule(judge)).toMatchObject({ + name: 'helpful', + enabled: false, + sample_rate: 0.25, + kind: 'judge', + filter: { agent_id: 'support', status: 'error', name_contains: 'refund' }, + config: { model: 'gpt-4o-mini', rubric: 'Is it helpful?' }, + }); + }); + }); + + it('400s an unknown kind, naming the allowed kinds', async () => { + await withDb(async (database) => { + await expect(create(database, { ...codeRule, kind: 'regex' })).rejects.toMatchObject({ + ...invalid, + message: expect.stringMatching(/code.*judge/), + }); + }); + }); + + it('400s a config that does not match the kind, and bad fields', async () => { + await withDb(async (database) => { + const bad: unknown[] = [ + { ...codeRule, config: judgeRule.config }, + { ...judgeRule, config: codeRule.config }, + { ...codeRule, config: { source: '' } }, + { ...codeRule, config: { source: 'x', extra: 1 } }, + { ...codeRule, sample_rate: 1.5 }, + { ...codeRule, sample_rate: -0.1 }, + { ...codeRule, filter: { tag: 'x' } }, + { ...codeRule, filter: { status: 'pending' } }, + { ...codeRule, filter: { agent_id: '' } }, + { ...codeRule, name: ' ' }, + { ...codeRule, unknown: true }, + { kind: 'code', config: codeRule.config }, + ]; + for (const body of bad) await expect(create(database, body)).rejects.toMatchObject(invalid); + }); + }); + + it('409s a duplicate name', async () => { + await withDb(async (database) => { + await create(database, codeRule); + const dup = await create(database, { ...judgeRule, name: 'length' }); + expect(dup.status).toBe(409); + expect(dup.body).toMatchObject(invalid); + }); + }); + + it('lists oldest first with paging and kind/enabled filters', async () => { + await withDb(async (database) => { + await create(database, { ...codeRule, name: 'a' }); + await create(database, { ...judgeRule, name: 'b', enabled: false }); + await create(database, { ...codeRule, name: 'c' }); + + const all = await call(database, 'GET', '/api/v1/evaluators'); + expect(all.status).toBe(200); + const body = all.body as { evaluators: EvaluatorRuleRow[]; total: number; limit: number; offset: number }; + expect(body.evaluators.map((r) => r.name)).toEqual(['a', 'b', 'c']); + expect(body).toMatchObject({ total: 3, limit: 100, offset: 0 }); + + const page = (await call(database, 'GET', '/api/v1/evaluators?limit=1&offset=1')).body as typeof body; + expect(page.evaluators.map((r) => r.name)).toEqual(['b']); + expect(page.total).toBe(3); + + const codes = (await call(database, 'GET', '/api/v1/evaluators?kind=code')).body as typeof body; + expect(codes.evaluators.map((r) => r.name)).toEqual(['a', 'c']); + expect(codes.total).toBe(2); + + const off = (await call(database, 'GET', '/api/v1/evaluators?enabled=false')).body as typeof body; + expect(off.evaluators.map((r) => r.name)).toEqual(['b']); + + await expect(call(database, 'GET', '/api/v1/evaluators?kind=regex')).rejects.toMatchObject(invalid); + await expect(call(database, 'GET', '/api/v1/evaluators?enabled=yes')).rejects.toMatchObject(invalid); + }); + }); + + it('gets one rule; 404s an unknown id', async () => { + await withDb(async (database) => { + const created = rule(await create(database, codeRule)); + const got = await call(database, 'GET', `/api/v1/evaluators/${created.rule_id}`); + expect(got.status).toBe(200); + expect(rule(got)).toEqual(created); + const missing = await call(database, 'GET', '/api/v1/evaluators/nope'); + expect(missing.status).toBe(404); + expect(missing.body).toMatchObject(invalid); + }); + }); + + it('PUT merges a partial update, bumps updated_at, and validates the merged rule', async () => { + await withDb(async (database) => { + const created = rule(await create(database, codeRule)); + await create(database, judgeRule); + await new Promise((resolve) => setTimeout(resolve, 5)); + + const toggled = await call(database, 'PUT', `/api/v1/evaluators/${created.rule_id}`, { enabled: false }); + expect(toggled.status).toBe(200); + expect(rule(toggled)).toMatchObject({ enabled: false, name: 'length', config: codeRule.config }); + expect(rule(toggled).created_at).toBe(created.created_at); + expect(rule(toggled).updated_at).toBeGreaterThan(created.updated_at); + + await expect( + call(database, 'PUT', `/api/v1/evaluators/${created.rule_id}`, { kind: 'judge' }) + ).rejects.toMatchObject(invalid); + const switched = await call(database, 'PUT', `/api/v1/evaluators/${created.rule_id}`, { + kind: 'judge', + config: judgeRule.config, + }); + expect(rule(switched)).toMatchObject({ kind: 'judge', config: judgeRule.config }); + + const renamed = await call(database, 'PUT', `/api/v1/evaluators/${created.rule_id}`, { name: 'helpful' }); + expect(renamed.status).toBe(409); + const missing = await call(database, 'PUT', '/api/v1/evaluators/nope', { enabled: true }); + expect(missing.status).toBe(404); + }); + }); + + it('deletes a rule with 204, then 404s it', async () => { + await withDb(async (database) => { + const created = rule(await create(database, codeRule)); + const deleted = await call(database, 'DELETE', `/api/v1/evaluators/${created.rule_id}`); + expect(deleted.status).toBe(204); + expect((await call(database, 'GET', `/api/v1/evaluators/${created.rule_id}`)).status).toBe(404); + expect((await call(database, 'DELETE', `/api/v1/evaluators/${created.rule_id}`)).status).toBe(404); + }); + }); +}); diff --git a/src/observe/routes/evaluators.ts b/src/observe/routes/evaluators.ts new file mode 100644 index 000000000..d47087eee --- /dev/null +++ b/src/observe/routes/evaluators.ts @@ -0,0 +1,164 @@ +/** + * Online evaluator rules (LangSmith "online evaluators", the config half): + * which traces to score and how. Rules are stored only; running them on new + * traces is a separate piece. + * + * - `GET /api/v1/evaluators?limit&offset&kind&enabled` -> + * `{ evaluators, total, limit, offset }`, oldest first (the runner's order). + * - `POST /api/v1/evaluators` `{ name, kind, config, filter?, sample_rate?, enabled? }` + * -> `201 { evaluator }` (a taken name is a 409). + * - `GET /api/v1/evaluators/:id` -> `{ evaluator }`. + * - `PUT /api/v1/evaluators/:id` any subset of the create fields; the merged + * rule is validated as a whole -> `{ evaluator }`. + * - `DELETE /api/v1/evaluators/:id` -> `204`. + * + * Auth is the daemon's global API-token check; nothing per-route. + */ + +import type * as http from 'node:http'; +import { z } from 'zod'; + +import { SDKError } from '../../execution/errors'; +import { + createEvaluatorRule, + deleteEvaluatorRule, + getEvaluatorRule, + listEvaluatorRules, + updateEvaluatorRule, + type EvaluatorRuleFields, +} from '../evaluators'; +import { intParam, requireJsonBody, sendJson, type Route } from './httpKit'; + +const RULE_BODY_LIMIT = 128 * 1024; + +const KINDS = ['code', 'judge'] as const; +const kindSchema = z.enum(KINDS, { + errorMap: () => ({ message: `must be one of ${KINDS.map((k) => `'${k}'`).join(', ')}` }), +}); + +const nonEmpty = z.string().min(1); + +const filterSchema = z + .object({ agent_id: nonEmpty.optional(), status: z.enum(['ok', 'error']).optional(), name_contains: nonEmpty.optional() }) + .strict(); + +const fieldShape = { + name: z.string().trim().min(1).max(200), + enabled: z.boolean(), + filter: filterSchema, + sample_rate: z.number().finite().min(0).max(1), + kind: kindSchema, + config: z.unknown(), +}; + +const createBody = z + .object({ + ...fieldShape, + enabled: fieldShape.enabled.default(true), + filter: filterSchema.default({}), + sample_rate: fieldShape.sample_rate.default(1), + }) + .strict(); + +const updateBody = z.object(fieldShape).partial().strict(); + +const configSchemas = { + code: z.object({ source: z.string().min(1).max(65_536) }).strict(), + judge: z.object({ model: z.string().min(1).max(200), rubric: z.string().min(1).max(20_000) }).strict(), +}; + +/** The rule as a whole, its `config` checked against its `kind`. */ +function withConfig(rule: z.infer): EvaluatorRuleFields { + const parsed = configSchemas[rule.kind].safeParse(rule.config); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + const where = ['config', ...(issue?.path ?? [])].join('.'); + throw new SDKError( + `Invalid evaluator rule body: ${where}: ${issue?.message ?? 'bad shape'} (for kind '${rule.kind}').`, + 'LOUSHO_REQUEST_INVALID', + { appendHelp: false } + ); + } + return { ...rule, config: parsed.data } as EvaluatorRuleFields; +} + +const notFound = (res: http.ServerResponse, id: string): void => + sendJson(res, 404, { error: `No evaluator rule '${id}'.`, code: 'LOUSHO_REQUEST_INVALID' }); + +const nameTaken = (res: http.ServerResponse, name: string): void => + sendJson(res, 409, { error: `An evaluator rule named '${name}' already exists.`, code: 'LOUSHO_REQUEST_INVALID' }); + +/** The list's `kind`/`enabled` query filters; a bad value throws `LOUSHO_REQUEST_INVALID`. */ +function listFilters(query: URLSearchParams): { kind?: 'code' | 'judge'; enabled?: boolean } { + const kind = query.get('kind'); + const enabled = query.get('enabled'); + if (kind !== null && !kindSchema.safeParse(kind).success) { + throw new SDKError(`Invalid evaluator list query: kind must be one of 'code', 'judge'.`, 'LOUSHO_REQUEST_INVALID', { + appendHelp: false, + }); + } + if (enabled !== null && enabled !== 'true' && enabled !== 'false') { + throw new SDKError(`Invalid evaluator list query: enabled must be 'true' or 'false'.`, 'LOUSHO_REQUEST_INVALID', { + appendHelp: false, + }); + } + return { + kind: kind === null ? undefined : (kind as 'code' | 'judge'), + enabled: enabled === null ? undefined : enabled === 'true', + }; +} + +export const evaluatorsRoutes: readonly Route[] = [ + { + method: 'GET', + path: '/api/v1/evaluators', + handle: ({ db, res, query }) => { + const limit = intParam(query, 'limit', 100, { min: 1, max: 1000 }); + const offset = intParam(query, 'offset', 0, { min: 0, max: Number.MAX_SAFE_INTEGER }); + sendJson(res, 200, { ...listEvaluatorRules(db, { limit, offset, ...listFilters(query) }), limit, offset }); + }, + }, + { + method: 'POST', + path: '/api/v1/evaluators', + handle: async ({ db, res, readBody }) => { + const fields = withConfig(requireJsonBody(createBody, await readBody(RULE_BODY_LIMIT), 'evaluator rule')); + const evaluator = createEvaluatorRule(db, fields); + if (evaluator === undefined) return nameTaken(res, fields.name); + sendJson(res, 201, { evaluator }); + }, + }, + { + method: 'GET', + path: '/api/v1/evaluators/:id', + handle: ({ db, res, params }) => { + const evaluator = getEvaluatorRule(db, params.id); + if (evaluator === undefined) return notFound(res, params.id); + sendJson(res, 200, { evaluator }); + }, + }, + { + method: 'PUT', + path: '/api/v1/evaluators/:id', + handle: async ({ db, res, params, readBody }) => { + const patch = requireJsonBody(updateBody, await readBody(RULE_BODY_LIMIT), 'evaluator rule'); + const current = getEvaluatorRule(db, params.id); + if (current === undefined) return notFound(res, params.id); + const { name, enabled, filter, sample_rate, kind, config } = current; + const fields = withConfig({ name, enabled, filter, sample_rate, kind, config, ...patch }); + const updated = updateEvaluatorRule(db, params.id, fields); + if (updated === 'missing') return notFound(res, params.id); + if (updated === 'name-taken') return nameTaken(res, fields.name); + sendJson(res, 200, { evaluator: updated }); + }, + }, + { + method: 'DELETE', + path: '/api/v1/evaluators/:id', + handle: ({ db, res, params }) => { + if (!deleteEvaluatorRule(db, params.id)) return notFound(res, params.id); + res.writeHead(204); + res.end(); + }, + }, +]; diff --git a/src/observe/routes/index.ts b/src/observe/routes/index.ts index d9e8af2af..de27bd5d1 100644 --- a/src/observe/routes/index.ts +++ b/src/observe/routes/index.ts @@ -1,6 +1,6 @@ /** * The daemon's route registry. One file per feature area (ingest, traces, - * insights, agents, scores, status, prompts, hub, datasets) exports a `routes` array; this index + * insights, agents, scores, status, prompts, hub, datasets, evaluators) exports a `routes` array; this index * concatenates them - so a later child spec (prompts, hub, alerts) adds a * file here and never touches the daemon's dispatch internals. * @@ -12,6 +12,7 @@ import { agentsRoutes } from './agents'; import { datasetsRoutes } from './datasets'; +import { evaluatorsRoutes } from './evaluators'; import { hubRoutes } from './hub'; import { ingestRoutes } from './ingest'; import { insightsRoutes } from './insights'; @@ -40,6 +41,7 @@ const routes: readonly Route[] = [ ...promptsRoutes, ...hubRoutes, ...datasetsRoutes, + ...evaluatorsRoutes, ]; interface CompiledRoute { diff --git a/src/observe/schema.ts b/src/observe/schema.ts index 23d49bc58..7b197e9de 100644 --- a/src/observe/schema.ts +++ b/src/observe/schema.ts @@ -232,6 +232,23 @@ export const OBSERVE_MIGRATIONS: readonly string[] = [ ); CREATE INDEX dataset_examples_by_dataset ON dataset_examples (dataset_id, created_at); `, + ` + -- Online evaluator rules: which traces to score (filter JSON, sample_rate + -- in [0, 1]) and how (kind 'code' with config { source }, or 'judge' with + -- config { model, rubric }). Stored only; the runner reads them in + -- created_at order. + CREATE TABLE evaluator_rules ( + rule_id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + enabled INTEGER NOT NULL DEFAULT 1, + filter TEXT NOT NULL DEFAULT '{}', + sample_rate REAL NOT NULL DEFAULT 1, + kind TEXT NOT NULL CHECK(kind IN ('code','judge')), + config TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + `, ]; /** From 12800279a0f39e2a133e919e6a9ebf682b475444 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 29 Oct 2026 00:05:40 +0300 Subject: [PATCH 2/2] fix(observe): evaluator kind validation works on zod 4 z.enum's errorMap option is zod-3 only; the { message } form typechecks and names the allowed kinds under both zod 3 and zod 4. Co-Authored-By: Claude Opus 5.5 --- src/observe/routes/evaluators.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/observe/routes/evaluators.ts b/src/observe/routes/evaluators.ts index d47087eee..ee62fb7da 100644 --- a/src/observe/routes/evaluators.ts +++ b/src/observe/routes/evaluators.ts @@ -32,9 +32,8 @@ import { intParam, requireJsonBody, sendJson, type Route } from './httpKit'; const RULE_BODY_LIMIT = 128 * 1024; const KINDS = ['code', 'judge'] as const; -const kindSchema = z.enum(KINDS, { - errorMap: () => ({ message: `must be one of ${KINDS.map((k) => `'${k}'`).join(', ')}` }), -}); +// `{ message }` is the custom-error form zod 3 and zod 4 both accept. +const kindSchema = z.enum(KINDS, { message: `must be one of ${KINDS.map((k) => `'${k}'`).join(', ')}` }); const nonEmpty = z.string().min(1);