feat(evals): run evals over a registry dataset

api/index.api.md

@@ -1550,6 +1550,15 @@ interface CustomWebhookAuth {
     verify(req: IncomingMessage, rawBody: Buffer): boolean | Promise<boolean>;
 }
 
+// @public
+export interface DatasetCase {
+    example_id: string;
+    expected: unknown | null;
+    input: unknown;
+    label: string;
+    source_trace_id: string | null;
+}
+
 // @public
 export const dayNameTool: ToolDescriptor;
 
@@ -1638,6 +1647,12 @@ export interface DefinedTool<S extends StandardSchemaV1 = StandardSchemaV1, O =
 // @public
 export function defineEval(config: EvalConfig): void;
 
+// @public (undocumented)
+export function defineEval(config: TrajectoryEvalConfig<DatasetCase> & {
+    dataset: string;
+    cases?: undefined;
+}): void;
+
 // @public (undocumented)
 export function defineEval<C = Record<string, never>>(config: TrajectoryEvalConfig<C>): void;
 
@@ -1938,6 +1953,7 @@ export const ERROR_CODES: {
     readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example.";
     readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target.";
     readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose.";
+    readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it.";
     readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured).";
     readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation.";
     readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names.";
@@ -7144,6 +7160,7 @@ export interface TrajectoryComparison {
 export interface TrajectoryEvalConfig<C = Record<string, never>> extends RepeatConfig {
     agent?: AgentSource;
     cases?: readonly C[];
+    dataset?: string;
     judge?: EvalJudgeConfig;
     name: string;
     tags?: string[];

api/mcp.api.md

@@ -575,6 +575,7 @@ const ERROR_CODES: {
     readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example.";
     readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target.";
     readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose.";
+    readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it.";
     readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured).";
     readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation.";
     readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names.";

api/testing.api.md

@@ -97,6 +97,7 @@ const ERROR_CODES: {
     readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example.";
     readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target.";
     readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose.";
+    readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it.";
     readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured).";
     readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation.";
     readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names.";

api/tools.api.md

@@ -671,6 +671,7 @@ const ERROR_CODES: {
     readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example.";
     readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target.";
     readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose.";
+    readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it.";
     readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured).";
     readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation.";
     readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names.";

api/worker.api.md

@@ -1312,6 +1312,7 @@ const ERROR_CODES: {
     readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example.";
     readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target.";
     readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose.";
+    readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it.";
     readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured).";
     readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation.";
     readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names.";

docs/errors.md

@@ -99,7 +99,7 @@ Find a code by area:
 | [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied), [`LOUSHO_SANDBOX_FAILED`](#lousho_sandbox_failed) | Asking a sandbox for something it cannot do on the current platform, writing outside its root, or a code-sandbox boundary (timeout / OOM / driver) failing a run. |
 | [Agent directories, skills and flows](#agent-directories-skills-and-flows) | [`LOUSHO_AGENT_DIR_INVALID`](#lousho_agent_dir_invalid), [`LOUSHO_SKILL_INVALID`](#lousho_skill_invalid), [`LOUSHO_FLOW_INVALID`](#lousho_flow_invalid), [`LOUSHO_FLOW_TOOL_DENIED`](#lousho_flow_tool_denied), [`LOUSHO_FLOW_CHANGED`](#lousho_flow_changed) | Loading an agent directory, a skill, or a flow definition, and gating a flow's tool calls. |
 | [Storage, deployment and integrations](#storage-deployment-and-integrations) | [`LOUSHO_STORAGE_FAILED`](#lousho_storage_failed), [`LOUSHO_STORAGE_BUSY`](#lousho_storage_busy), [`LOUSHO_TRIGGER_INVALID`](#lousho_trigger_invalid), [`LOUSHO_CHANNEL_REQUEST_FAILED`](#lousho_channel_request_failed), [`LOUSHO_DEPLOY_FAILED`](#lousho_deploy_failed), [`LOUSHO_SERVER_UNPROTECTED`](#lousho_server_unprotected) | A storage backend, a trigger, a channel request, `lousho build` or the built server's start-up. |
-| [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. |
+| [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_DATASET_UNAVAILABLE`](#lousho_dataset_unavailable), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. |
 | [General](#general) | [`LOUSHO_GENERIC_ERROR`](#lousho_generic_error), [`LOUSHO_AGENT_EXECUTION_FAILED`](#lousho_agent_execution_failed), [`LOUSHO_FLOW_EXECUTION_FAILED`](#lousho_flow_execution_failed), [`LOUSHO_VALIDATION_FAILED`](#lousho_validation_failed), [`LOUSHO_OPERATION_TIMEOUT`](#lousho_operation_timeout), [`LOUSHO_OUTPUT_INVALID`](#lousho_output_invalid), [`LOUSHO_BUDGET_EXCEEDED`](#lousho_budget_exceeded), [`LOUSHO_GUARDRAIL_TRIPPED`](#lousho_guardrail_tripped) | Run-level failures: a timeout, a budget or guardrail stop, invalid output, and the catch-all codes. |
 | [Auth](#auth) | [`LOUSHO_AUTH_CONFIG_INVALID`](#lousho_auth_config_invalid) | A route auth helper (`jwt()`, `oidc()`, `basic()`, `apiToken()`) got options it cannot verify with. |
 | [OAuth](#oauth) | [`LOUSHO_TOKEN_KEY_MISSING`](#lousho_token_key_missing), [`LOUSHO_TOKEN_DECRYPT_FAILED`](#lousho_token_decrypt_failed), [`LOUSHO_OAUTH_PRINCIPAL_REQUIRED`](#lousho_oauth_principal_required), [`LOUSHO_OAUTH_APP_SIGNIN_REQUIRED`](#lousho_oauth_app_signin_required), [`LOUSHO_OAUTH_STORE_MISSING`](#lousho_oauth_store_missing), [`LOUSHO_OAUTH_STATE_INVALID`](#lousho_oauth_state_invalid), [`LOUSHO_SIGNIN_PENDING`](#lousho_signin_pending), [`LOUSHO_OAUTH_TOKEN_EXCHANGE_FAILED`](#lousho_oauth_token_exchange_failed), [`LOUSHO_MCP_AUTH_REQUIRED`](#lousho_mcp_auth_required) | Storing or reading OAuth tokens in a file, SQLite or KV store, sign-in and token exchange for OAuth tools and MCP servers. |
@@ -1141,6 +1141,23 @@ judge runner.
 
 **Example:** calling `t.judge('polite')` before `t.send('hi')`.
 
+### LOUSHO_DATASET_UNAVAILABLE
+
+**Means:** `defineEval({ dataset: 'name' })` could not get the dataset's examples
+from the observe daemon when vitest collected the file: `LOUSHO_OBSERVE` is not
+set, the daemon is unreachable or did not answer in 10 seconds, it answered an
+error (a `404` says the dataset was not found), its answer had no `examples`,
+or the dataset is empty. The eval then fails as one test named after it, so it
+never passes green having run nothing. The message names the dataset and the
+daemon URL.
+
+**Fix:** start the daemon (`lousho observe`) and set `LOUSHO_OBSERVE` to its URL
+(and `LOUSHO_OBSERVE_TOKEN` for a `--require-token` daemon), then create the
+dataset and add examples to it (`POST /api/v1/datasets`, `POST
+/api/v1/datasets/:name/examples`). See [Evals](./evals.md#from-the-daemon).
+
+**Example:** `lousho eval` with `dataset: 'refunds'` and no `LOUSHO_OBSERVE` set.
+
 ### LOUSHO_TEST_FAILED
 
 **Means:** a check a test helper makes did not hold: an eval did not pass, or a

docs/evals.md

@@ -202,6 +202,51 @@ Pass a **factory** (`agent: () => createAgent(...)`) so each case gets its own
 agent and its own `mockModel` script. With a single shared agent, a scripted
 model runs out of turns on the second case.
 
+#### From the daemon
+
+The cases can also live in the observe daemon's dataset registry (see
+[Observability](./observability.md), `/api/v1/datasets`): create a dataset, add examples to it
+by hand or copy them from a trace you liked, then pass `dataset: 'name'` in
+place of `cases`.
+
+```bash
+curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets" -H 'content-type: application/json' -d '{"name":"refunds"}'
+curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets/refunds/examples" -H 'content-type: application/json' \
+  -d '{"input":"Refund order 42","expected":"refunded"}'
+curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets/refunds/examples" -H 'content-type: application/json' \
+  -d '{"from_trace_id":"<a trace id>"}'
+```
+
+```ts no-run
+import { createAgent, defineEval, includes } from '@lousho/build-ai-agent';
+
+defineEval({
+  name: 'refunds',
+  dataset: 'refunds',
+  agent: () => createAgent({ instructions: 'You handle refunds.' }),
+  async test(t, c) {
+    await t.send(String(c.input));
+    t.completed();
+    if (typeof c.expected === 'string') t.soft('mentions the outcome', t.reply, includes(c.expected));
+  },
+});
+```
+
+When vitest collects the file, `defineEval()` fetches every example of the
+dataset from the daemon `LOUSHO_OBSERVE` points at (sending
+`LOUSHO_OBSERVE_TOKEN` when set) and registers one test per example, oldest
+first: `refunds [#1]`, `refunds [#2]`, .... Each `c` is a `DatasetCase`:
+`{ input, expected, label, example_id, source_trace_id }`, where `input` and
+`expected` are whatever JSON the example holds (`expected` is `null` when
+unset). `LOUSHO_OBSERVE` is read when `defineEval()` is called.
+
+The eval fails with
+[`LOUSHO_DATASET_UNAVAILABLE`](./errors.md#lousho_dataset_unavailable) when
+`LOUSHO_OBSERVE` is not set, the daemon cannot be reached in 10 seconds, the
+dataset does not exist, or it has no examples: an eval that runs nothing never
+passes. Passing both `cases` and `dataset` throws `LOUSHO_EVALS_INVALID`. A
+dataset eval that `lousho eval --tag` filters out is skipped without a fetch.
+
 ## Judge evals
 
 `t.judge(rubric)` grades the latest reply with an LLM and returns a score from