From d55c8b78c11538752a2cc45d420fdfb31494567e Mon Sep 17 00:00:00 2001 From: Kem-Ada <283358351+Kem-Ada@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:48:51 +0100 Subject: [PATCH] feat(validation): add tool definition and message content validators Port identark/validation.py to src/validation.ts: - validateToolDefinitions: OpenAI function-calling schema checks, throws ConfigurationError on malformed tools - validateToolResultJson: warns (console.warn) on non-JSON tool results - validateMessageContent: string | content-block array | null Wire validateToolDefinitions into DirectGateway invokeLlm/invokeLlmStream (as Python's DirectGateway does) and export all three from the package index. MockGateway is left unvalidated, matching the Python MockGateway. ControlPlaneGateway wiring is deferred to PR #8. Co-Authored-By: Claude --- src/gateways/direct.ts | 3 + src/index.ts | 7 ++ src/validation.ts | 135 ++++++++++++++++++++++ tests/validation.test.ts | 242 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 387 insertions(+) create mode 100644 src/validation.ts create mode 100644 tests/validation.test.ts diff --git a/src/gateways/direct.ts b/src/gateways/direct.ts index 311a15f..2fbd5a1 100644 --- a/src/gateways/direct.ts +++ b/src/gateways/direct.ts @@ -32,6 +32,7 @@ import { } from "../errors.js"; import { estimateCost } from "../pricing.js"; +import { validateToolDefinitions } from "../validation.js"; /** * Local development implementation of AgentGateway. @@ -105,6 +106,7 @@ export class DirectGateway implements AgentGateway { toolChoice: string | Record = "auto", ): Promise { this._checkCostCap(); + validateToolDefinitions(tools); const messages = this._buildMessages(newMessages); let response: LLMResponse; @@ -164,6 +166,7 @@ export class DirectGateway implements AgentGateway { tools?: Record[], toolChoice: string | Record = "auto", ): AsyncGenerator { + validateToolDefinitions(tools); if (this.costCap !== undefined && this.totalCost >= this.costCap) { throw new CostCapExceededError( `Cost cap of $${this.costCap.toFixed(4)} reached.`, diff --git a/src/index.ts b/src/index.ts index 7c32a79..3ea4e6e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -76,6 +76,13 @@ export { ConfigurationError, } from "./errors.js"; +// Validation +export { + validateToolDefinitions, + validateToolResultJson, + validateMessageContent, +} from "./validation.js"; + // Testing export { MockGateway } from "./testing/mock-gateway.js"; diff --git a/src/validation.ts b/src/validation.ts new file mode 100644 index 0000000..05f9fcb --- /dev/null +++ b/src/validation.ts @@ -0,0 +1,135 @@ +/** + * identark/validation + * ~~~~~~~~~~~~~~~~~~~ + * Input validation utilities for tool definitions and messages. + */ + +import { ConfigurationError } from "./errors.js"; + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function typeName(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + return typeof value; +} + +/** + * Validate tool definitions follow the OpenAI function calling schema. + * + * @throws ConfigurationError if tools are malformed. + */ +export function validateToolDefinitions(tools: unknown): void { + if (tools === undefined || tools === null) { + return; + } + + if (!Array.isArray(tools)) { + throw new ConfigurationError(`tools must be an array, got ${typeName(tools)}`); + } + + tools.forEach((tool: unknown, i: number) => { + if (!isPlainObject(tool)) { + throw new ConfigurationError(`tools[${i}] must be an object, got ${typeName(tool)}`); + } + + // Required: type field + const toolType = tool.type; + if (toolType === undefined || toolType === null) { + throw new ConfigurationError(`tools[${i}] missing required 'type' field`); + } + if (toolType !== "function") { + throw new ConfigurationError(`tools[${i}].type must be 'function', got '${String(toolType)}'`); + } + + // Required: function field + const func = tool.function; + if (func === undefined || func === null) { + throw new ConfigurationError(`tools[${i}] missing required 'function' field`); + } + if (!isPlainObject(func)) { + throw new ConfigurationError( + `tools[${i}].function must be an object, got ${typeName(func)}`, + ); + } + + // Required: function.name + const name = func.name; + if (name === undefined || name === null) { + throw new ConfigurationError(`tools[${i}].function missing required 'name' field`); + } + if (typeof name !== "string" || !name) { + throw new ConfigurationError(`tools[${i}].function.name must be a non-empty string`); + } + + // Optional but common: function.description + if ("description" in func && typeof func.description !== "string") { + throw new ConfigurationError(`tools[${i}].function.description must be a string`); + } + + // Optional: function.parameters (JSON Schema). A missing 'type' is + // tolerated — providers assume 'object'. + const params = func.parameters; + if (params !== undefined && params !== null && !isPlainObject(params)) { + throw new ConfigurationError( + `tools[${i}].function.parameters must be an object (JSON Schema)`, + ); + } + }); +} + +/** + * Validate that a tool result is valid JSON. + * + * Warns but does not throw if content is not valid JSON. + * Tool results should be JSON-serializable for best LLM understanding. + */ +export function validateToolResultJson(content: string, toolName: string): void { + if (!content) { + return; + } + + try { + JSON.parse(content); + } catch { + console.warn( + `Tool result for '${toolName}' is not valid JSON. ` + + "LLMs work best when tool results are JSON-formatted.", + ); + } +} + +/** + * Validate message content structure. + * + * Content can be: + * - string: Plain text + * - array of objects: Multimodal content blocks (images, etc.) + * - null/undefined: Empty content (valid for tool calls) + * + * @throws ConfigurationError if content is malformed. + */ +export function validateMessageContent(content: unknown): void { + if (content === undefined || content === null || typeof content === "string") { + return; + } + + if (!Array.isArray(content)) { + throw new ConfigurationError( + `Message content must be string, array, or null, got ${typeName(content)}`, + ); + } + + content.forEach((block: unknown, i: number) => { + if (!isPlainObject(block)) { + throw new ConfigurationError( + `Message content[${i}] must be an object, got ${typeName(block)}`, + ); + } + if (!("type" in block)) { + throw new ConfigurationError(`Message content[${i}] missing 'type' field`); + } + }); +} diff --git a/tests/validation.test.ts b/tests/validation.test.ts new file mode 100644 index 0000000..c44a738 --- /dev/null +++ b/tests/validation.test.ts @@ -0,0 +1,242 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { + validateMessageContent, + validateToolDefinitions, + validateToolResultJson, +} from "../src/validation.js"; +import { ConfigurationError } from "../src/errors.js"; +import { DirectGateway } from "../src/gateways/direct.js"; +import { MockGateway } from "../src/testing/mock-gateway.js"; +import { Role, LLMResponse, Message } from "../src/types.js"; +import * as sdk from "../src/index.js"; + +describe("validateToolDefinitions", () => { + it("accepts undefined and null tools", () => { + expect(() => validateToolDefinitions(undefined)).not.toThrow(); + expect(() => validateToolDefinitions(null)).not.toThrow(); + }); + + it("accepts an empty array", () => { + expect(() => validateToolDefinitions([])).not.toThrow(); + }); + + it("accepts a well-formed tool definition", () => { + const tools = [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather for a location", + parameters: { + type: "object", + properties: { location: { type: "string" } }, + required: ["location"], + }, + }, + }, + ]; + expect(() => validateToolDefinitions(tools)).not.toThrow(); + }); + + it("accepts a minimal tool (just type and name)", () => { + expect(() => + validateToolDefinitions([{ type: "function", function: { name: "do_something" } }]), + ).not.toThrow(); + }); + + it("accepts parameters without a 'type' field", () => { + expect(() => + validateToolDefinitions([ + { type: "function", function: { name: "t", parameters: { properties: {} } } }, + ]), + ).not.toThrow(); + }); + + it("rejects non-array tools", () => { + expect(() => validateToolDefinitions({ type: "function" })).toThrow(ConfigurationError); + expect(() => validateToolDefinitions({ type: "function" })).toThrow(/must be an array/); + }); + + it("rejects a tool that is not an object", () => { + expect(() => validateToolDefinitions(["not a dict"])).toThrow(/tools\[0\] must be an object/); + expect(() => validateToolDefinitions([["nested"]])).toThrow(/must be an object, got array/); + }); + + it("rejects a tool missing the 'type' field", () => { + expect(() => validateToolDefinitions([{ function: { name: "test" } }])).toThrow( + /missing required 'type'/, + ); + }); + + it("rejects a non-function type", () => { + expect(() => + validateToolDefinitions([{ type: "retrieval", function: { name: "test" } }]), + ).toThrow(/must be 'function', got 'retrieval'/); + }); + + it("rejects a tool missing the 'function' field", () => { + expect(() => validateToolDefinitions([{ type: "function" }])).toThrow( + /missing required 'function'/, + ); + }); + + it("rejects a non-object function field", () => { + expect(() => validateToolDefinitions([{ type: "function", function: "not a dict" }])).toThrow( + /function must be an object/, + ); + }); + + it("rejects a function missing 'name'", () => { + expect(() => validateToolDefinitions([{ type: "function", function: {} }])).toThrow( + /missing required 'name'/, + ); + }); + + it("rejects an empty or non-string function name", () => { + expect(() => validateToolDefinitions([{ type: "function", function: { name: "" } }])).toThrow( + /non-empty string/, + ); + expect(() => validateToolDefinitions([{ type: "function", function: { name: 42 } }])).toThrow( + /non-empty string/, + ); + }); + + it("rejects a non-string description", () => { + expect(() => + validateToolDefinitions([{ type: "function", function: { name: "test", description: 123 } }]), + ).toThrow(/description must be a string/); + }); + + it("rejects non-object parameters", () => { + expect(() => + validateToolDefinitions([ + { type: "function", function: { name: "test", parameters: "invalid" } }, + ]), + ).toThrow(/parameters must be an object/); + expect(() => + validateToolDefinitions([{ type: "function", function: { name: "test", parameters: [] } }]), + ).toThrow(/parameters must be an object/); + }); + + it("reports the index of the offending tool", () => { + const tools = [ + { type: "function", function: { name: "ok" } }, + { type: "function", function: { name: "" } }, + ]; + expect(() => validateToolDefinitions(tools)).toThrow(/tools\[1\]\.function\.name/); + }); +}); + +describe("validateToolResultJson", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("passes silently for valid JSON", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + validateToolResultJson('{"result": "success"}', "test_tool"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("passes silently for empty content", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + validateToolResultJson("", "test_tool"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("warns but does not throw for invalid JSON", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(() => validateToolResultJson("not json", "test_tool")).not.toThrow(); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain("not valid JSON"); + expect(warn.mock.calls[0]?.[0]).toContain("test_tool"); + }); +}); + +describe("validateMessageContent", () => { + it("accepts string content", () => { + expect(() => validateMessageContent("Hello, world!")).not.toThrow(); + }); + + it("accepts null and undefined content", () => { + expect(() => validateMessageContent(null)).not.toThrow(); + expect(() => validateMessageContent(undefined)).not.toThrow(); + }); + + it("accepts multimodal content blocks", () => { + const content = [ + { type: "text", text: "What's in this image?" }, + { type: "image_url", image_url: { url: "https://example.com/img.png" } }, + ]; + expect(() => validateMessageContent(content)).not.toThrow(); + }); + + it("rejects invalid content types", () => { + expect(() => validateMessageContent(123)).toThrow(ConfigurationError); + expect(() => validateMessageContent(123)).toThrow(/must be string, array, or null/); + expect(() => validateMessageContent({ type: "text" })).toThrow(/got object/); + }); + + it("rejects content blocks that are not objects", () => { + expect(() => validateMessageContent(["not a dict"])).toThrow(/content\[0\] must be an object/); + }); + + it("rejects content blocks missing 'type'", () => { + expect(() => validateMessageContent([{ text: "no type field" }])).toThrow(/missing 'type'/); + }); +}); + +describe("gateway wiring", () => { + const badTools = [{ type: "retrieval", function: { name: "x" } }]; + const messages: Message[] = [{ role: Role.USER, content: "hi" }]; + const response: LLMResponse = { + message: { role: Role.ASSISTANT, content: "ok" }, + cost_usd: 0.001, + model: "test", + finish_reason: "stop", + }; + + it("exports the validators from the package index", () => { + expect(sdk.validateToolDefinitions).toBe(validateToolDefinitions); + expect(sdk.validateToolResultJson).toBe(validateToolResultJson); + expect(sdk.validateMessageContent).toBe(validateMessageContent); + }); + + it("DirectGateway.invokeLlm rejects malformed tools before calling the provider", async () => { + const create = vi.fn(); + const gateway = new DirectGateway({ chat: { completions: { create } } }, "gpt-4o"); + await expect(gateway.invokeLlm(messages, badTools)).rejects.toThrow(ConfigurationError); + expect(create).not.toHaveBeenCalled(); + expect(gateway.getHistory()).toHaveLength(0); + }); + + it("DirectGateway.invokeLlmStream rejects malformed tools before calling the provider", async () => { + const create = vi.fn(); + const gateway = new DirectGateway({ chat: { completions: { create } } }, "gpt-4o"); + const consume = async () => { + for await (const _chunk of gateway.invokeLlmStream(messages, badTools)) { + // drain + } + }; + await expect(consume()).rejects.toThrow(/must be 'function'/); + expect(create).not.toHaveBeenCalled(); + }); + + it("DirectGateway.invokeLlm passes valid tools through to the provider", async () => { + const create = vi.fn().mockResolvedValue({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const gateway = new DirectGateway({ chat: { completions: { create } } }, "gpt-4o"); + const tools = [{ type: "function", function: { name: "get_weather" } }]; + await gateway.invokeLlm(messages, tools); + expect(create).toHaveBeenCalledTimes(1); + expect(create.mock.calls[0]?.[0].tools).toEqual(tools); + }); + + it("MockGateway does not validate tools, matching the Python MockGateway", async () => { + const mock = new MockGateway([response]); + await expect(mock.invokeLlm(messages, badTools)).resolves.toBe(response); + expect(mock.lastRequest?.tools).toEqual(badTools); + }); +});