From a228b85fcb5569955e4432a3262b801174f23c9e Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:30:50 +0530 Subject: [PATCH 01/10] test: add ConnectorBench MCP conformance checks --- tests/connectorbench.py | 111 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/connectorbench.py diff --git a/tests/connectorbench.py b/tests/connectorbench.py new file mode 100644 index 0000000..b412a01 --- /dev/null +++ b/tests/connectorbench.py @@ -0,0 +1,111 @@ +"""Reusable, dependency-light conformance checks for MCP tool catalogs. + +ConnectorBench validates the contract exposed by a server's list_tools result. +It intentionally does not invoke tools: inspection must be safe for connectors +whose calls can mutate a model, launch a solver, or consume a commercial license. +""" +from __future__ import annotations + +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + + +_NAME = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def _field(value: Any, name: str, default: Any = None) -> Any: + """Read an MCP model field from either an object or a plain mapping.""" + if isinstance(value, Mapping): + return value.get(name, default) + return getattr(value, name, default) + + +def inspect_tool_contracts(tools: Sequence[Any]) -> dict[str, Any]: + """Inspect an MCP tool catalog and return a stable JSON-serializable report. + + Errors indicate malformed or ambiguous contracts. Missing descriptions are + warnings because MCP permits them, but they reduce discoverability. + No connector tool is called and no optional engine is imported here. + """ + errors: list[dict[str, str]] = [] + warnings: list[dict[str, str]] = [] + seen: set[str] = set() + catalog: list[dict[str, Any]] = [] + + def issue(target: str, code: str, message: str, *, warning: bool = False) -> None: + item = {"target": target, "code": code, "message": message} + (warnings if warning else errors).append(item) + + for index, tool in enumerate(tools): + name = _field(tool, "name") + target = name if isinstance(name, str) and name else f"tools[{index}]" + + if not isinstance(name, str) or not name.strip(): + issue(target, "invalid_name", "Tool name must be a non-empty string.") + elif not _NAME.fullmatch(name): + issue(target, "invalid_name", "Tool name may contain only letters, digits, '.', '_' and '-'.") + elif name in seen: + issue(target, "duplicate_name", f"Tool name {name!r} is registered more than once.") + else: + seen.add(name) + + description = _field(tool, "description") + if not isinstance(description, str) or not description.strip(): + issue( + target, + "missing_description", + "Tool has no non-empty description; agents may struggle to discover its purpose.", + warning=True, + ) + + schema = _field(tool, "inputSchema", _field(tool, "input_schema")) + if not isinstance(schema, Mapping): + issue(target, "invalid_input_schema", "Tool inputSchema must be a JSON Schema object.") + catalog.append({"name": target, "schema_valid": False}) + continue + + if schema.get("type") not in (None, "object"): + issue(target, "invalid_schema_root_type", "Tool inputSchema root type must be 'object'.") + + properties = schema.get("properties", {}) + if not isinstance(properties, Mapping): + issue(target, "invalid_schema_properties", "inputSchema.properties must be an object.") + properties = {} + + required = schema.get("required", []) + if not isinstance(required, list) or any(not isinstance(key, str) for key in required): + issue(target, "invalid_schema_required", "inputSchema.required must be a list of strings.") + required = [] + else: + unknown = sorted(set(required) - set(properties)) + if unknown: + issue( + target, + "unknown_required_property", + "Required properties are not declared in properties: " + ", ".join(unknown), + ) + + try: + json.dumps(schema, allow_nan=False) + except (TypeError, ValueError): + issue(target, "non_json_schema", "inputSchema must be JSON-serializable without NaN or Infinity.") + + catalog.append( + { + "name": target, + "schema_valid": not any(error["target"] == target for error in errors), + "property_count": len(properties), + "required_count": len(required), + } + ) + + return { + "schema_version": 1, + "tool_count": len(tools), + "tools": catalog, + "errors": errors, + "warnings": warnings, + "passed": not errors, + } From 7088cd7ffccda6ef11693387184689e8139d4358 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:30:52 +0530 Subject: [PATCH 02/10] test: add ConnectorBench MCP conformance checks --- tests/test_connectorbench.py | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_connectorbench.py diff --git a/tests/test_connectorbench.py b/tests/test_connectorbench.py new file mode 100644 index 0000000..1ab5c7f --- /dev/null +++ b/tests/test_connectorbench.py @@ -0,0 +1,72 @@ +"""Unit tests for ConnectorBench's safe, read-only MCP catalog checks.""" +from __future__ import annotations + +from connectorbench import inspect_tool_contracts + + +def _tool(name="solve", schema=None, description="Run a solver"): + return { + "name": name, + "description": description, + "inputSchema": schema if schema is not None else { + "type": "object", + "properties": {"case": {"type": "string"}}, + "required": ["case"], + }, + } + + +def test_valid_catalog_passes_and_is_json_serializable(): + report = inspect_tool_contracts([_tool(), _tool("inspect", {"type": "object"})]) + + assert report["passed"] is True + assert report["tool_count"] == 2 + assert report["errors"] == [] + assert report["tools"][0]["required_count"] == 1 + + +def test_duplicate_tool_names_are_reported(): + report = inspect_tool_contracts([_tool(), _tool()]) + + assert report["passed"] is False + assert any(item["code"] == "duplicate_name" for item in report["errors"]) + + +def test_required_properties_must_be_declared(): + report = inspect_tool_contracts([ + _tool(schema={"type": "object", "properties": {}, "required": ["bus_id"]}) + ]) + + assert report["passed"] is False + assert any(item["code"] == "unknown_required_property" for item in report["errors"]) + + +def test_schema_root_must_be_an_object(): + report = inspect_tool_contracts([_tool(schema={"type": "array"})]) + + assert report["passed"] is False + assert any(item["code"] == "invalid_schema_root_type" for item in report["errors"]) + + +def test_missing_description_is_a_warning_not_a_failure(): + report = inspect_tool_contracts([_tool(description=" ")]) + + assert report["passed"] is True + assert report["errors"] == [] + assert any(item["code"] == "missing_description" for item in report["warnings"]) + + +def test_non_json_values_are_rejected(): + report = inspect_tool_contracts([ + _tool(schema={"type": "object", "properties": {"value": {"default": float("nan")}}}) + ]) + + assert report["passed"] is False + assert any(item["code"] == "non_json_schema" for item in report["errors"]) + + +def test_tool_without_input_schema_fails_closed(): + report = inspect_tool_contracts([{"name": "solve", "description": "Run a solver"}]) + + assert report["passed"] is False + assert any(item["code"] == "invalid_input_schema" for item in report["errors"]) From 7a937cdac0e671e77353574fcfd4523698dca16c Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:30:58 +0530 Subject: [PATCH 03/10] test: apply ConnectorBench to connector smoke tests --- tests/server_entry_smoke.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/server_entry_smoke.py b/tests/server_entry_smoke.py index fd9536d..ddc02e0 100644 --- a/tests/server_entry_smoke.py +++ b/tests/server_entry_smoke.py @@ -124,6 +124,13 @@ def record_run(self, *args, **kwargs): tools = asyncio.run(server.list_tools()) assert tools, f"{tool_name}: server registered no tools" + # Contract checks are read-only: they inspect catalog metadata but never + # invoke a model-mutating tool or launch a vendor engine. + from connectorbench import inspect_tool_contracts + + report = inspect_tool_contracts(tools) + assert report["passed"], f"{tool_name}: ConnectorBench failures: {report['errors']}" + if __name__ == "__main__": main(sys.argv[1]) From 5d2d9933066873e64b3d2837ca0882dc5ad2e59c Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:41:43 +0530 Subject: [PATCH 04/10] fix: isolate ConnectorBench schema validity from name errors --- tests/connectorbench.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/connectorbench.py b/tests/connectorbench.py index b412a01..0e9bcee 100644 --- a/tests/connectorbench.py +++ b/tests/connectorbench.py @@ -60,6 +60,8 @@ def issue(target: str, code: str, message: str, *, warning: bool = False) -> Non warning=True, ) + # Keep schema validity independent from catalog-level name errors. + schema_error_count = len(errors) schema = _field(tool, "inputSchema", _field(tool, "input_schema")) if not isinstance(schema, Mapping): issue(target, "invalid_input_schema", "Tool inputSchema must be a JSON Schema object.") @@ -95,7 +97,7 @@ def issue(target: str, code: str, message: str, *, warning: bool = False) -> Non catalog.append( { "name": target, - "schema_valid": not any(error["target"] == target for error in errors), + "schema_valid": len(errors) == schema_error_count, "property_count": len(properties), "required_count": len(required), } From a1509e2c088732e203f712ab2435203ae8943e6d Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:41:47 +0530 Subject: [PATCH 05/10] test: distinguish catalog errors from schema validity --- tests/test_connectorbench.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_connectorbench.py b/tests/test_connectorbench.py index 1ab5c7f..db5b9a3 100644 --- a/tests/test_connectorbench.py +++ b/tests/test_connectorbench.py @@ -30,6 +30,8 @@ def test_duplicate_tool_names_are_reported(): assert report["passed"] is False assert any(item["code"] == "duplicate_name" for item in report["errors"]) + # Duplicate registration is a catalog error, not a schema error. + assert all(item["schema_valid"] for item in report["tools"]) def test_required_properties_must_be_declared(): From 307bf588449f7810b8a85d142aaec7e1d84b2693 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:02:11 +0530 Subject: [PATCH 06/10] fix: accept valid composed JSON Schema requirements --- tests/connectorbench.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/connectorbench.py b/tests/connectorbench.py index 0e9bcee..67232e3 100644 --- a/tests/connectorbench.py +++ b/tests/connectorbench.py @@ -80,15 +80,7 @@ def issue(target: str, code: str, message: str, *, warning: bool = False) -> Non if not isinstance(required, list) or any(not isinstance(key, str) for key in required): issue(target, "invalid_schema_required", "inputSchema.required must be a list of strings.") required = [] - else: - unknown = sorted(set(required) - set(properties)) - if unknown: - issue( - target, - "unknown_required_property", - "Required properties are not declared in properties: " + ", ".join(unknown), - ) - + # JSON Schema permits required names to be constrained through allOf,\n # $ref, patternProperties, or additionalProperties rather than root properties.\n try: json.dumps(schema, allow_nan=False) except (TypeError, ValueError): From f5546e6043406c095232832479f4710b1b9c6ac1 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:02:17 +0530 Subject: [PATCH 07/10] fix: correct composed-schema compatibility comment From 0c4b77b5bc606d02ed09a8011f697624bc968144 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:02:22 +0530 Subject: [PATCH 08/10] test: allow required fields in composed schemas --- tests/test_connectorbench.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_connectorbench.py b/tests/test_connectorbench.py index db5b9a3..06f5eb3 100644 --- a/tests/test_connectorbench.py +++ b/tests/test_connectorbench.py @@ -34,13 +34,17 @@ def test_duplicate_tool_names_are_reported(): assert all(item["schema_valid"] for item in report["tools"]) -def test_required_properties_must_be_declared(): +def test_required_properties_can_be_defined_by_composed_schema(): report = inspect_tool_contracts([ - _tool(schema={"type": "object", "properties": {}, "required": ["bus_id"]}) + _tool(schema={ + "type": "object", + "allOf": [{"properties": {"bus_id": {"type": "string"}}}], + "required": ["bus_id"], + }) ]) - assert report["passed"] is False - assert any(item["code"] == "unknown_required_property" for item in report["errors"]) + assert report["passed"] is True + assert report["errors"] == [] def test_schema_root_must_be_an_object(): From 209db7639713c4ec052ca4b06ec7ed0f25ef19b9 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:02:36 +0530 Subject: [PATCH 09/10] fix: format composed-schema compatibility comment --- tests/connectorbench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/connectorbench.py b/tests/connectorbench.py index 67232e3..5178fab 100644 --- a/tests/connectorbench.py +++ b/tests/connectorbench.py @@ -80,8 +80,8 @@ def issue(target: str, code: str, message: str, *, warning: bool = False) -> Non if not isinstance(required, list) or any(not isinstance(key, str) for key in required): issue(target, "invalid_schema_required", "inputSchema.required must be a list of strings.") required = [] - # JSON Schema permits required names to be constrained through allOf,\n # $ref, patternProperties, or additionalProperties rather than root properties.\n - try: + # JSON Schema permits required names to be constrained through allOf, + # $ref, patternProperties, or additionalProperties rather than root properties. json.dumps(schema, allow_nan=False) except (TypeError, ValueError): issue(target, "non_json_schema", "inputSchema must be JSON-serializable without NaN or Infinity.") From 43fe6e1a1df61813c8ee5e43702da950c84bcf2a Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:02:44 +0530 Subject: [PATCH 10/10] fix: restore schema serialization validation block --- tests/connectorbench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/connectorbench.py b/tests/connectorbench.py index 5178fab..16bc4c3 100644 --- a/tests/connectorbench.py +++ b/tests/connectorbench.py @@ -80,8 +80,8 @@ def issue(target: str, code: str, message: str, *, warning: bool = False) -> Non if not isinstance(required, list) or any(not isinstance(key, str) for key in required): issue(target, "invalid_schema_required", "inputSchema.required must be a list of strings.") required = [] - # JSON Schema permits required names to be constrained through allOf, - # $ref, patternProperties, or additionalProperties rather than root properties. + # JSON Schema permits required names to be constrained by composed schemas. + try: json.dumps(schema, allow_nan=False) except (TypeError, ValueError): issue(target, "non_json_schema", "inputSchema must be JSON-serializable without NaN or Infinity.")