diff --git a/docs/pyagentspec/source/changelog.rst b/docs/pyagentspec/source/changelog.rst index ecdeb2643..9a96acbb4 100644 --- a/docs/pyagentspec/source/changelog.rst +++ b/docs/pyagentspec/source/changelog.rst @@ -36,6 +36,16 @@ Improvements Bug fixes ^^^^^^^^^ +* **Validation error messages preserved during deserialization** + + Deserializing an invalid component with ``from_dict``, ``from_json`` or ``from_yaml`` failed with + ``TypeError: ValueError: 'error' required in context`` whenever a custom validator rejected the + component (for example a ``Flow`` with an unreachable ``EndNode`` or an ``Agent`` declaring an + input that is not in its prompt), hiding the actual problem. Deserialization now raises a + ``ValidationError`` carrying the original error type and message. + + We thank @fede-kamel for the contribution! + * **LangGraph adapter: agent outputs answered as plain text** When an agent with declared outputs ended its run with a plain message instead of calling diff --git a/pyagentspec/src/pyagentspec/serialization/pydanticdeserializationplugin.py b/pyagentspec/src/pyagentspec/serialization/pydanticdeserializationplugin.py index dcc340c15..7ffbaa117 100644 --- a/pyagentspec/src/pyagentspec/serialization/pydanticdeserializationplugin.py +++ b/pyagentspec/src/pyagentspec/serialization/pydanticdeserializationplugin.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Mapping, Tuple, Type, cast from pydantic import BaseModel, ValidationError -from pydantic_core import InitErrorDetails +from pydantic_core import InitErrorDetails, PydanticCustomError from pyagentspec.component import Component from pyagentspec.serialization.deserializationcontext import DeserializationContext @@ -57,7 +57,11 @@ def deserialize( if len(validation_errors) > 0: line_errors = [ InitErrorDetails( - type=e.type, + # Several pydantic error types (e.g. ``value_error`` raised by custom + # validators) cannot be rebuilt from their type alone, they need the context + # of the original error to render their message. Wrapping the type and the + # message in a ``PydanticCustomError`` preserves both without the context. + type=PydanticCustomError(e.type, e.msg), loc=e.loc, input=(), ) diff --git a/pyagentspec/tests/serialization/test_serialization.py b/pyagentspec/tests/serialization/test_serialization.py index 6e547f2ea..7984abfea 100644 --- a/pyagentspec/tests/serialization/test_serialization.py +++ b/pyagentspec/tests/serialization/test_serialization.py @@ -4,11 +4,12 @@ # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. import json -from typing import Any, Dict, List, Tuple, Type, Union, cast +from typing import Any, Callable, Dict, List, Tuple, Type, Union, cast from unittest.mock import patch import pytest import yaml +from pydantic import ValidationError from pyagentspec.agent import Agent from pyagentspec.component import Component @@ -18,6 +19,7 @@ from pyagentspec.flows.nodes.endnode import EndNode from pyagentspec.flows.nodes.llmnode import LlmNode from pyagentspec.flows.nodes.startnode import StartNode +from pyagentspec.llms.openaiconfig import OpenAiConfig from pyagentspec.llms.vllmconfig import VllmConfig from pyagentspec.serialization import AgentSpecDeserializer, AgentSpecSerializer from pyagentspec.serialization.pydanticdeserializationplugin import ( @@ -799,3 +801,80 @@ def test_union_mismatch_paths_use_valueerror_and_resolution_succeeds( s = AgentSpecSerializer(plugins=[ser_plugin]).to_json(instance) out = cast(type(instance), AgentSpecDeserializer(plugins=[deser_plugin]).from_json(s)) assert out.value == "keep-this-string" + + +@pytest.fixture +def serialized_agent_with_placeholder() -> Dict[str, Any]: + agent = Agent( + id="agent", + name="agent", + llm_config=OpenAiConfig(id="llm", name="llm", model_id="some_model"), + system_prompt="Hello {{who}}", + ) + return agent.to_dict() + + +@pytest.mark.parametrize( + "deserialize", + [ + lambda content: Component.from_dict(content), + lambda content: Component.from_json(json.dumps(content)), + lambda content: Component.from_yaml(yaml.safe_dump(content)), + lambda content: AgentSpecDeserializer().from_dict(content), + ], + ids=["from_dict", "from_json", "from_yaml", "deserializer_from_dict"], +) +def test_deserialization_reports_custom_validation_errors( + deserialize: Callable[[Dict[str, Any]], Component], +) -> None: + start_node = StartNode(id="start", name="start") + end_node = EndNode(id="end", name="end") + flow = Flow( + id="flow", + name="flow", + start_node=start_node, + nodes=[start_node, end_node], + control_flow_connections=[ + ControlFlowEdge(id="edge", name="edge", from_node=start_node, to_node=end_node) + ], + ) + serialized_flow = flow.to_dict() + # An end node without any incoming control flow edge is rejected by a custom Flow validator + serialized_end_node = serialized_flow["$referenced_components"]["end"] + serialized_flow["nodes"].append( + {**serialized_end_node, "id": "unreachable_end", "name": "unreachable_end"} + ) + + with pytest.raises( + ValidationError, + match="Found an end node without any incoming control flow edge.*'unreachable_end'", + ) as exc_info: + deserialize(serialized_flow) + (error,) = exc_info.value.errors() + assert error["type"] == "value_error" + + +def test_deserialization_reports_extra_io_property_errors( + serialized_agent_with_placeholder: Dict[str, Any], +) -> None: + serialized_agent_with_placeholder["inputs"].append({"title": "not_in_prompt", "type": "string"}) + + with pytest.raises( + ValidationError, + match="The Agent component received a property titled `not_in_prompt`", + ) as exc_info: + Component.from_dict(serialized_agent_with_placeholder) + (error,) = exc_info.value.errors() + assert error["type"] == "value_error" + + +def test_deserialization_reports_field_type_errors( + serialized_agent_with_placeholder: Dict[str, Any], +) -> None: + serialized_agent_with_placeholder["system_prompt"] = 123 + + with pytest.raises(ValidationError, match="Input should be a valid string") as exc_info: + Component.from_dict(serialized_agent_with_placeholder) + (error,) = exc_info.value.errors() + assert error["type"] == "string_type" + assert error["loc"] == ("system_prompt",)