Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/pyagentspec/source/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,16 @@ Improvements
Bug fixes
^^^^^^^^^

* **Validation error messages preserved during deserialization**

Deserializing an invalid component with ``from_dict``, ``from_json`` or ``from_yaml`` failed with
``TypeError: ValueError: 'error' required in context`` whenever a custom validator rejected the
component (for example a ``Flow`` with an unreachable ``EndNode`` or an ``Agent`` declaring an
input that is not in its prompt), hiding the actual problem. Deserialization now raises a
``ValidationError`` carrying the original error type and message.

We thank @fede-kamel for the contribution!

* **LangGraph adapter: agent outputs answered as plain text**

When an agent with declared outputs ended its run with a plain message instead of calling
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from typing import Any, Dict, List, Mapping, Tuple, Type, cast

from pydantic import BaseModel, ValidationError
from pydantic_core import InitErrorDetails
from pydantic_core import InitErrorDetails, PydanticCustomError

from pyagentspec.component import Component
from pyagentspec.serialization.deserializationcontext import DeserializationContext
Expand Down Expand Up @@ -57,7 +57,11 @@ def deserialize(
if len(validation_errors) > 0:
line_errors = [
InitErrorDetails(
type=e.type,
# Several pydantic error types (e.g. ``value_error`` raised by custom
# validators) cannot be rebuilt from their type alone, they need the context
# of the original error to render their message. Wrapping the type and the
# message in a ``PydanticCustomError`` preserves both without the context.
type=PydanticCustomError(e.type, e.msg),
loc=e.loc,
input=(),
)
Expand Down
81 changes: 80 additions & 1 deletion pyagentspec/tests/serialization/test_serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License
# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option.
import json
from typing import Any, Dict, List, Tuple, Type, Union, cast
from typing import Any, Callable, Dict, List, Tuple, Type, Union, cast
from unittest.mock import patch

import pytest
import yaml
from pydantic import ValidationError

from pyagentspec.agent import Agent
from pyagentspec.component import Component
Expand All @@ -18,6 +19,7 @@
from pyagentspec.flows.nodes.endnode import EndNode
from pyagentspec.flows.nodes.llmnode import LlmNode
from pyagentspec.flows.nodes.startnode import StartNode
from pyagentspec.llms.openaiconfig import OpenAiConfig
from pyagentspec.llms.vllmconfig import VllmConfig
from pyagentspec.serialization import AgentSpecDeserializer, AgentSpecSerializer
from pyagentspec.serialization.pydanticdeserializationplugin import (
Expand Down Expand Up @@ -799,3 +801,80 @@ def test_union_mismatch_paths_use_valueerror_and_resolution_succeeds(
s = AgentSpecSerializer(plugins=[ser_plugin]).to_json(instance)
out = cast(type(instance), AgentSpecDeserializer(plugins=[deser_plugin]).from_json(s))
assert out.value == "keep-this-string"


@pytest.fixture
def serialized_agent_with_placeholder() -> Dict[str, Any]:
agent = Agent(
id="agent",
name="agent",
llm_config=OpenAiConfig(id="llm", name="llm", model_id="some_model"),
system_prompt="Hello {{who}}",
)
return agent.to_dict()


@pytest.mark.parametrize(
"deserialize",
[
lambda content: Component.from_dict(content),
lambda content: Component.from_json(json.dumps(content)),
lambda content: Component.from_yaml(yaml.safe_dump(content)),
lambda content: AgentSpecDeserializer().from_dict(content),
],
ids=["from_dict", "from_json", "from_yaml", "deserializer_from_dict"],
)
def test_deserialization_reports_custom_validation_errors(
deserialize: Callable[[Dict[str, Any]], Component],
) -> None:
start_node = StartNode(id="start", name="start")
end_node = EndNode(id="end", name="end")
flow = Flow(
id="flow",
name="flow",
start_node=start_node,
nodes=[start_node, end_node],
control_flow_connections=[
ControlFlowEdge(id="edge", name="edge", from_node=start_node, to_node=end_node)
],
)
serialized_flow = flow.to_dict()
# An end node without any incoming control flow edge is rejected by a custom Flow validator
serialized_end_node = serialized_flow["$referenced_components"]["end"]
serialized_flow["nodes"].append(
{**serialized_end_node, "id": "unreachable_end", "name": "unreachable_end"}
)

with pytest.raises(
ValidationError,
match="Found an end node without any incoming control flow edge.*'unreachable_end'",
) as exc_info:
deserialize(serialized_flow)
(error,) = exc_info.value.errors()
assert error["type"] == "value_error"


def test_deserialization_reports_extra_io_property_errors(
serialized_agent_with_placeholder: Dict[str, Any],
) -> None:
serialized_agent_with_placeholder["inputs"].append({"title": "not_in_prompt", "type": "string"})

with pytest.raises(
ValidationError,
match="The Agent component received a property titled `not_in_prompt`",
) as exc_info:
Component.from_dict(serialized_agent_with_placeholder)
(error,) = exc_info.value.errors()
assert error["type"] == "value_error"


def test_deserialization_reports_field_type_errors(
serialized_agent_with_placeholder: Dict[str, Any],
) -> None:
serialized_agent_with_placeholder["system_prompt"] = 123

with pytest.raises(ValidationError, match="Input should be a valid string") as exc_info:
Component.from_dict(serialized_agent_with_placeholder)
(error,) = exc_info.value.errors()
assert error["type"] == "string_type"
assert error["loc"] == ("system_prompt",)
Loading