diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d4d77..95ec907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,11 @@ when the result of `read()` method fails to be decoded as a utf8 byte sequence. * The `yajl2_c` backend now releases the result returned by the user-provided async `read()` method when parsing that buffer fails. +* The `yajl2_c` backend correctly propagates exceptions raised by `items()` + and `kvitems()`. It used to keep parsing with the error pending, which + later surfaced as an unrelated `SystemError` or crashed the interpreter. +* The `yajl2_c` backend no longer leaks the event value when an error is raised + while `parse()`, `items()` or `kvitems()` process it. ## [3.6.0] diff --git a/src/ijson/backends/ext/_yajl2/basic_parse_basecoro.c b/src/ijson/backends/ext/_yajl2/basic_parse_basecoro.c index 9812831..eb2e105 100644 --- a/src/ijson/backends/ext/_yajl2/basic_parse_basecoro.c +++ b/src/ijson/backends/ext/_yajl2/basic_parse_basecoro.c @@ -26,8 +26,9 @@ static inline int add_event_and_value(void *ctx, PyObject *evt_name, PyObject *val) { PyObject *target_send = ((yajl_parse_context *)ctx)->target_send; if (ParseBasecoro_Check(target_send)) { - Z_N(parse_basecoro_send_impl(target_send, evt_name, val)); + PyObject *res = parse_basecoro_send_impl(target_send, evt_name, val); Py_DECREF(val); + Z_N(res); return 1; } PyObject *tuple; diff --git a/src/ijson/backends/ext/_yajl2/parse_basecoro.c b/src/ijson/backends/ext/_yajl2/parse_basecoro.c index 3705a84..525e2bc 100644 --- a/src/ijson/backends/ext/_yajl2/parse_basecoro.c +++ b/src/ijson/backends/ext/_yajl2/parse_basecoro.c @@ -101,10 +101,10 @@ PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *va } if (KVItemsBasecoro_Check(gen->target_send)) { - kvitems_basecoro_send_impl(gen->target_send, prefix, event, value); + N_N(kvitems_basecoro_send_impl(gen->target_send, prefix, event, value)); } else if (ItemsBasecoro_Check(gen->target_send)) { - items_basecoro_send_impl(gen->target_send, prefix, event, value); + N_N(items_basecoro_send_impl(gen->target_send, prefix, event, value)); } else { PyObject *res; diff --git a/tests/test_base.py b/tests/test_base.py index f510f3b..434aa41 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -279,6 +279,13 @@ (b'[{"abc": [0, 1]},', {'abc': [0, 1]}), ] +class NoDuplicateKeysDict(dict): + """A map_type that rejects duplicate keys""" + def __setitem__(self, key, value): + if key in self: + raise ValueError(f"duplicate key {key!r}") + super().__setitem__(key, value) + items_test_case = collections.namedtuple('items_test_case', 'json, prefix, kvitems, items') EMPTY_MEMBER_TEST_CASES = { 'simple': items_test_case( diff --git a/tests/test_items.py b/tests/test_items.py index 2bca8e6..4fa206a 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -5,7 +5,7 @@ import pytest -from .test_base import ARRAY_JSON, ARRAY_JSON_OBJECT, EMPTY_MEMBER_TEST_CASES, JSON, JSON_OBJECT +from .test_base import ARRAY_JSON, ARRAY_JSON_OBJECT, EMPTY_MEMBER_TEST_CASES, JSON, JSON_OBJECT, NoDuplicateKeysDict from ijson import JSONError @@ -46,6 +46,12 @@ def test_map_type(adaptor): assert isinstance(obj, collections.OrderedDict) +def test_map_type_error(adaptor): + """Errors raised by map_type propagate -- yajl2_c used to drop them and crash""" + with pytest.raises(ValueError, match="duplicate key"): + adaptor.items(b'[{"a": 1, "a": 2}, {}]', 'item', map_type=NoDuplicateKeysDict) + + @pytest.mark.parametrize("test_case", [ pytest.param(value, id=name) for name, value in EMPTY_MEMBER_TEST_CASES.items() ]) diff --git a/tests/test_kvitems.py b/tests/test_kvitems.py index a822620..6eb6c2c 100644 --- a/tests/test_kvitems.py +++ b/tests/test_kvitems.py @@ -1,6 +1,6 @@ """Tests for the ijson.kvitems method""" -from .test_base import ARRAY_JSON, EMPTY_MEMBER_TEST_CASES, JSON, JSON_KVITEMS, JSON_KVITEMS_META, JSON_OBJECT +from .test_base import ARRAY_JSON, EMPTY_MEMBER_TEST_CASES, JSON, JSON_KVITEMS, JSON_KVITEMS_META, JSON_OBJECT, NoDuplicateKeysDict import sys @@ -47,6 +47,12 @@ def test_kvitems_empty_member(adaptor, test_case): assert test_case.kvitems == adaptor.kvitems(test_case.json, test_case.prefix) +def test_map_type_error(adaptor): + """Errors raised by map_type propagate -- yajl2_c used to drop them and crash""" + with pytest.raises(ValueError, match="duplicate key"): + adaptor.kvitems(b'{"x": {"a": 1, "a": 2}, "y": {}}', '', map_type=NoDuplicateKeysDict) + + def test_coro_needs_input_with_three_elements(backend): int_element_parse_events = list(backend.parse(b'{"a": 0}')) # all good