Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@
when the result of `read()` method fails to be decoded as a utf8 byte sequence.
* The `yajl2_c` backend now releases the result returned by the user-provided async `read()` method
when parsing that buffer fails.
* The `yajl2_c` backend correctly propagates exceptions raised by `items()`
and `kvitems()`. It used to keep parsing with the error pending, which
later surfaced as an unrelated `SystemError` or crashed the interpreter.
* The `yajl2_c` backend no longer leaks the event value when an error is raised
while `parse()`, `items()` or `kvitems()` process it.

## [3.6.0]

Expand Down
3 changes: 2 additions & 1 deletion src/ijson/backends/ext/_yajl2/basic_parse_basecoro.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ static inline
int add_event_and_value(void *ctx, PyObject *evt_name, PyObject *val) {
PyObject *target_send = ((yajl_parse_context *)ctx)->target_send;
if (ParseBasecoro_Check(target_send)) {
Z_N(parse_basecoro_send_impl(target_send, evt_name, val));
PyObject *res = parse_basecoro_send_impl(target_send, evt_name, val);
Py_DECREF(val);
Z_N(res);
Comment thread
rtobar marked this conversation as resolved.
Resolved
return 1;
}
PyObject *tuple;
Expand Down
4 changes: 2 additions & 2 deletions src/ijson/backends/ext/_yajl2/parse_basecoro.c
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,10 @@ PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *va
}

if (KVItemsBasecoro_Check(gen->target_send)) {
kvitems_basecoro_send_impl(gen->target_send, prefix, event, value);
N_N(kvitems_basecoro_send_impl(gen->target_send, prefix, event, value));
Comment thread
rtobar marked this conversation as resolved.
Resolved
Comment thread
rtobar marked this conversation as resolved.
Resolved
Comment thread
rtobar marked this conversation as resolved.
Resolved
}
else if (ItemsBasecoro_Check(gen->target_send)) {
items_basecoro_send_impl(gen->target_send, prefix, event, value);
N_N(items_basecoro_send_impl(gen->target_send, prefix, event, value));
}
else {
PyObject *res;
Expand Down
7 changes: 7 additions & 0 deletions tests/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,13 @@
(b'[{"abc": [0, 1]},', {'abc': [0, 1]}),
]

class NoDuplicateKeysDict(dict):
"""A map_type that rejects duplicate keys"""
def __setitem__(self, key, value):
if key in self:
raise ValueError(f"duplicate key {key!r}")
super().__setitem__(key, value)

items_test_case = collections.namedtuple('items_test_case', 'json, prefix, kvitems, items')
EMPTY_MEMBER_TEST_CASES = {
'simple': items_test_case(
Expand Down
8 changes: 7 additions & 1 deletion tests/test_items.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import pytest

from .test_base import ARRAY_JSON, ARRAY_JSON_OBJECT, EMPTY_MEMBER_TEST_CASES, JSON, JSON_OBJECT
from .test_base import ARRAY_JSON, ARRAY_JSON_OBJECT, EMPTY_MEMBER_TEST_CASES, JSON, JSON_OBJECT, NoDuplicateKeysDict

from ijson import JSONError

Expand Down Expand Up @@ -46,6 +46,12 @@ def test_map_type(adaptor):
assert isinstance(obj, collections.OrderedDict)


def test_map_type_error(adaptor):
"""Errors raised by map_type propagate -- yajl2_c used to drop them and crash"""
with pytest.raises(ValueError, match="duplicate key"):
adaptor.items(b'[{"a": 1, "a": 2}, {}]', 'item', map_type=NoDuplicateKeysDict)


@pytest.mark.parametrize("test_case", [
pytest.param(value, id=name) for name, value in EMPTY_MEMBER_TEST_CASES.items()
])
Expand Down
8 changes: 7 additions & 1 deletion tests/test_kvitems.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Tests for the ijson.kvitems method"""

from .test_base import ARRAY_JSON, EMPTY_MEMBER_TEST_CASES, JSON, JSON_KVITEMS, JSON_KVITEMS_META, JSON_OBJECT
from .test_base import ARRAY_JSON, EMPTY_MEMBER_TEST_CASES, JSON, JSON_KVITEMS, JSON_KVITEMS_META, JSON_OBJECT, NoDuplicateKeysDict

import sys

Expand Down Expand Up @@ -47,6 +47,12 @@ def test_kvitems_empty_member(adaptor, test_case):
assert test_case.kvitems == adaptor.kvitems(test_case.json, test_case.prefix)


def test_map_type_error(adaptor):
"""Errors raised by map_type propagate -- yajl2_c used to drop them and crash"""
with pytest.raises(ValueError, match="duplicate key"):
adaptor.kvitems(b'{"x": {"a": 1, "a": 2}, "y": {}}', '', map_type=NoDuplicateKeysDict)


def test_coro_needs_input_with_three_elements(backend):
int_element_parse_events = list(backend.parse(b'{"a": 0}'))
# all good
Expand Down
Loading