Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@
later surfaced as an unrelated `SystemError` or crashed the interpreter.
* The `yajl2_c` backend no longer leaks the event value when an error is raised
while `parse()`, `items()` or `kvitems()` process it.
* The `python` backend now rejects Unicode whitespace such as `U+00A0`
or `U+2028` between tokens, as the `yajl` backends already do.
It used to skip anything Python's `re` module considers whitespace.
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
Vertical tab and form feed are still accepted: they aren't JSON whitespace
either, but `yajl` skips them, so all backends keep doing so for consistency.

## [3.6.0]

Expand Down
6 changes: 5 additions & 1 deletion src/ijson/backends/python.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@
import codecs


LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S')
# Only skip the whitespace yajl skips; re's \s also matches Unicode spaces.
# \v and \f aren't JSON whitespace per RFC 8259 (and the json module rejects
# them), but yajl skips them and system or dynamically loaded yajl builds
# can't be fixed from here, so they stay to keep all backends consistent.
LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|[^ \t\n\v\f\r]')
INTEGER_RE = re.compile(r'-?[0-9]+')
UNARY_LEXEMES = set('[]{},')
EOF = -1, None
Expand Down
7 changes: 7 additions & 0 deletions tests/test_basic_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,13 @@ def test_leading_plus_sign(adaptor, json):
_raises_json_error(adaptor, json)


@pytest.mark.parametrize("separator", ('\x1c', '\x1f', '\x85', '\xa0', '\N{LINE SEPARATOR}', '\N{IDEOGRAPHIC SPACE}'))
def test_non_json_whitespace(adaptor, separator):
"""characters outside JSON's whitespace can't separate tokens"""
json = '{"a":%s[1,%s2]}' % (separator, separator)
_raises_json_error(adaptor, json.encode('utf-8'))


def test_incomplete(adaptor):
for json in INCOMPLETE_JSONS:
_raises_incomplete_json_error(adaptor, json)
Expand Down