diff --git a/CHANGELOG.md b/CHANGELOG.md index 95ec907..358c93a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,11 @@ later surfaced as an unrelated `SystemError` or crashed the interpreter. * The `yajl2_c` backend no longer leaks the event value when an error is raised while `parse()`, `items()` or `kvitems()` process it. +* The `python` backend now rejects Unicode whitespace such as `U+00A0` + or `U+2028` between tokens, as the `yajl` backends already do. + It used to skip anything Python's `re` module considers whitespace. + Vertical tab and form feed are still accepted: they aren't JSON whitespace + either, but `yajl` skips them, so all backends keep doing so for consistency. ## [3.6.0] diff --git a/src/ijson/backends/python.py b/src/ijson/backends/python.py index 4d53f60..7d344f1 100644 --- a/src/ijson/backends/python.py +++ b/src/ijson/backends/python.py @@ -8,7 +8,11 @@ import codecs -LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S') +# Only skip the whitespace yajl skips; re's \s also matches Unicode spaces. +# \v and \f aren't JSON whitespace per RFC 8259 (and the json module rejects +# them), but yajl skips them and system or dynamically loaded yajl builds +# can't be fixed from here, so they stay to keep all backends consistent. +LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|[^ \t\n\v\f\r]') INTEGER_RE = re.compile(r'-?[0-9]+') UNARY_LEXEMES = set('[]{},') EOF = -1, None diff --git a/tests/test_basic_parse.py b/tests/test_basic_parse.py index bcb0782..0a5fcc0 100644 --- a/tests/test_basic_parse.py +++ b/tests/test_basic_parse.py @@ -150,6 +150,13 @@ def test_leading_plus_sign(adaptor, json): _raises_json_error(adaptor, json) +@pytest.mark.parametrize("separator", ('\x1c', '\x1f', '\x85', '\xa0', '\N{LINE SEPARATOR}', '\N{IDEOGRAPHIC SPACE}')) +def test_non_json_whitespace(adaptor, separator): + """characters outside JSON's whitespace can't separate tokens""" + json = '{"a":%s[1,%s2]}' % (separator, separator) + _raises_json_error(adaptor, json.encode('utf-8')) + + def test_incomplete(adaptor): for json in INCOMPLETE_JSONS: _raises_incomplete_json_error(adaptor, json)