From b4abdb35f309bd8003cdeba1aa07475958ec1f44 Mon Sep 17 00:00:00 2001 From: nabeel a Date: Sat, 14 Nov 2026 20:33:10 +0530 Subject: [PATCH 1/2] reject Unicode whitespace between tokens in the python backend The python backend's Lexer skips everything between LEXEME_RE matches, and the regex ended in \S, so any character Python's re module treats as whitespace separated tokens. That includes U+001C-U+001F, U+0085, U+00A0, U+1680, U+2000-U+200A, U+2028, U+2029, U+202F, U+205F and U+3000, none of which are JSON whitespace, so a document like [1,2] parsed with the python backend while the yajl backends and the json module reject it. Only skip the characters yajl's lexer skips (space, \t, \n, \v, \f and \r). Anything else is now a lexeme of its own, which parse_value rejects. --- CHANGELOG.md | 3 +++ src/ijson/backends/python.py | 3 ++- tests/test_basic_parse.py | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95ec907..b9c6ef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ later surfaced as an unrelated `SystemError` or crashed the interpreter. * The `yajl2_c` backend no longer leaks the event value when an error is raised while `parse()`, `items()` or `kvitems()` process it. +* The `python` backend now rejects Unicode whitespace such as `U+00A0` + or `U+2028` between tokens, as the `yajl` backends already do. + It used to skip anything Python's `re` module considers whitespace. ## [3.6.0] diff --git a/src/ijson/backends/python.py b/src/ijson/backends/python.py index 4d53f60..cae36a9 100644 --- a/src/ijson/backends/python.py +++ b/src/ijson/backends/python.py @@ -8,7 +8,8 @@ import codecs -LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S') +# Only skip the whitespace yajl skips; re's \s also matches Unicode spaces +LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|[^ \t\n\v\f\r]') INTEGER_RE = re.compile(r'-?[0-9]+') UNARY_LEXEMES = set('[]{},') EOF = -1, None diff --git a/tests/test_basic_parse.py b/tests/test_basic_parse.py index bcb0782..0a5fcc0 100644 --- a/tests/test_basic_parse.py +++ b/tests/test_basic_parse.py @@ -150,6 +150,13 @@ def test_leading_plus_sign(adaptor, json): _raises_json_error(adaptor, json) +@pytest.mark.parametrize("separator", ('\x1c', '\x1f', '\x85', '\xa0', '\N{LINE SEPARATOR}', '\N{IDEOGRAPHIC SPACE}')) +def test_non_json_whitespace(adaptor, separator): + """characters outside JSON's whitespace can't separate tokens""" + json = '{"a":%s[1,%s2]}' % (separator, separator) + _raises_json_error(adaptor, json.encode('utf-8')) + + def test_incomplete(adaptor): for json in INCOMPLETE_JSONS: _raises_incomplete_json_error(adaptor, json) From b4ca6b09199f686080de0c900d80490447c04a80 Mon Sep 17 00:00:00 2001 From: nabeel a Date: Mon, 16 Nov 2026 16:04:05 +0530 Subject: [PATCH 2/2] explain why the python lexer still skips \v and \f --- CHANGELOG.md | 2 ++ src/ijson/backends/python.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c6ef8..358c93a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ * The `python` backend now rejects Unicode whitespace such as `U+00A0` or `U+2028` between tokens, as the `yajl` backends already do. It used to skip anything Python's `re` module considers whitespace. + Vertical tab and form feed are still accepted: they aren't JSON whitespace + either, but `yajl` skips them, so all backends keep doing so for consistency. ## [3.6.0] diff --git a/src/ijson/backends/python.py b/src/ijson/backends/python.py index cae36a9..7d344f1 100644 --- a/src/ijson/backends/python.py +++ b/src/ijson/backends/python.py @@ -8,7 +8,10 @@ import codecs -# Only skip the whitespace yajl skips; re's \s also matches Unicode spaces +# Only skip the whitespace yajl skips; re's \s also matches Unicode spaces. +# \v and \f aren't JSON whitespace per RFC 8259 (and the json module rejects +# them), but yajl skips them and system or dynamically loaded yajl builds +# can't be fixed from here, so they stay to keep all backends consistent. LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|[^ \t\n\v\f\r]') INTEGER_RE = re.compile(r'-?[0-9]+') UNARY_LEXEMES = set('[]{},')