From f54fc473881b4be599e4223c092d20f360b816d3 Mon Sep 17 00:00:00 2001 From: lipengyu Date: Mon, 27 Jul 2026 17:39:49 +0800 Subject: [PATCH 1/3] Preserve trailing whitespace in t-string interpolation expressions --- Lib/test/test_annotationlib.py | 2 +- Lib/test/test_tstring.py | 37 ++++++++++++++++++- Lib/test/test_unparse.py | 5 +++ ...-07-27-17-20-49.gh-issue-154719.eI88Gs.rst | 4 ++ Parser/action_helpers.c | 23 +++++++----- Parser/lexer/string.c | 11 +++++- 6 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 5087c3ca425f1fc..4892d34a7623279 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1862,7 +1862,7 @@ def nested(): self.assertEqual(type_repr(t'''{ 0 & 1 | 2 - }'''), 't"""{ 0\n & 1\n | 2}"""') + }'''), 't"""{ 0\n & 1\n | 2\n }"""') self.assertEqual( type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" ) diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 74653c77c55de17..bceaca178965556 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -136,10 +136,45 @@ def test_debug_specifier(self): # Test white space in debug specifier t = t"Value: {value = }" self.assertTStringEqual( - t, ("Value: value = ", ""), [(value, "value", "r")] + t, ("Value: value = ", ""), [(value, "value ", "r")] ) self.assertEqual(fstring(t), "Value: value = 42") + def test_interpolation_expression_whitespace(self): + x = 42 + for template, expected in ( + (t"{x}", "x"), + (t"{x }", "x "), + (t"{ x}", " x"), + (t"{ x }", " x "), + (t"{ x }", " x "), + (t"""{ + x +}""", "\n x\n"), + (t"{ x !r}", " x "), + (t"{ x :.2f}", " x "), + (t"{ x = }", " x "), + (t"{ x = !r}", " x "), + (t"{ x = :.2f}", " x "), + (t"{x == 42 = }", "x == 42 "), + ): + with self.subTest(template=template): + self.assertEqual( + template.interpolations[0].expression, + expected, + ) + + def test_interpolation_expression_with_reconstructed_metadata(self): + regular = t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")}''' + debug = t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")=}''' + expected = '(\n 1, \n "\\"#")' + self.assertEqual(regular.interpolations[0].expression, expected) + self.assertEqual(debug.interpolations[0].expression, expected) + def test_raw_tstrings(self): path = r"C:\Users" t = rt"{path}\Documents" diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index dcaad49ffab5d26..82d8eddc7de54b0 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -216,6 +216,11 @@ def test_tstrings(self): self.check_ast_roundtrip('t""') self.check_ast_roundtrip("t'{(lambda x: x)}'") self.check_ast_roundtrip("t'{t'{x}'}'") + self.check_ast_roundtrip( + r"""t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")}'''""" + ) def test_tstring_with_nonsensical_str_field(self): # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst new file mode 100644 index 000000000000000..e0f3d7f2c7286b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst @@ -0,0 +1,4 @@ +Whitespace immediately before ``}``, ``!``, ``:``, or ``=`` is now included +in :attr:`string.templatelib.Interpolation.expression` for interpolations +created from t-string literals. Preserve escaped quotes while reconstructing +interpolation expression metadata containing comments. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 809f3c0a7b270e8..80cb09bb2ec5971 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1540,21 +1540,24 @@ _get_interpolation_conversion(Parser *p, Token *debug, ResultTokenWithMetadata * } static PyObject * -_strip_interpolation_expr(PyObject *exprstr) +_strip_interpolation_debug_expr(PyObject *exprstr) { Py_ssize_t len = PyUnicode_GET_LENGTH(exprstr); - for (Py_ssize_t i = len - 1; i >= 0; i--) { - Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, i); - if (_PyUnicode_IsWhitespace(c) || c == '=') { - len--; - } - else { + /* Discard whitespace after the debug "=" but preserve whitespace before it. */ + while (len > 0) { + Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, len - 1); + if (!_PyUnicode_IsWhitespace(c)) { break; } + len--; + } + /* The debug marker may be absent from reconstructed lexer metadata. */ + if (len == 0 || PyUnicode_READ_CHAR(exprstr, len - 1) != '=') { + return Py_NewRef(exprstr); } - return PyUnicode_Substring(exprstr, 0, len); + return PyUnicode_Substring(exprstr, 0, len - 1); } expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, ResultTokenWithMetadata *conversion, @@ -1585,7 +1588,9 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu } assert(exprstr != NULL); - PyObject *final_exprstr = _strip_interpolation_expr(exprstr); + PyObject *final_exprstr = debug + ? _strip_interpolation_debug_expr(exprstr) + : Py_NewRef(exprstr); if (!final_exprstr || _PyArena_AddPyObject(arena, final_exprstr) < 0) { Py_XDECREF(final_exprstr); return NULL; diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index e546954ba5c0d82..a801b8ce93cf219 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -74,8 +74,17 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { char ch = tok_mode->last_expr_buffer[i]; + // Copy escaped characters without interpreting the escaped + // character as a quote or comment marker. + if (ch == '\\') { + result[j++] = ch; + i++; + if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + result[j++] = tok_mode->last_expr_buffer[i]; + } + } // Handle string quotes - if (ch == '"' || ch == '\'') { + else if (ch == '"' || ch == '\'') { // See comment above to understand this part if (!in_string) { in_string = 1; From 9f5a8e022886c88cd9607331ef185ecc8ebd188c Mon Sep 17 00:00:00 2001 From: lipengyu Date: Tue, 28 Jul 2026 10:47:22 +0800 Subject: [PATCH 2/3] address review --- Lib/test/test_fstring.py | 8 ++++ Lib/test/test_tstring.py | 37 +++++++++++++++++++ Lib/test/test_unparse.py | 4 ++ ...-07-27-17-20-49.gh-issue-154719.eI88Gs.rst | 5 ++- Parser/action_helpers.c | 22 +++++++++-- 5 files changed, 70 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 45890f8fb35dcb7..3b1473a299a0bf5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1678,6 +1678,14 @@ def __repr__(self): self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + result = f'''{( + 1, # Force lexer metadata reconstruction. + "\"#")=}''' + self.assertEqual( + result, + '(\n 1, \n "\\"#")=(1, \'"#\')', + ) + self.assertEqual(f'{ # some comment goes here """hello"""=}', ' \n """hello"""=\'hello\'') self.assertEqual(f'{"""# this is not a comment diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index bceaca178965556..046cc1e58346089 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -140,6 +140,43 @@ def test_debug_specifier(self): ) self.assertEqual(fstring(t), "Value: value = 42") + # Explicit line continuations after the debug marker are part of + # the debug text, not the interpolation expression. + for template, strings, interpolation, rendered in ( + ( + t"""Value: {value =\ +}""", + ("Value: value =\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n42", + ), + ( + t"""Value: {value =\ +!r}""", + ("Value: value =\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n42", + ), + ( + t"""Value: {value =\ +:04}""", + ("Value: value =\\\n", ""), + (value, "value ", None, "04"), + "Value: value =\\\n0042", + ), + ( + t"""Value: {value =\ +\ +}""", + ("Value: value =\\\n\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n\\\n42", + ), + ): + with self.subTest(template=template): + self.assertTStringEqual(template, strings, [interpolation]) + self.assertEqual(fstring(template), rendered) + def test_interpolation_expression_whitespace(self): x = 42 for template, expected in ( diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 82d8eddc7de54b0..28faede2e2a1a5c 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -221,6 +221,10 @@ def test_tstrings(self): 1, # Force lexer metadata reconstruction. "\"#")}'''""" ) + self.check_ast_roundtrip( + r'''t"""Value: {value =\ +}"""''' + ) def test_tstring_with_nonsensical_str_field(self): # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst index e0f3d7f2c7286b4..a60e63bbad8223b 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst @@ -1,4 +1,5 @@ Whitespace immediately before ``}``, ``!``, ``:``, or ``=`` is now included in :attr:`string.templatelib.Interpolation.expression` for interpolations -created from t-string literals. Preserve escaped quotes while reconstructing -interpolation expression metadata containing comments. +created from t-string literals. Preserve escaped quotes while reconstructing +t-string interpolation metadata and f-string debug expression text containing +comments. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 80cb09bb2ec5971..71c4eca8f5cf77e 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1544,15 +1544,29 @@ _strip_interpolation_debug_expr(PyObject *exprstr) { Py_ssize_t len = PyUnicode_GET_LENGTH(exprstr); - /* Discard whitespace after the debug "=" but preserve whitespace before it. */ + /* Discard whitespace and explicit line continuations after the debug "=" + but preserve whitespace before it. */ while (len > 0) { - Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, len - 1); - if (!_PyUnicode_IsWhitespace(c)) { + int has_newline = 0; + while (len > 0) { + Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, len - 1); + if (!_PyUnicode_IsWhitespace(c)) { + break; + } + if (c == '\r' || c == '\n') { + has_newline = 1; + } + len--; + } + if (!has_newline || len == 0 || + PyUnicode_READ_CHAR(exprstr, len - 1) != '\\') + { break; } len--; } - /* The debug marker may be absent from reconstructed lexer metadata. */ + + /* Preserve unexpected metadata instead of dropping source text. */ if (len == 0 || PyUnicode_READ_CHAR(exprstr, len - 1) != '=') { return Py_NewRef(exprstr); } From 0cdcfbd920a5d8750ecc6a2867663a30e360e40a Mon Sep 17 00:00:00 2001 From: lipengyu Date: Wed, 29 Jul 2026 19:45:56 +0800 Subject: [PATCH 3/3] Update 2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst --- .../2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst index a60e63bbad8223b..950d3ef0039f3d3 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst @@ -1,5 +1,7 @@ -Whitespace immediately before ``}``, ``!``, ``:``, or ``=`` is now included -in :attr:`string.templatelib.Interpolation.expression` for interpolations -created from t-string literals. Preserve escaped quotes while reconstructing -t-string interpolation metadata and f-string debug expression text containing -comments. +Trailing whitespace in a t-string interpolation expression is now preserved +in :attr:`string.templatelib.Interpolation.expression`, up to the closing ``}`` +or the conversion (``!``), format (``:``), or debug (``=``) delimiter. +Explicit line continuations following a debug ``=`` remain part of the debug +text and are excluded from :attr:`~string.templatelib.Interpolation.expression`. +Escaped quotes are now preserved while reconstructing t-string interpolation +metadata and f-string debug text containing comments.