From 3ba7336bf8a86d638d176c56927b02ac3cdb38ec Mon Sep 17 00:00:00 2001 From: xxyzz Date: Fri, 24 Jul 2026 14:17:40 +0800 Subject: [PATCH] gh-154577: Fix data race in `Modules/cjkcodecs/multibytecodec.c` Prevent race by adding `@critical_section` to functions that have `MultibyteIncrementalEncoderObject` or `MultibyteIncrementalDecoderObject` argument Fix TSAN-0001 from gh-153852 --- ...-07-24-14-13-38.gh-issue-154577.2d3roU.rst | 1 + Modules/cjkcodecs/clinic/multibytecodec.c.h | 72 ++++++++++++++++--- Modules/cjkcodecs/multibytecodec.c | 45 ++++++++---- 3 files changed, 95 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-24-14-13-38.gh-issue-154577.2d3roU.rst diff --git a/Misc/NEWS.d/next/Library/2026-07-24-14-13-38.gh-issue-154577.2d3roU.rst b/Misc/NEWS.d/next/Library/2026-07-24-14-13-38.gh-issue-154577.2d3roU.rst new file mode 100644 index 000000000000000..28756a4f21be2a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-24-14-13-38.gh-issue-154577.2d3roU.rst @@ -0,0 +1 @@ +Fix data race of incremental encoder/decoder objects diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 32588b0561e1acc..ffd4ad1afbc5e3a 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, @@ -258,7 +259,9 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(PyObject *self, PyObject *con goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteIncrementalEncoder_encode_impl((MultibyteIncrementalEncoderObject *)self, input, final); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -278,7 +281,13 @@ _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEn static PyObject * _multibytecodec_MultibyteIncrementalEncoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalEncoder_getstate_impl((MultibyteIncrementalEncoderObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multibytecodec_MultibyteIncrementalEncoder_getstate_impl((MultibyteIncrementalEncoderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalEncoder_setstate__doc__, @@ -304,7 +313,9 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate(PyObject *self, PyObject *a goto exit; } statelong = (PyLongObject *)arg; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteIncrementalEncoder_setstate_impl((MultibyteIncrementalEncoderObject *)self, statelong); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -324,7 +335,13 @@ _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncod static PyObject * _multibytecodec_MultibyteIncrementalEncoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalEncoder_reset_impl((MultibyteIncrementalEncoderObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multibytecodec_MultibyteIncrementalEncoder_reset_impl((MultibyteIncrementalEncoderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_decode__doc__, @@ -392,7 +409,9 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(PyObject *self, PyObject *con goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteIncrementalDecoder_decode_impl((MultibyteIncrementalDecoderObject *)self, &input, final); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for input */ @@ -417,7 +436,13 @@ _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDe static PyObject * _multibytecodec_MultibyteIncrementalDecoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalDecoder_getstate_impl((MultibyteIncrementalDecoderObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multibytecodec_MultibyteIncrementalDecoder_getstate_impl((MultibyteIncrementalDecoderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_setstate__doc__, @@ -443,7 +468,9 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate(PyObject *self, PyObject *a goto exit; } state = arg; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteIncrementalDecoder_setstate_impl((MultibyteIncrementalDecoderObject *)self, state); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -463,7 +490,13 @@ _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecod static PyObject * _multibytecodec_MultibyteIncrementalDecoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalDecoder_reset_impl((MultibyteIncrementalDecoderObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multibytecodec_MultibyteIncrementalDecoder_reset_impl((MultibyteIncrementalDecoderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_multibytecodec_MultibyteStreamReader_read__doc__, @@ -492,7 +525,9 @@ _multibytecodec_MultibyteStreamReader_read(PyObject *self, PyObject *const *args } sizeobj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteStreamReader_read_impl((MultibyteStreamReaderObject *)self, sizeobj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -524,7 +559,9 @@ _multibytecodec_MultibyteStreamReader_readline(PyObject *self, PyObject *const * } sizeobj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteStreamReader_readline_impl((MultibyteStreamReaderObject *)self, sizeobj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -556,7 +593,9 @@ _multibytecodec_MultibyteStreamReader_readlines(PyObject *self, PyObject *const } sizehintobj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteStreamReader_readlines_impl((MultibyteStreamReaderObject *)self, sizehintobj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -576,7 +615,13 @@ _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *se static PyObject * _multibytecodec_MultibyteStreamReader_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteStreamReader_reset_impl((MultibyteStreamReaderObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multibytecodec_MultibyteStreamReader_reset_impl((MultibyteStreamReaderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_multibytecodec_MultibyteStreamWriter_write__doc__, @@ -618,7 +663,9 @@ _multibytecodec_MultibyteStreamWriter_write(PyObject *self, PyTypeObject *cls, P goto exit; } strobj = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteStreamWriter_write_impl((MultibyteStreamWriterObject *)self, cls, strobj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -663,7 +710,9 @@ _multibytecodec_MultibyteStreamWriter_writelines(PyObject *self, PyTypeObject *c goto exit; } lines = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multibytecodec_MultibyteStreamWriter_writelines_impl((MultibyteStreamWriterObject *)self, cls, lines); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -684,11 +733,18 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se static PyObject * _multibytecodec_MultibyteStreamWriter_reset(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reset() takes no arguments"); - return NULL; + goto exit; } - return _multibytecodec_MultibyteStreamWriter_reset_impl((MultibyteStreamWriterObject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multibytecodec_MultibyteStreamWriter_reset_impl((MultibyteStreamWriterObject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_multibytecodec___create_codec__doc__, @@ -698,4 +754,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=a84b1544d7d01abb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cb747fb3277c7d72 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 32c96c9d2cb3cde..38efe9adce3b911 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -931,6 +931,7 @@ decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalEncoder.encode input: object @@ -941,18 +942,19 @@ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self, PyObject *input, int final) -/*[clinic end generated code: output=123361b6c505e2c1 input=bd5f7d40d43e99b0]*/ +/*[clinic end generated code: output=123361b6c505e2c1 input=56f866366a67e48c]*/ { return encoder_encode_stateful(STATEFUL_ECTX(self), input, final); } /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalEncoder.getstate [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self) -/*[clinic end generated code: output=9794a5ace70d7048 input=4a2a82874ffa40bb]*/ +/*[clinic end generated code: output=9794a5ace70d7048 input=672e617242e38889]*/ { /* state made up of 1 byte for buffer size, up to MAXENCPENDING*4 bytes for UTF-8 encoded buffer (each character can use up to 4 @@ -1001,6 +1003,7 @@ _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEn } /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalEncoder.setstate state as statelong: object(type='PyLongObject *', subclass_of='&PyLong_Type') / @@ -1009,7 +1012,7 @@ _multibytecodec.MultibyteIncrementalEncoder.setstate static PyObject * _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEncoderObject *self, PyLongObject *statelong) -/*[clinic end generated code: output=4e5e98ac1f4039ca input=c80fb5830d4d2f76]*/ +/*[clinic end generated code: output=4e5e98ac1f4039ca input=88d7ab04360fe581]*/ { PyObject *pending = NULL; unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)]; @@ -1044,12 +1047,13 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEn } /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalEncoder.reset [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncoderObject *self) -/*[clinic end generated code: output=b4125d8f537a253f input=930f06760707b6ea]*/ +/*[clinic end generated code: output=b4125d8f537a253f input=0d3cef28b3588e3d]*/ { /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */ unsigned char buffer[4], *outbuf; @@ -1164,6 +1168,7 @@ static PyType_Spec encoder_spec = { /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalDecoder.decode input: Py_buffer @@ -1174,7 +1179,7 @@ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDecoderObject *self, Py_buffer *input, int final) -/*[clinic end generated code: output=b9b9090e8a9ce2ba input=8795fbb20860027a]*/ +/*[clinic end generated code: output=b9b9090e8a9ce2ba input=876dd4cd3ab21a2d]*/ { MultibyteDecodeBuffer buf; char *data, *wdata = NULL; @@ -1247,12 +1252,13 @@ _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDeco } /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalDecoder.getstate [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDecoderObject *self) -/*[clinic end generated code: output=255009c4713b7f82 input=4006aa49bddbaa75]*/ +/*[clinic end generated code: output=255009c4713b7f82 input=a8f7c7110e2a02de]*/ { PyObject *buffer; PyObject *statelong; @@ -1276,6 +1282,7 @@ _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDe } /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalDecoder.setstate state: object(subclass_of='&PyTuple_Type') / @@ -1284,7 +1291,7 @@ _multibytecodec.MultibyteIncrementalDecoder.setstate static PyObject * _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDecoderObject *self, PyObject *state) -/*[clinic end generated code: output=106b2fbca3e2dcc2 input=e5d794e8baba1a47]*/ +/*[clinic end generated code: output=106b2fbca3e2dcc2 input=efec15718b7365e7]*/ { PyObject *buffer; PyLongObject *statelong; @@ -1333,12 +1340,13 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe } /*[clinic input] +@critical_section _multibytecodec.MultibyteIncrementalDecoder.reset [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecoderObject *self) -/*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/ +/*[clinic end generated code: output=da423b1782c23ed1 input=349aae10fdf5e262]*/ { if (self->codec->decreset != NULL && self->codec->decreset(&self->state, self->codec) != 0) @@ -1548,6 +1556,7 @@ mbstreamreader_iread(MultibyteStreamReaderObject *self, } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamReader.read sizeobj: object = None @@ -1557,7 +1566,7 @@ mbstreamreader_iread(MultibyteStreamReaderObject *self, static PyObject * _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *self, PyObject *sizeobj) -/*[clinic end generated code: output=35621eb75355d5b8 input=015b0d3ff2fca485]*/ +/*[clinic end generated code: output=35621eb75355d5b8 input=fff49594f51bd797]*/ { Py_ssize_t size; @@ -1577,6 +1586,7 @@ _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *sel } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamReader.readline sizeobj: object = None @@ -1586,7 +1596,7 @@ _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *sel static PyObject * _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject *self, PyObject *sizeobj) -/*[clinic end generated code: output=4fbfaae1ed457a11 input=41ccc64f9bb0cec3]*/ +/*[clinic end generated code: output=4fbfaae1ed457a11 input=678318056105659a]*/ { Py_ssize_t size; @@ -1606,6 +1616,7 @@ _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamReader.readlines sizehintobj: object = None @@ -1615,7 +1626,7 @@ _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject static PyObject * _multibytecodec_MultibyteStreamReader_readlines_impl(MultibyteStreamReaderObject *self, PyObject *sizehintobj) -/*[clinic end generated code: output=e7c4310768ed2ad4 input=54932f5d4d88e880]*/ +/*[clinic end generated code: output=e7c4310768ed2ad4 input=3588f523f00a50f3]*/ { PyObject *r, *sr; Py_ssize_t sizehint; @@ -1642,12 +1653,13 @@ _multibytecodec_MultibyteStreamReader_readlines_impl(MultibyteStreamReaderObject } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamReader.reset [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *self) -/*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/ +/*[clinic end generated code: output=138490370a680abc input=100515334705cfe5]*/ { if (self->codec->decreset != NULL && self->codec->decreset(&self->state, self->codec) != 0) @@ -1784,6 +1796,7 @@ mbstreamwriter_iwrite(MultibyteStreamWriterObject *self, } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamWriter.write cls: defining_class @@ -1795,7 +1808,7 @@ static PyObject * _multibytecodec_MultibyteStreamWriter_write_impl(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *strobj) -/*[clinic end generated code: output=68ade3aea26410ac input=199f26f68bd8425a]*/ +/*[clinic end generated code: output=68ade3aea26410ac input=8626f7b437b3cb84]*/ { module_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -1806,6 +1819,7 @@ _multibytecodec_MultibyteStreamWriter_write_impl(MultibyteStreamWriterObject *se } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamWriter.writelines cls: defining_class @@ -1817,7 +1831,7 @@ static PyObject * _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *lines) -/*[clinic end generated code: output=b4c99d2cf23ffb88 input=a6d5fe7c74972a34]*/ +/*[clinic end generated code: output=b4c99d2cf23ffb88 input=3a582d57986e2d4b]*/ { PyObject *strobj; int i, r; @@ -1849,6 +1863,7 @@ _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObjec } /*[clinic input] +@critical_section _multibytecodec.MultibyteStreamWriter.reset cls: defining_class @@ -1859,7 +1874,7 @@ _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObjec static PyObject * _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/ +/*[clinic end generated code: output=32ef224c2a38aa3d input=e3ea4012eb54f582]*/ { PyObject *pwrt;