diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 2c6d6daa01994e..36faef0620b52d 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -225,6 +225,29 @@ extern int _Py_GetForceASCII(void); encoding. */ extern void _Py_ResetForceASCII(void); +#ifdef HAVE_LANGINFO_H +# include // CODESET +#endif + +/* On FreeBSD, NetBSD, DragonFly BSD and macOS, wchar_t values are not + Unicode code points in non-UTF-8 locales. Locale-encoded text is converted + with iconv() (see Python/fileutils.c), and the wide character + functions which use the locale (such as wcscoll()) cannot be used + with Unicode wchar_t. */ +#if (defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) \ + || defined(__APPLE__)) \ + && defined(HAVE_ICONV) \ + && defined(HAVE_LANGINFO_H) && defined(CODESET) && SIZEOF_WCHAR_T == 4 +# define _Py_NON_UNICODE_WCHAR_T + +// Convert between Unicode code points and the native wchar_t form for +// C library functions which use the latter (such as the curses library). +// Export for '_curses'. +PyAPI_FUNC(int) _Py_LocaleNeedsWcharConversion(void); +PyAPI_FUNC(int) _Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size); +PyAPI_FUNC(void) _Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size); +#endif + extern int _Py_GetLocaleconvNumeric( struct lconv *lc, diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 145e66de9984ca..f1dccd82c5b5b7 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -52,6 +52,12 @@ struct pyhash_runtime_state { struct _fileutils_state { int force_ascii; + /* Memoized result of _Py_LocaleNeedsWcharConversion() with the locale + encoding it was computed for (see Python/fileutils.c). */ + struct _Py_wchar_conversion_state { + char codeset[32]; + int needed; + } wchar; }; #include "pycore_interpframe_structs.h" // _PyInterpreterFrame diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 8057c35f540841..f51879bd23f596 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -350,6 +350,53 @@ def test_strxfrm(self): self.assertRaises(ValueError, locale.strxfrm, 'a\0') +class TestCollationAcrossEncodings(unittest.TestCase): + # gh-154682: collation must give the same results in locales which + # only differ in the encoding. + + @support.thread_unsafe('setlocale is not thread-safe') + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") + @unittest.skipIf(sys.platform.startswith("dragonfly") or + sys.platform == "darwin", + "no collation data for non-UTF-8 locales") + def test_across_encodings(self): + def sign(x): + return (x > 0) - (x < 0) + cases = [ + ('uk_UA.UTF-8', 'uk_UA.KOI8-U', 'гґдиіїя'), + ('uk_UA.UTF-8', 'uk_UA.CP1251', 'гґдиіїя'), + ('el_GR.UTF-8', 'el_GR.ISO8859-7', 'αβδω'), + ('de_DE.UTF-8', 'de_DE.ISO8859-1', 'aäbößz'), + ('tr_TR.UTF-8', 'tr_TR.ISO8859-9', 'cçdıisşz'), + ('ca_ES.UTF-8', 'ca_ES.ISO8859-1', 'cçde'), + ] + oldloc = locale.setlocale(locale.LC_ALL) + self.addCleanup(locale.setlocale, locale.LC_ALL, oldloc) + tested = False + for utf8_loc, legacy_loc, chars in cases: + try: + locale.setlocale(locale.LC_ALL, utf8_loc) + except locale.Error: + continue + expected_order = sorted(chars, key=locale.strxfrm) + expected_signs = [[sign(locale.strcoll(a, b)) for b in chars] + for a in chars] + try: + locale.setlocale(locale.LC_ALL, legacy_loc) + except locale.Error: + continue + with self.subTest(locale=legacy_loc): + self.assertEqual(sorted(chars, key=locale.strxfrm), + expected_order) + signs = [[sign(locale.strcoll(a, b)) for b in chars] + for a in chars] + self.assertEqual(signs, expected_signs) + tested = True + if not tested: + self.skipTest('no suitable locales') + + class TestEnUSCollation(BaseLocalizedTest, TestCollation): # Test string collation functions with a real English locale diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 11ec57a259a9ab..70b6190dd2a446 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -125,6 +125,10 @@ def test_write_read_append(self): readline.append_history_file(-2147483648, hfilename) def test_nonascii_history(self): + try: + "entrée".encode(locale.getencoding()) + except UnicodeEncodeError as err: + self.skipTest("Locale cannot encode test data: " + format(err)) readline.clear_history() try: readline.add_history("entrée 1") diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 985984b55123ce..f36c6a4c8a264a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -3,6 +3,7 @@ import decimal import enum import fractions +import locale import math import platform import sys @@ -232,6 +233,41 @@ def test_strftime(self): self.assertRaises(TypeError, time.strftime, b'%S', tt) + @support.thread_unsafe('setlocale is not thread-safe') + def test_strftime_locale_encodings(self): + # gh-154682: the result must be the same in locales which only + # differ in the encoding. + pairs = [ + ('uk_UA.UTF-8', 'uk_UA.KOI8-U'), + ('uk_UA.UTF-8', 'uk_UA.CP1251'), + ('el_GR.UTF-8', 'el_GR.ISO8859-7'), + ('ja_JP.UTF-8', 'ja_JP.eucJP'), + ('de_DE.UTF-8', 'de_DE.ISO8859-1'), + ('tr_TR.UTF-8', 'tr_TR.ISO8859-9'), + ('ca_ES.UTF-8', 'ca_ES.ISO8859-1'), + ] + oldloc = locale.setlocale(locale.LC_ALL) + self.addCleanup(locale.setlocale, locale.LC_ALL, oldloc) + # Wednesday, March: non-ASCII in all tested locales + # (including Turkish and German) + tt = (2026, 3, 4, 12, 34, 56, 2, 63, 0) + tested = False + for utf8_loc, legacy_loc in pairs: + try: + locale.setlocale(locale.LC_ALL, utf8_loc) + except locale.Error: + continue + expected = time.strftime('%A, %B', tt) + try: + locale.setlocale(locale.LC_ALL, legacy_loc) + except locale.Error: + continue + with self.subTest(locale=legacy_loc): + self.assertEqual(time.strftime('%A, %B', tt), expected) + tested = True + if not tested: + self.skipTest('no suitable locales') + def test_strftime_invalid_format(self): tt = time.gmtime(self.t) with SuppressCrashReport(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst new file mode 100644 index 00000000000000..cfdc812d7c6c24 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst @@ -0,0 +1,10 @@ +Fix corruption of locale-encoded text on FreeBSD, NetBSD, DragonFly BSD +and macOS +in non-UTF-8 locales, +e.g. in the results of :func:`time.strftime` and :func:`locale.nl_langinfo`, +in localized error messages, +and in decoding of the command line arguments, +fix wrong collation in :func:`locale.strcoll` and :func:`locale.strxfrm`, +and fix display and input of non-ASCII characters in :mod:`curses`. +``wchar_t`` values are not Unicode code points on these platforms, +so ``iconv()`` is now used instead of ``mbstowcs()`` and ``wcstombs()``. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index b2d745332317a3..73a9a8ad664d76 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -527,6 +527,11 @@ PyCurses_ConvertToWideCell(PyObject *obj, wchar_t *wch) (int)CCHARW_MAX, PyUnicode_GET_LENGTH(obj)); return -1; } +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(wch, nch) < 0) { + return -1; + } +#endif /* A lone control character is allowed (like addch(ord('\n'))), but in a multi-character cell the base must be a printable spacing character and the rest zero-width combining characters. Check explicitly: otherwise @@ -622,6 +627,13 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, *wstr = PyUnicode_AsWideCharString(obj, NULL); if (*wstr == NULL) return 0; +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(*wstr, wcslen(*wstr)) < 0) { + PyMem_Free(*wstr); + *wstr = NULL; + return 0; + } +#endif return 2; #else assert (wstr == NULL); @@ -880,6 +892,9 @@ curses_cell_text(cursesmodule_state *state, const curses_cell_t *cell) PyErr_SetString(state->error, "getcchar() returned ERR"); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(wstr, wcslen(wstr)); +#endif return PyUnicode_FromWideChar(wstr, -1); #else char ch = (char)(*cell & A_CHARTEXT); @@ -1296,6 +1311,12 @@ complexstr_from_string(cursesmodule_state *state, PyObject *str, if (wbuf == NULL) { return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(wbuf, n) < 0) { + PyMem_Free(wbuf); + return NULL; + } +#endif curses_cell_t *cells = n > 0 ? PyMem_New(curses_cell_t, n) : NULL; if (n > 0 && cells == NULL) { PyMem_Free(wbuf); @@ -1585,6 +1606,9 @@ complexstr_str(PyObject *self) } pos += wcslen(buf + pos); } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(buf, pos); +#endif PyObject *res = PyUnicode_FromWideChar(buf, pos); PyMem_Free(buf); return res; @@ -3275,6 +3299,13 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, rtn += 256; } #endif +#endif +#ifdef _Py_NON_UNICODE_WCHAR_T + { + wchar_t wch = (wchar_t)rtn; + _Py_LocaleWcharToUnicode_InPlace(&wch, 1); + return PyUnicode_FromOrdinal(wch); + } #endif return PyUnicode_FromOrdinal(rtn); } else { @@ -3325,8 +3356,16 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, } if (ct == KEY_CODE_YES) return PyLong_FromLong(rtn); +#ifdef _Py_NON_UNICODE_WCHAR_T + else { + wchar_t wch = (wchar_t)rtn; + _Py_LocaleWcharToUnicode_InPlace(&wch, 1); + return PyUnicode_FromOrdinal(wch); + } +#else else return PyUnicode_FromOrdinal(rtn); +#endif #else /* Without the wide library, read one key with wgetch(): a value above 255 is a function key (returned as an int); a byte is decoded with the @@ -3801,6 +3840,9 @@ PyCursesWindow_get_wstr(PyObject *op, PyObject *args) for (Py_ssize_t i = 0; i < len; i++) { wbuf[i] = (wchar_t)buf[i]; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(wbuf, len); +#endif PyObject *res = PyUnicode_FromWideChar(wbuf, len); PyMem_Free(wbuf); PyMem_Free(buf); @@ -3866,6 +3908,9 @@ PyCursesWindow_in_wstr(PyObject *op, PyObject *args) PyMem_Free(buf); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(buf, wcslen(buf)); +#endif PyObject *res = PyUnicode_FromWideChar(buf, -1); PyMem_Free(buf); return res; @@ -5854,6 +5899,9 @@ _curses_erasewchar_impl(PyObject *module) curses_set_error(module, "erasewchar", NULL); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(&ch, 1); +#endif return PyUnicode_FromWideChar(&ch, 1); #else /* Without the wide library, decode the single-byte erase character @@ -7076,6 +7124,9 @@ _curses_killwchar_impl(PyObject *module) curses_set_error(module, "killwchar", NULL); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(&ch, 1); +#endif return PyUnicode_FromWideChar(&ch, 1); #else /* Without the wide library, decode the single-byte kill character @@ -8018,6 +8069,19 @@ _curses_wunctrl(PyObject *module, PyObject *ch) curses_set_null_error(module, "wunctrl", NULL); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + { + /* wunctrl() returns a pointer to a static buffer; make a copy + before converting in place. */ + wchar_t wbuf[8]; + size_t len = wcslen(res); + if (len < Py_ARRAY_LENGTH(wbuf)) { + wcscpy(wbuf, res); + _Py_LocaleWcharToUnicode_InPlace(wbuf, len); + return PyUnicode_FromWideChar(wbuf, len); + } + } +#endif return PyUnicode_FromWideChar(res, -1); #else /* Without the wide library, fall back to the single-byte unctrl() and @@ -8107,6 +8171,10 @@ _curses_ungetch(PyObject *module, PyObject *ch) wchar_t wch; if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(&wch, 1) < 0) + return NULL; +#endif return curses_check_err(module, unget_wch(wch), "unget_wch", "ungetch"); } #endif @@ -8138,6 +8206,10 @@ _curses_unget_wch(PyObject *module, PyObject *ch) if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; #ifdef HAVE_NCURSESW +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(&wch, 1) < 0) + return NULL; +#endif return curses_check_err(module, unget_wch(wch), "unget_wch", NULL); #else /* Without the wide library there is no unget_wch(): encode the character as @@ -8226,6 +8298,12 @@ _curses_slk_set_impl(PyObject *module, int labnum, PyObject *label, if (wstr == NULL) { return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(wstr, wcslen(wstr)) < 0) { + PyMem_Free(wstr); + return NULL; + } +#endif rtn = slk_wset(labnum, wstr, justify); PyMem_Free(wstr); #else diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 6fbce0ed85e695..4f88ad7f1b5474 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3657,20 +3657,20 @@ dotsep_as_utf8(const char *s) { PyObject *utf8; PyObject *tmp; - wchar_t buf[2]; - size_t n; - n = mbstowcs(buf, s, 2); - if (n != 1) { /* Issue #7442 */ + /* Do not use mbstowcs(): wchar_t values are not Unicode code points + in non-UTF-8 locales on some platforms (e.g. the BSDs and macOS). */ + tmp = PyUnicode_DecodeLocale(s, NULL); + if (tmp == NULL) { + return NULL; + } + if (PyUnicode_GET_LENGTH(tmp) != 1) { /* Issue #7442 */ + Py_DECREF(tmp); PyErr_SetString(PyExc_ValueError, "invalid decimal point or unsupported " "combination of LC_CTYPE and LC_NUMERIC"); return NULL; } - tmp = PyUnicode_FromWideChar(buf, n); - if (tmp == NULL) { - return NULL; - } utf8 = PyUnicode_AsUTF8String(tmp); Py_DECREF(tmp); return utf8; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 20783dc00f90d3..5043069dad047f 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -406,6 +406,28 @@ static PyObject * _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) /*[clinic end generated code: output=82ddc6d62c76d618 input=693cd02bcbf38dd8]*/ { +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_LocaleNeedsWcharConversion()) { + /* wchar_t is not Unicode in non-UTF-8 locales and the collation + tables are keyed by the raw locale values, so wcscoll() cannot + be used with Unicode wchar_t. Encode to the locale encoding + and use strcoll(). */ + PyObject *result = NULL; + PyObject *b1 = PyUnicode_EncodeLocale(os1, NULL); + if (b1 == NULL) { + return NULL; + } + PyObject *b2 = PyUnicode_EncodeLocale(os2, NULL); + if (b2 != NULL) { + result = PyLong_FromLong(strcoll(PyBytes_AS_STRING(b1), + PyBytes_AS_STRING(b2))); + Py_DECREF(b2); + } + Py_DECREF(b1); + return result; + } +#endif + { PyObject *result = NULL; wchar_t *ws1 = NULL, *ws2 = NULL; @@ -423,6 +445,7 @@ _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) if (ws1) PyMem_Free(ws1); if (ws2) PyMem_Free(ws2); return result; + } } #endif @@ -441,6 +464,47 @@ static PyObject * _locale_strxfrm_impl(PyObject *module, PyObject *str) /*[clinic end generated code: output=3081866ebffc01af input=1378bbe6a88b4780]*/ { +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_LocaleNeedsWcharConversion()) { + /* wchar_t is not Unicode in non-UTF-8 locales and the collation + tables are keyed by the raw locale values, so wcsxfrm() cannot + be used with Unicode wchar_t. Encode to the locale encoding + and use strxfrm(). The result is decoded as Latin-1: it maps + bytes to code points in the same order, so comparison of the + resulting strings compares the byte sequences. */ + PyObject *bytes = PyUnicode_EncodeLocale(str, NULL); + if (bytes == NULL) { + return NULL; + } + const char *s = PyBytes_AS_STRING(bytes); + PyObject *result = NULL; + char dummy[1]; + errno = 0; + size_t n = strxfrm(dummy, s, 1); + if (errno && errno != ERANGE) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + char *buf = PyMem_Malloc(n + 1); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } + errno = 0; + n = strxfrm(buf, s, n + 1); + if (errno) { + PyErr_SetFromErrno(PyExc_OSError); + } + else { + result = PyUnicode_DecodeLatin1(buf, n, NULL); + } + PyMem_Free(buf); +error: + Py_DECREF(bytes); + return result; + } +#endif + { Py_ssize_t n1; wchar_t *s = NULL, *buf = NULL; wchar_t dummy[1]; @@ -528,6 +592,7 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) PyMem_Free(buf); PyMem_Free(s); return result; + } } #endif diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 70d7e1b3713687..38c2703a927805 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -770,6 +770,15 @@ the C library strftime function.\n" # undef HAVE_WCSFTIME #endif +// On FreeBSD, NetBSD, DragonFly BSD and macOS, wchar_t values are not +// Unicode code points in non-UTF-8 locales, so the result of wcsftime() +// cannot be used directly. Use strftime() and decode its result with +// PyUnicode_DecodeLocaleAndSize() (which handles such platforms). +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) \ + || defined(__APPLE__) +# undef HAVE_WCSFTIME +#endif + #ifdef HAVE_WCSFTIME #define time_char wchar_t #define format_time wcsftime diff --git a/Python/fileutils.c b/Python/fileutils.c index 5cbfd8e6ce4fa0..c858e6e2604c31 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -454,6 +454,398 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, } #endif /* !HAVE_MBRTOWC */ +#ifdef _Py_NON_UNICODE_WCHAR_T +/* wchar_t is not Unicode in non-UTF-8 locales (see pycore_fileutils.h), + so mbstowcs() and wcstombs() cannot be used to convert locale-encoded + text. Use iconv() instead. */ +# include +# include // MB_LEN_MAX +/* Native-endian UTF-32: a raw array of Unicode code points (unlike + "UCS-4", it also validates the converted values). */ +# ifdef WORDS_BIGENDIAN +# define _Py_UTF32_NATIVE "UTF-32BE" +# else +# define _Py_UTF32_NATIVE "UTF-32LE" +# endif +/* Fall back to mbstowcs()/wcstombs(). */ +# define _Py_ICONV_FALLBACK (-9) + +/* Return non-zero if wchar_t is not Unicode in the current locale, i.e. if + the locale encoding is neither UTF-8 nor ASCII. The result is memoized: + this is called before every conversion, and in the common case (a UTF-8 + locale) it is the only overhead. */ +int +_Py_LocaleNeedsWcharConversion(void) +{ + const char *codeset = nl_langinfo(CODESET); + if (codeset == NULL) { + return 0; + } + struct _Py_wchar_conversion_state *state = &_PyRuntime.fileutils.wchar; + if (strcmp(codeset, state->codeset) == 0) { + return state->needed; + } + int needed = (*codeset != '\0' + && strcmp(codeset, "UTF-8") != 0 + && strcmp(codeset, "US-ASCII") != 0 + && strcmp(codeset, "646") != 0); + if (strlen(codeset) < sizeof(state->codeset)) { + memcpy(state->codeset, codeset, strlen(codeset) + 1); + state->needed = needed; + } + return needed; +} + +static iconv_t +locale_iconv_open(int decode) +{ + const char *codeset = nl_langinfo(CODESET); + if (!_Py_LocaleNeedsWcharConversion()) { + /* mbstowcs() and wcstombs() are correct for these encodings */ + return (iconv_t)-1; + } + return decode ? iconv_open(_Py_UTF32_NATIVE, codeset) + : iconv_open(codeset, _Py_UTF32_NATIVE); +} + +static int +decode_locale_iconv(const char *arg, wchar_t **wstr, size_t *wlen, + const char **reason, _Py_error_handler errors) +{ + if (!_Py_LocaleNeedsWcharConversion()) { + return _Py_ICONV_FALLBACK; + } + + int surrogateescape; + if (get_surrogateescape(errors, &surrogateescape) < 0) { + return -3; + } + + size_t argsize = strlen(arg); + int ascii = 1; + for (size_t i = 0; i < argsize; i++) { + if ((unsigned char)arg[i] >= 0x80) { + ascii = 0; + break; + } + } + if (ascii) { + return _Py_ICONV_FALLBACK; + } + + iconv_t cd = locale_iconv_open(1); + if (cd == (iconv_t)-1) { + return _Py_ICONV_FALLBACK; + } + + if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { + iconv_close(cd); + return -1; + } + /* the output cannot be longer than the number of input bytes */ + wchar_t *res = PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t)); + if (res == NULL) { + iconv_close(cd); + return -1; + } + + char *in = (char *)arg; + size_t inleft = argsize; + wchar_t *wout = res; + while (inleft > 0) { + char *out = (char *)wout; + size_t outleft = (res + argsize - wout) * sizeof(wchar_t); + /* Cast through void*: iconv() declares its second argument as + "char **" on most systems, but "const char **" on some. */ + size_t converted = iconv(cd, (void *)&in, &inleft, &out, &outleft); + wout = (wchar_t *)out; + if (converted == (size_t)-1) { + if (errno != EILSEQ && errno != EINVAL) { + /* unexpected error: fall back to mbstowcs() */ + PyMem_RawFree(res); + iconv_close(cd); + return _Py_ICONV_FALLBACK; + } + if (!surrogateescape) { + /* Leave it to mbstowcs() to decide whether the input is + valid: it can be more permissive than iconv(), and a + broken locale should not make the conversion fail. */ + PyMem_RawFree(res); + iconv_close(cd); + return _Py_ICONV_FALLBACK; + } + /* Escape the byte as UTF-8b, and start over in the initial + shift state. */ + *wout++ = 0xdc00 + (unsigned char)*in++; + inleft--; + iconv(cd, NULL, NULL, NULL, NULL); + } + } + iconv_close(cd); + *wout = L'\0'; + *wstr = res; + if (wlen != NULL) { + *wlen = wout - res; + } + return 0; +} + +static int +encode_locale_iconv(const wchar_t *text, char **str, + size_t *error_pos, const char **reason, + int raw_malloc, _Py_error_handler errors) +{ + if (!_Py_LocaleNeedsWcharConversion()) { + return _Py_ICONV_FALLBACK; + } + + int surrogateescape; + if (get_surrogateescape(errors, &surrogateescape) < 0) { + return -3; + } + + size_t len = wcslen(text); + int ascii = 1; + for (size_t i = 0; i < len; i++) { + if ((unsigned long)text[i] >= 0x80) { + ascii = 0; + break; + } + } + if (ascii) { + return _Py_ICONV_FALLBACK; + } + + iconv_t cd = locale_iconv_open(0); + if (cd == (iconv_t)-1) { + return _Py_ICONV_FALLBACK; + } + + char *result = NULL; + int res = -1; + if (len > (PY_SSIZE_T_MAX - MB_LEN_MAX - 1) / MB_LEN_MAX) { + goto done; + } + /* reserve room for the final shift sequence and the null byte */ + size_t size = len * MB_LEN_MAX + MB_LEN_MAX + 1; + result = raw_malloc ? PyMem_RawMalloc(size) : PyMem_Malloc(size); + if (result == NULL) { + goto done; + } + + const wchar_t *win = text; + const wchar_t *wend = text + len; + char *out = result; + size_t outleft = size - 1; + while (win < wend) { + wchar_t c = *win; + if (c >= 0xdc80 && c <= 0xdcff) { + if (!surrogateescape) { + goto encode_error; + } + /* UTF-8b surrogate: write the raw byte, and start over in + the initial shift state */ + *out++ = (char)(c - 0xdc00); + outleft--; + win++; + iconv(cd, NULL, NULL, NULL, NULL); + continue; + } + /* Note: iconv() can replace a character which cannot be encoded + with an approximation (e.g. "'e" for "é" or "?"), depending + on the implementation. */ + char *in = (char *)win; + size_t inleft = (wend - win) * sizeof(wchar_t); + /* Cast through void*: iconv() declares its second argument as + "char **" on most systems, but "const char **" on some. */ + size_t converted = iconv(cd, (void *)&in, &inleft, &out, &outleft); + win = (const wchar_t *)in; + if (converted == (size_t)-1) { + if (errno != EILSEQ && errno != EINVAL) { + /* unexpected error: fall back to wcstombs() */ + goto fallback; + } + /* iconv() stopped at the first character which cannot be + encoded; an escaped surrogate is handled at the start of + the next iteration */ + if (!(*win >= 0xdc80 && *win <= 0xdcff)) { + goto encode_error; + } + } + } + /* write the final shift sequence for stateful encodings */ + if (iconv(cd, NULL, NULL, &out, &outleft) == (size_t)-1) { + goto fallback; + } + *out = '\0'; + *str = result; + result = NULL; + res = 0; + goto done; + +fallback: + res = _Py_ICONV_FALLBACK; + goto done; + +encode_error: + res = -2; + if (error_pos != NULL) { + *error_pos = win - text; + } + if (reason != NULL) { + *reason = "encoding error"; + } + +done: + if (result != NULL) { + if (raw_malloc) { + PyMem_RawFree(result); + } + else { + PyMem_Free(result); + } + } + iconv_close(cd); + return res; +} + +/* Convert a wide character string in place from Unicode code points to + the native wchar_t form, for passing to C library functions which + expect the latter (such as the curses library). + Return 0 on success. Return -1 with an exception set if a character + cannot be represented in the locale encoding. */ +int +_Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size) +{ + if (!_Py_LocaleNeedsWcharConversion()) { + return 0; + } + + iconv_t cd = (iconv_t)-1; + int res = 0; + for (Py_ssize_t i = 0; i < size; i++) { + wchar_t wc = str[i]; + if ((unsigned long)wc < 0x80 + || ((unsigned long)wc >= 0xdc80 && (unsigned long)wc <= 0xdcff)) + { + continue; + } + if (cd == (iconv_t)-1) { + cd = locale_iconv_open(0); + if (cd == (iconv_t)-1) { + /* wchar_t is Unicode for this locale encoding, or no + conversion is possible */ + return 0; + } + } + else { + /* reset cd to the initial shift state */ + iconv(cd, NULL, NULL, NULL, NULL); + } + Py_UCS4 ch = (Py_UCS4)wc; + char buf[MB_LEN_MAX]; + char *in = (char *)&ch; + size_t inleft = sizeof(ch); + char *out = buf; + size_t outleft = sizeof(buf); + mbstate_t mbs; + wchar_t converted; + size_t blen; + /* A non-zero result means that the character was replaced + with a substitute. */ + if (iconv(cd, (void *)&in, &inleft, &out, &outleft) != 0 + || inleft != 0) + { + goto encode_error; + } + /* locale bytes -> native form: exactly one wchar_t */ + blen = sizeof(buf) - outleft; + memset(&mbs, 0, sizeof(mbs)); + if (mbrtowc(&converted, buf, blen, &mbs) != blen) { + goto encode_error; + } + str[i] = converted; + continue; + +encode_error: + { + PyObject *obj = PyUnicode_FromOrdinal((int)wc); + if (obj != NULL) { + PyObject *exc = PyObject_CallFunction( + PyExc_UnicodeEncodeError, "sOnns", + nl_langinfo(CODESET), obj, + (Py_ssize_t)0, (Py_ssize_t)1, + "character maps to "); + if (exc != NULL) { + PyCodec_StrictErrors(exc); + Py_DECREF(exc); + } + Py_DECREF(obj); + } + } + res = -1; + break; + } + if (cd != (iconv_t)-1) { + iconv_close(cd); + } + return res; +} + +/* The reverse conversion: from the native wchar_t form to Unicode + code points. Values which cannot be converted are left unchanged. */ +void +_Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size) +{ + if (!_Py_LocaleNeedsWcharConversion()) { + return; + } + + iconv_t cd = (iconv_t)-1; + for (Py_ssize_t i = 0; i < size; i++) { + wchar_t wc = str[i]; + if ((unsigned long)wc < 0x80 + || ((unsigned long)wc >= 0xdc80 && (unsigned long)wc <= 0xdcff)) + { + continue; + } + /* native form -> locale bytes */ + char buf[MB_LEN_MAX]; + mbstate_t mbs; + memset(&mbs, 0, sizeof(mbs)); + size_t blen = wcrtomb(buf, wc, &mbs); + if (blen == (size_t)-1) { + continue; + } + /* locale bytes -> Unicode */ + if (cd == (iconv_t)-1) { + cd = locale_iconv_open(1); + if (cd == (iconv_t)-1) { + return; + } + } + else { + /* reset cd to the initial shift state */ + iconv(cd, NULL, NULL, NULL, NULL); + } + char *in = buf; + size_t inleft = blen; + Py_UCS4 ch; + char *out = (char *)&ch; + size_t outleft = sizeof(ch); + if (iconv(cd, (void *)&in, &inleft, &out, &outleft) == (size_t)-1 + || inleft != 0 || outleft != 0) + { + continue; + } + str[i] = (wchar_t)ch; + } + if (cd != (iconv_t)-1) { + iconv_close(cd); + } +} +#endif /* _Py_NON_UNICODE_WCHAR_T */ + static int decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) @@ -467,6 +859,13 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, mbstate_t mbs; #endif +#ifdef _Py_NON_UNICODE_WCHAR_T + int iconv_res = decode_locale_iconv(arg, wstr, wlen, reason, errors); + if (iconv_res != _Py_ICONV_FALLBACK) { + return iconv_res; + } +#endif + int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { return -3; @@ -683,6 +1082,14 @@ encode_current_locale(const wchar_t *text, char **str, size_t i, size, converted; wchar_t c, buf[2]; +#ifdef _Py_NON_UNICODE_WCHAR_T + int iconv_res = encode_locale_iconv(text, str, error_pos, reason, + raw_malloc, errors); + if (iconv_res != _Py_ICONV_FALLBACK) { + return iconv_res; + } +#endif + int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { return -3;