bpo-31900: Fix localeconv() encoding for LC_NUMERIC (#4174) · pythoncapi/cpython@cb064fc · GitHub
Skip to content

Commit cb064fc

Browse files
vstinnervstinner
authored
bpo-31900: Fix localeconv() encoding for LC_NUMERIC (python#4174)
* Add _Py_GetLocaleconvNumeric() function: decode decimal_point and thousands_sep fields of localeconv() from the LC_NUMERIC encoding, rather than decoding from the LC_CTYPE encoding. * Modify locale.localeconv() and "n" formatter of str.format() (for int, float and complex to use _Py_GetLocaleconvNumeric() internally.
1 parent 7ed7aea commit cb064fc

8 files changed

Lines changed: 151 additions & 19 deletions

File tree

‎Doc/library/locale.rst‎

Lines changed: 10 additions & 0 deletions

‎Doc/library/stdtypes.rst‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1599,6 +1599,20 @@ expression support in the :mod:`re` module).
15991599
See :ref:`formatstrings` for a description of the various formatting options
16001600
that can be specified in format strings.
16011601

1602+
.. note::
1603+
When formatting a number (:class:`int`, :class:`float`, :class:`float`
1604+
and subclasses) with the ``n`` type (ex: ``'{:n}'.format(1234)``), the
1605+
function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC``
1606+
locale to decode ``decimal_point`` and ``thousands_sep`` fields of
1607+
:c:func:`localeconv` if they are non-ASCII or longer than 1 byte, and the
1608+
``LC_NUMERIC`` locale is different than the ``LC_CTYPE`` locale. This
1609+
temporary change affects other threads.
1610+
1611+
.. versionchanged:: 3.7
1612+
When formatting a number with the ``n`` type, the function sets
1613+
temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some
1614+
cases.
1615+
16021616

16031617
.. method:: str.format_map(mapping)
16041618

‎Doc/whatsnew/3.7.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -866,6 +866,9 @@ Changes in Python behavior
866866
Changes in the Python API
867867
-------------------------
868868

869+
* The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE``
870+
locale to the ``LC_NUMERIC`` locale in some cases.
871+
869872
* The ``asyncio.windows_utils.socketpair()`` function has been
870873
removed: use directly :func:`socket.socketpair` which is available on all
871874
platforms since Python 3.5 (before, it wasn't available on Windows).

‎Include/fileutils.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,11 @@ PyAPI_FUNC(int) _Py_get_blocking(int fd);
160160
PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking);
161161
#endif /* !MS_WINDOWS */
162162

163+
PyAPI_FUNC(int) _Py_GetLocaleconvNumeric(
164+
PyObject **decimal_point,
165+
PyObject **thousands_sep,
166+
const char **grouping);
167+
163168
#endif /* Py_LIMITED_API */
164169

165170
#ifdef __cplusplus

‎Misc/NEWS.d/next/Library/2017-10-30-15-55-32.bpo-31900.-S9xc4.rst‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE``
2+
locale to the ``LC_NUMERIC`` locale to decode ``decimal_point`` and
3+
``thousands_sep`` byte strings if they are non-ASCII or longer than 1 byte, and
4+
the ``LC_NUMERIC`` locale is different than the ``LC_CTYPE`` locale. This
5+
temporary change affects other threads.
6+
7+
Same change for the :meth:`str.format` method when formatting a number
8+
(:class:`int`, :class:`float`, :class:`float` and subclasses) with the ``n``
9+
type (ex: ``'{:n}'.format(1234)``).

‎Modules/_localemodule.c‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,9 @@ PyLocale_localeconv(PyObject* self)
139139
PyObject *x;
140140

141141
result = PyDict_New();
142-
if (!result)
142+
if (!result) {
143143
return NULL;
144+
}
144145

145146
/* if LC_NUMERIC is different in the C library, use saved value */
146147
l = localeconv();
@@ -171,12 +172,6 @@ PyLocale_localeconv(PyObject* self)
171172
RESULT(#i, x); \
172173
} while (0)
173174

174-
/* Numeric information */
175-
RESULT_STRING(decimal_point);
176-
RESULT_STRING(thousands_sep);
177-
x = copy_grouping(l->grouping);
178-
RESULT("grouping", x);
179-
180175
/* Monetary information */
181176
RESULT_STRING(int_curr_symbol);
182177
RESULT_STRING(currency_symbol);
@@ -195,10 +190,36 @@ PyLocale_localeconv(PyObject* self)
195190
RESULT_INT(n_sep_by_space);
196191
RESULT_INT(p_sign_posn);
197192
RESULT_INT(n_sign_posn);
193+
194+
/* Numeric information */
195+
PyObject *decimal_point, *thousands_sep;
196+
const char *grouping;
197+
if (_Py_GetLocaleconvNumeric(&decimal_point,
198+
&thousands_sep,
199+
&grouping) < 0) {
200+
goto failed;
201+
}
202+
203+
if (PyDict_SetItemString(result, "decimal_point", decimal_point) < 0) {
204+
Py_DECREF(decimal_point);
205+
Py_DECREF(thousands_sep);
206+
goto failed;
207+
}
208+
Py_DECREF(decimal_point);
209+
210+
if (PyDict_SetItemString(result, "thousands_sep", thousands_sep) < 0) {
211+
Py_DECREF(thousands_sep);
212+
goto failed;
213+
}
214+
Py_DECREF(thousands_sep);
215+
216+
x = copy_grouping(grouping);
217+
RESULT("grouping", x);
218+
198219
return result;
199220

200221
failed:
201-
Py_XDECREF(result);
222+
Py_DECREF(result);
202223
return NULL;
203224
}
204225

‎Python/fileutils.c‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1746,3 +1746,80 @@ _Py_set_blocking(int fd, int blocking)
17461746
return -1;
17471747
}
17481748
#endif
1749+
1750+
1751+
int
1752+
_Py_GetLocaleconvNumeric(PyObject **decimal_point, PyObject **thousands_sep,
1753+
const char **grouping)
1754+
{
1755+
int res = -1;
1756+
1757+
struct lconv *lc = localeconv();
1758+
1759+
int change_locale = 0;
1760+
if (decimal_point != NULL &&
1761+
(strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127))
1762+
{
1763+
change_locale = 1;
1764+
}
1765+
if (thousands_sep != NULL &&
1766+
(strlen(lc->thousands_sep) > 1 || ((unsigned char)lc->thousands_sep[0]) > 127))
1767+
{
1768+
change_locale = 1;
1769+
}
1770+
1771+
/* Keep a copy of the LC_CTYPE locale */
1772+
char *oldloc = NULL, *loc = NULL;
1773+
if (change_locale) {
1774+
oldloc = setlocale(LC_CTYPE, NULL);
1775+
if (!oldloc) {
1776+
PyErr_SetString(PyExc_RuntimeWarning, "faild to get LC_CTYPE locale");
1777+
return -1;
1778+
}
1779+
1780+
oldloc = _PyMem_Strdup(oldloc);
1781+
if (!oldloc) {
1782+
PyErr_NoMemory();
1783+
return -1;
1784+
}
1785+
1786+
loc = setlocale(LC_NUMERIC, NULL);
1787+
if (loc != NULL && strcmp(loc, oldloc) == 0) {
1788+
loc = NULL;
1789+
}
1790+
1791+
if (loc != NULL) {
1792+
/* Only set the locale temporarilty the LC_CTYPE locale
1793+
if LC_NUMERIC locale is different than LC_CTYPE locale and
1794+
decimal_point and/or thousands_sep are non-ASCII or longer than
1795+
1 byte */
1796+
setlocale(LC_CTYPE, loc);
1797+
}
1798+
}
1799+
1800+
if (decimal_point != NULL) {
1801+
*decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL);
1802+
if (*decimal_point == NULL) {
1803+
goto error;
1804+
}
1805+
}
1806+
if (thousands_sep != NULL) {
1807+
*thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL);
1808+
if (*thousands_sep == NULL) {
1809+
goto error;
1810+
}
1811+
}
1812+
1813+
if (grouping != NULL) {
1814+
*grouping = lc->grouping;
1815+
}
1816+
1817+
res = 0;
1818+
1819+
error:
1820+
if (loc != NULL) {
1821+
setlocale(LC_CTYPE, oldloc);
1822+
}
1823+
PyMem_Free(oldloc);
1824+
return res;
1825+
}

‎Python/formatter_unicode.c‎

Lines changed: 4 additions & 11 deletions

0 commit comments

Comments
 (0)