bpo-34523: Py_DecodeLocale() use UTF-8 on Windows (GH-8998) · pythoncapi/cpython@c5989cd · GitHub
Skip to content

Commit c5989cd

Browse files
vstinnervstinner
authored
bpo-34523: Py_DecodeLocale() use UTF-8 on Windows (pythonGH-8998)
Py_DecodeLocale() and Py_EncodeLocale() now use the UTF-8 encoding on Windows if Py_LegacyWindowsFSEncodingFlag is zero. pymain_read_conf() now sets Py_LegacyWindowsFSEncodingFlag in its loop, but restore its value at exit.
1 parent 70fead2 commit c5989cd

5 files changed

Lines changed: 55 additions & 30 deletions

File tree

‎Doc/c-api/sys.rst‎

Lines changed: 11 additions & 4 deletions

‎Lib/test/test_embed.py‎

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -268,10 +268,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
268268
'dump_refs': 0,
269269
'malloc_stats': 0,
270270

271-
# None means that the default encoding is read at runtime:
272-
# see get_locale_encoding().
271+
# None means that the value is get by get_locale_encoding()
273272
'filesystem_encoding': None,
274-
'filesystem_errors': sys.getfilesystemencodeerrors(),
273+
'filesystem_errors': None,
274+
275275
'utf8_mode': 0,
276276
'coerce_c_locale': 0,
277277
'coerce_c_locale_warn': 0,
@@ -294,7 +294,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
294294
'quiet': 0,
295295
'user_site_directory': 1,
296296
'buffered_stdio': 1,
297-
# None means that check_config() gets the expected encoding at runtime
297+
298+
# None means that the value is get by get_stdio_encoding()
298299
'stdio_encoding': None,
299300
'stdio_errors': None,
300301

@@ -303,7 +304,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
303304
'_frozen': 0,
304305
}
305306

306-
307307
def get_stdio_encoding(self, env):
308308
code = 'import sys; print(sys.stdout.encoding, sys.stdout.errors)'
309309
args = (sys.executable, '-c', code)
@@ -315,18 +315,12 @@ def get_stdio_encoding(self, env):
315315
out = proc.stdout.rstrip()
316316
return out.split()
317317

318-
def get_locale_encoding(self, isolated):
319-
if sys.platform in ('win32', 'darwin') or support.is_android:
320-
# Windows, macOS and Android use UTF-8
321-
return "utf-8"
322-
323-
code = ('import codecs, locale, sys',
324-
'locale.setlocale(locale.LC_CTYPE, "")',
325-
'enc = locale.nl_langinfo(locale.CODESET)',
326-
'enc = codecs.lookup(enc).name',
327-
'print(enc)')
328-
args = (sys.executable, '-c', '; '.join(code))
329-
env = dict(os.environ)
318+
def get_filesystem_encoding(self, isolated, env):
319+
code = ('import codecs, locale, sys; '
320+
'print(sys.getfilesystemencoding(), '
321+
'sys.getfilesystemencodeerrors())')
322+
args = (sys.executable, '-c', code)
323+
env = dict(env)
330324
if not isolated:
331325
env['PYTHONCOERCECLOCALE'] = '0'
332326
env['PYTHONUTF8'] = '0'
@@ -336,7 +330,8 @@ def get_locale_encoding(self, isolated):
336330
if proc.returncode:
337331
raise Exception(f"failed to get the locale encoding: "
338332
f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
339-
return proc.stdout.rstrip()
333+
out = proc.stdout.rstrip()
334+
return out.split()
340335

341336
def check_config(self, testname, expected):
342337
expected = dict(self.DEFAULT_CONFIG, **expected)
@@ -356,8 +351,12 @@ def check_config(self, testname, expected):
356351
expected['stdio_encoding'] = res[0]
357352
if expected['stdio_errors'] is None:
358353
expected['stdio_errors'] = res[1]
359-
if expected['filesystem_encoding'] is None:
360-
expected['filesystem_encoding'] = self.get_locale_encoding(expected['isolated'])
354+
if expected['filesystem_encoding'] is None or expected['filesystem_errors'] is None:
355+
res = self.get_filesystem_encoding(expected['isolated'], env)
356+
if expected['filesystem_encoding'] is None:
357+
expected['filesystem_encoding'] = res[0]
358+
if expected['filesystem_errors'] is None:
359+
expected['filesystem_errors'] = res[1]
361360
for key, value in expected.items():
362361
expected[key] = str(value)
363362

‎Misc/NEWS.d/next/C API/2018-08-29-18-48-47.bpo-34523.lLQ8rh.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Py_DecodeLocale() and Py_EncodeLocale() now use the UTF-8 encoding on
2+
Windows if Py_LegacyWindowsFSEncodingFlag is zero.

‎Modules/main.c‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1287,6 +1287,9 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config,
12871287
_PyCmdline *cmdline)
12881288
{
12891289
int init_utf8_mode = Py_UTF8Mode;
1290+
#ifdef MS_WINDOWS
1291+
int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag;
1292+
#endif
12901293
_PyCoreConfig save_config = _PyCoreConfig_INIT;
12911294
int res = -1;
12921295

@@ -1313,9 +1316,12 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config,
13131316
goto done;
13141317
}
13151318

1316-
/* bpo-34207: Py_DecodeLocale(), Py_EncodeLocale() and similar
1317-
functions depend on Py_UTF8Mode. */
1319+
/* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend
1320+
on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */
13181321
Py_UTF8Mode = config->utf8_mode;
1322+
#ifdef MS_WINDOWS
1323+
Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding;
1324+
#endif
13191325

13201326
if (pymain_init_cmdline_argv(pymain, config, cmdline) < 0) {
13211327
goto done;
@@ -1380,6 +1386,9 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config,
13801386
done:
13811387
_PyCoreConfig_Clear(&save_config);
13821388
Py_UTF8Mode = init_utf8_mode ;
1389+
#ifdef MS_WINDOWS
1390+
Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding;
1391+
#endif
13831392
return res;
13841393
}
13851394

‎Python/fileutils.c‎

Lines changed: 12 additions & 4 deletions

0 commit comments

Comments
 (0)