Issue #12281: Rewrite the MBCS codec to handle correctly replace and … · pythoncapi/cpython@3a50e70 · GitHub
Skip to content

Commit 3a50e70

Browse files
author
Victor Stinner
committed
Issue python#12281: Rewrite the MBCS codec to handle correctly replace and ignore
error handlers on all Windows versions. The MBCS codec is now supporting all error handlers, instead of only replace to encode and ignore to decode.
1 parent 1e73a24 commit 3a50e70

8 files changed

Lines changed: 888 additions & 147 deletions

File tree

‎Doc/library/codecs.rst‎

Lines changed: 4 additions & 3 deletions

‎Doc/whatsnew/3.3.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,11 @@ The :mod:`array` module supports the :c:type:`long long` type using ``q`` and
197197
codecs
198198
------
199199

200+
The :mod:`~encodings.mbcs` codec has be rewritten to handle correclty
201+
``replace`` and ``ignore`` error handlers on all Windows versions. The
202+
:mod:`~encodings.mbcs` codec is now supporting all error handlers, instead of
203+
only ``replace`` to encode and ``ignore`` to decode.
204+
200205
Multibyte CJK decoders now resynchronize faster. They only ignore the first
201206
byte of an invalid byte sequence. For example, ``b'\xff\n'.decode('gb2312',
202207
'replace')`` now returns a ``\n`` after the replacement character.

‎Include/unicodeobject.h‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1466,18 +1466,32 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
14661466
Py_ssize_t *consumed /* bytes consumed */
14671467
);
14681468

1469+
PyAPI_FUNC(PyObject*) PyUnicode_DecodeCodePageStateful(
1470+
int code_page, /* code page number */
1471+
const char *string, /* encoded string */
1472+
Py_ssize_t length, /* size of string */
1473+
const char *errors, /* error handling */
1474+
Py_ssize_t *consumed /* bytes consumed */
1475+
);
1476+
14691477
PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
14701478
PyObject *unicode /* Unicode object */
14711479
);
14721480

14731481
#ifndef Py_LIMITED_API
14741482
PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
14751483
const Py_UNICODE *data, /* Unicode char buffer */
1476-
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1484+
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
14771485
const char *errors /* error handling */
14781486
);
14791487
#endif
14801488

1489+
PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage(
1490+
int code_page, /* code page number */
1491+
PyObject *unicode, /* Unicode object */
1492+
const char *errors /* error handling */
1493+
);
1494+
14811495
#endif /* HAVE_MBCS */
14821496

14831497
/* --- Decimal Encoder ---------------------------------------------------- */

‎Lib/test/test_codecs.py‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,6 +1744,203 @@ def test_readline(self):
17441744
self.assertEqual(sout, b"\x80")
17451745

17461746

1747+
class CodePageTest(unittest.TestCase):
1748+
CP_UTF8 = 65001
1749+
vista_or_later = (sys.getwindowsversion().major >= 6)
1750+
1751+
def test_invalid_code_page(self):
1752+
self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a')
1753+
self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a')
1754+
self.assertRaises(WindowsError, codecs.code_page_encode, 123, 'a')
1755+
self.assertRaises(WindowsError, codecs.code_page_decode, 123, b'a')
1756+
1757+
def test_code_page_name(self):
1758+
self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
1759+
codecs.code_page_encode, 932, '\xff')
1760+
self.assertRaisesRegex(UnicodeDecodeError, 'cp932',
1761+
codecs.code_page_decode, 932, b'\x81\x00')
1762+
self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8',
1763+
codecs.code_page_decode, self.CP_UTF8, b'\xff')
1764+
1765+
def check_decode(self, cp, tests):
1766+
for raw, errors, expected in tests:
1767+
if expected is not None:
1768+
try:
1769+
decoded = codecs.code_page_decode(cp, raw, errors)
1770+
except UnicodeDecodeError as err:
1771+
self.fail('Unable to decode %a from "cp%s" with '
1772+
'errors=%r: %s' % (raw, cp, errors, err))
1773+
self.assertEqual(decoded[0], expected,
1774+
'%a.decode("cp%s", %r)=%a != %a'
1775+
% (raw, cp, errors, decoded[0], expected))
1776+
# assert 0 <= decoded[1] <= len(raw)
1777+
self.assertGreaterEqual(decoded[1], 0)
1778+
self.assertLessEqual(decoded[1], len(raw))
1779+
else:
1780+
self.assertRaises(UnicodeDecodeError,
1781+
codecs.code_page_decode, cp, raw, errors)
1782+
1783+
def check_encode(self, cp, tests):
1784+
for text, errors, expected in tests:
1785+
if expected is not None:
1786+
try:
1787+
encoded = codecs.code_page_encode(cp, text, errors)
1788+
except UnicodeEncodeError as err:
1789+
self.fail('Unable to encode %a to "cp%s" with '
1790+
'errors=%r: %s' % (text, cp, errors, err))
1791+
self.assertEqual(encoded[0], expected,
1792+
'%a.encode("cp%s", %r)=%a != %a'
1793+
% (text, cp, errors, encoded[0], expected))
1794+
self.assertEqual(encoded[1], len(text))
1795+
else:
1796+
self.assertRaises(UnicodeEncodeError,
1797+
codecs.code_page_encode, cp, text, errors)
1798+
1799+
def test_cp932(self):
1800+
self.check_encode(932, (
1801+
('abc', 'strict', b'abc'),
1802+
('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
1803+
# not encodable
1804+
('\xff', 'strict', None),
1805+
('[\xff]', 'ignore', b'[]'),
1806+
('[\xff]', 'replace', b'[y]'),
1807+
('[\u20ac]', 'replace', b'[?]'),
1808+
))
1809+
tests = [
1810+
(b'abc', 'strict', 'abc'),
1811+
(b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'),
1812+
# invalid bytes
1813+
(b'\xff', 'strict', None),
1814+
(b'\xff', 'ignore', ''),
1815+
(b'\xff', 'replace', '\ufffd'),
1816+
(b'\x81\x00abc', 'strict', None),
1817+
(b'\x81\x00abc', 'ignore', '\x00abc'),
1818+
]
1819+
if self.vista_or_later:
1820+
tests.append((b'\x81\x00abc', 'replace', '\ufffd\x00abc'))
1821+
else:
1822+
tests.append((b'\x81\x00abc', 'replace', '\x00\x00abc'))
1823+
self.check_decode(932, tests)
1824+
1825+
def test_cp1252(self):
1826+
self.check_encode(1252, (
1827+
('abc', 'strict', b'abc'),
1828+
('\xe9\u20ac', 'strict', b'\xe9\x80'),
1829+
('\xff', 'strict', b'\xff'),
1830+
('\u0141', 'strict', None),
1831+
('\u0141', 'ignore', b''),
1832+
('\u0141', 'replace', b'L'),
1833+
))
1834+
self.check_decode(1252, (
1835+
(b'abc', 'strict', 'abc'),
1836+
(b'\xe9\x80', 'strict', '\xe9\u20ac'),
1837+
(b'\xff', 'strict', '\xff'),
1838+
))
1839+
1840+
def test_cp_utf7(self):
1841+
cp = 65000
1842+
self.check_encode(cp, (
1843+
('abc', 'strict', b'abc'),
1844+
('\xe9\u20ac', 'strict', b'+AOkgrA-'),
1845+
('\U0010ffff', 'strict', b'+2//f/w-'),
1846+
('\udc80', 'strict', b'+3IA-'),
1847+
('\ufffd', 'strict', b'+//0-'),
1848+
))
1849+
self.check_decode(cp, (
1850+
(b'abc', 'strict', 'abc'),
1851+
(b'+AOkgrA-', 'strict', '\xe9\u20ac'),
1852+
(b'+2//f/w-', 'strict', '\U0010ffff'),
1853+
(b'+3IA-', 'strict', '\udc80'),
1854+
(b'+//0-', 'strict', '\ufffd'),
1855+
# invalid bytes
1856+
(b'[+/]', 'strict', '[]'),
1857+
(b'[\xff]', 'strict', '[\xff]'),
1858+
))
1859+
1860+
def test_cp_utf8(self):
1861+
cp = self.CP_UTF8
1862+
1863+
tests = [
1864+
('abc', 'strict', b'abc'),
1865+
('\xe9\u20ac', 'strict', b'\xc3\xa9\xe2\x82\xac'),
1866+
('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'),
1867+
]
1868+
if self.vista_or_later:
1869+
tests.append(('\udc80', 'strict', None))
1870+
tests.append(('\udc80', 'ignore', b''))
1871+
tests.append(('\udc80', 'replace', b'?'))
1872+
else:
1873+
tests.append(('\udc80', 'strict', b'\xed\xb2\x80'))
1874+
self.check_encode(cp, tests)
1875+
1876+
tests = [
1877+
(b'abc', 'strict', 'abc'),
1878+
(b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'),
1879+
(b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'),
1880+
(b'\xef\xbf\xbd', 'strict', '\ufffd'),
1881+
(b'[\xc3\xa9]', 'strict', '[\xe9]'),
1882+
# invalid bytes
1883+
(b'[\xff]', 'strict', None),
1884+
(b'[\xff]', 'ignore', '[]'),
1885+
(b'[\xff]', 'replace', '[\ufffd]'),
1886+
]
1887+
if self.vista_or_later:
1888+
tests.extend((
1889+
(b'[\xed\xb2\x80]', 'strict', None),
1890+
(b'[\xed\xb2\x80]', 'ignore', '[]'),
1891+
(b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'),
1892+
))
1893+
else:
1894+
tests.extend((
1895+
(b'[\xed\xb2\x80]', 'strict', '[\udc80]'),
1896+
))
1897+
self.check_decode(cp, tests)
1898+
1899+
def test_error_handlers(self):
1900+
self.check_encode(932, (
1901+
('\xff', 'backslashreplace', b'\\xff'),
1902+
('\xff', 'xmlcharrefreplace', b'&#255;'),
1903+
))
1904+
self.check_decode(932, (
1905+
(b'\xff', 'surrogateescape', '\udcff'),
1906+
))
1907+
if self.vista_or_later:
1908+
self.check_encode(self.CP_UTF8, (
1909+
('\udc80', 'surrogatepass', b'\xed\xb2\x80'),
1910+
))
1911+
1912+
def test_multibyte_encoding(self):
1913+
self.check_decode(932, (
1914+
(b'\x84\xe9\x80', 'ignore', '\u9a3e'),
1915+
(b'\x84\xe9\x80', 'replace', '\ufffd\u9a3e'),
1916+
))
1917+
self.check_decode(self.CP_UTF8, (
1918+
(b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'),
1919+
(b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'),
1920+
))
1921+
if self.vista_or_later:
1922+
self.check_encode(self.CP_UTF8, (
1923+
('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'),
1924+
('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'),
1925+
))
1926+
1927+
def test_incremental(self):
1928+
decoded = codecs.code_page_decode(932,
1929+
b'\xe9\x80\xe9', 'strict',
1930+
False)
1931+
self.assertEqual(decoded, ('\u9a3e', 2))
1932+
1933+
decoded = codecs.code_page_decode(932,
1934+
b'\xe9\x80\xe9\x80', 'strict',
1935+
False)
1936+
self.assertEqual(decoded, ('\u9a3e\u9a3e', 4))
1937+
1938+
decoded = codecs.code_page_decode(932,
1939+
b'abc', 'strict',
1940+
False)
1941+
self.assertEqual(decoded, ('abc', 3))
1942+
1943+
17471944
def test_main():
17481945
support.run_unittest(
17491946
UTF32Test,
@@ -1772,6 +1969,7 @@ def test_main():
17721969
SurrogateEscapeTest,
17731970
BomTest,
17741971
TransformCodecTest,
1972+
CodePageTest,
17751973
)
17761974

17771975

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ What's New in Python 3.3 Alpha 1?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #12281: Rewrite the MBCS codec to handle correctly replace and ignore
14+
error handlers on all Windows versions. The MBCS codec is now supporting all
15+
error handlers, instead of only replace to encode and ignore to decode.
16+
1317
- Issue #13188: When called without an explicit traceback argument,
1418
generator.throw() now gets the traceback from the passed exception's
1519
``__traceback__`` attribute. Patch by Petri Lehtinen.

‎Modules/_codecsmodule.c‎

Lines changed: 50 additions & 0 deletions

0 commit comments

Comments
 (0)