gh-88886: Remove excessive encoding name normalization (GH-137167) · python/cpython@af58a6f · GitHub
Skip to content

Commit af58a6f

Browse files
gh-88886: Remove excessive encoding name normalization (GH-137167)
The codecs lookup function now performs only minimal normalization of the encoding name before passing it to the search functions: all ASCII letters are converted to lower case, spaces are replaced with hyphens. Excessive normalization broke third-party codecs providers, like python-iconv. Revert "bpo-37751: Fix codecs.lookup() normalization (GH-15092)" This reverts commit 20f59fe.
1 parent 6b7b9d0 commit af58a6f

5 files changed

Lines changed: 54 additions & 45 deletions

File tree

‎Doc/library/codecs.rst‎

Lines changed: 14 additions & 7 deletions

‎Lib/test/test_capi/test_codecs.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -630,7 +630,6 @@ def test_codec_known_encoding(self):
630630
for name in [
631631
encoding_name,
632632
encoding_name.upper(),
633-
encoding_name.replace('_', '-'),
634633
]:
635634
with self.subTest(name):
636635
self.assertTrue(_testcapi.codec_known_encoding(name))

‎Lib/test/test_codecs.py‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3873,26 +3873,22 @@ def test_rot13_func(self):
38733873
class CodecNameNormalizationTest(unittest.TestCase):
38743874
"""Test codec name normalization"""
38753875
def test_codecs_lookup(self):
3876-
FOUND = (1, 2, 3, 4)
3877-
NOT_FOUND = (None, None, None, None)
38783876
def search_function(encoding):
3879-
if encoding == "aaa_8":
3880-
return FOUND
3877+
if encoding.startswith("test."):
3878+
return (encoding, 2, 3, 4)
38813879
else:
3882-
return NOT_FOUND
3880+
return None
38833881

38843882
codecs.register(search_function)
38853883
self.addCleanup(codecs.unregister, search_function)
3886-
self.assertEqual(FOUND, codecs.lookup('aaa_8'))
3887-
self.assertEqual(FOUND, codecs.lookup('AAA-8'))
3888-
self.assertEqual(FOUND, codecs.lookup('AAA---8'))
3889-
self.assertEqual(FOUND, codecs.lookup('AAA 8'))
3890-
self.assertEqual(FOUND, codecs.lookup('aaa\xe9\u20ac-8'))
3891-
self.assertEqual(NOT_FOUND, codecs.lookup('AAA.8'))
3892-
self.assertEqual(NOT_FOUND, codecs.lookup('AAA...8'))
3893-
self.assertEqual(NOT_FOUND, codecs.lookup('BBB-8'))
3894-
self.assertEqual(NOT_FOUND, codecs.lookup('BBB.8'))
3895-
self.assertEqual(NOT_FOUND, codecs.lookup('a\xe9\u20ac-8'))
3884+
self.assertEqual(codecs.lookup('test.aaa_8'), ('test.aaa_8', 2, 3, 4))
3885+
self.assertEqual(codecs.lookup('TEST.AAA-8'), ('test.aaa-8', 2, 3, 4))
3886+
self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa-8', 2, 3, 4))
3887+
self.assertEqual(codecs.lookup('TEST.AAA---8'), ('test.aaa---8', 2, 3, 4))
3888+
self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa---8', 2, 3, 4))
3889+
self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4))
3890+
self.assertEqual(codecs.lookup('TEST.AAA.8'), ('test.aaa.8', 2, 3, 4))
3891+
self.assertEqual(codecs.lookup('TEST.AAA...8'), ('test.aaa...8', 2, 3, 4))
38963892

38973893
def test_encodings_normalize_encoding(self):
38983894
# encodings.normalize_encoding() ignores non-ASCII characters.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The codecs lookup function now again performs only minimal normalization of
2+
the encoding name before passing it to the search functions: all ASCII
3+
letters are converted to lower case, spaces are replaced with hyphens.
4+
This restores the pre-Python 3.9 behavior.

‎Python/codecs.c‎

Lines changed: 25 additions & 22 deletions

0 commit comments

Comments
 (0)