Issue #10335: Add tokenize.open(), detect the file encoding using · python/cpython@58c0752 · GitHub
Skip to content

Commit 58c0752

Browse files
author
Victor Stinner
committed
Issue #10335: Add tokenize.open(), detect the file encoding using
tokenize.detect_encoding() and open it in read only mode.
1 parent ae4836d commit 58c0752

8 files changed

Lines changed: 54 additions & 22 deletions

File tree

‎Doc/library/tokenize.rst‎

Lines changed: 9 additions & 8 deletions

‎Lib/linecache.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,7 @@ def updatecache(filename, module_globals=None):
123123
else:
124124
return []
125125
try:
126-
with open(fullname, 'rb') as fp:
127-
coding, line = tokenize.detect_encoding(fp.readline)
128-
with open(fullname, 'r', encoding=coding) as fp:
126+
with tokenize.open(fullname) as fp:
129127
lines = fp.readlines()
130128
except IOError:
131129
return []

‎Lib/py_compile.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,7 @@ def compile(file, cfile=None, dfile=None, doraise=False):
104104
byte-compile all installed files (or all files in selected
105105
directories).
106106
"""
107-
with open(file, "rb") as f:
108-
encoding = tokenize.detect_encoding(f.readline)[0]
109-
with open(file, encoding=encoding) as f:
107+
with tokenize.open(file) as f:
110108
try:
111109
timestamp = int(os.fstat(f.fileno()).st_mtime)
112110
except AttributeError:

‎Lib/tabnanny.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,8 @@ def check(file):
9393
check(fullname)
9494
return
9595

96-
with open(file, 'rb') as f:
97-
encoding, lines = tokenize.detect_encoding(f.readline)
98-
9996
try:
100-
f = open(file, encoding=encoding)
97+
f = tokenize.open(file)
10198
except IOError as msg:
10299
errprint("%r: I/O Error: %s" % (file, msg))
103100
return

‎Lib/test/test_tokenize.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,8 @@
564564

565565
from test import support
566566
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
567-
STRING, ENDMARKER, tok_name, detect_encoding)
567+
STRING, ENDMARKER, tok_name, detect_encoding,
568+
open as tokenize_open)
568569
from io import BytesIO
569570
from unittest import TestCase
570571
import os, sys, glob
@@ -857,6 +858,26 @@ def test_short_files(self):
857858
readline = self.get_readline((b'# coding: bad\n',))
858859
self.assertRaises(SyntaxError, detect_encoding, readline)
859860

861+
def test_open(self):
862+
filename = support.TESTFN + '.py'
863+
self.addCleanup(support.unlink, filename)
864+
865+
# test coding cookie
866+
for encoding in ('iso-8859-15', 'utf-8'):
867+
with open(filename, 'w', encoding=encoding) as fp:
868+
print("# coding: %s" % encoding, file=fp)
869+
print("print('euro:\u20ac')", file=fp)
870+
with tokenize_open(filename) as fp:
871+
assert fp.encoding == encoding
872+
assert fp.mode == 'r'
873+
874+
# test BOM (no coding cookie)
875+
with open(filename, 'w', encoding='utf-8-sig') as fp:
876+
print("print('euro:\u20ac')", file=fp)
877+
with tokenize_open(filename) as fp:
878+
assert fp.encoding == 'utf-8-sig'
879+
assert fp.mode == 'r'
880+
860881
class TestTokenize(TestCase):
861882

862883
def test_tokenize(self):

‎Lib/tokenize.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from token import *
3030
from codecs import lookup, BOM_UTF8
3131
import collections
32+
from io import TextIOWrapper
3233
cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
3334

3435
import token
@@ -335,6 +336,20 @@ def find_cookie(line):
335336
return default, [first, second]
336337

337338

339+
_builtin_open = open
340+
341+
def open(filename):
342+
"""Open a file in read only mode using the encoding detected by
343+
detect_encoding().
344+
"""
345+
buffer = _builtin_open(filename, 'rb')
346+
encoding, lines = detect_encoding(buffer.readline)
347+
buffer.seek(0)
348+
text = TextIOWrapper(buffer, encoding, line_buffering=True)
349+
text.mode = 'r'
350+
return text
351+
352+
338353
def tokenize(readline):
339354
"""
340355
The tokenize() generator requires one argment, readline, which

‎Lib/trace.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -432,10 +432,9 @@ def find_strings(filename, encoding=None):
432432
def find_executable_linenos(filename):
433433
"""Return dict where keys are line numbers in the line number table."""
434434
try:
435-
with io.FileIO(filename, 'r') as file:
436-
encoding, lines = tokenize.detect_encoding(file.readline)
437-
with open(filename, "r", encoding=encoding) as f:
435+
with tokenize.open(filename) as f:
438436
prog = f.read()
437+
encoding = f.encoding
439438
except IOError as err:
440439
print(("Not printing coverage data for %r: %s"
441440
% (filename, err)), file=sys.stderr)

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions

0 commit comments

Comments
 (0)