gh-153569: Base the tokenizer API on source offsets · python/cpython@bce3caa · GitHub
Skip to content

Commit bce3caa

Browse files
pablogsalpablogsal
committed
gh-153569: Base the tokenizer API on source offsets
1 parent a2d3787 commit bce3caa

50 files changed

Lines changed: 5266 additions & 3241 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Lib/test/test_codeop.py‎

Lines changed: 72 additions & 1 deletion

‎Lib/test/test_source_encoding.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,45 @@ def test_truncated_utf8_at_eof(self):
8282
with self.subTest(seq=seq):
8383
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
8484

85+
def test_invalid_utf8_offset_after_non_ascii(self):
86+
with self.assertRaises(SyntaxError) as caught:
87+
compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
88+
error = caught.exception
89+
self.assertEqual(
90+
(error.lineno, error.offset, error.end_lineno, error.end_offset),
91+
(1, 6, 1, 6),
92+
)
93+
94+
def test_unbounded_bom_conflict_message(self):
95+
encoding = "x" * 400
96+
source = b"\xef\xbb\xbf# coding:" + encoding.encode() + b"\n"
97+
with self.assertRaises(SyntaxError) as caught:
98+
compile(source, "<test>", "exec")
99+
self.assertEqual(
100+
caught.exception.msg,
101+
f"encoding problem: {encoding} with BOM",
102+
)
103+
104+
@support.requires_subprocess()
105+
def test_stateful_file_decoder_spans_lines(self):
106+
encoded_name = "変数".encode("iso2022_jp")
107+
payload = encoded_name[3:-3]
108+
source = (
109+
b"# coding: iso2022_jp\n"
110+
b"# \x1b$B" + payload + b"\n"
111+
+ payload + b"\x1b(B = 1\n"
112+
)
113+
with tempfile.TemporaryDirectory() as directory:
114+
filename = os.path.join(directory, "stateful.py")
115+
with open(filename, "wb") as file:
116+
file.write(source)
117+
process = subprocess.run(
118+
[sys.executable, filename],
119+
stdout=subprocess.PIPE,
120+
stderr=subprocess.PIPE,
121+
)
122+
self.assertEqual(process.returncode, 0, process.stderr)
123+
85124
@support.requires_subprocess()
86125
def test_20731(self):
87126
sub = subprocess.Popen([sys.executable,

‎Lib/test/test_syntax.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3088,6 +3088,24 @@ def test_expression_with_assignment(self):
30883088
def test_curly_brace_after_primary_raises_immediately(self):
30893089
self._check_error("f{}", "invalid syntax", mode="single")
30903090

3091+
def test_tokenizer_eof_error_offsets_after_non_ascii(self):
3092+
self._check_error(
3093+
"é + (",
3094+
re.escape("'(' was never closed"),
3095+
lineno=1,
3096+
offset=5,
3097+
end_lineno=1,
3098+
end_offset=0,
3099+
)
3100+
self._check_error(
3101+
"é + \\\n",
3102+
"unexpected EOF while parsing",
3103+
lineno=1,
3104+
offset=6,
3105+
end_lineno=1,
3106+
end_offset=-1,
3107+
)
3108+
30913109
def test_assign_call(self):
30923110
self._check_error("f() = 1", "assign")
30933111

‎Lib/test/test_tokenize.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
import token
88
import tokenize
99
import unittest
10+
import warnings
11+
import weakref
12+
import _tokenize
1013
from io import BytesIO, StringIO
1114
from textwrap import dedent
1215
from unittest import TestCase, mock
@@ -2243,6 +2246,38 @@ def _get_tokens(source, *, extra_tokens=False):
22432246
extra_tokens=extra_tokens,
22442247
))
22452248

2249+
def test_readline_reentry_is_rejected(self):
2250+
def make_cycle():
2251+
iterator = None
2252+
2253+
def readline():
2254+
next(iterator)
2255+
2256+
iterator = _tokenize.TokenizerIter(readline, extra_tokens=False)
2257+
with self.assertRaisesRegex(RuntimeError, "already executing"):
2258+
next(iterator)
2259+
return weakref.ref(readline)
2260+
2261+
readline_ref = make_cycle()
2262+
support.gc_collect()
2263+
self.assertIsNone(readline_ref())
2264+
2265+
def test_warning_reentry_is_rejected(self):
2266+
iterator = None
2267+
2268+
def showwarning(*args, **kwargs):
2269+
next(iterator)
2270+
2271+
with warnings.catch_warnings():
2272+
warnings.simplefilter("always")
2273+
with mock.patch.object(warnings, "showwarning", showwarning):
2274+
iterator = _tokenize.TokenizerIter(
2275+
StringIO("1if\n").readline,
2276+
extra_tokens=False,
2277+
)
2278+
with self.assertRaisesRegex(RuntimeError, "already executing"):
2279+
next(iterator)
2280+
22462281
def check_tokenize(self, s, expected):
22472282
# Format the tokens in s in a table format.
22482283
# The ENDMARKER and final NEWLINE are omitted.
@@ -2273,6 +2308,71 @@ def readline(encoding):
22732308
))
22742309
self.assertEqual(tokens, expected)
22752310

2311+
def test_stateful_encoding_across_readline_calls(self):
2312+
encoded_name = "変数".encode("iso2022_jp")
2313+
payload = encoded_name[3:-3]
2314+
lines = iter([
2315+
b"# \x1b$B" + payload + b"\n",
2316+
payload + b"\x1b(B\n",
2317+
b"",
2318+
])
2319+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2320+
lines.__next__,
2321+
extra_tokens=True,
2322+
encoding="iso2022_jp",
2323+
))
2324+
self.assertEqual(
2325+
[(tok.type, tok.string) for tok in tokens],
2326+
[
2327+
(token.COMMENT, "# 変数"),
2328+
(token.NL, "\n"),
2329+
(token.NAME, "変数"),
2330+
(token.NEWLINE, "\n"),
2331+
(token.ENDMARKER, ""),
2332+
],
2333+
)
2334+
2335+
def test_utf8_decoder_spans_readline_calls(self):
2336+
lines = iter([b"\xc3", b"\xa9\n", b""])
2337+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2338+
lines.__next__,
2339+
extra_tokens=True,
2340+
encoding="utf-8",
2341+
))
2342+
self.assertEqual(
2343+
[(tok.type, tok.string, tok.start, tok.end) for tok in tokens],
2344+
[
2345+
(token.NAME, "é", (1, 0), (1, 1)),
2346+
(token.NEWLINE, "\n", (1, 1), (1, 2)),
2347+
(token.ENDMARKER, "", (2, 0), (2, 0)),
2348+
],
2349+
)
2350+
2351+
def test_encoded_readline_replaces_invalid_bytes(self):
2352+
lines = iter([b"\xff\n", b""])
2353+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2354+
lines.__next__,
2355+
extra_tokens=True,
2356+
encoding="utf-8",
2357+
))
2358+
self.assertEqual(
2359+
[(tok.type, tok.string) for tok in tokens],
2360+
[
2361+
(token.NAME, "�"),
2362+
(token.NEWLINE, "\n"),
2363+
(token.ENDMARKER, ""),
2364+
],
2365+
)
2366+
2367+
def test_encoded_readline_codec_lookup_is_lazy(self):
2368+
iterator = _tokenize.TokenizerIter(
2369+
lambda: b"",
2370+
extra_tokens=True,
2371+
encoding="missing-tokenizer-codec",
2372+
)
2373+
with self.assertRaises(LookupError):
2374+
next(iterator)
2375+
22762376
def test_extra_tokens_relaxes_lexer_errors(self):
22772377
cases = [
22782378
(
@@ -2407,6 +2507,38 @@ def test_escaped_fstring_brace_has_a_position_gap(self):
24072507
],
24082508
)
24092509

2510+
def test_unclosed_parenthesis_error_uses_buffer_relative_position(self):
2511+
for extra_tokens in (False, True):
2512+
with self.subTest(extra_tokens=extra_tokens):
2513+
with self.assertRaises(tokenize.TokenError) as caught:
2514+
self._get_tokens("é = (\n", extra_tokens=extra_tokens)
2515+
self.assertEqual(
2516+
caught.exception.args,
2517+
("unexpected EOF in multi-line statement", (1, 0)),
2518+
)
2519+
2520+
def test_line_continuation_error_uses_logical_line_position(self):
2521+
cases = [
2522+
("é \\'f\n", (1, 6)),
2523+
("x1\\\n==\\_", (2, 9)),
2524+
]
2525+
for extra_tokens in (False, True):
2526+
for source, position in cases:
2527+
with self.subTest(
2528+
source=source,
2529+
extra_tokens=extra_tokens,
2530+
):
2531+
with self.assertRaises(tokenize.TokenError) as caught:
2532+
self._get_tokens(source, extra_tokens=extra_tokens)
2533+
self.assertEqual(
2534+
caught.exception.args,
2535+
(
2536+
"unexpected character after line continuation "
2537+
"character",
2538+
position,
2539+
),
2540+
)
2541+
24102542
def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
24112543
with self.assertRaises(tokenize.TokenError) as caught:
24122544
self._get_tokens('bé )tf"2 ', extra_tokens=True)

‎Makefile.pre.in‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -393,15 +393,17 @@ PEGEN_OBJS= \
393393
Parser/peg_api.o
394394

395395
TOKENIZER_OBJS= \
396-
Parser/lexer/buffer.o \
396+
Parser/tokenizer/source.o \
397+
Parser/tokenizer/api.o \
398+
Parser/tokenizer/errors.o \
399+
Parser/tokenizer/seam.o \
400+
Parser/tokenizer/cursor.o \
397401
Parser/lexer/lexer.o \
398402
Parser/lexer/number.o \
399-
Parser/lexer/state.o \
400403
Parser/lexer/string.o \
401-
Parser/tokenizer/file_tokenizer.o \
402-
Parser/tokenizer/readline_tokenizer.o \
403-
Parser/tokenizer/string_tokenizer.o \
404-
Parser/tokenizer/utf8_tokenizer.o \
404+
Parser/lexer/state.o \
405+
Parser/tokenizer/decoder.o \
406+
Parser/tokenizer/reader.o \
405407
Parser/tokenizer/helpers.o
406408

407409
PEGEN_HEADERS= \
@@ -410,11 +412,16 @@ PEGEN_HEADERS= \
410412
$(srcdir)/Parser/string_parser.h
411413

412414
TOKENIZER_HEADERS= \
413-
Parser/lexer/buffer.h \
415+
Parser/tokenizer/source.h \
416+
Parser/tokenizer/tokenizer.h \
417+
Parser/tokenizer/errors.h \
418+
Parser/tokenizer/seam.h \
419+
Parser/tokenizer/cursor.h \
420+
Parser/tokenizer/reader.h \
414421
Parser/lexer/lexer.h \
415422
Parser/lexer/lexer_internal.h \
416423
Parser/lexer/state.h \
417-
Parser/tokenizer/tokenizer.h \
424+
Parser/tokenizer/reader_internal.h \
418425
Parser/tokenizer/helpers.h
419426

420427
POBJS= \

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 7 additions & 5 deletions

0 commit comments

Comments
 (0)