gh-153569: finish tokenizer offsets and the opaque API cutover · python/cpython@9240806 · GitHub
Skip to content

Commit 9240806

Browse files
pablogsalpablogsal
committed
gh-153569: finish tokenizer offsets and the opaque API cutover
Use persistent decoded-source offsets for scanner positions and explicit reporting locations and text for diagnostics, including incomplete input. Move pegen and _tokenize behind tokenizer.h operations and remove their dependence on lexer layout. Remove the obsolete cursor, source lookup, and compatibility APIs after the consumer cutover.
1 parent 3ef307f commit 9240806

35 files changed

Lines changed: 693 additions & 1158 deletions

‎Lib/test/test_capi/test_tokenizer.py‎

Lines changed: 0 additions & 3 deletions

‎Lib/test/test_codeop.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,17 @@ def test_valid(self, compiler):
113113
av("def f():\n pass\n#foo\n")
114114
av("@a.b.c\ndef f():\n pass\n")
115115

116+
@subTests('symbol', ('single', 'exec'))
117+
@subTests('prefix', ('', 'f', 't'))
118+
def test_incomplete_string_diagnostics(self, symbol, prefix):
119+
opening = f' á = {prefix}"""first\n'
120+
source = 'if True:\n' + opening + 'second'
121+
with self.assertRaises(_IncompleteInputError) as cm:
122+
Compile()(source, '<input>', symbol)
123+
text = opening + 'second' + ('\n' if symbol == 'exec' else '')
124+
self.assertEqual(cm.exception.args, (
125+
'incomplete input', ('<input>', 2, 9, text, 2, -1)))
126+
116127
@subTests('compiler', COMPILERS)
117128
def test_incomplete(self, compiler):
118129
ai = functools.partial(self.assertIncomplete, compiler=compiler)

‎Lib/test/test_source_encoding.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import unittest
44
from test import support
55
from test.support import script_helper
6-
from test.support.os_helper import TESTFN, unlink, rmtree
7-
from test.support.import_helper import unload
6+
from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
7+
from test.support.import_helper import import_module, unload
88
import importlib
99
import os
1010
import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
8383
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
8484

8585
def test_invalid_utf8_offset_after_non_ascii(self):
86+
for name in ('é', 'éé', '𝒜'):
87+
with self.subTest(name=name):
88+
source = ('x = ' + name).encode() + b'\xff\n'
89+
with self.assertRaises(SyntaxError) as caught:
90+
compile(source, '<test>', 'exec')
91+
error = caught.exception
92+
self.assertEqual(
93+
(error.lineno, error.offset, error.end_lineno, error.end_offset),
94+
(1, 5 + len(name), 1, 5 + len(name)),
95+
)
96+
97+
@support.cpython_only
98+
def test_invalid_utf8_file_offset_after_non_ascii(self):
99+
_testcapi = import_module('_testcapi')
100+
self.addCleanup(unlink, TESTFN_ASCII)
101+
with open(TESTFN_ASCII, 'wb') as f:
102+
f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
86103
with self.assertRaises(SyntaxError) as caught:
87-
compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
104+
_testcapi.run_file(
105+
os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
88106
error = caught.exception
89107
self.assertEqual(
90108
(error.lineno, error.offset, error.end_lineno, error.end_offset),
91-
(1, 6, 1, 6),
109+
(2, 7, 2, 7),
92110
)
93111

94112
def test_long_bom_conflict_message_is_not_truncated(self):

‎Makefile.pre.in‎

Lines changed: 3 additions & 4 deletions

0 commit comments

Comments
 (0)