gh-153569: hide tokenizer state behind token and source operations (#… · python/cpython@7bfa97f · GitHub
Skip to content

Commit 7bfa97f

Browse files
pablogsalpablogsal
authored
gh-153569: hide tokenizer state behind token and source operations (#156654)
Make pegen and _tokenize consume tokenizer operations instead of accessing the tokenizer layout. Return token kind, span, location, and raw-string context together, so parser actions use the context of the cached token. Represent persistent scanner positions as offsets and remove pointer save/restore bookkeeping when input storage grows.
1 parent 227b5f0 commit 7bfa97f

31 files changed

Lines changed: 657 additions & 511 deletions

‎Lib/test/test_fstring.py‎

Lines changed: 3 additions & 0 deletions

‎Lib/test/test_tstring.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,14 @@ def test_raw_tstrings(self):
159159
t = tr"{path}\Documents"
160160
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
161161

162+
value = 42
163+
t = rt"{value:{f'\xFF'}}\n"
164+
self.assertTStringEqual(
165+
t, ("", "\\n"), [(value, "value", None, 'ÿ')])
166+
t = t"{value:{rf'\xFF'}}\n"
167+
self.assertTStringEqual(
168+
t, ("", "\n"), [(value, "value", None, '\\xFF')])
169+
162170
def test_template_concatenation(self):
163171
# Test template + template
164172
t1 = t"Hello, "
@@ -226,6 +234,10 @@ def test_syntax_errors(self):
226234
("t'{x=!}'", "t-string: missing conversion character"),
227235
("t'{x!z}'", "t-string: invalid conversion character 'z': "
228236
"expected 's', 'r', or 'a'"),
237+
("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
238+
"expected 's', 'r', or 'a'"),
239+
("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
240+
"expected 's', 'r', or 'a'"),
229241
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
230242
"without parentheses"),
231243
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),

‎Makefile.pre.in‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -394,13 +394,13 @@ PEGEN_OBJS= \
394394
Parser/peg_api.o
395395

396396
TOKENIZER_OBJS= \
397-
Parser/lexer/buffer.o \
398397
Parser/lexer/lexer.o \
399398
Parser/lexer/number.o \
400399
Parser/lexer/state.o \
401400
Parser/lexer/string.o \
402401
Parser/tokenizer/cursor.o \
403402
Parser/tokenizer/decoder.o \
403+
Parser/tokenizer/api.o \
404404
Parser/tokenizer/reader.o \
405405
Parser/tokenizer/source.o \
406406
Parser/tokenizer/helpers.o
@@ -411,14 +411,14 @@ PEGEN_HEADERS= \
411411
$(srcdir)/Parser/string_parser.h
412412

413413
TOKENIZER_HEADERS= \
414-
Parser/lexer/buffer.h \
415414
Parser/lexer/lexer.h \
416415
Parser/lexer/lexer_internal.h \
417416
Parser/lexer/state.h \
418417
Parser/tokenizer/cursor.h \
419418
Parser/tokenizer/reader.h \
420419
Parser/tokenizer/reader_internal.h \
421420
Parser/tokenizer/source.h \
421+
Parser/tokenizer/types.h \
422422
Parser/tokenizer/tokenizer.h \
423423
Parser/tokenizer/helpers.h
424424

@@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
34633463
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
34643464
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
34653465
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
3466-
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
3466+
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
34673467
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
34683468
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
34693469

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,12 +181,12 @@
181181
<ClCompile Include="..\Parser\action_helpers.c" />
182182
<ClCompile Include="..\Parser\string_parser.c" />
183183
<ClCompile Include="..\Parser\token.c" />
184-
<ClCompile Include="..\Parser\lexer\buffer.c" />
185184
<ClCompile Include="..\Parser\lexer\state.c" />
186185
<ClCompile Include="..\Parser\lexer\lexer.c" />
187186
<ClCompile Include="..\Parser\lexer\number.c" />
188187
<ClCompile Include="..\Parser\lexer\string.c" />
189188
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
189+
<ClCompile Include="..\Parser\tokenizer\api.c" />
190190
<ClCompile Include="..\Parser\tokenizer\reader.c" />
191191
<ClCompile Include="..\Parser\tokenizer\source.c" />
192192
<ClCompile Include="..\Parser\tokenizer\helpers.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -469,15 +469,15 @@
469469
<ClCompile Include="..\Parser\lexer\string.c">
470470
<Filter>Source Files</Filter>
471471
</ClCompile>
472-
<ClCompile Include="..\Parser\lexer\buffer.c">
473-
<Filter>Source Files</Filter>
474-
</ClCompile>
475472
<ClCompile Include="..\Parser\lexer\state.c">
476473
<Filter>Source Files</Filter>
477474
</ClCompile>
478475
<ClCompile Include="..\Parser\tokenizer\decoder.c">
479476
<Filter>Source Files</Filter>
480477
</ClCompile>
478+
<ClCompile Include="..\Parser\tokenizer\api.c">
479+
<Filter>Source Files</Filter>
480+
</ClCompile>
481481
<ClCompile Include="..\Parser\tokenizer\reader.c">
482482
<Filter>Source Files</Filter>
483483
</ClCompile>

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -423,10 +423,10 @@
423423
<ClInclude Include="..\Parser\lexer\state.h" />
424424
<ClInclude Include="..\Parser\lexer\lexer.h" />
425425
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
426-
<ClInclude Include="..\Parser\lexer\buffer.h" />
427426
<ClInclude Include="..\Parser\tokenizer\cursor.h" />
428427
<ClInclude Include="..\Parser\tokenizer\reader.h" />
429428
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
429+
<ClInclude Include="..\Parser\tokenizer\types.h" />
430430
<ClInclude Include="..\Parser\tokenizer\source.h" />
431431
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
432432
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -593,10 +593,10 @@
593593
<ClCompile Include="..\Parser\lexer\lexer.c" />
594594
<ClCompile Include="..\Parser\lexer\number.c" />
595595
<ClCompile Include="..\Parser\lexer\string.c" />
596-
<ClCompile Include="..\Parser\lexer\buffer.c" />
597596
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
598597
<ClCompile Include="..\Parser\tokenizer\source.c" />
599598
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
599+
<ClCompile Include="..\Parser\tokenizer\api.c" />
600600
<ClCompile Include="..\Parser\tokenizer\reader.c" />
601601
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
602602
<ClCompile Include="..\Parser\token.c" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -330,9 +330,6 @@
330330
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
331331
<Filter>Parser</Filter>
332332
</ClInclude>
333-
<ClInclude Include="..\Parser\lexer\buffer.h">
334-
<Filter>Parser</Filter>
335-
</ClInclude>
336333
<ClInclude Include="..\Parser\tokenizer\cursor.h">
337334
<Filter>Parser</Filter>
338335
</ClInclude>
@@ -342,6 +339,9 @@
342339
<ClInclude Include="..\Parser\tokenizer\reader_internal.h">
343340
<Filter>Parser</Filter>
344341
</ClInclude>
342+
<ClInclude Include="..\Parser\tokenizer\types.h">
343+
<Filter>Parser</Filter>
344+
</ClInclude>
345345
<ClInclude Include="..\Parser\tokenizer\source.h">
346346
<Filter>Parser</Filter>
347347
</ClInclude>
@@ -1361,9 +1361,6 @@
13611361
<ClCompile Include="..\Parser\lexer\state.c">
13621362
<Filter>Parser</Filter>
13631363
</ClCompile>
1364-
<ClCompile Include="..\Parser\lexer\buffer.c">
1365-
<Filter>Parser</Filter>
1366-
</ClCompile>
13671364
<ClCompile Include="..\Parser\tokenizer\cursor.c">
13681365
<Filter>Parser</Filter>
13691366
</ClCompile>
@@ -1373,6 +1370,9 @@
13731370
<ClCompile Include="..\Parser\tokenizer\decoder.c">
13741371
<Filter>Parser</Filter>
13751372
</ClCompile>
1373+
<ClCompile Include="..\Parser\tokenizer\api.c">
1374+
<Filter>Parser</Filter>
1375+
</ClCompile>
13761376
<ClCompile Include="..\Parser\tokenizer\reader.c">
13771377
<Filter>Parser</Filter>
13781378
</ClCompile>

‎Parser/action_helpers.c‎

Lines changed: 23 additions & 11 deletions

‎Parser/lexer/buffer.c‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎Parser/lexer/buffer.h‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

0 commit comments

Comments
 (0)