gh-153569: keep character scanning inline with source offsets · python/cpython@0b47836 · GitHub
Skip to content

Commit 0b47836

Browse files
pablogsalpablogsal
committed
gh-153569: keep character scanning inline with source offsets
1 parent c8e18ce commit 0b47836

5 files changed

Lines changed: 51 additions & 39 deletions

File tree

‎Lib/test/test_tstring.py‎

Lines changed: 4 additions & 0 deletions

‎Makefile.pre.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
34633463
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
34643464
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
34653465
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
3466-
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
3466+
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
34673467
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
34683468
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
34693469

‎Parser/lexer/lexer.c‎

Lines changed: 22 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -23,43 +23,33 @@ contains_null_bytes(const char* str, size_t size)
2323
return memchr(str, 0, size) != NULL;
2424
}
2525

26-
/* Get next char, updating state; error code goes into tok->done */
2726
int
28-
_PyLexer_nextc(struct tok_state *tok)
27+
_PyLexer_refill(struct tok_state *tok)
2928
{
30-
int rc;
31-
for (;;) {
32-
if (tok->cur != tok->inp) {
33-
if (tok->cur - tok->line_start >= INT_MAX) {
34-
tok->done = E_COLUMNOVERFLOW;
35-
return EOF;
36-
}
37-
return Py_CHARMASK(tok->source.bytes[tok->cur++ - tok->source.base_offset]); /* Fast path */
38-
}
39-
if (tok->done != E_OK) {
40-
return EOF;
41-
}
42-
rc = _PyTok_ReaderUnderflow(tok);
29+
if (tok->done != E_OK) {
30+
return 0;
31+
}
32+
int rc = _PyTok_ReaderUnderflow(tok);
4333
#if defined(Py_DEBUG)
44-
if (tok->debug) {
45-
fprintf(stderr, "line[%d] = ", tok->lineno);
46-
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur), tok->inp - tok->cur);
47-
fprintf(stderr, " tok->done = %d\n", tok->done);
48-
}
34+
if (tok->debug) {
35+
fprintf(stderr, "line[%d] = ", tok->lineno);
36+
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur),
37+
tok->inp - tok->cur);
38+
fprintf(stderr, " tok->done = %d\n", tok->done);
39+
}
4940
#endif
50-
if (!rc) {
51-
tok->cur = tok->inp;
52-
return EOF;
53-
}
54-
tok->line_start = tok->cur;
55-
56-
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start), tok->inp - tok->line_start)) {
57-
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
58-
tok->cur = tok->inp;
59-
return EOF;
60-
}
41+
if (!rc) {
42+
tok->cur = tok->inp;
43+
return 0;
6144
}
62-
Py_UNREACHABLE();
45+
tok->line_start = tok->cur;
46+
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
47+
tok->inp - tok->line_start)) {
48+
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
49+
tok->cur = tok->inp;
50+
return 0;
51+
}
52+
return 1;
6353
}
6454

6555
/* Back-up one character */

‎Parser/lexer/lexer_internal.h‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
(_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
2222
#define FTSTRING_END(state) \
2323
(_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END)
24-
#define tok_nextc _PyLexer_nextc
2524
#define tok_backup _PyLexer_backup
2625

2726
static inline int
@@ -31,7 +30,26 @@ tok_failed(const struct tok_state *tok)
3130
tok->done != E_INTERACT_STOP;
3231
}
3332

34-
int _PyLexer_nextc(struct tok_state *);
33+
int _PyLexer_refill(struct tok_state *);
34+
35+
static inline int
36+
tok_nextc(struct tok_state *tok)
37+
{
38+
while (tok->cur == tok->inp) {
39+
if (!_PyLexer_refill(tok)) {
40+
return EOF;
41+
}
42+
}
43+
assert(tok->cur >= tok->source.base_offset);
44+
assert(tok->cur - tok->source.base_offset < tok->source.len);
45+
if (tok->cur - tok->line_start >= INT_MAX) {
46+
tok->done = E_COLUMNOVERFLOW;
47+
return EOF;
48+
}
49+
return Py_CHARMASK(
50+
tok->source.bytes[tok->cur++ - tok->source.base_offset]);
51+
}
52+
3553
void _PyLexer_backup(struct tok_state *, int);
3654
int _PyLexer_record_ftstring_comment(
3755
struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off);

‎Parser/lexer/state.h‎

Lines changed: 4 additions & 4 deletions

0 commit comments

Comments
 (0)