gh-153569: assert tokenizer state invariants · python/cpython@841b57e · GitHub
Skip to content

Commit 841b57e

Browse files
committed
gh-153569: assert tokenizer state invariants
Check formatted-string stack transitions, decoder cursor bounds, and input chunks at the points where the tokenizer relies on them. These checks stay debug-only and make state corruption fail close to its source.
1 parent ea554f8 commit 841b57e

4 files changed

Lines changed: 12 additions & 2 deletions

File tree

Parser/lexer/state.c

Lines changed: 1 addition & 0 deletions

Parser/lexer/state.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ static inline ftstring_state *
115115
_PyLexer_CurrentFTString(struct tok_state *tok)
116116
{
117117
assert(tok->ftstring_stack != NULL);
118-
assert(tok->ftstring_depth <= tok->ftstring_capacity);
118+
assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity);
119119
if (tok->ftstring_depth == 0) {
120120
return NULL;
121121
}

Parser/lexer/string.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ int
2020
_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
2121
const char *start, const char *end)
2222
{
23+
assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
2324
if (state->expr_span.end >= 0) {
2425
return 0;
2526
}
@@ -57,7 +58,8 @@ int
5758
_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
5859
struct token *token)
5960
{
60-
assert(token != NULL);
61+
assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
62+
assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
6163

6264
if (state->expr_span.end >= 0) {
6365
return 0;
@@ -338,6 +340,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
338340
int
339341
_PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct token *token)
340342
{
343+
assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
344+
assert((current->quote_size == 1 || current->quote_size == 3) &&
345+
current->replacement_depth <= MAX_EXPR_NESTING);
341346
const char *p_start = NULL;
342347
const char *p_end = NULL;
343348
int end_quote_size = 0;

Parser/tokenizer/reader.c

Lines changed: 4 additions & 0 deletions

0 commit comments

Comments
 (0)