python
diff --git a/‎Parser/lexer/lexer.c‎
Lines changed: 26 additions & 34 deletions b/‎Parser/lexer/lexer.c‎
Lines changed: 26 additions & 34 deletions
diff --git a/‎Parser/lexer/lexer_internal.h‎
Lines changed: 5 additions & 2 deletions b/‎Parser/lexer/lexer_internal.h‎
Lines changed: 5 additions & 2 deletions
diff --git a/‎Parser/lexer/string.c‎
Lines changed: 81 additions & 15 deletions b/‎Parser/lexer/string.c‎
Lines changed: 81 additions & 15 deletions
@@ -552,26 +552,14 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
     /* Punctuation character */
     int is_punctuation = (c == ':' || c == '}' || c == '!');
     if (is_punctuation && current != NULL) {
-        int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
-        int at_expression_boundary =
-            bracket_depth == current->replacement_depth;
-        if (at_expression_boundary && c == '!') {
-            int c2 = tok_nextc(tok);
-            if (c2 == '=') {
-                at_expression_boundary = 0;
-            }
-            tok_backup(tok, c2);
-        }
-        if (at_expression_boundary &&
-                _PyLexer_finish_ftstring_expr(tok, current, token)) {
+        int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
+        if (type < 0) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
-
-        if (c == ':' && at_expression_boundary) {
-            current->mode = FTSTRING_MODE_FORMAT_SPEC;
+        if (type != 0) {
             p_start = tok->start;
             p_end = tok->cur;
-            return MAKE_TOKEN(_PyToken_OneChar(c));
+            return MAKE_TOKEN(type);
         }
     }
 
@@ -656,18 +644,9 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
                 }
             }
         }
-
-        if (current != NULL) {
-            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
-            if (bracket_depth < 0) {
-                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
-                    _PyLexer_StringPrefix(current->kind), c));
-            }
-            if (c == '}' && bracket_depth == current->replacement_depth - 1) {
-                current->replacement_depth--;
-                current->mode = FTSTRING_MODE_MIDDLE;
-                current->debug_expr = 0;
-            }
+        if (current != NULL &&
+                _PyLexer_close_ftstring_expr(tok, current, c) < 0) {
+            return MAKE_TOKEN(ERRORTOKEN);
         }
         break;
     default:
@@ -678,9 +657,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
     }
 
-    if (c == '=' && current != NULL &&
-            _PyLexer_FTStringBracketDepth(tok, current) == current->replacement_depth) {
-        current->debug_expr = 1;
+    if (c == '=' && current != NULL) {
+        _PyLexer_mark_ftstring_debug(tok, current);
     }
 
     /* Punctuation character */
@@ -694,9 +672,23 @@ int
 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
 {
     ftstring_state *current = _PyLexer_CurrentFTString(tok);
-    int result = current == NULL || current->mode == FTSTRING_MODE_EXPRESSION
-        ? _PyLexer_get_normal(tok, current, token)
-        : _PyLexer_get_ftstring(tok, current, token);
+    int result;
+    if (current == NULL) {
+        result = _PyLexer_get_normal(tok, NULL, token);
+    }
+    else {
+        switch (current->mode) {
+        case FTSTRING_MODE_EXPRESSION:
+            result = _PyLexer_get_normal(tok, current, token);
+            break;
+        case FTSTRING_MODE_MIDDLE:
+        case FTSTRING_MODE_FORMAT_SPEC:
+            result = _PyLexer_get_ftstring(tok, current, token);
+            break;
+        default:
+            Py_UNREACHABLE();
+        }
+    }
     if (tok_failed(tok)) {
         result = ERRORTOKEN;
     }
 
@@ -35,8 +35,11 @@ int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
 int _PyLexer_record_ftstring_comment(
     struct tok_state *, ftstring_state *, const char *, const char *);
-int _PyLexer_finish_ftstring_expr(
-    struct tok_state *, ftstring_state *, struct token *);
+int _PyLexer_ftstring_punctuation(
+    struct tok_state *, ftstring_state *, struct token *, int);
+int _PyLexer_close_ftstring_expr(
+    struct tok_state *, ftstring_state *, int);
+void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
 
@@ -54,9 +54,9 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
     return 0;
 }
 
-int
-_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
-                              struct token *token)
+static int
+finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                     struct token *token)
 {
     assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
     assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
@@ -127,6 +127,82 @@ _PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
     return 0;
 }
 
+static int
+begin_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                    _PyTok_Off expr_start)
+{
+    assert(state->mode != FTSTRING_MODE_EXPRESSION);
+    state->expr_span = (_PyTok_Span){expr_start, -1};
+    if (state->comments != NULL) {
+        state->comments->count = 0;
+    }
+    if (state->replacement_depth >= MAX_EXPR_NESTING) {
+        _PyTokenizer_syntaxerror(
+            tok, "%c-string: expressions nested too deeply",
+            _PyLexer_StringPrefix(state->kind));
+        return -1;
+    }
+    state->replacement_depth++;
+    state->mode = FTSTRING_MODE_EXPRESSION;
+    state->debug_expr = 0;
+    return 0;
+}
+
+int
+_PyLexer_ftstring_punctuation(struct tok_state *tok, ftstring_state *state,
+                              struct token *token, int c)
+{
+    assert(state->mode == FTSTRING_MODE_EXPRESSION);
+    assert(c == ':' || c == '}' || c == '!');
+    if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
+        return 0;
+    }
+    if (c == '!') {
+        int next = tok_nextc(tok);
+        tok_backup(tok, next);
+        if (next == '=') {
+            return 0;
+        }
+    }
+    if (finish_ftstring_expr(tok, state, token) < 0) {
+        return -1;
+    }
+    if (c == ':') {
+        state->mode = FTSTRING_MODE_FORMAT_SPEC;
+        return COLON;
+    }
+    return 0;
+}
+
+int
+_PyLexer_close_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                             int c)
+{
+    assert(state->mode == FTSTRING_MODE_EXPRESSION);
+    assert(c == ')' || c == ']' || c == '}');
+    int depth = _PyLexer_FTStringBracketDepth(tok, state);
+    if (depth < 0) {
+        _PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
+                                _PyLexer_StringPrefix(state->kind), c);
+        return -1;
+    }
+    if (c == '}' && depth == state->replacement_depth - 1) {
+        state->replacement_depth--;
+        state->mode = FTSTRING_MODE_MIDDLE;
+        state->debug_expr = 0;
+    }
+    return 0;
+}
+
+void
+_PyLexer_mark_ftstring_debug(struct tok_state *tok, ftstring_state *state)
+{
+    assert(state->mode == FTSTRING_MODE_EXPRESSION);
+    if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
+        state->debug_expr = 1;
+    }
+}
+
 int
 _PyLexer_check_string_prefixes(struct tok_state *tok,
                                              int saw_b, int saw_r, int saw_u,
@@ -408,21 +484,11 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
-                current->expr_span = (_PyTok_Span){
-                    _PyLexer_BufferOffset(tok, tok->cur), -1};
-                if (current->comments != NULL) {
-                    current->comments->count = 0;
-                }
+                _PyTok_Off expr_start = _PyLexer_BufferOffset(tok, tok->cur);
                 tok_backup(tok, c);
-                if (current->replacement_depth >= MAX_EXPR_NESTING) {
-                    _PyTokenizer_syntaxerror(
-                        tok, "%c-string: expressions nested too deeply",
-                        _PyLexer_StringPrefix(current->kind));
+                if (begin_ftstring_expr(tok, current, expr_start) < 0) {
                     return MAKE_TOKEN(ERRORTOKEN);
                 }
-                current->replacement_depth++;
-                current->mode = FTSTRING_MODE_EXPRESSION;
-                current->debug_expr = 0;
                 p_start = tok->start;
                 p_end = tok->cur;
                 if (p_start == p_end) {