gh-116042: Fix location for SyntaxErrors of invalid escapes in the to… · python/cpython@56eda25 · GitHub
Skip to content

Commit 56eda25

Browse files
authored
gh-116042: Fix location for SyntaxErrors of invalid escapes in the tokenizer (#116049)
1 parent 49b1103 commit 56eda25

5 files changed

Lines changed: 80 additions & 17 deletions

File tree

‎Lib/test/test_cmd_line_script.py‎

Lines changed: 1 addition & 1 deletion

‎Lib/test/test_string_literals.py‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def test_eval_str_invalid_escape(self):
120120
r'Such sequences will not work in the future. '
121121
r'Did you mean "\\z"? A raw string is also an option.')
122122
self.assertEqual(w[0].filename, '<string>')
123-
self.assertEqual(w[0].lineno, 1)
123+
self.assertEqual(w[0].lineno, 2)
124124

125125
with warnings.catch_warnings(record=True) as w:
126126
warnings.simplefilter('error', category=SyntaxWarning)
@@ -131,7 +131,7 @@ def test_eval_str_invalid_escape(self):
131131
self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
132132
r'Did you mean "\\z"? A raw string is also an option.')
133133
self.assertEqual(exc.filename, '<string>')
134-
self.assertEqual(exc.lineno, 1)
134+
self.assertEqual(exc.lineno, 2)
135135
self.assertEqual(exc.offset, 1)
136136

137137
# Check that the warning is raised only once if there are syntax errors
@@ -160,7 +160,7 @@ def test_eval_str_invalid_octal_escape(self):
160160
r'Such sequences will not work in the future. '
161161
r'Did you mean "\\407"? A raw string is also an option.')
162162
self.assertEqual(w[0].filename, '<string>')
163-
self.assertEqual(w[0].lineno, 1)
163+
self.assertEqual(w[0].lineno, 2)
164164

165165
with warnings.catch_warnings(record=True) as w:
166166
warnings.simplefilter('error', category=SyntaxWarning)
@@ -171,9 +171,32 @@ def test_eval_str_invalid_octal_escape(self):
171171
self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
172172
r'Did you mean "\\407"? A raw string is also an option.')
173173
self.assertEqual(exc.filename, '<string>')
174-
self.assertEqual(exc.lineno, 1)
174+
self.assertEqual(exc.lineno, 2)
175175
self.assertEqual(exc.offset, 1)
176176

177+
def test_invalid_escape_locations_with_offset(self):
178+
with warnings.catch_warnings(record=True) as w:
179+
warnings.simplefilter('always', category=SyntaxWarning)
180+
eval("\"'''''''''''''''''''''invalid\ Escape\"")
181+
self.assertEqual(len(w), 1)
182+
self.assertEqual(str(w[0].message),
183+
r'"\ " is an invalid escape sequence. Such sequences '
184+
r'will not work in the future. Did you mean "\\ "? '
185+
r'A raw string is also an option.')
186+
self.assertEqual(w[0].filename, '<string>')
187+
self.assertEqual(w[0].lineno, 1)
188+
189+
with warnings.catch_warnings(record=True) as w:
190+
warnings.simplefilter('always', category=SyntaxWarning)
191+
eval("\"''Incorrect \ logic?\"")
192+
self.assertEqual(len(w), 1)
193+
self.assertEqual(str(w[0].message),
194+
r'"\ " is an invalid escape sequence. Such sequences '
195+
r'will not work in the future. Did you mean "\\ "? '
196+
r'A raw string is also an option.')
197+
self.assertEqual(w[0].filename, '<string>')
198+
self.assertEqual(w[0].lineno, 1)
199+
177200
def test_eval_str_raw(self):
178201
self.assertEqual(eval(""" r'x' """), 'x')
179202
self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01')
@@ -215,7 +238,7 @@ def test_eval_bytes_invalid_escape(self):
215238
r'Such sequences will not work in the future. '
216239
r'Did you mean "\\z"? A raw string is also an option.')
217240
self.assertEqual(w[0].filename, '<string>')
218-
self.assertEqual(w[0].lineno, 1)
241+
self.assertEqual(w[0].lineno, 2)
219242

220243
with warnings.catch_warnings(record=True) as w:
221244
warnings.simplefilter('error', category=SyntaxWarning)
@@ -226,7 +249,7 @@ def test_eval_bytes_invalid_escape(self):
226249
self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
227250
r'Did you mean "\\z"? A raw string is also an option.')
228251
self.assertEqual(exc.filename, '<string>')
229-
self.assertEqual(exc.lineno, 1)
252+
self.assertEqual(exc.lineno, 2)
230253

231254
def test_eval_bytes_invalid_octal_escape(self):
232255
for i in range(0o400, 0o1000):
@@ -241,7 +264,7 @@ def test_eval_bytes_invalid_octal_escape(self):
241264
r'Such sequences will not work in the future. '
242265
r'Did you mean "\\407"? A raw string is also an option.')
243266
self.assertEqual(w[0].filename, '<string>')
244-
self.assertEqual(w[0].lineno, 1)
267+
self.assertEqual(w[0].lineno, 2)
245268

246269
with warnings.catch_warnings(record=True) as w:
247270
warnings.simplefilter('error', category=SyntaxWarning)
@@ -252,7 +275,7 @@ def test_eval_bytes_invalid_octal_escape(self):
252275
self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
253276
r'Did you mean "\\407"? A raw string is also an option.')
254277
self.assertEqual(exc.filename, '<string>')
255-
self.assertEqual(exc.lineno, 1)
278+
self.assertEqual(exc.lineno, 2)
256279

257280
def test_eval_bytes_raw(self):
258281
self.assertEqual(eval(""" br'x' """), b'x')
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix location for SyntaxErrors of invalid escapes in the tokenizer. Patch by
2+
Pablo Galindo

‎Parser/pegen_errors.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,8 +352,8 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
352352
assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
353353

354354
if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
355-
Py_ssize_t size = p->tok->inp - p->tok->buf;
356-
error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
355+
Py_ssize_t size = p->tok->inp - p->tok->line_start;
356+
error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
357357
}
358358
else if (p->tok->fp == NULL || p->tok->fp == stdin) {
359359
error_line = get_error_line_from_tokenizer_buffers(p, lineno);

‎Parser/string_parser.c‎

Lines changed: 44 additions & 6 deletions

0 commit comments

Comments
 (0)