gh-153569: normalize prepared tokenizer input once · python/cpython@b2bc8d1 · GitHub
Skip to content

Commit b2bc8d1

Browse files
pablogsalpablogsal
committed
gh-153569: normalize prepared tokenizer input once
Prepared string input is split, normalized, and appended one line at a time, while the source object keeps line tables that the lexer no longer queries. Normalize prepared input once and transfer that allocation to the tokenizer source. The reader scans lines from the byte buffer and keeps the implicit-newline flag with the prepared input.
1 parent 1504171 commit b2bc8d1

7 files changed

Lines changed: 80 additions & 223 deletions

File tree

‎Modules/_testinternalcapi/tokenizer.c‎

Lines changed: 13 additions & 47 deletions

‎Parser/lexer/state.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ struct tok_state {
7474
_PyTok_Off buf_offset; /* Logical offset of buf[0]. */
7575
const char *start; /* Start of current token if not NULL */
7676
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
77-
/* NB If done != E_OK, cur must be == inp!!! */
7877
FILE *fp; /* Rest of input; NULL if tokenizing a string */
7978
int indent; /* Current indentation index */
8079
int indstack[MAXINDENT]; /* Stack of indents */
@@ -146,7 +145,7 @@ _PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
146145
return tok->buf_offset + offset;
147146
}
148147

149-
static inline char *
148+
static inline const char *
150149
_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
151150
{
152151
assert(tok->buf != NULL);

‎Parser/tokenizer/decoder.c‎

Lines changed: 28 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
103103
}
104104
result[write] = '\0';
105105
*out_len = write;
106-
*implicit_newline = implicit;
106+
if (implicit_newline != NULL) {
107+
*implicit_newline = implicit;
108+
}
107109
return result;
108110
}
109111

@@ -297,51 +299,34 @@ static int
297299
store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
298300
int preserve_crlf, int add_final_newline)
299301
{
300-
Py_ssize_t pos = 0;
301-
while (pos < len) {
302-
Py_ssize_t raw_line_len;
303-
if (preserve_crlf) {
304-
const char *newline = memchr(data + pos, '\n', len - pos);
305-
raw_line_len = newline == NULL
306-
? len - pos : newline - data - pos + 1;
307-
}
308-
else {
309-
raw_line_len = raw_line_length(data + pos, len - pos);
310-
}
311-
int terminated = preserve_crlf
312-
? data[pos + raw_line_len - 1] == '\n'
313-
: data[pos + raw_line_len - 1] == '\n' ||
314-
data[pos + raw_line_len - 1] == '\r';
315-
int add_newline = add_final_newline &&
316-
pos + raw_line_len == len && !terminated;
317-
int normalize = add_newline ||
318-
(!preserve_crlf &&
319-
memchr(data + pos, '\r', raw_line_len) != NULL);
320-
321-
const char *line = data + pos;
322-
Py_ssize_t line_len = raw_line_len;
323-
char *normalized = NULL;
324-
int implicit = 0;
325-
if (normalize) {
326-
normalized = _PyTok_NormalizeNewlines(
327-
line, line_len, preserve_crlf, add_newline,
328-
&line_len, &implicit);
329-
if (normalized == NULL) {
330-
tok->done = E_NOMEM;
331-
return -1;
332-
}
333-
line = normalized;
334-
}
335-
_PyTok_Off appended = _PyTok_SourceAppendLine(
336-
&tok->source, line, line_len, implicit);
337-
PyMem_Free(normalized);
338-
if (appended < 0) {
339-
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
340-
? E_NOMEM : E_ERROR;
302+
int normalize = (!preserve_crlf && memchr(data, '\r', len) != NULL) ||
303+
(add_final_newline && len > 0 && data[len - 1] != '\n');
304+
char *normalized = NULL;
305+
int implicit = 0;
306+
if (normalize) {
307+
normalized = _PyTok_NormalizeNewlines(
308+
data, len, preserve_crlf, add_final_newline,
309+
&len, &implicit);
310+
if (normalized == NULL) {
311+
tok->done = E_NOMEM;
341312
return -1;
342313
}
343-
pos += raw_line_len;
314+
data = normalized;
315+
}
316+
if (normalized != NULL) {
317+
assert(tok->source.bytes == NULL);
318+
tok->source = (_PyTok_SourceText){
319+
.bytes = normalized,
320+
.len = len,
321+
.cap = len + 1,
322+
};
323+
}
324+
else if (_PyTok_SourceAppend(&tok->source, data, len) < 0) {
325+
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
326+
? E_NOMEM : E_ERROR;
327+
return -1;
344328
}
329+
tok->reader->prepared_implicit_newline = implicit;
345330
return 0;
346331
}
347332

‎Parser/tokenizer/reader.c‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -191,20 +191,22 @@ chunk_is_line(const _PyTok_Chunk *chunk)
191191
static _PyTok_ReadResult
192192
next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
193193
{
194-
int lineno = tok->lineno + 1;
195-
if (lineno > tok->source.nlines) {
194+
if (tok->source.len == 0) {
195+
return _PYTOK_READ_EOF;
196+
}
197+
assert(tok->source.bytes != NULL);
198+
const char *source_end = tok->source.bytes + tok->source.len;
199+
if (tok->inp == source_end) {
196200
return _PYTOK_READ_EOF;
197201
}
198202
const char *start = tok->inp;
199-
const char *newline = memchr(
200-
start, '\n', tok->source.bytes + tok->source.len - start);
201-
_PyTok_Off end = newline != NULL
202-
? newline - tok->source.bytes + 1 : tok->source.len;
203+
const char *newline = memchr(start, '\n', source_end - start);
204+
const char *end = newline != NULL ? newline + 1 : source_end;
203205
chunk->data = (char *)start;
204-
chunk->len = tok->source.bytes + end - start;
206+
chunk->len = end - start;
205207
chunk->ownership = _PYTOK_CHUNK_BORROWED;
206-
chunk->implicit_newline = _PyTok_SourceLineIsImplicit(
207-
&tok->source, lineno);
208+
chunk->implicit_newline = end == source_end &&
209+
tok->reader->prepared_implicit_newline;
208210
return _PYTOK_READ_LINE;
209211
}
210212

@@ -557,10 +559,9 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
557559
}
558560
chunk->data = _PyTok_NormalizeNewlines(
559561
decoded.data, decoded.len, 0, 0,
560-
&chunk->len, &chunk->implicit_newline);
562+
&chunk->len, NULL);
561563
_PyTok_ChunkClear(&decoded);
562564
if (chunk->data == NULL) {
563-
PyErr_NoMemory();
564565
tok->done = E_NOMEM;
565566
return _PYTOK_READ_ERROR;
566567
}
@@ -647,12 +648,15 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
647648
}
648649
return 0;
649650
}
651+
if (tok->lineno == INT_MAX) {
652+
_PyTok_ChunkClear(&chunk);
653+
PyErr_SetString(PyExc_OverflowError,
654+
"too many tokenizer source lines");
655+
tok->done = E_ERROR;
656+
return 0;
657+
}
650658

651659
Py_ssize_t scan_len = chunk.len;
652-
if (kind == _PYTOK_READER_INTERACTIVE &&
653-
chunk.implicit_newline) {
654-
scan_len--;
655-
}
656660
if (streaming) {
657661
if (reset_buffer) {
658662
reset_streaming_buffer(tok);
@@ -677,9 +681,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
677681
if (!reset_buffer) {
678682
offsets = save_buffer_offsets(tok, tok->source.bytes);
679683
}
680-
_PyTok_Off source_start = _PyTok_SourceAppendLine(
681-
&tok->source, chunk.data, chunk.len,
682-
chunk.implicit_newline);
684+
_PyTok_Off source_start = _PyTok_SourceAppend(
685+
&tok->source, chunk.data, chunk.len);
683686
if (source_start < 0) {
684687
_PyTok_ChunkClear(&chunk);
685688
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)

‎Parser/tokenizer/reader_internal.h‎

Lines changed: 3 additions & 2 deletions

0 commit comments

Comments
 (0)