gh-153569: normalize prepared tokenizer input once · python/cpython@d9b2b16 · GitHub
Skip to content

Commit d9b2b16

Browse files
pablogsalpablogsal
committed
gh-153569: normalize prepared tokenizer input once
Prepared string input is split, normalized, and appended one line at a time, while the source object keeps line tables that the lexer no longer queries. Normalize prepared input once and transfer its storage to the tokenizer source. The reader scans lines from that byte buffer and keeps the implicit-newline flag with the prepared input.
1 parent 64522b9 commit d9b2b16

7 files changed

Lines changed: 79 additions & 221 deletions

File tree

‎Modules/_testinternalcapi/tokenizer.c‎

Lines changed: 13 additions & 47 deletions

‎Parser/lexer/state.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ struct tok_state {
7474
_PyTok_Off buf_offset; /* Logical offset of buf[0]. */
7575
const char *start; /* Start of current token if not NULL */
7676
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
77-
/* NB If done != E_OK, cur must be == inp!!! */
7877
FILE *fp; /* Rest of input; NULL if tokenizing a string */
7978
int indent; /* Current indentation index */
8079
int indstack[MAXINDENT]; /* Stack of indents */
@@ -146,7 +145,7 @@ _PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
146145
return tok->buf_offset + offset;
147146
}
148147

149-
static inline char *
148+
static inline const char *
150149
_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
151150
{
152151
assert(tok->buf != NULL);

‎Parser/tokenizer/decoder.c‎

Lines changed: 28 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
103103
}
104104
result[write] = '\0';
105105
*out_len = write;
106-
*implicit_newline = implicit;
106+
if (implicit_newline != NULL) {
107+
*implicit_newline = implicit;
108+
}
107109
return result;
108110
}
109111

@@ -297,51 +299,35 @@ static int
297299
store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
298300
int preserve_crlf, int add_final_newline)
299301
{
300-
Py_ssize_t pos = 0;
301-
while (pos < len) {
302-
Py_ssize_t raw_line_len;
303-
if (preserve_crlf) {
304-
const char *newline = memchr(data + pos, '\n', len - pos);
305-
raw_line_len = newline == NULL
306-
? len - pos : newline - data - pos + 1;
307-
}
308-
else {
309-
raw_line_len = raw_line_length(data + pos, len - pos);
310-
}
311-
int terminated = preserve_crlf
312-
? data[pos + raw_line_len - 1] == '\n'
313-
: data[pos + raw_line_len - 1] == '\n' ||
314-
data[pos + raw_line_len - 1] == '\r';
315-
int add_newline = add_final_newline &&
316-
pos + raw_line_len == len && !terminated;
317-
int normalize = add_newline ||
318-
(!preserve_crlf &&
319-
memchr(data + pos, '\r', raw_line_len) != NULL);
320-
321-
const char *line = data + pos;
322-
Py_ssize_t line_len = raw_line_len;
323-
char *normalized = NULL;
324-
int implicit = 0;
325-
if (normalize) {
326-
normalized = _PyTok_NormalizeNewlines(
327-
line, line_len, preserve_crlf, add_newline,
328-
&line_len, &implicit);
329-
if (normalized == NULL) {
330-
tok->done = E_NOMEM;
331-
return -1;
332-
}
333-
line = normalized;
302+
int normalize = (!preserve_crlf && memchr(data, '\r', len) != NULL) ||
303+
(add_final_newline && len > 0 && data[len - 1] != '\n');
304+
char *normalized = NULL;
305+
int implicit = 0;
306+
if (normalize) {
307+
normalized = _PyTok_NormalizeNewlines(
308+
data, len, preserve_crlf, add_final_newline,
309+
&len, &implicit);
310+
if (normalized == NULL) {
311+
tok->done = E_NOMEM;
312+
return -1;
334313
}
335-
_PyTok_Off appended = _PyTok_SourceAppendLine(
336-
&tok->source, line, line_len, implicit);
337-
PyMem_Free(normalized);
338-
if (appended < 0) {
339-
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
340-
? E_NOMEM : E_ERROR;
314+
}
315+
char *stored = normalized;
316+
if (stored == NULL && len > 0) {
317+
stored = _PyTok_CopyBytes(data, len);
318+
if (stored == NULL) {
319+
tok->done = E_NOMEM;
341320
return -1;
342321
}
343-
pos += raw_line_len;
344322
}
323+
assert(tok->source.bytes == NULL);
324+
assert(tok->source.len == 0 && tok->source.cap == 0);
325+
tok->source = (_PyTok_SourceText){
326+
.bytes = stored,
327+
.len = len,
328+
.cap = stored != NULL ? len + 1 : 0,
329+
};
330+
tok->reader->prepared_implicit_newline = implicit;
345331
return 0;
346332
}
347333

‎Parser/tokenizer/reader.c‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -191,20 +191,22 @@ chunk_is_line(const _PyTok_Chunk *chunk)
191191
static _PyTok_ReadResult
192192
next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
193193
{
194-
int lineno = tok->lineno + 1;
195-
if (lineno > tok->source.nlines) {
194+
if (tok->source.len == 0) {
195+
return _PYTOK_READ_EOF;
196+
}
197+
assert(tok->source.bytes != NULL);
198+
const char *source_end = tok->source.bytes + tok->source.len;
199+
if (tok->inp == source_end) {
196200
return _PYTOK_READ_EOF;
197201
}
198202
const char *start = tok->inp;
199-
const char *newline = memchr(
200-
start, '\n', tok->source.bytes + tok->source.len - start);
201-
_PyTok_Off end = newline != NULL
202-
? newline - tok->source.bytes + 1 : tok->source.len;
203+
const char *newline = memchr(start, '\n', source_end - start);
204+
const char *end = newline != NULL ? newline + 1 : source_end;
203205
chunk->data = (char *)start;
204-
chunk->len = tok->source.bytes + end - start;
206+
chunk->len = end - start;
205207
chunk->ownership = _PYTOK_CHUNK_BORROWED;
206-
chunk->implicit_newline = _PyTok_SourceLineIsImplicit(
207-
&tok->source, lineno);
208+
chunk->implicit_newline = end == source_end &&
209+
tok->reader->prepared_implicit_newline;
208210
return _PYTOK_READ_LINE;
209211
}
210212

@@ -557,10 +559,9 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
557559
}
558560
chunk->data = _PyTok_NormalizeNewlines(
559561
decoded.data, decoded.len, 0, 0,
560-
&chunk->len, &chunk->implicit_newline);
562+
&chunk->len, NULL);
561563
_PyTok_ChunkClear(&decoded);
562564
if (chunk->data == NULL) {
563-
PyErr_NoMemory();
564565
tok->done = E_NOMEM;
565566
return _PYTOK_READ_ERROR;
566567
}
@@ -647,12 +648,15 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
647648
}
648649
return 0;
649650
}
651+
if (tok->lineno == INT_MAX) {
652+
_PyTok_ChunkClear(&chunk);
653+
PyErr_SetString(PyExc_OverflowError,
654+
"too many tokenizer source lines");
655+
tok->done = E_ERROR;
656+
return 0;
657+
}
650658

651659
Py_ssize_t scan_len = chunk.len;
652-
if (kind == _PYTOK_READER_INTERACTIVE &&
653-
chunk.implicit_newline) {
654-
scan_len--;
655-
}
656660
if (streaming) {
657661
if (reset_buffer) {
658662
reset_streaming_buffer(tok);
@@ -680,9 +684,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
680684
if (!reset_buffer && source_will_grow) {
681685
offsets = save_buffer_offsets(tok, tok->source.bytes);
682686
}
683-
_PyTok_Off source_start = _PyTok_SourceAppendLine(
684-
&tok->source, chunk.data, chunk.len,
685-
chunk.implicit_newline);
687+
_PyTok_Off source_start = _PyTok_SourceAppend(
688+
&tok->source, chunk.data, chunk.len);
686689
if (source_start < 0) {
687690
_PyTok_ChunkClear(&chunk);
688691
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)

‎Parser/tokenizer/reader_internal.h‎

Lines changed: 1 addition & 0 deletions

0 commit comments

Comments
 (0)