Skip to content

Commit 739ad12

Browse files
committed
gh-153568: Skip newline translation for source without carriage returns
The tokenizer copied every input byte by byte to normalize newlines, but source without a carriage return needs no translation and can be copied verbatim.
1 parent e2118b0 commit 739ad12

2 files changed

Lines changed: 12 additions & 1 deletion

File tree

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Speed up the tokenizer by copying source text verbatim when it contains no
2+
carriage returns.

Parser/tokenizer/helpers.c

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,8 @@ char *
265265
_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf,
266266
struct tok_state *tok) {
267267
int skip_next_lf = 0;
268-
size_t needed_length = strlen(s) + 2, final_length;
268+
size_t input_length = strlen(s);
269+
size_t needed_length = input_length + 2, final_length;
269270
char *buf, *current;
270271
char c = '\0';
271272
buf = PyMem_Malloc(needed_length);
@@ -274,6 +275,13 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf
274275
PyErr_NoMemory();
275276
return NULL;
276277
}
278+
if (memchr(s, '\r', input_length) == NULL) {
279+
// No carriage returns: nothing to translate, copy verbatim.
280+
memcpy(buf, s, input_length);
281+
current = buf + input_length;
282+
c = input_length ? s[input_length - 1] : '\0';
283+
goto add_final_newline;
284+
}
277285
for (current = buf; *s; s++, current++) {
278286
c = *s;
279287
if (skip_next_lf) {
@@ -290,6 +298,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf
290298
}
291299
*current = c;
292300
}
301+
add_final_newline:
293302
/* If this is exec input, add a newline to the end of the string if
294303
there isn't one already. */
295304
if (exec_input && c != '\n' && c != '\0') {

0 commit comments

Comments
 (0)