diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst new file mode 100644 index 00000000000000..deaa82c96fd3ac --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst @@ -0,0 +1,2 @@ +Speed up the tokenizer by copying source text verbatim when it contains no +carriage returns. diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 5b588572f1fb7e..ab8ae393d03032 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -87,15 +87,22 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, return NULL; } Py_ssize_t write = 0; - for (Py_ssize_t read = 0; read < len; read++) { - char c = data[read]; - if (!preserve_crlf && c == '\r') { - if (read + 1 < len && data[read + 1] == '\n') { - read++; + if (memchr(data, '\r', len) == NULL) { + // No carriage returns: nothing to translate, copy verbatim. + memcpy(result, data, len); + write = len; + } + else { + for (Py_ssize_t read = 0; read < len; read++) { + char c = data[read]; + if (!preserve_crlf && c == '\r') { + if (read + 1 < len && data[read + 1] == '\n') { + read++; + } + c = '\n'; } - c = '\n'; + result[write++] = c; } - result[write++] = c; } int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; if (implicit) {