From 91c20f5a298ea50318e95c1d06686b766d7ee0c6 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sat, 5 Oct 2024 23:50:30 +0200 Subject: [PATCH 1/4] Fix tokenizer roundtrip with escape characters --- Lib/test/test_tokenize.py | 10 ++++++++++ Lib/tokenize.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index de0e0b430a21bf..c758f8734182e6 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1919,6 +1919,16 @@ def test_roundtrip(self): self.check_roundtrip(r"f'\\\\N{{'") self.check_roundtrip(r"f'\\\\\\N{{'") self.check_roundtrip(r"f'\\\\\\\\N{{'") + + self.check_roundtrip(r"f'\n{{foo}}'") + self.check_roundtrip(r"f'\\n{{foo}}'") + self.check_roundtrip(r"f'\\\n{{foo}}'") + self.check_roundtrip(r"f'\\\\n{{foo}}'") + + self.check_roundtrip(r"f'\t{{foo}}'") + self.check_roundtrip(r"f'\\t{{foo}}'") + self.check_roundtrip(r"f'\\\t{{foo}}'") + self.check_roundtrip(r"f'\\\\t{{foo}}'") cases = [ """ if 1: diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7f418bb7a1b37f..4b4c3cfe16999b 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -200,7 +200,7 @@ def escape_brackets(self, token): characters[-2::-1] ) ) - if n_backslashes % 2 == 0: + if n_backslashes % 2 == 0 or characters[-1] != "N": characters.append(character) else: consume_until_next_bracket = True From b2b5f17f55cd222991632c76fc0509d962952cd6 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sat, 5 Oct 2024 23:53:10 +0200 Subject: [PATCH 2/4] Add news entry --- .../2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst new file mode 100644 index 00000000000000..8971e052860225 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst @@ -0,0 +1,2 @@ +Fix :func:`tokenize.untokenize` producing invalid syntax for +double braces preceded by certain escape characters. From 10ed775089190b757064a384385e7b79345c8571 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sun, 6 Oct 2024 12:43:53 +0200 Subject: [PATCH 3/4] Add tests with raw strings --- Lib/test/test_tokenize.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index c758f8734182e6..0d7c8b81bc1fae 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1929,6 +1929,11 @@ def test_roundtrip(self): self.check_roundtrip(r"f'\\t{{foo}}'") self.check_roundtrip(r"f'\\\t{{foo}}'") self.check_roundtrip(r"f'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\t{{foo}}'") + self.check_roundtrip(r"rf'\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\\t{{foo}}'") cases = [ """ if 1: From 633264e7ab32dbcb8b27eace45322c7f47cde984 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sun, 6 Oct 2024 13:26:01 +0200 Subject: [PATCH 4/4] Add tests for backslashes --- Lib/test/test_tokenize.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 0d7c8b81bc1fae..75710db7d05375 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1934,6 +1934,11 @@ def test_roundtrip(self): self.check_roundtrip(r"rf'\\t{{foo}}'") self.check_roundtrip(r"rf'\\\t{{foo}}'") self.check_roundtrip(r"rf'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\{{foo}}'") + self.check_roundtrip(r"f'\\{{foo}}'") + self.check_roundtrip(r"rf'\\\{{foo}}'") + self.check_roundtrip(r"f'\\\\{{foo}}'") cases = [ """ if 1: