From c9a687628449be3a172be6b354ab7f3e6679cc57 Mon Sep 17 00:00:00 2001
From: Tian Gao <gaogaotiantian@hotmail.com>
Date: Wed, 17 Jan 2024 06:50:31 -0800
Subject: [PATCH 1/3] gh-112343: pdb: Use tokenize to replace convenience
 variables (#112380)

(cherry picked from commit 5c351fc85afd2ed167694a7dfe066741a5cdab53)
---
 Lib/pdb.py                                    | 37 ++++++++++++++++++-
 Lib/test/test_pdb.py                          |  9 +++++
 ...-11-24-19-08-50.gh-issue-112343.RarGFC.rst |  1 +
 3 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 2e048ac5ba6e6a..a838a26b038df6 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -76,6 +76,7 @@
 import dis
 import code
 import glob
+import token
 import pprint
 import signal
 import inspect
@@ -467,6 +468,39 @@ def default(self, line):
         except:
             self._error_exc()
 
+    def _replace_convenience_variables(self, line):
+        """Replace the convenience variables in 'line' with their values.
+           e.g. $foo is replaced by __pdb_convenience_variables["foo"].
+           Note: such pattern in string literals will be skipped"""
+
+        if "$" not in line:
+            return line
+
+        dollar_start = dollar_end = -1
+        replace_variables = []
+        try:
+            for t in tokenize.generate_tokens(io.StringIO(line).readline):
+                token_type, token_string, start, end, _ = t
+                if token_type == token.OP and token_string == '$':
+                    dollar_start, dollar_end = start, end
+                elif start == dollar_end and token_type == token.NAME:
+                    # line is a one-line command so we only care about column
+                    replace_variables.append((dollar_start[1], end[1], token_string))
+        except tokenize.TokenError:
+            return line
+
+        if not replace_variables:
+            return line
+
+        last_end = 0
+        line_pieces = []
+        for start, end, name in replace_variables:
+            line_pieces.append(line[last_end:start] + f'__pdb_convenience_variables["{name}"]')
+            last_end = end
+        line_pieces.append(line[last_end:])
+
+        return ''.join(line_pieces)
+
     def precmd(self, line):
         """Handle alias expansion and ';;' separator."""
         if not line.strip():
@@ -492,7 +526,8 @@ def precmd(self, line):
                 line = line[:marker].rstrip()
 
         # Replace all the convenience variables
-        line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line)
+        line = self._replace_convenience_variables(line)
+
         return line
 
     def onecmd(self, line):
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 08b2867266e3c4..2778293e3f617d 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -770,9 +770,12 @@ def test_convenience_variables():
 
     >>> with PdbTestInput([  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
     ...     '$_frame.f_lineno', # Check frame convenience variable
+    ...     '$ _frame',         # This should be a syntax error
     ...     '$a = 10',          # Set a convenience variable
     ...     '$a',               # Print its value
+    ...     'p "$a"',           # Print the string $a
     ...     'p $a + 2',         # Do some calculation
+    ...     'p f"$a = {$a}"',   # Make sure $ in string is not converted and f-string works
     ...     'u',                # Switch frame
     ...     '$_frame.f_lineno', # Make sure the frame changed
     ...     '$a',               # Make sure the value persists
@@ -792,11 +795,17 @@ def test_convenience_variables():
     -> try:
     (Pdb) $_frame.f_lineno
     3
+    (Pdb) $ _frame
+    *** SyntaxError: invalid syntax
     (Pdb) $a = 10
     (Pdb) $a
     10
+    (Pdb) p "$a"
+    '$a'
     (Pdb) p $a + 2
     12
+    (Pdb) p f"$a = {$a}"
+    '$a = 10'
     (Pdb) u
     > <doctest test.test_pdb.test_convenience_variables[1]>(2)test_function()
     -> util_function()
diff --git a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst
new file mode 100644
index 00000000000000..aaa50fce3ac962
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst
@@ -0,0 +1 @@
+Improve handling of pdb convenience variables to avoid replacing string contents.

From cb4ae8fbb86253b2eea43698e8249b5ff3cf387a Mon Sep 17 00:00:00 2001
From: Tian Gao <gaogaotiantian@hotmail.com>
Date: Wed, 17 Jan 2024 10:44:30 -0800
Subject: [PATCH 2/3] [3.12] gh-112343: pdb: Use tokenize to replace
 convenience variables (GH-112380) (cherry picked from commit
 5c351fc85afd2ed167694a7dfe066741a5cdab53)

Co-authored-by: Tian Gao <gaogaotiantian@hotmail.com>

From 78a71fda6073cac9e5563c0fd0156284646f32b7 Mon Sep 17 00:00:00 2001
From: Tian Gao <gaogaotiantian@hotmail.com>
Date: Wed, 17 Jan 2024 10:53:52 -0800
Subject: [PATCH 3/3] Fix pdb test

---
 Lib/test/test_pdb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 2778293e3f617d..56ee6c4c1e2847 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -389,7 +389,7 @@ def test_pdb_breakpoints_preserved_across_interactive_sessions():
     1   breakpoint   keep yes   at ...test_pdb.py:...
     2   breakpoint   keep yes   at ...test_pdb.py:...
     (Pdb) break pdb.find_function
-    Breakpoint 3 at ...pdb.py:97
+    Breakpoint 3 at ...pdb.py:...
     (Pdb) break
     Num Type         Disp Enb   Where
     1   breakpoint   keep yes   at ...test_pdb.py:...