Merge branch 'main' into shared-testcase

asvetlov · asvetlov · commit 09b416154a92 · 2022-03-09T16:31:41.000+02:00
diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
@@ -286,7 +286,7 @@ Directory and files operations
    .. versionadded:: 3.8
       The *dirs_exist_ok* parameter.
 
-.. function:: rmtree(path, ignore_errors=False, onerror=None)
+.. function:: rmtree(path, ignore_errors=False, onerror=None, *, dir_fd=None)
 
    .. index:: single: directory; deleting
 
@@ -296,6 +296,9 @@ Directory and files operations
    handled by calling a handler specified by *onerror* or, if that is omitted,
    they raise an exception.
 
+   This function can support :ref:`paths relative to directory descriptors
+   <dir_fd>`.
+
    .. note::
 
       On platforms that support the necessary fd-based functions a symlink
@@ -315,7 +318,7 @@ Directory and files operations
    *excinfo*, will be the exception information returned by
    :func:`sys.exc_info`.  Exceptions raised by *onerror* will not be caught.
 
-   .. audit-event:: shutil.rmtree path shutil.rmtree
+   .. audit-event:: shutil.rmtree path,dir_fd shutil.rmtree
 
    .. versionchanged:: 3.3
       Added a symlink attack resistant version that is used automatically
@@ -325,6 +328,9 @@ Directory and files operations
       On Windows, will no longer delete the contents of a directory junction
       before removing the junction.
 
+   .. versionchanged:: 3.11
+      The *dir_fd* parameter.
+
    .. attribute:: rmtree.avoids_symlink_attacks
 
       Indicates whether the current platform and implementation provides a
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
@@ -560,6 +560,9 @@ Connection Objects
 
       Passing :const:`None` as *trace_callback* will disable the trace callback.
 
+      For SQLite 3.14.0 and newer, bound parameters are expanded in the passed
+      statement string.
+
       .. note::
          Exceptions raised in the trace callback are not propagated. As a
          development and debugging aid, use
@@ -568,6 +571,9 @@ Connection Objects
 
       .. versionadded:: 3.3
 
+      .. versionchanged:: 3.11
+         Added support for expanded SQL statements.
+
 
    .. method:: enable_load_extension(enabled)
 
@@ -836,11 +842,11 @@ Cursor Objects
 
    .. method:: setinputsizes(sizes)
 
-      Required by the DB-API. Is a no-op in :mod:`sqlite3`.
+      Required by the DB-API. Does nothing in :mod:`sqlite3`.
 
    .. method:: setoutputsize(size [, column])
 
-      Required by the DB-API. Is a no-op in :mod:`sqlite3`.
+      Required by the DB-API. Does nothing in :mod:`sqlite3`.
 
    .. attribute:: rowcount
 
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
@@ -65,10 +65,15 @@ Summary -- Release highlights
 
 .. PEP-sized items next.
 
-PEP-654: Exception Groups and ``except*``.
-(Contributed by Irit Katriel in :issue:`45292`.)
-PEP-673: ``Self`` Type.
-(Contributed by James Hilton-Balfe and Pradeep Kumar in :issue:`30924`.)
+New syntax features:
+
+* :pep:`654`: Exception Groups and ``except*``.
+  (Contributed by Irit Katriel in :issue:`45292`.)
+
+New typing features:
+
+* :pep:`673`: ``Self`` Type.
+  (Contributed by James Hilton-Balfe and Pradeep Kumar in :issue:`30924`.)
 
 New Features
 ============
@@ -278,6 +283,13 @@ os
   (Contributed by Dong-hee Na in :issue:`44611`.)
 
 
+shutil
+------
+
+* Add optional parameter *dir_fd* in :func:`shutil.rmtree`.
+  (Contributed by Serhiy Storchaka in :issue:`46245`.)
+
+
 socket
 ------
 
@@ -322,6 +334,10 @@ sqlite3
   Instead we leave it to the SQLite library to handle these cases.
   (Contributed by Erlend E. Aasland in :issue:`44092`.)
 
+* For SQLite 3.14.0 and newer, bound parameters are expanded in the statement
+  string passed to the trace callback. See :meth:`~sqlite3.Connection.set_trace_callback`.
+  (Contributed by Erlend E. Aasland in :issue:`45138`.)
+
 
 sys
 ---
diff --git a/Lib/shutil.py b/Lib/shutil.py
@@ -684,9 +684,14 @@ def _rmtree_safe_fd(topfd, path, onerror):
                      os.scandir in os.supports_fd and
                      os.stat in os.supports_follow_symlinks)
 
-def rmtree(path, ignore_errors=False, onerror=None):
+def rmtree(path, ignore_errors=False, onerror=None, *, dir_fd=None):
     """Recursively delete a directory tree.
 
+    If dir_fd is not None, it should be a file descriptor open to a directory;
+    path will then be relative to that directory.
+    dir_fd may not be implemented on your platform.
+    If it is unavailable, using it will raise a NotImplementedError.
+
     If ignore_errors is set, errors are ignored; otherwise, if onerror
     is set, it is called to handle the error with arguments (func,
     path, exc_info) where func is platform and implementation dependent;
@@ -695,7 +700,7 @@ def rmtree(path, ignore_errors=False, onerror=None):
     is false and onerror is None, an exception is raised.
 
     """
-    sys.audit("shutil.rmtree", path)
+    sys.audit("shutil.rmtree", path, dir_fd)
     if ignore_errors:
         def onerror(*args):
             pass
@@ -709,12 +714,12 @@ def onerror(*args):
         # Note: To guard against symlink races, we use the standard
         # lstat()/open()/fstat() trick.
         try:
-            orig_st = os.lstat(path)
+            orig_st = os.lstat(path, dir_fd=dir_fd)
         except Exception:
             onerror(os.lstat, path, sys.exc_info())
             return
         try:
-            fd = os.open(path, os.O_RDONLY)
+            fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd)
             fd_closed = False
         except Exception:
             onerror(os.open, path, sys.exc_info())
@@ -725,7 +730,7 @@ def onerror(*args):
                 try:
                     os.close(fd)
                     fd_closed = True
-                    os.rmdir(path)
+                    os.rmdir(path, dir_fd=dir_fd)
                 except OSError:
                     onerror(os.rmdir, path, sys.exc_info())
             else:
@@ -738,6 +743,8 @@ def onerror(*args):
             if not fd_closed:
                 os.close(fd)
     else:
+        if dir_fd is not None:
+            raise NotImplementedError("dir_fd unavailable on this platform")
         try:
             if _rmtree_islink(path):
                 # symlinks to directories are forbidden, see bug #1669
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
@@ -426,6 +426,9 @@ def test_invalid_operations(self):
             self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR)
             self.assertRaises(exc, fp.seek, -1, self.SEEK_END)
 
+    @unittest.skipIf(
+        support.is_emscripten, "fstat() of a pipe fd is not supported"
+    )
     def test_optional_abilities(self):
         # Test for OSError when optional APIs are not supported
         # The purpose of this test is to try fileno(), reading, writing and
@@ -3971,6 +3974,9 @@ def test_removed_u_mode(self):
                 self.open(os_helper.TESTFN, mode)
             self.assertIn('invalid mode', str(cm.exception))
 
+    @unittest.skipIf(
+        support.is_emscripten, "fstat() of a pipe fd is not supported"
+    )
     def test_open_pipe_with_append(self):
         # bpo-27805: Ignore ESPIPE from lseek() in open().
         r, w = os.pipe()
@@ -4134,9 +4140,15 @@ def test_pickling(self):
                 with self.open(os_helper.TESTFN, **kwargs) as f:
                     self.assertRaises(TypeError, pickle.dumps, f, protocol)
 
+    @unittest.skipIf(
+        support.is_emscripten, "fstat() of a pipe fd is not supported"
+    )
     def test_nonblock_pipe_write_bigbuf(self):
         self._test_nonblock_pipe_write(16*1024)
 
+    @unittest.skipIf(
+        support.is_emscripten, "fstat() of a pipe fd is not supported"
+    )
     def test_nonblock_pipe_write_smallbuf(self):
         self._test_nonblock_pipe_write(1024)
 
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
@@ -405,6 +405,27 @@ def _raiser(*args, **kwargs):
             self.assertFalse(shutil._use_fd_functions)
             self.assertFalse(shutil.rmtree.avoids_symlink_attacks)
 
+    @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported")
+    def test_rmtree_with_dir_fd(self):
+        tmp_dir = self.mkdtemp()
+        victim = 'killme'
+        fullname = os.path.join(tmp_dir, victim)
+        dir_fd = os.open(tmp_dir, os.O_RDONLY)
+        self.addCleanup(os.close, dir_fd)
+        os.mkdir(fullname)
+        os.mkdir(os.path.join(fullname, 'subdir'))
+        write_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo')
+        self.assertTrue(os.path.exists(fullname))
+        shutil.rmtree(victim, dir_fd=dir_fd)
+        self.assertFalse(os.path.exists(fullname))
+
+    @unittest.skipIf(shutil._use_fd_functions, "dir_fd is supported")
+    def test_rmtree_with_dir_fd_unsupported(self):
+        tmp_dir = self.mkdtemp()
+        with self.assertRaises(NotImplementedError):
+            shutil.rmtree(tmp_dir, dir_fd=0)
+        self.assertTrue(os.path.exists(tmp_dir))
+
     def test_rmtree_dont_delete_file(self):
         # When called on a file instead of a directory, don't delete it.
         handle, path = tempfile.mkstemp(dir=self.mkdtemp())
diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py
@@ -20,12 +20,16 @@
 #    misrepresented as being the original software.
 # 3. This notice may not be removed or altered from any source distribution.
 
-import unittest
+import contextlib
 import sqlite3 as sqlite
+import unittest
 
 from test.support.os_helper import TESTFN, unlink
+
+from test.test_sqlite3.test_dbapi import memory_database, cx_limit
 from test.test_sqlite3.test_userfunctions import with_tracebacks
 
+
 class CollationTests(unittest.TestCase):
     def test_create_collation_not_string(self):
         con = sqlite.connect(":memory:")
@@ -224,6 +228,16 @@ def bad_progress():
 
 
 class TraceCallbackTests(unittest.TestCase):
+    @contextlib.contextmanager
+    def check_stmt_trace(self, cx, expected):
+        try:
+            traced = []
+            cx.set_trace_callback(lambda stmt: traced.append(stmt))
+            yield
+        finally:
+            self.assertEqual(traced, expected)
+            cx.set_trace_callback(None)
+
     def test_trace_callback_used(self):
         """
         Test that the trace callback is invoked once it is set.
@@ -289,6 +303,51 @@ def trace(statement):
             con2.close()
         self.assertEqual(traced_statements, queries)
 
+    @unittest.skipIf(sqlite.sqlite_version_info < (3, 14, 0),
+                     "Requires SQLite 3.14.0 or newer")
+    def test_trace_expanded_sql(self):
+        expected = [
+            "create table t(t)",
+            "BEGIN ",
+            "insert into t values(0)",
+            "insert into t values(1)",
+            "insert into t values(2)",
+            "COMMIT",
+        ]
+        with memory_database() as cx, self.check_stmt_trace(cx, expected):
+            with cx:
+                cx.execute("create table t(t)")
+                cx.executemany("insert into t values(?)", ((v,) for v in range(3)))
+
+    @with_tracebacks(
+        sqlite.DataError,
+        regex="Expanded SQL string exceeds the maximum string length"
+    )
+    def test_trace_too_much_expanded_sql(self):
+        # If the expanded string is too large, we'll fall back to the
+        # unexpanded SQL statement. The resulting string length is limited by
+        # SQLITE_LIMIT_LENGTH.
+        template = "select 'b' as \"a\" from sqlite_master where \"a\"="
+        category = sqlite.SQLITE_LIMIT_LENGTH
+        with memory_database() as cx, cx_limit(cx, category=category) as lim:
+            nextra = lim - (len(template) + 2) - 1
+            ok_param = "a" * nextra
+            bad_param = "a" * (nextra + 1)
+
+            unexpanded_query = template + "?"
+            with self.check_stmt_trace(cx, [unexpanded_query]):
+                cx.execute(unexpanded_query, (bad_param,))
+
+            expanded_query = f"{template}'{ok_param}'"
+            with self.check_stmt_trace(cx, [expanded_query]):
+                cx.execute(unexpanded_query, (ok_param,))
+
+    @with_tracebacks(ZeroDivisionError, regex="division by zero")
+    def test_trace_bad_handler(self):
+        with memory_database() as cx:
+            cx.set_trace_callback(lambda stmt: 5/0)
+            cx.execute("select 1")
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -1869,6 +1869,7 @@ LIBSUBDIRS=	asyncio \
 		site-packages \
 		sqlite3 \
 		tkinter \
+		tomllib \
 		turtledemo \
 		unittest \
 		urllib \
diff --git a/Misc/NEWS.d/next/Library/2021-09-08-16-21-03.bpo-45138.yghUrK.rst b/Misc/NEWS.d/next/Library/2021-09-08-16-21-03.bpo-45138.yghUrK.rst
@@ -0,0 +1,3 @@
+For SQLite 3.14.0 and newer, bound parameters are expanded in the statement
+string passed to the :mod:`sqlite3` trace callback. Patch by Erlend E.
+Aasland.
diff --git a/Misc/NEWS.d/next/Library/2022-01-03-20-12-14.bpo-46245.3w4RlA.rst b/Misc/NEWS.d/next/Library/2022-01-03-20-12-14.bpo-46245.3w4RlA.rst
@@ -0,0 +1 @@
+Add optional parameter *dir_fd* in :func:`shutil.rmtree`.
diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+For SQLite 3.14.0 and newer, bound parameters are expanded in the statement`
	`2`	+string passed to the :mod:`sqlite3` trace callback. Patch by Erlend E.
	`3`	`+Aasland.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Add optional parameter dir_fd in :func:`shutil.rmtree`.