Merge branch 'python:main' into fix-issue-116180

NGRsoftlab · web-flow · commit 59747572c340 · 2024-04-17T13:58:11.000+03:00
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
@@ -221,13 +221,14 @@ For convenience, some of these functions will always return a
 
 .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr)
 
-   This is a convenience function to raise :exc:`WindowsError`. If called with
+   This is a convenience function to raise :exc:`OSError`. If called with
    *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError`
    is used instead.  It calls the Win32 function :c:func:`!FormatMessage` to retrieve
    the Windows description of error code given by *ierr* or :c:func:`!GetLastError`,
-   then it constructs a tuple object whose first item is the *ierr* value and whose
-   second item is the corresponding error message (gotten from
-   :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError,
+   then it constructs a :exc:`OSError` object with the :attr:`~OSError.winerror`
+   attribute set to the error code, the :attr:`~OSError.strerror` attribute
+   set to the corresponding error message (gotten from
+   :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_OSError,
    object)``. This function always returns ``NULL``.
 
    .. availability:: Windows.
diff --git a/Include/cpython/object.h b/Include/cpython/object.h
@@ -448,8 +448,8 @@ without deallocating anything (and so unbounded call-stack depth is avoided).
 When the call stack finishes unwinding again, code generated by the END macro
 notices this, and calls another routine to deallocate all the objects that
 may have been added to the list of deferred deallocations.  In effect, a
-chain of N deallocations is broken into (N-1)/(_PyTrash_UNWIND_LEVEL-1) pieces,
-with the call stack never exceeding a depth of _PyTrash_UNWIND_LEVEL.
+chain of N deallocations is broken into (N-1)/(Py_TRASHCAN_HEADROOM-1) pieces,
+with the call stack never exceeding a depth of Py_TRASHCAN_HEADROOM.
 
 Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base
 class, we need to ensure that the trashcan is only triggered on the tp_dealloc
@@ -461,30 +461,33 @@ passed as second argument to Py_TRASHCAN_BEGIN().
 /* Python 3.9 private API, invoked by the macros below. */
 PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op);
 PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate);
+
+PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op);
+PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
+
+
 /* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */
-PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc);
 
-#define Py_TRASHCAN_BEGIN_CONDITION(op, cond) \
-    do { \
-        PyThreadState *_tstate = NULL; \
-        /* If "cond" is false, then _tstate remains NULL and the deallocator \
-         * is run normally without involving the trashcan */ \
-        if (cond) { \
-            _tstate = PyThreadState_GetUnchecked(); \
-            if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \
-                break; \
-            } \
-        }
-        /* The body of the deallocator is here. */
-#define Py_TRASHCAN_END \
-        if (_tstate) { \
-            _PyTrash_end(_tstate); \
-        } \
-    } while (0);
+/* To avoid raising recursion errors during dealloc trigger trashcan before we reach
+ * recursion limit. To avoid trashing, we don't attempt to empty the trashcan until
+ * we have headroom above the trigger limit */
+#define Py_TRASHCAN_HEADROOM 50
 
 #define Py_TRASHCAN_BEGIN(op, dealloc) \
-    Py_TRASHCAN_BEGIN_CONDITION((op), \
-        _PyTrash_cond(_PyObject_CAST(op), (destructor)(dealloc)))
+do { \
+    PyThreadState *tstate = PyThreadState_Get(); \
+    if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
+        _PyTrash_thread_deposit_object(tstate, (PyObject *)op); \
+        break; \
+    } \
+    tstate->c_recursion_remaining--;
+    /* The body of the deallocator is here. */
+#define Py_TRASHCAN_END \
+    tstate->c_recursion_remaining++; \
+    if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \
+        _PyTrash_thread_destroy_chain(tstate); \
+    } \
+} while (0);
 
 
 PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj);
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
@@ -56,11 +56,6 @@ typedef struct _stack_chunk {
     PyObject * data[1]; /* Variable sized */
 } _PyStackChunk;
 
-struct _py_trashcan {
-    int delete_nesting;
-    PyObject *delete_later;
-};
-
 struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
 
@@ -152,7 +147,7 @@ struct _ts {
      */
     unsigned long native_thread_id;
 
-    struct _py_trashcan trash;
+    PyObject *delete_later;
 
     /* Tagged pointer to top-most critical section, or zero if there is no
      * active critical section. Critical sections are only used in
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -615,7 +615,6 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op)
     return (PyWeakReference **)((char *)op + offset);
 }
 
-
 // Fast inlined version of PyObject_IS_GC()
 static inline int
 _PyObject_IS_GC(PyObject *obj)
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
@@ -1213,7 +1213,7 @@ def get_bare_quoted_string(value):
     value is the text between the quote marks, with whitespace
     preserved and quoted pairs decoded.
     """
-    if value[0] != '"':
+    if not value or value[0] != '"':
         raise errors.HeaderParseError(
             "expected '\"' but found '{}'".format(value))
     bare_quoted_string = BareQuotedString()
@@ -1454,7 +1454,7 @@ def get_local_part(value):
     """
     local_part = LocalPart()
     leader = None
-    if value[0] in CFWS_LEADER:
+    if value and value[0] in CFWS_LEADER:
         leader, value = get_cfws(value)
     if not value:
         raise errors.HeaderParseError(
@@ -1613,7 +1613,7 @@ def get_domain(value):
     """
     domain = Domain()
     leader = None
-    if value[0] in CFWS_LEADER:
+    if value and value[0] in CFWS_LEADER:
         leader, value = get_cfws(value)
     if not value:
         raise errors.HeaderParseError(
@@ -1689,6 +1689,8 @@ def get_obs_route(value):
         if value[0] in CFWS_LEADER:
             token, value = get_cfws(value)
             obs_route.append(token)
+        if not value:
+            break
         if value[0] == '@':
             obs_route.append(RouteComponentMarker)
             token, value = get_domain(value[1:])
@@ -1707,7 +1709,7 @@ def get_angle_addr(value):
 
     """
     angle_addr = AngleAddr()
-    if value[0] in CFWS_LEADER:
+    if value and value[0] in CFWS_LEADER:
         token, value = get_cfws(value)
         angle_addr.append(token)
     if not value or value[0] != '<':
@@ -1717,7 +1719,7 @@ def get_angle_addr(value):
     value = value[1:]
     # Although it is not legal per RFC5322, SMTP uses '<>' in certain
     # circumstances.
-    if value[0] == '>':
+    if value and value[0] == '>':
         angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
         angle_addr.defects.append(errors.InvalidHeaderDefect(
             "null addr-spec in angle-addr"))
@@ -1769,6 +1771,9 @@ def get_name_addr(value):
     name_addr = NameAddr()
     # Both the optional display name and the angle-addr can start with cfws.
     leader = None
+    if not value:
+        raise errors.HeaderParseError(
+            "expected name-addr but found '{}'".format(value))
     if value[0] in CFWS_LEADER:
         leader, value = get_cfws(value)
         if not value:
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
@@ -1637,11 +1637,11 @@ def bold(self, text):
 
 # --------------------------------------------------------- user interfaces
 
-def pager(text):
+def pager(text, title=''):
     """The first time this is called, determine what kind of pager to use."""
     global pager
     pager = getpager()
-    pager(text)
+    pager(text, title)
 
 def getpager():
     """Decide what method to use for paging through text."""
@@ -1656,24 +1656,24 @@ def getpager():
     use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
     if use_pager:
         if sys.platform == 'win32': # pipes completely broken in Windows
-            return lambda text: tempfilepager(plain(text), use_pager)
+            return lambda text, title='': tempfilepager(plain(text), use_pager)
         elif os.environ.get('TERM') in ('dumb', 'emacs'):
-            return lambda text: pipepager(plain(text), use_pager)
+            return lambda text, title='': pipepager(plain(text), use_pager, title)
         else:
-            return lambda text: pipepager(text, use_pager)
+            return lambda text, title='': pipepager(text, use_pager, title)
     if os.environ.get('TERM') in ('dumb', 'emacs'):
         return plainpager
     if sys.platform == 'win32':
-        return lambda text: tempfilepager(plain(text), 'more <')
+        return lambda text, title='': tempfilepager(plain(text), 'more <')
     if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
-        return lambda text: pipepager(text, 'less')
+        return lambda text, title='': pipepager(text, 'less', title)
 
     import tempfile
     (fd, filename) = tempfile.mkstemp()
     os.close(fd)
     try:
         if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
-            return lambda text: pipepager(text, 'more')
+            return lambda text, title='': pipepager(text, 'more', title)
         else:
             return ttypager
     finally:
@@ -1683,12 +1683,18 @@ def plain(text):
     """Remove boldface formatting from text."""
     return re.sub('.\b', '', text)
 
-def pipepager(text, cmd):
+def escape_less(s):
+    return re.sub(r'([?:.%\\])', r'\\\1', s)
+
+def pipepager(text, cmd, title=''):
     """Page through text by feeding it to another program."""
     import subprocess
     env = os.environ.copy()
+    if title:
+        title += ' '
+    esc_title = escape_less(title)
     prompt_string = (
-        ' '
+        f' {esc_title}' +
         '?ltline %lt?L/%L.'
         ':byte %bB?s/%s.'
         '.'
@@ -1716,7 +1722,7 @@ def pipepager(text, cmd):
             # left running and the terminal is in raw mode and unusable.
             pass
 
-def tempfilepager(text, cmd):
+def tempfilepager(text, cmd, title=''):
     """Page through text by invoking a program on a temporary file."""
     import tempfile
     with tempfile.TemporaryDirectory() as tempdir:
@@ -1733,7 +1739,7 @@ def _escape_stdout(text):
     encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
     return text.encode(encoding, 'backslashreplace').decode(encoding)
 
-def ttypager(text):
+def ttypager(text, title=''):
     """Page through text on a text terminal."""
     lines = plain(_escape_stdout(text)).split('\n')
     try:
@@ -1777,7 +1783,7 @@ def ttypager(text):
         if tty:
             tty.tcsetattr(fd, tty.TCSAFLUSH, old)
 
-def plainpager(text):
+def plainpager(text, title=''):
     """Simply print unformatted text.  This is the ultimate fallback."""
     sys.stdout.write(plain(_escape_stdout(text)))
 
@@ -1879,7 +1885,8 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0,
     """Display text documentation, given an object or a path to an object."""
     if output is None:
         try:
-            pager(render_doc(thing, title, forceload))
+            what = thing if isinstance(thing, str) else type(thing).__name__
+            pager(render_doc(thing, title, forceload), f'Help on {what!s}')
         except ImportError as exc:
             if is_cli:
                 raise
@@ -2253,7 +2260,7 @@ def showtopic(self, topic, more_xrefs=''):
             text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n'
             wrapped_text = textwrap.wrap(text, 72)
             doc += '\n%s\n' % '\n'.join(wrapped_text)
-        pager(doc)
+        pager(doc, f'Help on {topic!s}')
 
     def _gettopic(self, topic, more_xrefs=''):
         """Return unbuffered tuple of (topic, xrefs).
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
@@ -801,6 +801,10 @@ def test_get_quoted_string_header_ends_in_qcontent(self):
         self.assertEqual(qs.content, 'bob')
         self.assertEqual(qs.quoted_value, ' "bob"')
 
+    def test_get_quoted_string_cfws_only_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_quoted_string(' (foo) ')
+
     def test_get_quoted_string_no_quoted_string(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_quoted_string(' (ab) xyz')
@@ -1135,6 +1139,10 @@ def test_get_local_part_complex_obsolete_invalid(self):
             '@python.org')
         self.assertEqual(local_part.local_part, 'Fred.A.Johnson and  dogs')
 
+    def test_get_local_part_empty_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_local_part('')
+
     def test_get_local_part_no_part_raises(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_local_part(' (foo) ')
@@ -1387,6 +1395,10 @@ def test_get_domain_obsolete(self):
                                   '')
         self.assertEqual(domain.domain, 'example.com')
 
+    def test_get_domain_empty_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_domain("")
+
     def test_get_domain_no_non_cfws_raises(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_domain("  (foo)\t")
@@ -1512,6 +1524,10 @@ def test_get_obs_route_no_route_before_end_raises(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_obs_route('(foo) @example.com,')
 
+    def test_get_obs_route_no_route_before_end_raises2(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_obs_route('(foo) @example.com, (foo) ')
+
     def test_get_obs_route_no_route_before_special_raises(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_obs_route('(foo) [abc],')
@@ -1520,6 +1536,14 @@ def test_get_obs_route_no_route_before_special_raises2(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_obs_route('(foo) @example.com [abc],')
 
+    def test_get_obs_route_no_domain_after_at_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_obs_route('@')
+
+    def test_get_obs_route_no_domain_after_at_raises2(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_obs_route('@example.com, @')
+
     # get_angle_addr
 
     def test_get_angle_addr_simple(self):
@@ -1646,6 +1670,14 @@ def test_get_angle_addr_ends_at_special(self):
         self.assertIsNone(angle_addr.route)
         self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
 
+    def test_get_angle_addr_empty_raise(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_angle_addr('')
+
+    def test_get_angle_addr_left_angle_only_raise(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_angle_addr('<')
+
     def test_get_angle_addr_no_angle_raise(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_angle_addr('(foo) ')
@@ -1857,6 +1889,10 @@ def test_get_name_addr_ends_at_special(self):
         self.assertIsNone(name_addr.route)
         self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
 
+    def test_get_name_addr_empty_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_name_addr('')
+
     def test_get_name_addr_no_content_raises(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_name_addr(' (foo) ')
@@ -2732,6 +2768,10 @@ def test_get_msg_id_empty_id_right(self):
         with self.assertRaises(errors.HeaderParseError):
             parser.get_msg_id("<simplelocal@>")
 
+    def test_get_msg_id_no_id_right(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_msg_id("<simplelocal@")
+
     def test_get_msg_id_with_brackets(self):
         # Microsof Outlook generates non-standard one-off addresses:
         # https://learn.microsoft.com/en-us/office/client-developer/outlook/mapi/one-off-addresses
diff --git a/Misc/NEWS.d/next/Library/2020-06-10-19-24-17.bpo-40943.vjiiN_.rst b/Misc/NEWS.d/next/Library/2020-06-10-19-24-17.bpo-40943.vjiiN_.rst
@@ -0,0 +1 @@
+Fix several IndexError when parse emails with truncated Message-ID, address, routes, etc, e.g. ``example@``.
diff --git a/Objects/object.c b/Objects/object.c
diff --git a/Python/pystate.c b/Python/pystate.c

Original file line number	Diff line number	Diff line change
`@@ -615,7 +615,6 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op)`
`615`	`615`	`return (PyWeakReference *)((char )op + offset);`
`616`	`616`	`}`
`617`	`617`
`618`		`-`
`619`	`618`	`// Fast inlined version of PyObject_IS_GC()`
`620`	`619`	`static inline int`
`621`	`620`	`_PyObject_IS_GC(PyObject *obj)`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fix several IndexError when parse emails with truncated Message-ID, address, routes, etc, e.g. ``example@``.