From c952b12673837648478005d5e05ceb014b62cf90 Mon Sep 17 00:00:00 2001 From: Akshat Gupta Date: Thu, 8 May 2025 19:46:50 +0000 Subject: [PATCH 1/8] Fix inconsistent set ordering in annotations --- .github/workflows/mypy.yml | 3 + Doc/deprecations/pending-removal-in-3.14.rst | 2 +- Doc/using/windows.rst | 13 + Lib/test/test_dict.py | 23 +- Lib/test/test_free_threading/test_io.py | 109 +++++++ Lib/test/test_generated_cases.py | 183 ++++++++++++ ...-05-08-13-48-02.gh-issue-132762.tKbygC.rst | 1 + ...-04-16-21-02-57.gh-issue-132551.Psa7pL.rst | 1 + Modules/_io/bytesio.c | 278 ++++++++++++------ Modules/_io/clinic/bytesio.c.h | 66 ++++- Modules/_zstd/_zstdmodule.c | 6 +- Modules/_zstd/_zstdmodule.h | 2 +- Modules/_zstd/compressor.c | 2 +- Modules/_zstd/decompressor.c | 4 +- Modules/_zstd/zstddict.c | 2 +- Objects/dictobject.c | 7 +- Python/optimizer_bytecodes.c | 120 ++++---- Python/optimizer_cases.c.h | 161 +++++----- Tools/build/mypy.ini | 4 + Tools/build/verify_ensurepip_wheels.py | 6 +- Tools/cases_generator/optimizer_generator.py | 44 ++- 21 files changed, 786 insertions(+), 251 deletions(-) create mode 100644 Lib/test/test_free_threading/test_io.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-13-48-02.gh-issue-132762.tKbygC.rst create mode 100644 Misc/NEWS.d/next/Library/2025-04-16-21-02-57.gh-issue-132551.Psa7pL.rst diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 4683848ab5e68d..be63fcb48bb4d1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,7 +13,10 @@ on: - "Lib/test/libregrtest/**" - "Lib/tomllib/**" - "Misc/mypy/**" + - "Tools/build/compute-changes.py" - "Tools/build/generate_sbom.py" + - "Tools/build/verify_ensurepip_wheels.py" + - "Tools/build/update_file.py" - "Tools/cases_generator/**" - "Tools/clinic/**" - "Tools/jit/**" diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 6159fa48848285..9aac10840a663f 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -78,7 +78,7 @@ Pending removal in Python 3.14 :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 74d6db5d7d1a98..c084392ca66d55 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -95,6 +95,9 @@ as ``python3.14.exe``) to be available. The directory will be administrator. Click Start and search for "Edit environment variables for your account" for the system settings page to add the path. +Each Python runtime you install will have its own directory for scripts. These +also need to be added to :envvar:`PATH` if you want to use them. + The Python install manager will be automatically updated to new releases. This does not affect any installs of Python runtimes. Uninstalling the Python install manager does not uninstall any Python runtimes. @@ -713,6 +716,16 @@ default). your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. " + "``pip`` gives me a ""command not found"" error when I type it in my + terminal.","Have you activated a virtual environment? Run the + ``.venv\Scripts\activate`` script in your terminal to activate. + " + "","The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead, or alternatively + the ``python -m pip install --force pip`` command will recreate the + executables and show you the path to add to :envvar:`PATH`. These scripts are + separated for each runtime, and so you may need to add multiple paths. + " .. _windows-embeddable: diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3104cbc66cb115..69f1a098920b94 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -338,17 +338,34 @@ def __setitem__(self, key, value): self.assertRaises(Exc, baddict2.fromkeys, [1]) # test fast path for dictionary inputs + res = dict(zip(range(6), [0]*6)) d = dict(zip(range(6), range(6))) - self.assertEqual(dict.fromkeys(d, 0), dict(zip(range(6), [0]*6))) - + self.assertEqual(dict.fromkeys(d, 0), res) + # test fast path for set inputs + d = set(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + # test slow path for other iterable inputs + d = list(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + + # test fast path when object's constructor returns large non-empty dict class baddict3(dict): def __new__(cls): return d - d = {i : i for i in range(10)} + d = {i : i for i in range(1000)} res = d.copy() res.update(a=None, b=None, c=None) self.assertEqual(baddict3.fromkeys({"a", "b", "c"}), res) + # test slow path when object is a proper subclass of dict + class baddict4(dict): + def __init__(self): + dict.__init__(self, d) + d = {i : i for i in range(1000)} + res = d.copy() + res.update(a=None, b=None, c=None) + self.assertEqual(baddict4.fromkeys({"a", "b", "c"}), res) + def test_copy(self): d = {1: 1, 2: 2, 3: 3} self.assertIsNot(d.copy(), d) diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py new file mode 100644 index 00000000000000..f9bec740ddff50 --- /dev/null +++ b/Lib/test/test_free_threading/test_io.py @@ -0,0 +1,109 @@ +import threading +from unittest import TestCase +from test.support import threading_helper +from random import randint +from io import BytesIO +from sys import getsizeof + + +class TestBytesIO(TestCase): + # Test pretty much everything that can break under free-threading. + # Non-deterministic, but at least one of these things will fail if + # BytesIO object is not free-thread safe. + + def check(self, funcs, *args): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_free_threading(self): + """Test for segfaults and aborts.""" + + def write(barrier, b, *ignore): + barrier.wait() + try: b.write(b'0' * randint(100, 1000)) + except ValueError: pass # ignore write fail to closed file + + def writelines(barrier, b, *ignore): + barrier.wait() + b.write(b'0\n' * randint(100, 1000)) + + def truncate(barrier, b, *ignore): + barrier.wait() + try: b.truncate(0) + except: BufferError # ignore exported buffer + + def read(barrier, b, *ignore): + barrier.wait() + b.read() + + def read1(barrier, b, *ignore): + barrier.wait() + b.read1() + + def readline(barrier, b, *ignore): + barrier.wait() + b.readline() + + def readlines(barrier, b, *ignore): + barrier.wait() + b.readlines() + + def readinto(barrier, b, into, *ignore): + barrier.wait() + b.readinto(into) + + def close(barrier, b, *ignore): + barrier.wait() + b.close() + + def getvalue(barrier, b, *ignore): + barrier.wait() + b.getvalue() + + def getbuffer(barrier, b, *ignore): + barrier.wait() + b.getbuffer() + + def iter(barrier, b, *ignore): + barrier.wait() + list(b) + + def getstate(barrier, b, *ignore): + barrier.wait() + b.__getstate__() + + def setstate(barrier, b, st, *ignore): + barrier.wait() + b.__setstate__(st) + + def sizeof(barrier, b, *ignore): + barrier.wait() + getsizeof(b) + + self.check([write] * 10, BytesIO()) + self.check([writelines] * 10, BytesIO()) + self.check([write] * 10 + [truncate] * 10, BytesIO()) + self.check([truncate] + [read] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [read1] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [readline] * 10, BytesIO(b'0\n'*20480)) + self.check([truncate] + [readlines] * 10, BytesIO(b'0\n'*20480)) + self.check([truncate] + [readinto] * 10, BytesIO(b'0\n'*204800), bytearray(b'0\n'*204800)) + self.check([close] + [write] * 10, BytesIO()) + self.check([truncate] + [getvalue] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [getbuffer] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [iter] * 10, BytesIO(b'0\n'*20480)) + self.check([truncate] + [getstate] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [setstate] * 10, BytesIO(b'0\n'*204800), (b'123', 0, None)) + self.check([truncate] + [sizeof] * 10, BytesIO(b'0\n'*204800)) + + # no tests for seek or tell because they don't break anything diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index d481cb07f757d8..a71ddc01d1c045 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2069,6 +2069,189 @@ def test_missing_override_failure(self): with self.assertRaisesRegex(AssertionError, "All abstract uops"): self.run_cases_test(input, input2, output) + def test_validate_uop_input_length_mismatch(self): + input = """ + op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1, arg2 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Must have the same number of inputs"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_length_mismatch(self): + input = """ + op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1 -- out1, out2)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Must have the same number of outputs"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_name_mismatch(self): + input = """ + op(OP, (foo -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (bar -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal names"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_name_mismatch(self): + input = """ + op(OP, (arg1 -- foo)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1 -- bar)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Outputs must have equal names"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_input(self): + input = """ + op(OP, (unused -- )) { + } + """ + input2 = """ + op(OP, (foo -- )) { + } + """ + output = """ + case OP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + """ + self.run_cases_test(input, input2, output) + + input = """ + op(OP, (foo -- )) { + } + """ + input2 = """ + op(OP, (unused -- )) { + } + """ + output = """ + case OP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_output(self): + input = """ + op(OP, ( -- unused)) { + } + """ + input2 = """ + op(OP, ( -- foo)) { + foo = NULL; + } + """ + output = """ + case OP: { + JitOptSymbol *foo; + foo = NULL; + stack_pointer[0] = foo; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + """ + self.run_cases_test(input, input2, output) + + input = """ + op(OP, ( -- foo)) { + foo = NULL; + } + """ + input2 = """ + op(OP, ( -- unused)) { + } + """ + output = """ + case OP: { + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_size_mismatch(self): + input = """ + op(OP, (arg1[2] -- )) { + } + """ + input2 = """ + op(OP, (arg1[4] -- )) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_size_mismatch(self): + input = """ + op(OP, ( -- out[2])) { + } + """ + input2 = """ + op(OP, ( -- out[4])) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Outputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_size_mismatch(self): + input = """ + op(OP, (foo[2] -- )) { + } + """ + input2 = """ + op(OP, (unused[4] -- )) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal sizes"): + self.run_cases_test(input, input2, output) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-13-48-02.gh-issue-132762.tKbygC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-13-48-02.gh-issue-132762.tKbygC.rst new file mode 100644 index 00000000000000..80b830ebd786f3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-13-48-02.gh-issue-132762.tKbygC.rst @@ -0,0 +1 @@ +:meth:`~dict.fromkeys` no longer loops forever when adding a small set of keys to a large base dict. Patch by Angela Liss. diff --git a/Misc/NEWS.d/next/Library/2025-04-16-21-02-57.gh-issue-132551.Psa7pL.rst b/Misc/NEWS.d/next/Library/2025-04-16-21-02-57.gh-issue-132551.Psa7pL.rst new file mode 100644 index 00000000000000..c8743d49ca01d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-16-21-02-57.gh-issue-132551.Psa7pL.rst @@ -0,0 +1 @@ +Make :class:`io.BytesIO` safe in :term:`free-threaded ` build. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index e45a2d1a16dcba..4cde5f87032a3f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,8 +1,10 @@ #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" -#include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_sysmodule.h" // _PySys_GetSizeOf() -#include // offsetof() +#include // offsetof() #include "_iomodule.h" /*[clinic input] @@ -50,7 +52,7 @@ check_closed(bytesio *self) static int check_exports(bytesio *self) { - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return 1; @@ -68,15 +70,17 @@ check_exports(bytesio *self) return NULL; \ } -#define SHARED_BUF(self) (Py_REFCNT((self)->buf) > 1) +#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) /* Internal routine to get a line from the buffer of a BytesIO object. Returns the length between the current position to the next newline character. */ static Py_ssize_t -scan_eol(bytesio *self, Py_ssize_t len) +scan_eol_lock_held(bytesio *self, Py_ssize_t len) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *start, *n; Py_ssize_t maxlen; @@ -109,11 +113,13 @@ scan_eol(bytesio *self, Py_ssize_t len) The caller should ensure that the 'size' argument is non-negative and not lesser than self->string_size. Returns 0 on success, -1 otherwise. */ static int -unshare_buffer(bytesio *self, size_t size) +unshare_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + PyObject *new_buf; assert(SHARED_BUF(self)); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); assert(size >= (size_t)self->string_size); new_buf = PyBytes_FromStringAndSize(NULL, size); if (new_buf == NULL) @@ -128,10 +134,12 @@ unshare_buffer(bytesio *self, size_t size) The caller should ensure that the 'size' argument is non-negative. Returns 0 on success, -1 otherwise. */ static int -resize_buffer(bytesio *self, size_t size) +resize_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + assert(self->buf != NULL); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ @@ -160,7 +168,7 @@ resize_buffer(bytesio *self, size_t size) } if (SHARED_BUF(self)) { - if (unshare_buffer(self, alloc) < 0) + if (unshare_buffer_lock_held(self, alloc) < 0) return -1; } else { @@ -181,8 +189,10 @@ resize_buffer(bytesio *self, size_t size) Inlining is disabled because it's significantly decreases performance of writelines() in PGO build. */ Py_NO_INLINE static Py_ssize_t -write_bytes(bytesio *self, PyObject *b) +write_bytes_lock_held(bytesio *self, PyObject *b) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + if (check_closed(self)) { return -1; } @@ -202,13 +212,13 @@ write_bytes(bytesio *self, PyObject *b) assert(self->pos >= 0); size_t endpos = (size_t)self->pos + len; if (endpos > (size_t)PyBytes_GET_SIZE(self->buf)) { - if (resize_buffer(self, endpos) < 0) { + if (resize_buffer_lock_held(self, endpos) < 0) { len = -1; goto done; } } else if (SHARED_BUF(self)) { - if (unshare_buffer(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { + if (unshare_buffer_lock_held(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { len = -1; goto done; } @@ -245,13 +255,17 @@ write_bytes(bytesio *self, PyObject *b) static PyObject * bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) { + PyObject *ret; bytesio *self = bytesio_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); if (self->buf == NULL) { - Py_RETURN_TRUE; + ret = Py_True; } else { - Py_RETURN_FALSE; + ret = Py_False; } + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] @@ -311,6 +325,7 @@ _io_BytesIO_flush_impl(bytesio *self) } /*[clinic input] +@critical_section _io.BytesIO.getbuffer cls: defining_class @@ -321,7 +336,7 @@ Get a read-write view over the contents of the BytesIO object. static PyObject * _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) -/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ +/*[clinic end generated code: output=045091d7ce87fe4e input=8295764061be77fd]*/ { _PyIO_State *state = get_io_state_by_cls(cls); PyTypeObject *type = state->PyBytesIOBuffer_Type; @@ -340,6 +355,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _io.BytesIO.getvalue Retrieve the entire contents of the BytesIO object. @@ -347,16 +363,16 @@ Retrieve the entire contents of the BytesIO object. static PyObject * _io_BytesIO_getvalue_impl(bytesio *self) -/*[clinic end generated code: output=b3f6a3233c8fd628 input=4b403ac0af3973ed]*/ +/*[clinic end generated code: output=b3f6a3233c8fd628 input=c91bff398df0c352]*/ { CHECK_CLOSED(self); - if (self->string_size <= 1 || self->exports > 0) + if (self->string_size <= 1 || FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) return PyBytes_FromStringAndSize(PyBytes_AS_STRING(self->buf), self->string_size); if (self->string_size != PyBytes_GET_SIZE(self->buf)) { if (SHARED_BUF(self)) { - if (unshare_buffer(self, self->string_size) < 0) + if (unshare_buffer_lock_held(self, self->string_size) < 0) return NULL; } else { @@ -384,6 +400,7 @@ _io_BytesIO_isatty_impl(bytesio *self) } /*[clinic input] +@critical_section _io.BytesIO.tell Current file position, an integer. @@ -391,22 +408,24 @@ Current file position, an integer. static PyObject * _io_BytesIO_tell_impl(bytesio *self) -/*[clinic end generated code: output=b54b0f93cd0e5e1d input=b106adf099cb3657]*/ +/*[clinic end generated code: output=b54b0f93cd0e5e1d input=2c7b0e8f82e05c4d]*/ { CHECK_CLOSED(self); return PyLong_FromSsize_t(self->pos); } static PyObject * -read_bytes(bytesio *self, Py_ssize_t size) +read_bytes_lock_held(bytesio *self, Py_ssize_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *output; assert(self->buf != NULL); assert(size <= self->string_size); if (size > 1 && self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && - self->exports == 0) { + FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { self->pos += size; return Py_NewRef(self->buf); } @@ -417,6 +436,7 @@ read_bytes(bytesio *self, Py_ssize_t size) } /*[clinic input] +@critical_section _io.BytesIO.read size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -429,7 +449,7 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9cc025f21c75bdd2 input=74344a39f431c3d7]*/ +/*[clinic end generated code: output=9cc025f21c75bdd2 input=9e2f7ff3075fdd39]*/ { Py_ssize_t n; @@ -443,11 +463,12 @@ _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) size = 0; } - return read_bytes(self, size); + return read_bytes_lock_held(self, size); } /*[clinic input] +@critical_section _io.BytesIO.read1 size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -460,12 +481,13 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=440a395bf9129ef5]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=a08fc9e507ab380c]*/ { return _io_BytesIO_read_impl(self, size); } /*[clinic input] +@critical_section _io.BytesIO.readline size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -479,18 +501,19 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_readline_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=4bff3c251df8ffcd input=e7c3fbd1744e2783]*/ +/*[clinic end generated code: output=4bff3c251df8ffcd input=db09d47e23cf2c9e]*/ { Py_ssize_t n; CHECK_CLOSED(self); - n = scan_eol(self, size); + n = scan_eol_lock_held(self, size); - return read_bytes(self, n); + return read_bytes_lock_held(self, n); } /*[clinic input] +@critical_section _io.BytesIO.readlines size as arg: object = None / @@ -504,7 +527,7 @@ total number of bytes in the lines returned. static PyObject * _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) -/*[clinic end generated code: output=09b8e34c880808ff input=691aa1314f2c2a87]*/ +/*[clinic end generated code: output=09b8e34c880808ff input=5c57d7d78e409985]*/ { Py_ssize_t maxsize, size, n; PyObject *result, *line; @@ -533,7 +556,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) return NULL; output = PyBytes_AS_STRING(self->buf) + self->pos; - while ((n = scan_eol(self, -1)) != 0) { + while ((n = scan_eol_lock_held(self, -1)) != 0) { self->pos += n; line = PyBytes_FromStringAndSize(output, n); if (!line) @@ -556,6 +579,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) } /*[clinic input] +@critical_section _io.BytesIO.readinto buffer: Py_buffer(accept={rwbuffer}) / @@ -568,7 +592,7 @@ is set not to block and has no data to read. static PyObject * _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) -/*[clinic end generated code: output=a5d407217dcf0639 input=1424d0fdce857919]*/ +/*[clinic end generated code: output=a5d407217dcf0639 input=093a8d330de3fcd1]*/ { Py_ssize_t len, n; @@ -592,8 +616,9 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) } /*[clinic input] +@critical_section _io.BytesIO.truncate - size: Py_ssize_t(accept={int, NoneType}, c_default="((bytesio *)self)->pos") = None + size: object = None / Truncate the file to at most size bytes. @@ -603,44 +628,68 @@ The current file position is unchanged. Returns the new size. [clinic start generated code]*/ static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9ad17650c15fa09b input=dae4295e11c1bbb4]*/ +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size) +/*[clinic end generated code: output=ab42491b4824f384 input=b4acb5f80481c053]*/ { CHECK_CLOSED(self); CHECK_EXPORTS(self); - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative size value %zd", size); - return NULL; + Py_ssize_t new_size; + + if (size == Py_None) { + new_size = self->pos; + } + else { + new_size = PyLong_AsLong(size); + if (new_size == -1 && PyErr_Occurred()) { + return NULL; + } + if (new_size < 0) { + PyErr_Format(PyExc_ValueError, + "negative size value %zd", new_size); + return NULL; + } } - if (size < self->string_size) { - self->string_size = size; - if (resize_buffer(self, size) < 0) + if (new_size < self->string_size) { + self->string_size = new_size; + if (resize_buffer_lock_held(self, new_size) < 0) return NULL; } - return PyLong_FromSsize_t(size); + return PyLong_FromSsize_t(new_size); } static PyObject * -bytesio_iternext(PyObject *op) +bytesio_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + Py_ssize_t n; bytesio *self = bytesio_CAST(op); CHECK_CLOSED(self); - n = scan_eol(self, -1); + n = scan_eol_lock_held(self, -1); if (n == 0) return NULL; - return read_bytes(self, n); + return read_bytes_lock_held(self, n); +} + +static PyObject * +bytesio_iternext(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] +@critical_section _io.BytesIO.seek pos: Py_ssize_t whence: int = 0 @@ -657,7 +706,7 @@ Returns the new absolute position. static PyObject * _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=c26204a68e9190e4 input=1e875e6ebc652948]*/ +/*[clinic end generated code: output=c26204a68e9190e4 input=20f05ddf659255df]*/ { CHECK_CLOSED(self); @@ -700,6 +749,7 @@ _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) } /*[clinic input] +@critical_section _io.BytesIO.write b: object / @@ -711,13 +761,14 @@ Return the number of bytes written. static PyObject * _io_BytesIO_write_impl(bytesio *self, PyObject *b) -/*[clinic end generated code: output=d3e46bcec8d9e21c input=f5ec7c8c64ed720a]*/ +/*[clinic end generated code: output=d3e46bcec8d9e21c input=46c0c17eac7474a4]*/ { - Py_ssize_t n = write_bytes(self, b); + Py_ssize_t n = write_bytes_lock_held(self, b); return n >= 0 ? PyLong_FromSsize_t(n) : NULL; } /*[clinic input] +@critical_section _io.BytesIO.writelines lines: object / @@ -731,7 +782,7 @@ each element. static PyObject * _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=03a43a75773bc397 input=e972539176fc8fc1]*/ +/*[clinic end generated code: output=03a43a75773bc397 input=5d6a616ae39dc9ca]*/ { PyObject *it, *item; @@ -742,7 +793,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) return NULL; while ((item = PyIter_Next(it)) != NULL) { - Py_ssize_t ret = write_bytes(self, item); + Py_ssize_t ret = write_bytes_lock_held(self, item); Py_DECREF(item); if (ret < 0) { Py_DECREF(it); @@ -759,6 +810,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) } /*[clinic input] +@critical_section _io.BytesIO.close Disable all I/O operations. @@ -766,7 +818,7 @@ Disable all I/O operations. static PyObject * _io_BytesIO_close_impl(bytesio *self) -/*[clinic end generated code: output=1471bb9411af84a0 input=37e1f55556e61f60]*/ +/*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/ { CHECK_EXPORTS(self); Py_CLEAR(self->buf); @@ -788,35 +840,49 @@ _io_BytesIO_close_impl(bytesio *self) function to use the efficient instance representation of PEP 307. */ + static PyObject * + bytesio_getstate_lock_held(PyObject *op) + { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + + bytesio *self = bytesio_CAST(op); + PyObject *initvalue = _io_BytesIO_getvalue_impl(self); + PyObject *dict; + PyObject *state; + + if (initvalue == NULL) + return NULL; + if (self->dict == NULL) { + dict = Py_NewRef(Py_None); + } + else { + dict = PyDict_Copy(self->dict); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + } + + state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); + Py_DECREF(initvalue); + return state; +} + static PyObject * bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) { - bytesio *self = bytesio_CAST(op); - PyObject *initvalue = _io_BytesIO_getvalue_impl(self); - PyObject *dict; - PyObject *state; - - if (initvalue == NULL) - return NULL; - if (self->dict == NULL) { - dict = Py_NewRef(Py_None); - } - else { - dict = PyDict_Copy(self->dict); - if (dict == NULL) { - Py_DECREF(initvalue); - return NULL; - } - } - - state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); - Py_DECREF(initvalue); - return state; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_getstate_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * -bytesio_setstate(PyObject *op, PyObject *state) +bytesio_setstate_lock_held(PyObject *op, PyObject *state) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + PyObject *result; PyObject *position_obj; PyObject *dict; @@ -890,13 +956,23 @@ bytesio_setstate(PyObject *op, PyObject *state) Py_RETURN_NONE; } +static PyObject * +bytesio_setstate(PyObject *op, PyObject *state) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_setstate_lock_held(op, state); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesio_dealloc(PyObject *op) { bytesio *self = bytesio_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_SystemError, "deallocated BytesIO object has exported buffers"); PyErr_Print(); @@ -932,6 +1008,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@critical_section _io.BytesIO.__init__ initial_bytes as initvalue: object(c_default="NULL") = b'' @@ -940,13 +1017,13 @@ Buffered I/O implementation using an in-memory bytes buffer. static int _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) -/*[clinic end generated code: output=65c0c51e24c5b621 input=aac7f31b67bf0fb6]*/ +/*[clinic end generated code: output=65c0c51e24c5b621 input=3da5a74ee4c4f1ac]*/ { /* In case, __init__ is called multiple times. */ self->string_size = 0; self->pos = 0; - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return -1; @@ -970,8 +1047,10 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) } static PyObject * -bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +bytesio_sizeof_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + bytesio *self = bytesio_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { @@ -984,6 +1063,16 @@ bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) return PyLong_FromSize_t(res); } +static PyObject * +bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_sizeof_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int bytesio_traverse(PyObject *op, visitproc visit, void *arg) { @@ -999,7 +1088,7 @@ bytesio_clear(PyObject *op) { bytesio *self = bytesio_CAST(op); Py_CLEAR(self->dict); - if (self->exports == 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { Py_CLEAR(self->buf); } return 0; @@ -1077,18 +1166,15 @@ PyType_Spec bytesio_spec = { */ static int -bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - if (view == NULL) { - PyErr_SetString(PyExc_BufferError, - "bytesiobuf_getbuffer: view==NULL argument is obsolete"); - return -1; - } - if (b->exports == 0 && SHARED_BUF(b)) { - if (unshare_buffer(b, b->string_size) < 0) + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b); + + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) { + if (unshare_buffer_lock_held(b, b->string_size) < 0) return -1; } @@ -1096,16 +1182,32 @@ bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) (void)PyBuffer_FillInfo(view, op, PyBytes_AS_STRING(b->buf), b->string_size, 0, flags); - b->exports++; + FT_ATOMIC_ADD_SSIZE(b->exports, 1); return 0; } +static int +bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + if (view == NULL) { + PyErr_SetString(PyExc_BufferError, + "bytesiobuf_getbuffer: view==NULL argument is obsolete"); + return -1; + } + + int ret; + Py_BEGIN_CRITICAL_SECTION(bytesiobuf_CAST(op)->source); + ret = bytesiobuf_getbuffer_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - b->exports--; + FT_ATOMIC_ADD_SSIZE(b->exports, -1); } static int diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index aaf4884d1732a9..8553ed05f70d1b 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_io_BytesIO_readable__doc__, @@ -96,11 +97,18 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); - return NULL; + goto exit; } - return _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -118,7 +126,13 @@ _io_BytesIO_getvalue_impl(bytesio *self); static PyObject * _io_BytesIO_getvalue(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_getvalue_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getvalue_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_isatty__doc__, @@ -156,7 +170,13 @@ _io_BytesIO_tell_impl(bytesio *self); static PyObject * _io_BytesIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_tell_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_tell_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_read__doc__, @@ -190,7 +210,9 @@ _io_BytesIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -227,7 +249,9 @@ _io_BytesIO_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read1_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +289,9 @@ _io_BytesIO_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readline_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -301,7 +327,9 @@ _io_BytesIO_readlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } arg = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readlines_impl((bytesio *)self, arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,7 +360,9 @@ _io_BytesIO_readinto(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readinto_impl((bytesio *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -356,13 +386,13 @@ PyDoc_STRVAR(_io_BytesIO_truncate__doc__, {"truncate", _PyCFunction_CAST(_io_BytesIO_truncate), METH_FASTCALL, _io_BytesIO_truncate__doc__}, static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size); +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size); static PyObject * _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = ((bytesio *)self)->pos; + PyObject *size = Py_None; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -370,11 +400,11 @@ _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { - goto exit; - } + size = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_truncate_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -428,7 +458,9 @@ _io_BytesIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_seek_impl((bytesio *)self, pos, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -453,7 +485,9 @@ _io_BytesIO_write(PyObject *self, PyObject *b) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_write_impl((bytesio *)self, b); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -479,7 +513,9 @@ _io_BytesIO_writelines(PyObject *self, PyObject *lines) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_writelines_impl((bytesio *)self, lines); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -499,7 +535,13 @@ _io_BytesIO_close_impl(bytesio *self); static PyObject * _io_BytesIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_close_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_close_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO___init____doc__, @@ -558,9 +600,11 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } initvalue = fastargs[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO___init___impl((bytesio *)self, initvalue); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=6dbfd82f4e9d4ef3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=580205daa01def2e input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 4d046859a1540e..4b14315462b1c5 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -826,7 +826,7 @@ static int _zstd_exec(PyObject *module) { // ZstdDecompressor if (add_type_to_module(module, "ZstdDecompressor", - &ZstdDecompressor_type_spec, + &zstddecompressor_type_spec, &mod_state->ZstdDecompressor_type) < 0) { return -1; } @@ -890,9 +890,9 @@ _zstd_free(void *module) static struct PyModuleDef_Slot _zstd_slots[] = { {Py_mod_exec, _zstd_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0} + {0, NULL}, }; struct PyModuleDef _zstdmodule = { diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 1e9e4c75056831..120fe9f84c754d 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -32,7 +32,7 @@ get_zstd_state_from_type(PyTypeObject *type) { extern PyType_Spec zstddict_type_spec; extern PyType_Spec zstdcompressor_type_spec; -extern PyType_Spec ZstdDecompressor_type_spec; +extern PyType_Spec zstddecompressor_type_spec; struct _zstd_state { PyObject *empty_bytes; diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 9a3d8dedcf235e..fc1d3b9d210639 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -702,6 +702,6 @@ static PyType_Slot zstdcompressor_slots[] = { PyType_Spec zstdcompressor_type_spec = { .name = "_zstd.ZstdCompressor", .basicsize = sizeof(ZstdCompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .slots = zstdcompressor_slots, }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 748947af79c3f7..4ac28d9c987d82 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -883,9 +883,9 @@ static PyType_Slot ZstdDecompressor_slots[] = { {0} }; -PyType_Spec ZstdDecompressor_type_spec = { +PyType_Spec zstddecompressor_type_spec = { .name = "_zstd.ZstdDecompressor", .basicsize = sizeof(ZstdDecompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .slots = ZstdDecompressor_slots, }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index a19224c4a6403b..53c96b104104e0 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -281,6 +281,6 @@ static PyType_Slot zstddict_slots[] = { PyType_Spec zstddict_type_spec = { .name = "_zstd.ZstdDict", .basicsize = sizeof(ZstdDict), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .slots = zstddict_slots, }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 59b0cf1ce7d422..32356f0634db15 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3178,9 +3178,10 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, Py_ssize_t pos = 0; PyObject *key; Py_hash_t hash; - - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PySet_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(interp, mp, new_size, 0)) { Py_DECREF(mp); return NULL; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e99421a3aff4ba..238c370f475c28 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -104,18 +104,18 @@ dummy_func(void) { res = sym_new_null(ctx); } - op(_GUARD_TOS_INT, (tos -- tos)) { - if (sym_matches_type(tos, &PyLong_Type)) { + op(_GUARD_TOS_INT, (value -- value)) { + if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyLong_Type); + sym_set_type(value, &PyLong_Type); } - op(_GUARD_NOS_INT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyLong_Type)) { + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyLong_Type); + sym_set_type(left, &PyLong_Type); } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { @@ -141,25 +141,25 @@ dummy_func(void) { } } - op(_GUARD_TOS_FLOAT, (tos -- tos)) { - if (sym_matches_type(tos, &PyFloat_Type)) { + op(_GUARD_TOS_FLOAT, (value -- value)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); } - op(_GUARD_NOS_FLOAT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyFloat_Type)) { + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); } - op(_BINARY_OP, (left, right -- res)) { - bool lhs_int = sym_matches_type(left, &PyLong_Type); - bool rhs_int = sym_matches_type(right, &PyLong_Type); - bool lhs_float = sym_matches_type(left, &PyFloat_Type); - bool rhs_float = sym_matches_type(right, &PyFloat_Type); + op(_BINARY_OP, (lhs, rhs -- res)) { + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: res = sym_new_unknown(ctx); @@ -185,11 +185,11 @@ dummy_func(void) { // Case C: res = sym_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, right)) { + else if (!sym_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { // Case B: res = sym_new_type(ctx, &PyFloat_Type); } @@ -366,7 +366,7 @@ dummy_func(void) { ctx->done = true; } - op(_BINARY_OP_SUBSCR_STR_INT, (left, right -- res)) { + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } @@ -398,11 +398,11 @@ dummy_func(void) { } } - op(_TO_BOOL_BOOL, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res); + op(_TO_BOOL_BOOL, (value -- value)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, true); + value = sym_new_truthiness(ctx, value, true); } } @@ -493,20 +493,20 @@ dummy_func(void) { res = sym_new_type(ctx, &PyBool_Type); } - op(_IS_OP, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_IS_OP, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP_SET, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP_SET, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP_DICT, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP_DICT, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { @@ -707,10 +707,10 @@ dummy_func(void) { } } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -730,14 +730,14 @@ dummy_func(void) { ctx->done = true; } - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { init_frame = NULL; ctx->done = true; } @@ -789,21 +789,23 @@ dummy_func(void) { } } - op(_YIELD_VALUE, (unused -- res)) { - res = sym_new_unknown(ctx); + op(_YIELD_VALUE, (unused -- value)) { + value = sym_new_unknown(ctx); } - op(_FOR_ITER_GEN_FRAME, ( -- )) { + op(_FOR_ITER_GEN_FRAME, (unused -- unused, gen_frame: _Py_UOpsAbstractFrame*)) { + gen_frame = NULL; /* We are about to hit the end of the trace */ ctx->done = true; } - op(_SEND_GEN_FRAME, ( -- )) { + op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame: _Py_UOpsAbstractFrame *)) { + gen_frame = NULL; // We are about to hit the end of the trace: ctx->done = true; } - op(_CHECK_STACK_SPACE, ( --)) { + op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; } @@ -848,14 +850,16 @@ dummy_func(void) { corresponding_check_stack = NULL; } - op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { + (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } } - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) { + (void)top; /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { @@ -904,27 +908,27 @@ dummy_func(void) { sym_set_const(flag, Py_False); } - op(_GUARD_IS_NONE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + op(_GUARD_IS_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(flag, Py_None); + sym_set_const(val, Py_None); } - op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + op(_GUARD_IS_NOT_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -969,7 +973,7 @@ dummy_func(void) { list = sym_new_type(ctx, &PyList_Type); } - op(_BUILD_SLICE, (values[oparg] -- slice)) { + op(_BUILD_SLICE, (args[oparg] -- slice)) { slice = sym_new_type(ctx, &PySlice_Type); } @@ -977,7 +981,7 @@ dummy_func(void) { map = sym_new_type(ctx, &PyDict_Type); } - op(_BUILD_STRING, (values[oparg] -- str)) { + op(_BUILD_STRING, (pieces[oparg] -- str)) { str = sym_new_type(ctx, &PyUnicode_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 56b4b9983fbaaa..4c49d106fa8dee 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -171,14 +171,13 @@ case _TO_BOOL_BOOL: { JitOptSymbol *value; - JitOptSymbol *res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res); + int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, true); + value = sym_new_truthiness(ctx, value, true); } - stack_pointer[-1] = res; + stack_pointer[-1] = value; break; } @@ -292,22 +291,22 @@ } case _GUARD_NOS_INT: { - JitOptSymbol *nos; - nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyLong_Type)) { + JitOptSymbol *left; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyLong_Type); + sym_set_type(left, &PyLong_Type); break; } case _GUARD_TOS_INT: { - JitOptSymbol *tos; - tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyLong_Type)) { + JitOptSymbol *value; + value = stack_pointer[-1]; + if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyLong_Type); + sym_set_type(value, &PyLong_Type); break; } @@ -396,22 +395,22 @@ } case _GUARD_NOS_FLOAT: { - JitOptSymbol *nos; - nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyFloat_Type)) { + JitOptSymbol *left; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { - JitOptSymbol *tos; - tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyFloat_Type)) { + JitOptSymbol *value; + value = stack_pointer[-1]; + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); break; } @@ -811,14 +810,17 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { + _Py_UOpsAbstractFrame *gen_frame; + gen_frame = NULL; ctx->done = true; + stack_pointer[-1] = (JitOptSymbol *)gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *res; - res = sym_new_unknown(ctx); - stack_pointer[-1] = res; + JitOptSymbol *value; + value = sym_new_unknown(ctx); + stack_pointer[-1] = value; break; } @@ -858,7 +860,10 @@ case _UNPACK_SEQUENCE: { JitOptSymbol **values; + JitOptSymbol **top; values = &stack_pointer[-1]; + top = &stack_pointer[-1 + oparg]; + (void)top; for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } @@ -907,7 +912,10 @@ case _UNPACK_EX: { JitOptSymbol **values; + JitOptSymbol **top; values = &stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + (void)top; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { values[i] = sym_new_unknown(ctx); @@ -1376,18 +1384,18 @@ } case _IS_OP: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptSymbol *b; + b = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _CONTAINS_OP: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptSymbol *b; + b = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1405,18 +1413,18 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptSymbol *b; + b = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _CONTAINS_OP_DICT: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptSymbol *b; + b = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1600,7 +1608,12 @@ } case _FOR_ITER_GEN_FRAME: { + _Py_UOpsAbstractFrame *gen_frame; + gen_frame = NULL; ctx->done = true; + stack_pointer[0] = (JitOptSymbol *)gen_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1721,15 +1734,16 @@ case _MAYBE_EXPAND_METHOD: { JitOptSymbol **args; - JitOptSymbol *func; - JitOptSymbol *maybe_self; - args = &stack_pointer[-oparg]; + JitOptSymbol *self_or_null; + JitOptSymbol *callable; args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; (void)args; - func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; break; } @@ -2001,17 +2015,18 @@ case _CHECK_AND_ALLOCATE_OBJECT: { JitOptSymbol **args; - JitOptSymbol *self; - JitOptSymbol *init; - args = &stack_pointer[-oparg]; + JitOptSymbol *self_or_null; + JitOptSymbol *callable; args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)args; - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = self; - stack_pointer[-1 - oparg] = init; + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; break; } @@ -2270,15 +2285,15 @@ } case _BINARY_OP: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - bool lhs_int = sym_matches_type(left, &PyLong_Type); - bool rhs_int = sym_matches_type(right, &PyLong_Type); - bool lhs_float = sym_matches_type(left, &PyFloat_Type); - bool rhs_float = sym_matches_type(right, &PyFloat_Type); + JitOptSymbol *rhs; + JitOptSymbol *lhs; + JitOptSymbol *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { res = sym_new_unknown(ctx); } @@ -2289,10 +2304,10 @@ else if (lhs_float) { res = sym_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, right)) { + else if (!sym_is_const(ctx, rhs)) { res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { res = sym_new_type(ctx, &PyFloat_Type); } else { @@ -2375,33 +2390,33 @@ } case _GUARD_IS_NONE_POP: { - JitOptSymbol *flag; - flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + JitOptSymbol *val; + val = stack_pointer[-1]; + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(flag, Py_None); + sym_set_const(val, Py_None); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *flag; - flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + JitOptSymbol *val; + val = stack_pointer[-1]; + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } stack_pointer += -1; diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index db546c6fb3481c..fab35bf68904af 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -1,7 +1,11 @@ [mypy] + +# Please, when adding new files here, also add them to: +# .github/workflows/mypy.yml files = Tools/build/compute-changes.py, Tools/build/generate_sbom.py, + Tools/build/verify_ensurepip_wheels.py, Tools/build/update_file.py pretty = True diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index a37da2f70757e5..46c42916d9354d 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -20,13 +20,13 @@ GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" -def print_notice(file_path: str, message: str) -> None: +def print_notice(file_path: str | Path, message: str) -> None: if GITHUB_ACTIONS: message = f"::notice file={file_path}::{message}" print(message, end="\n\n") -def print_error(file_path: str, message: str) -> None: +def print_error(file_path: str | Path, message: str) -> None: if GITHUB_ACTIONS: message = f"::error file={file_path}::{message}" print(message, end="\n\n") @@ -67,6 +67,7 @@ def verify_wheel(package_name: str) -> bool: return False release_files = json.loads(raw_text)["releases"][package_version] + expected_digest = "" for release_info in release_files: if package_path.name != release_info["filename"]: continue @@ -95,6 +96,7 @@ def verify_wheel(package_name: str) -> bool: return True + if __name__ == "__main__": exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 7a32275347e896..fda022a44e59cc 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -30,16 +30,52 @@ def validate_uop(override: Uop, uop: Uop) -> None: - # To do - pass + """ + Check that the overridden uop (defined in 'optimizer_bytecodes.c') + has the same stack effects as the original uop (defined in 'bytecodes.c'). + + Ensure that: + - The number of inputs and outputs is the same. + - The names of the inputs and outputs are the same + (except for 'unused' which is ignored). + - The sizes of the inputs and outputs are the same. + """ + for stack_effect in ('inputs', 'outputs'): + orig_effects = getattr(uop.stack, stack_effect) + new_effects = getattr(override.stack, stack_effect) + + if len(orig_effects) != len(new_effects): + msg = ( + f"{uop.name}: Must have the same number of {stack_effect} " + "in bytecodes.c and optimizer_bytecodes.c " + f"({len(orig_effects)} != {len(new_effects)})" + ) + raise analysis_error(msg, override.body.open) + + for orig, new in zip(orig_effects, new_effects, strict=True): + if orig.name != new.name and orig.name != "unused" and new.name != "unused": + msg = ( + f"{uop.name}: {stack_effect.capitalize()} must have " + "equal names in bytecodes.c and optimizer_bytecodes.c " + f"({orig.name} != {new.name})" + ) + raise analysis_error(msg, override.body.open) + + if orig.size != new.size: + msg = ( + f"{uop.name}: {stack_effect.capitalize()} must have " + "equal sizes in bytecodes.c and optimizer_bytecodes.c " + f"({orig.size!r} != {new.size!r})" + ) + raise analysis_error(msg, override.body.open) def type_name(var: StackItem) -> str: if var.is_array(): - return f"JitOptSymbol **" + return "JitOptSymbol **" if var.type: return var.type - return f"JitOptSymbol *" + return "JitOptSymbol *" def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: From c0602f3a5cea56c57637c8fcd7e4b9a36d574ca7 Mon Sep 17 00:00:00 2001 From: Akshat Gupta Date: Thu, 8 May 2025 19:52:54 +0000 Subject: [PATCH 2/8] Fix inconsistent set ordering in annotations --- Lib/annotationlib.py | 42 ++++++++++++++++++++++++++-------- Lib/test/test_annotationlib.py | 11 +++++++++ 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 32b8553458930c..a900a9d4915fb3 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -371,11 +371,24 @@ def __convert_to_ast(self, other): elif type(other) in (list, tuple, set): extra_names = {} elts = [] - for elt in other: - new_elt, new_extra_names = self.__convert_to_ast(elt) - if new_extra_names is not None: - extra_names.update(new_extra_names) - elts.append(new_elt) + + # For sets of types, sort elements by name for consistent ordering. + if type(other) is set and all(isinstance(x, type) for x in other): + # Sort the elements by name to ensure deterministic output. + sorted_elts = sorted(other, key=lambda x: x.__name__) + for elt in sorted_elts: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + else: + # For lists, tuples, and other sets, preserve the original order. + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] return ast_class(elts), extra_names else: @@ -817,6 +830,10 @@ def _stringify_single(anno): return anno elif isinstance(anno, _Template): return ast.unparse(_template_to_ast(anno)) + elif isinstance(anno, set) and all(isinstance(x, type) for x in anno): + # Sort set elements by name to ensure consistent ordering. + sorted_elements = sorted(anno, key=lambda x: x.__name__) + return "{" + ", ".join(x.__name__ for x in sorted_elements) + "}" else: return repr(anno) @@ -1022,10 +1039,17 @@ def annotations_to_string(annotations): Always returns a fresh a dictionary. """ - return { - n: t if isinstance(t, str) else type_repr(t) - for n, t in annotations.items() - } + result = {} + for n, t in annotations.items(): + if isinstance(t, str): + result[n] = t + elif isinstance(t, set) and all(isinstance(x, type) for x in t): + # Sort set elements by name to ensure consistent ordering. + sorted_elements = sorted(t, key=lambda x: x.__name__) + result[n] = "{" + ", ".join(x.__name__ for x in sorted_elements) + "}" + else: + result[n] = type_repr(t) + return result def _get_and_call_annotate(obj, format): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 4af97c82de9d46..76343818b216d1 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -394,6 +394,17 @@ def g( }, ) + def test_set_ordering_consistency(self): + """Test that sets of types are consistently ordered in string representations.""" + # Test with direct set annotations + def g1(x: {int, str}): + pass + + anno_g1 = get_annotations(g1, format=Format.STRING) + + # The set should have elements sorted by name (int before str) + self.assertEqual(anno_g1["x"], "{int, str}") + def test_nested_expressions(self): def f( nested: list[Annotated[set[int], "set of ints", 4j]], From 99e948479050dbcd8954b483d82fcadccc22a4ab Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 20:07:26 +0000 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst b/Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst new file mode 100644 index 00000000000000..6517287703acd7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst @@ -0,0 +1 @@ +Fix inconsistent set ordering in annotations. Sets of types are now consistently ordered by name in string representations, making the output deterministic and easier to test. From ddd6642ab968ad4ad3596149c2833fe6ed5fdf1e Mon Sep 17 00:00:00 2001 From: Akshat Gupta Date: Fri, 9 May 2025 03:54:39 +0000 Subject: [PATCH 4/8] Revert --- Lib/annotationlib.py | 42 ++++++++-------------------------- Lib/test/test_annotationlib.py | 11 --------- 2 files changed, 9 insertions(+), 44 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index a900a9d4915fb3..32b8553458930c 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -371,24 +371,11 @@ def __convert_to_ast(self, other): elif type(other) in (list, tuple, set): extra_names = {} elts = [] - - # For sets of types, sort elements by name for consistent ordering. - if type(other) is set and all(isinstance(x, type) for x in other): - # Sort the elements by name to ensure deterministic output. - sorted_elts = sorted(other, key=lambda x: x.__name__) - for elt in sorted_elts: - new_elt, new_extra_names = self.__convert_to_ast(elt) - if new_extra_names is not None: - extra_names.update(new_extra_names) - elts.append(new_elt) - else: - # For lists, tuples, and other sets, preserve the original order. - for elt in other: - new_elt, new_extra_names = self.__convert_to_ast(elt) - if new_extra_names is not None: - extra_names.update(new_extra_names) - elts.append(new_elt) - + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] return ast_class(elts), extra_names else: @@ -830,10 +817,6 @@ def _stringify_single(anno): return anno elif isinstance(anno, _Template): return ast.unparse(_template_to_ast(anno)) - elif isinstance(anno, set) and all(isinstance(x, type) for x in anno): - # Sort set elements by name to ensure consistent ordering. - sorted_elements = sorted(anno, key=lambda x: x.__name__) - return "{" + ", ".join(x.__name__ for x in sorted_elements) + "}" else: return repr(anno) @@ -1039,17 +1022,10 @@ def annotations_to_string(annotations): Always returns a fresh a dictionary. """ - result = {} - for n, t in annotations.items(): - if isinstance(t, str): - result[n] = t - elif isinstance(t, set) and all(isinstance(x, type) for x in t): - # Sort set elements by name to ensure consistent ordering. - sorted_elements = sorted(t, key=lambda x: x.__name__) - result[n] = "{" + ", ".join(x.__name__ for x in sorted_elements) + "}" - else: - result[n] = type_repr(t) - return result + return { + n: t if isinstance(t, str) else type_repr(t) + for n, t in annotations.items() + } def _get_and_call_annotate(obj, format): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 76343818b216d1..4af97c82de9d46 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -394,17 +394,6 @@ def g( }, ) - def test_set_ordering_consistency(self): - """Test that sets of types are consistently ordered in string representations.""" - # Test with direct set annotations - def g1(x: {int, str}): - pass - - anno_g1 = get_annotations(g1, format=Format.STRING) - - # The set should have elements sorted by name (int before str) - self.assertEqual(anno_g1["x"], "{int, str}") - def test_nested_expressions(self): def f( nested: list[Annotated[set[int], "set of ints", 4j]], From d03fe55667cdc99ab1beec204e0be7c62ac775cb Mon Sep 17 00:00:00 2001 From: Akshat Gupta Date: Fri, 9 May 2025 03:55:51 +0000 Subject: [PATCH 5/8] Revert --- .../next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst b/Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst deleted file mode 100644 index 6517287703acd7..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-08-20-07-14.gh-issue-133682.B0F3Bx.rst +++ /dev/null @@ -1 +0,0 @@ -Fix inconsistent set ordering in annotations. Sets of types are now consistently ordered by name in string representations, making the output deterministic and easier to test. From fa3ea5e95146345e14ae823a160d05330867bd00 Mon Sep 17 00:00:00 2001 From: Akshat Gupta Date: Fri, 9 May 2025 04:08:53 +0000 Subject: [PATCH 6/8] Fixing test case --- Lib/test/test_annotationlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 4af97c82de9d46..73a821d15e3481 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -378,7 +378,7 @@ def f(x: a[[int, str], float]): def g( w: a[[int, str], float], - x: a[{int, str}, 3], + x: a[{int}, 3], y: a[{int: str}, 4], z: a[(int, str), 5], ): @@ -388,7 +388,7 @@ def g( anno, { "w": "a[[int, str], float]", - "x": "a[{int, str}, 3]", + "x": "a[{int}, 3]", "y": "a[{int: str}, 4]", "z": "a[(int, str), 5]", }, From e3ed191aaa0638f9c581ae976e654dc1dbac6cf3 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 04:11:07 +0000 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst diff --git a/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst b/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst new file mode 100644 index 00000000000000..a61511df0fce26 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst @@ -0,0 +1 @@ +Fixed test case `test.test_annotationlib.TestStringFormat.test_displays` which ensures proper handling of complex data structures (lists, sets, dictionaries, and tuples) in string annotations. From 6629dc49c678115b2bf3685d43d1c5262d028ac3 Mon Sep 17 00:00:00 2001 From: Akshat Gupta Date: Fri, 9 May 2025 09:45:29 +0530 Subject: [PATCH 8/8] Update 2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst --- .../next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst b/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst index a61511df0fce26..ebd17f73ca57f1 100644 --- a/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst +++ b/Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst @@ -1 +1 @@ -Fixed test case `test.test_annotationlib.TestStringFormat.test_displays` which ensures proper handling of complex data structures (lists, sets, dictionaries, and tuples) in string annotations. +Fixed test case ``test.test_annotationlib.TestStringFormat.test_displays`` which ensures proper handling of complex data structures (lists, sets, dictionaries, and tuples) in string annotations.