From bd780908cec25e68914d4d75446b669812b2146a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Thu, 12 Oct 2023 12:15:40 +0200 Subject: [PATCH] MAINT: Backport numpy._core stubs. Remove NumpyUnpickler --- .../upcoming_changes/24870.new_feature.rst | 6 --- .../upcoming_changes/24906.new_feature.rst | 6 +++ doc/source/reference/routines.io.rst | 1 - doc/source/user/how-to-io.rst | 6 +-- numpy/_core/__init__.py | 4 ++ numpy/_core/__init__.pyi | 0 numpy/_core/_dtype.py | 6 +++ numpy/_core/_dtype_ctypes.py | 6 +++ numpy/_core/_internal.py | 6 +++ numpy/_core/_multiarray_umath.py | 6 +++ numpy/_core/multiarray.py | 6 +++ numpy/_core/umath.py | 6 +++ numpy/core/tests/data/numpy_2_0_array.pkl | Bin 0 -> 718 bytes numpy/core/tests/test_numpy_2_0_compat.py | 48 ++++++++++++++++++ numpy/lib/format.py | 16 +----- numpy/lib/format.pyi | 3 -- numpy/lib/npyio.py | 2 +- numpy/meson.build | 1 + numpy/setup.py | 1 + 19 files changed, 101 insertions(+), 29 deletions(-) delete mode 100644 doc/release/upcoming_changes/24870.new_feature.rst create mode 100644 doc/release/upcoming_changes/24906.new_feature.rst create mode 100644 numpy/_core/__init__.py create mode 100644 numpy/_core/__init__.pyi create mode 100644 numpy/_core/_dtype.py create mode 100644 numpy/_core/_dtype_ctypes.py create mode 100644 numpy/_core/_internal.py create mode 100644 numpy/_core/_multiarray_umath.py create mode 100644 numpy/_core/multiarray.py create mode 100644 numpy/_core/umath.py create mode 100644 numpy/core/tests/data/numpy_2_0_array.pkl create mode 100644 numpy/core/tests/test_numpy_2_0_compat.py diff --git a/doc/release/upcoming_changes/24870.new_feature.rst b/doc/release/upcoming_changes/24870.new_feature.rst deleted file mode 100644 index 82d4015ff05f..000000000000 --- a/doc/release/upcoming_changes/24870.new_feature.rst +++ /dev/null @@ -1,6 +0,0 @@ -`numpy.lib.format.NumpyUnpickler` ---------------------------------- - -`numpy.lib.format.NumpyUnpickler` class was added -that provides a stable way for loading pickled arrays, -created with NumPy 2.0, with Numpy 1.26. diff --git a/doc/release/upcoming_changes/24906.new_feature.rst b/doc/release/upcoming_changes/24906.new_feature.rst new file mode 100644 index 000000000000..44c73a7b9490 --- /dev/null +++ b/doc/release/upcoming_changes/24906.new_feature.rst @@ -0,0 +1,6 @@ +`numpy._core` submodules' stubs +------------------------------- + +`numpy._core` submodules' stubs were added +to provide a stable way for loading pickled arrays, +created with NumPy 2.0, with Numpy 1.26. diff --git a/doc/source/reference/routines.io.rst b/doc/source/reference/routines.io.rst index 93a2da9495d0..1ec2ccb5eea4 100644 --- a/doc/source/reference/routines.io.rst +++ b/doc/source/reference/routines.io.rst @@ -14,7 +14,6 @@ NumPy binary files (NPY, NPZ) save savez savez_compressed - lib.format.NumpyUnpickler The format of these binary file types is documented in :py:mod:`numpy.lib.format` diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index 55070ff5e727..8d8bdd7d4f5b 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -319,10 +319,8 @@ Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, unless the array dtype includes Python objects, in which case pickling is required. -:func:`numpy.load` also supports unpickling files created with NumPy 2.0. -If you try to unpickle a 2.0 pickled array directly, you will get -an exception. Use :class:`numpy.lib.format.NumpyUnpickler` for -unpickling these files. +NumPy 1.26 also supports unpickling files created with NumPy 2.0, either +via :func:`numpy.load` or the pickle module directly. Convert from a pandas DataFrame to a NumPy array ================================================ diff --git a/numpy/_core/__init__.py b/numpy/_core/__init__.py new file mode 100644 index 000000000000..a2f096f3f174 --- /dev/null +++ b/numpy/_core/__init__.py @@ -0,0 +1,4 @@ +""" +This private module only contains stubs for interoperability with +NumPy 2.0 pickled arrays. It may not be used by the end user. +""" diff --git a/numpy/_core/__init__.pyi b/numpy/_core/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/numpy/_core/_dtype.py b/numpy/_core/_dtype.py new file mode 100644 index 000000000000..974d93d98cbb --- /dev/null +++ b/numpy/_core/_dtype.py @@ -0,0 +1,6 @@ +from numpy.core import _dtype + +_globals = globals() + +for item in _dtype.__dir__(): + _globals[item] = getattr(_dtype, item) diff --git a/numpy/_core/_dtype_ctypes.py b/numpy/_core/_dtype_ctypes.py new file mode 100644 index 000000000000..bfa16aabf423 --- /dev/null +++ b/numpy/_core/_dtype_ctypes.py @@ -0,0 +1,6 @@ +from numpy.core import _dtype_ctypes + +_globals = globals() + +for item in _dtype_ctypes.__dir__(): + _globals[item] = getattr(_dtype_ctypes, item) diff --git a/numpy/_core/_internal.py b/numpy/_core/_internal.py new file mode 100644 index 000000000000..52a8e907292e --- /dev/null +++ b/numpy/_core/_internal.py @@ -0,0 +1,6 @@ +from numpy.core import _internal + +_globals = globals() + +for item in _internal.__dir__(): + _globals[item] = getattr(_internal, item) diff --git a/numpy/_core/_multiarray_umath.py b/numpy/_core/_multiarray_umath.py new file mode 100644 index 000000000000..7ce48fcb258d --- /dev/null +++ b/numpy/_core/_multiarray_umath.py @@ -0,0 +1,6 @@ +from numpy.core import _multiarray_umath + +_globals = globals() + +for item in _multiarray_umath.__dir__(): + _globals[item] = getattr(_multiarray_umath, item) diff --git a/numpy/_core/multiarray.py b/numpy/_core/multiarray.py new file mode 100644 index 000000000000..6c37d1da9fe7 --- /dev/null +++ b/numpy/_core/multiarray.py @@ -0,0 +1,6 @@ +from numpy.core import multiarray + +_globals = globals() + +for item in multiarray.__dir__(): + _globals[item] = getattr(multiarray, item) diff --git a/numpy/_core/umath.py b/numpy/_core/umath.py new file mode 100644 index 000000000000..3d08c90332a3 --- /dev/null +++ b/numpy/_core/umath.py @@ -0,0 +1,6 @@ +from numpy.core import umath + +_globals = globals() + +for item in umath.__dir__(): + _globals[item] = getattr(umath, item) diff --git a/numpy/core/tests/data/numpy_2_0_array.pkl b/numpy/core/tests/data/numpy_2_0_array.pkl new file mode 100644 index 0000000000000000000000000000000000000000..958eee50c9a6a630844801fe190562b3f8d3477c GIT binary patch literal 718 zcmXYte=HPW7{~7%Y4=CX%wngtjLmI+rgc-$Ylz#oSo0yX=6*%cdGdoJVrU6DY_Py&vNB4dEw>%8`9-P zc@lQMCEwzlACJLfQiGK%kHu#OL`O$QY|;9+@fdPZ<>@dB1|vAmhjXkOkTt9+-sl2s zP~~+-679Ibb!UlV?h^sMF96f-6dZBr0JK*nMCqFVyPeJaxf;M)@hRcGHo#|VKQp`s z0o8806q_MHWlU}2?O{NUr^$KO2;iGmb8h@7ptt?reO^@8mJTHEp?ck;(dlM7cNGQ( zss>2M{i&d|53q%G_~c?IY-KO)WO z1&nep>p0j0Xm~ziU!w(VPFfxa)liVCY5LOGQNf8|Zl6ok$~5A`X2A^`BcP?jf`k8iz{Rrq z#?@(>&&4cdycw`>(AhZo1(2ua-fH^>=qT{LYD;Y0m@qYc1@x38u(FB5-4&`JqV^2; z<|6fb$+s@V6LlOzp_=|-YzHGhC)AH4FH^3}oJ*p4%B5YO8a+(;uU|2V%%saT-(}m) z0S=~?4H=e5SLW;eXAy9raZRTu-C>z+ae)DlY56_$C6VJYe!qVla5Y0Eu%4i~^)T9# zCIPKCtqz2Lr1>@-uLzt1bWQL{Ofdl}(r5dPr1QJIVSVKjU}V)S*NJ?()mO}{tOCkK G9_+u){Xawi literal 0 HcmV?d00001 diff --git a/numpy/core/tests/test_numpy_2_0_compat.py b/numpy/core/tests/test_numpy_2_0_compat.py new file mode 100644 index 000000000000..5224261fd29a --- /dev/null +++ b/numpy/core/tests/test_numpy_2_0_compat.py @@ -0,0 +1,48 @@ +from os import path +import pickle + +import numpy as np + + +class TestNumPy2Compatibility: + + data_dir = path.join(path.dirname(__file__), "data") + filename = path.join(data_dir, "numpy_2_0_array.pkl") + + def test_importable__core_stubs(self): + """ + Checks if stubs for `numpy._core` are importable. + """ + from numpy._core.multiarray import _reconstruct + from numpy._core.umath import cos + from numpy._core._multiarray_umath import exp + from numpy._core._internal import ndarray + from numpy._core._dtype import _construction_repr + from numpy._core._dtype_ctypes import dtype_from_ctypes_type + + def test_unpickle_numpy_2_0_file(self): + """ + Checks that NumPy 1.26 and pickle is able to load pickles + created with NumPy 2.0 without errors/warnings. + """ + with open(self.filename, mode="rb") as file: + content = file.read() + + # Let's make sure that the pickle object we're loading + # was built with NumPy 2.0. + assert b"numpy._core.multiarray" in content + + arr = pickle.loads(content, encoding="latin1") + + assert isinstance(arr, np.ndarray) + assert arr.shape == (73,) and arr.dtype == np.float64 + + def test_numpy_load_numpy_2_0_file(self): + """ + Checks that `numpy.load` for NumPy 1.26 is able to load pickles + created with NumPy 2.0 without errors/warnings. + """ + arr = np.load(self.filename, encoding="latin1", allow_pickle=True) + + assert isinstance(arr, np.ndarray) + assert arr.shape == (73,) and arr.dtype == np.float64 diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 05c51fe5635f..d5b3fbac23ab 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -169,7 +169,7 @@ ) -__all__ = ["NumpyUnpickler"] +__all__ = [] EXPECTED_KEYS = {'descr', 'fortran_order', 'shape'} @@ -797,7 +797,7 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None, *, if pickle_kwargs is None: pickle_kwargs = {} try: - array = NumpyUnpickler(fp, **pickle_kwargs).load() + array = pickle.load(fp, **pickle_kwargs) except UnicodeError as err: # Friendlier error message raise UnicodeError("Unpickling a python object failed: %r\n" @@ -974,15 +974,3 @@ def _read_bytes(fp, size, error_template="ran out of data"): raise ValueError(msg % (error_template, size, len(data))) else: return data - - -class NumpyUnpickler(pickle.Unpickler): - """ - A thin wrapper for :py:class:`pickle.Unpickler` that - allows to load 2.0 array pickles with numpy 1.26. - """ - - def find_class(self, module: str, name: str) -> object: - if module.startswith("numpy._core"): - module = module.replace("_core", "core", 1) - return pickle.Unpickler.find_class(self, module, name) diff --git a/numpy/lib/format.pyi b/numpy/lib/format.pyi index a433309c2583..a4468f52f464 100644 --- a/numpy/lib/format.pyi +++ b/numpy/lib/format.pyi @@ -1,4 +1,3 @@ -import pickle from typing import Any, Literal, Final __all__: list[str] @@ -21,5 +20,3 @@ def read_array_header_2_0(fp): ... def write_array(fp, array, version=..., allow_pickle=..., pickle_kwargs=...): ... def read_array(fp, allow_pickle=..., pickle_kwargs=...): ... def open_memmap(filename, mode=..., dtype=..., shape=..., fortran_order=..., version=...): ... - -class NumpyUnpickler(pickle.Unpickler): ... diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 73ef6dde900b..339b1dc62113 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -462,7 +462,7 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, raise ValueError("Cannot load file containing pickled data " "when allow_pickle=False") try: - return format.NumpyUnpickler(fid, **pickle_kwargs).load() + return pickle.load(fid, **pickle_kwargs) except Exception as e: raise pickle.UnpicklingError( f"Failed to interpret file {file!r} as a pickle") from e diff --git a/numpy/meson.build b/numpy/meson.build index fc10acb5eb5a..8da83ce61643 100644 --- a/numpy/meson.build +++ b/numpy/meson.build @@ -246,6 +246,7 @@ pure_subdirs = [ '_pyinstaller', '_typing', '_utils', + '_core', 'array_api', 'compat', 'doc', diff --git a/numpy/setup.py b/numpy/setup.py index b6f879bcca85..536224a81192 100644 --- a/numpy/setup.py +++ b/numpy/setup.py @@ -7,6 +7,7 @@ def configuration(parent_package='',top_path=None): config.add_subpackage('array_api') config.add_subpackage('compat') config.add_subpackage('core') + config.add_subpackage('_core') config.add_subpackage('distutils') config.add_subpackage('doc') config.add_subpackage('f2py')