From 50686ca0434cc8a34237a385973c081852cfb5c9 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:01:44 -0400 Subject: [PATCH 01/11] Remove tester to improve import time --- numpy/__init__.py | 4 ---- numpy/tests/test_public_api.py | 1 - 2 files changed, 5 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index ba88c733f42a..66f7f2a47c09 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -182,10 +182,6 @@ oldnumeric = 'removed' numarray = 'removed' - # We don't actually use this ourselves anymore, but I'm not 100% sure that - # no-one else in the world is using it (though I hope not) - from .testing import Tester - # Pytest testing from numpy._pytesttester import PytestTester test = PytestTester(__name__) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 807c98652b94..2fe99203866f 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -28,7 +28,6 @@ def check_dir(module, module_name=None): def test_numpy_namespace(): # None of these objects are publicly documented. undocumented = { - 'Tester': 'numpy.testing._private.nosetester.NoseTester', '_add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', 'add_docstring': 'numpy.core._multiarray_umath.add_docstring', 'add_newdoc': 'numpy.core.function_base.add_newdoc', From 625db1400d332b74fdaa98eb9e5c4c6b193e4293 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:20:10 -0400 Subject: [PATCH 02/11] Lazy import pickle --- numpy/compat/py3k.py | 16 ++++++++++------ numpy/core/_internal.py | 1 - numpy/core/_methods.py | 4 +++- numpy/core/numeric.py | 6 ++++-- numpy/core/tests/test_datetime.py | 3 ++- numpy/core/tests/test_dtype.py | 12 ++++++------ numpy/core/tests/test_multiarray.py | 15 ++++++++------- numpy/core/tests/test_overrides.py | 3 ++- numpy/core/tests/test_records.py | 4 ++-- numpy/core/tests/test_regression.py | 3 ++- numpy/core/tests/test_ufunc.py | 4 ++-- numpy/lib/format.py | 4 +++- numpy/lib/npyio.py | 6 ++++-- numpy/ma/core.py | 4 +++- numpy/ma/tests/test_core.py | 4 ++-- numpy/ma/tests/test_mrecords.py | 3 ++- numpy/ma/tests/test_old_ma.py | 3 ++- numpy/matrixlib/tests/test_masked_matrix.py | 3 ++- numpy/tests/test_reloading.py | 4 +++- 19 files changed, 62 insertions(+), 40 deletions(-) diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index c9ed9d52cce8..c2566fd9f325 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -8,7 +8,7 @@ 'unicode', 'asunicode', 'asbytes_nested', 'asunicode_nested', 'asstr', 'open_latin1', 'long', 'basestring', 'sixu', 'integer_types', 'is_pathlib_path', 'npy_load_module', 'Path', - 'pickle', 'contextlib_nullcontext', 'os_fspath', 'os_PathLike'] + 'get_pickle', 'contextlib_nullcontext', 'os_fspath', 'os_PathLike'] import sys import os @@ -20,10 +20,12 @@ if sys.version_info[0] >= 3: import io - try: - import pickle5 as pickle - except ImportError: - import pickle + def get_pickle(): + try: + import pickle5 as pickle + except ImportError: + import pickle + return pickle long = int integer_types = (int,) @@ -58,7 +60,9 @@ def sixu(s): strchar = 'U' else: - import cpickle as pickle + def get_pickle(): + import cpickle as pickle + return pickle bytes = str long = long diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index c70718cb6e1f..5f6fe960cbab 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -909,4 +909,3 @@ def __init__(self, func): self.func = func def __call__(self, *args, **kwargs): return self.func(self, *args, **kwargs) - diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 269e509b860e..b6639688ca3f 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -13,7 +13,7 @@ from numpy.core import numerictypes as nt from numpy.core import _exceptions from numpy._globals import _NoValue -from numpy.compat import pickle, os_fspath, contextlib_nullcontext +from numpy.compat import os_fspath, contextlib_nullcontext, get_pickle # save those O(100) nanoseconds! umr_maximum = um.maximum.reduce @@ -233,6 +233,7 @@ def _ptp(a, axis=None, out=None, keepdims=False): ) def _dump(self, file, protocol=2): + pickle = get_pickle() if hasattr(file, 'write'): ctx = contextlib_nullcontext(file) else: @@ -241,4 +242,5 @@ def _dump(self, file, protocol=2): pickle.dump(self, f, protocol=protocol) def _dumps(self, protocol=2): + pickle = get_pickle() return pickle.dumps(self, protocol=protocol) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index ea2ef900e0a9..53485632f852 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -9,7 +9,7 @@ import contextlib import numpy as np -from numpy.compat import pickle, basestring +from numpy.compat import get_pickle, basestring from . import multiarray from .multiarray import ( _fastCopyAndTranspose as fastCopyAndTranspose, ALLOW_THREADS, @@ -49,6 +49,7 @@ def loads(*args, **kwargs): + pickle = get_pickle() # NumPy 1.15.0, 2017-12-10 warnings.warn( "np.core.numeric.loads is deprecated, use pickle.loads instead", @@ -937,7 +938,7 @@ def tensordot(a, b, axes=2): Returns ------- output : ndarray - The tensor dot product of the input. + The tensor dot product of the input. See Also -------- @@ -2038,6 +2039,7 @@ def load(file): load, save """ + pickle = get_pickle() # NumPy 1.15.0, 2017-12-10 warnings.warn( "np.core.numeric.load is deprecated, use pickle.load instead", diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index f99c0f72b76d..2f7d4a801375 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -9,7 +9,8 @@ assert_, assert_equal, assert_raises, assert_warns, suppress_warnings, assert_raises_regex, ) -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() # Use pytz to test out various time zones if available try: diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index f60eab6965fd..5b52ddad0d10 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -10,7 +10,8 @@ from numpy.core._rational_tests import rational from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_raises, HAS_REFCOUNT) -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() from itertools import permutations def assert_dtype_equal(a, b): @@ -138,11 +139,11 @@ def test_bad_param(self): 'offsets':[0, 2]}, align=True) def test_field_order_equality(self): - x = np.dtype({'names': ['A', 'B'], - 'formats': ['i4', 'f4'], + x = np.dtype({'names': ['A', 'B'], + 'formats': ['i4', 'f4'], 'offsets': [0, 4]}) - y = np.dtype({'names': ['B', 'A'], - 'formats': ['f4', 'i4'], + y = np.dtype({'names': ['B', 'A'], + 'formats': ['f4', 'i4'], 'offsets': [4, 0]}) assert_equal(x == y, False) @@ -1279,4 +1280,3 @@ def test_pairs(self, pair): pair_type = np.dtype('{},{}'.format(*pair)) expected = np.dtype([('f0', pair[0]), ('f1', pair[1])]) assert_equal(pair_type, expected) - diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 53e538f7d1cc..95d8d86ec153 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -21,7 +21,8 @@ import pytest from contextlib import contextmanager -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() try: import pathlib @@ -114,7 +115,7 @@ def test_writeable_any_base(self): # Ensure that any base being writeable is sufficient to change flag; # this is especially interesting for arrays from an array interface. arr = np.arange(10) - + class subclass(np.ndarray): pass @@ -3967,13 +3968,13 @@ def test_subarray_int_shape(self): def test_datetime64_byteorder(self): original = np.array([['2015-02-24T00:00:00.000000000']], dtype='datetime64[ns]') - + original_byte_reversed = original.copy(order='K') original_byte_reversed.dtype = original_byte_reversed.dtype.newbyteorder('S') original_byte_reversed.byteswap(inplace=True) new = pickle.loads(pickle.dumps(original_byte_reversed)) - + assert_equal(original.dtype, new.dtype) @@ -4868,7 +4869,7 @@ def test_fromfile_offset(self): offset_bytes = self.dtype.itemsize z = np.fromfile(f, dtype=self.dtype, offset=offset_bytes) assert_array_equal(z, self.x.flat[offset_items+count_items+1:]) - + with open(self.filename, 'wb') as f: self.x.tofile(f, sep=",") @@ -6225,14 +6226,14 @@ def test_dot_equivalent(self, args): r3 = np.matmul(args[0].copy(), args[1].copy()) assert_equal(r1, r3) - + def test_matmul_object(self): import fractions f = np.vectorize(fractions.Fraction) def random_ints(): return np.random.randint(1, 1000, size=(10, 3, 3)) - M1 = f(random_ints(), random_ints()) + M1 = f(random_ints(), random_ints()) M2 = f(random_ints(), random_ints()) M3 = self.matmul(M1, M2) diff --git a/numpy/core/tests/test_overrides.py b/numpy/core/tests/test_overrides.py index 63b0e4539e9e..ece5aacf92c6 100644 --- a/numpy/core/tests/test_overrides.py +++ b/numpy/core/tests/test_overrides.py @@ -10,7 +10,8 @@ from numpy.core.overrides import ( _get_implementing_args, array_function_dispatch, verify_matching_signatures, ARRAY_FUNCTION_ENABLED) -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() import pytest diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index 14413224eeb9..d80dabdb0a86 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -17,8 +17,8 @@ assert_, assert_equal, assert_array_equal, assert_array_almost_equal, assert_raises, temppath ) -from numpy.compat import pickle - +from numpy.compat import get_pickle +pickle = get_pickle() class TestFromrecords(object): def test_fromrecords(self): diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index e564ae30021e..3f47a2f6e66f 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -16,7 +16,8 @@ assert_raises_regex, assert_warns, suppress_warnings, _assert_valid_refcount, HAS_REFCOUNT, ) -from numpy.compat import asbytes, asunicode, long, pickle +from numpy.compat import asbytes, asunicode, long, get_pickle +pickle = get_pickle() try: RecursionError diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 69fbc35e33d4..c0a6c04340ba 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -15,8 +15,8 @@ assert_almost_equal, assert_array_almost_equal, assert_no_warnings, assert_allclose, ) -from numpy.compat import pickle - +from numpy.compat import get_pickle +pickle = get_pickle() class TestUfuncKwargs(object): def test_kwarg_exact(self): diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 3bf818812b78..6a3678811459 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -169,7 +169,7 @@ import warnings from numpy.lib.utils import safe_eval from numpy.compat import ( - isfileobj, long, os_fspath, pickle + isfileobj, long, os_fspath, get_pickle ) @@ -656,6 +656,7 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None): "allow_pickle=False") if pickle_kwargs is None: pickle_kwargs = {} + pickle = get_pickle() pickle.dump(array, fp, protocol=3, **pickle_kwargs) elif array.flags.f_contiguous and not array.flags.c_contiguous: if isfileobj(fp): @@ -724,6 +725,7 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): if pickle_kwargs is None: pickle_kwargs = {} try: + pickle = get_pickle() array = pickle.load(fp, **pickle_kwargs) except UnicodeError as err: if sys.version_info[0] >= 3: diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 318dc434aa3d..5f049b8b072c 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -25,7 +25,7 @@ from numpy.compat import ( asbytes, asstr, asunicode, bytes, basestring, os_fspath, os_PathLike, - pickle, contextlib_nullcontext + get_pickle, contextlib_nullcontext ) if sys.version_info[0] >= 3: @@ -37,6 +37,7 @@ @set_module('numpy') def loads(*args, **kwargs): + pickle = get_pickle() # NumPy 1.15.0, 2017-12-10 warnings.warn( "np.loads is deprecated, use pickle.loads instead", @@ -457,6 +458,7 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, raise ValueError("Cannot load file containing pickled data " "when allow_pickle=False") try: + pickle = get_pickle() return pickle.load(fid, **pickle_kwargs) except Exception: raise IOError( @@ -680,7 +682,7 @@ def savez_compressed(file, *args, **kwds): The ``.npz`` file format is a zipped archive of files named after the variables they contain. The archive is compressed with ``zipfile.ZIP_DEFLATED`` and each file in the archive contains one variable - in ``.npy`` format. For a description of the ``.npy`` format, see + in ``.npy`` format. For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`. diff --git a/numpy/ma/core.py b/numpy/ma/core.py index f221b319ab16..0b44d2b4998f 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -46,7 +46,7 @@ from numpy import expand_dims from numpy.core.numeric import normalize_axis_tuple from numpy.core._internal import recursive -from numpy.compat import pickle +from numpy.compat import get_pickle __all__ = [ @@ -7919,6 +7919,7 @@ def dump(a, F): """ _pickle_warn('dump') + pickle = get_pickle() if not hasattr(F, 'readline'): with open(F, 'w') as F: pickle.dump(a, F) @@ -7940,6 +7941,7 @@ def dumps(a): """ _pickle_warn('dumps') + pickle = get_pickle() return pickle.dumps(a) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 9fe550ef8902..babc711725b2 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -49,8 +49,8 @@ ravel, repeat, reshape, resize, shape, sin, sinh, sometrue, sort, sqrt, subtract, sum, take, tan, tanh, transpose, where, zeros, ) -from numpy.compat import pickle - +from numpy.compat import get_pickle +pickle = get_pickle() pi = np.pi diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py index 94e772d5571d..ad7ccbb2f5be 100644 --- a/numpy/ma/tests/test_mrecords.py +++ b/numpy/ma/tests/test_mrecords.py @@ -23,7 +23,8 @@ assert_, assert_equal, assert_equal_records, ) -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() class TestMRecords(object): diff --git a/numpy/ma/tests/test_old_ma.py b/numpy/ma/tests/test_old_ma.py index 7100eccbbe82..888fa9eb969e 100644 --- a/numpy/ma/tests/test_old_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -21,7 +21,8 @@ repeat, resize, shape, sin, sinh, sometrue, sort, sqrt, subtract, sum, take, tan, tanh, transpose, where, zeros, ) -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() pi = np.pi diff --git a/numpy/matrixlib/tests/test_masked_matrix.py b/numpy/matrixlib/tests/test_masked_matrix.py index d3911d2e1053..355246f81084 100644 --- a/numpy/matrixlib/tests/test_masked_matrix.py +++ b/numpy/matrixlib/tests/test_masked_matrix.py @@ -7,7 +7,8 @@ MaskType, getmask, MaskedArray, nomask, log, add, hypot, divide) from numpy.ma.extras import mr_ -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() class MMatrix(MaskedArray, np.matrix,): diff --git a/numpy/tests/test_reloading.py b/numpy/tests/test_reloading.py index e378d1463f4b..e045efe396a1 100644 --- a/numpy/tests/test_reloading.py +++ b/numpy/tests/test_reloading.py @@ -3,7 +3,9 @@ import sys from numpy.testing import assert_raises, assert_, assert_equal -from numpy.compat import pickle +from numpy.compat import get_pickle +pickle = get_pickle() + if sys.version_info[:2] >= (3, 4): from importlib import reload From b76631a34405959056d58b6f005fd4e6c5df8acb Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:25:42 -0400 Subject: [PATCH 03/11] Don't import pathlib --- numpy/compat/py3k.py | 11 ++++++----- numpy/core/tests/test_memmap.py | 2 +- numpy/core/tests/test_records.py | 2 +- numpy/lib/tests/test_io.py | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index c2566fd9f325..0df78cf81ab2 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -7,15 +7,11 @@ __all__ = ['bytes', 'asbytes', 'isfileobj', 'getexception', 'strchar', 'unicode', 'asunicode', 'asbytes_nested', 'asunicode_nested', 'asstr', 'open_latin1', 'long', 'basestring', 'sixu', - 'integer_types', 'is_pathlib_path', 'npy_load_module', 'Path', + 'integer_types', 'is_pathlib_path', 'npy_load_module', 'get_pickle', 'contextlib_nullcontext', 'os_fspath', 'os_PathLike'] import sys import os -try: - from pathlib import Path, PurePath -except ImportError: - Path = PurePath = None if sys.version_info[0] >= 3: import io @@ -108,6 +104,7 @@ def is_pathlib_path(obj): Prefer using `isinstance(obj, os_PathLike)` instead of this function. """ + from pathlib import Path return Path is not None and isinstance(obj, Path) # from Python 3.7 @@ -205,6 +202,10 @@ def npy_load_module(name, fn, info=None): def _PurePath__fspath__(self): return str(self) + # Pathlib isn't exactly a cheap import. Lazy import it here in case + # users are running python older 3.6 + from pathlib import Pathlike + class os_PathLike(abc_ABC): """Abstract base class for implementing the file system path protocol.""" diff --git a/numpy/core/tests/test_memmap.py b/numpy/core/tests/test_memmap.py index d2ae564b24da..3071d679eb31 100644 --- a/numpy/core/tests/test_memmap.py +++ b/numpy/core/tests/test_memmap.py @@ -9,7 +9,7 @@ from numpy import ( memmap, sum, average, product, ndarray, isscalar, add, subtract, multiply) -from numpy.compat import Path +from pathlib import Path from numpy import arange, allclose, asarray from numpy.testing import ( diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index d80dabdb0a86..5af8a0049ec4 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -12,7 +12,7 @@ import pytest import numpy as np -from numpy.compat import Path +from pathlib import Path from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_array_almost_equal, assert_raises, temppath diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 38d4c19fac43..18b45a7f6115 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -17,7 +17,8 @@ import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes, Path +from numpy.compat import asbytes, bytes +from pathlib import Path from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, From 04aeae134604e515d3e16c799f85fee503b201ae Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:27:22 -0400 Subject: [PATCH 04/11] lazy import shutils --- numpy/lib/_datasource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index 0d71375c28c3..6b11714344de 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -39,7 +39,6 @@ import os import sys import warnings -import shutil import io from contextlib import closing @@ -330,6 +329,7 @@ def __init__(self, destpath=os.curdir): self._istmpdest = True def __del__(self): + import shutil # Remove temp directories if hasattr(self, '_istmpdest') and self._istmpdest: shutil.rmtree(self._destpath) @@ -413,6 +413,7 @@ def _cache(self, path): os.makedirs(os.path.dirname(upath)) # TODO: Doesn't handle compressed files! + import shutil if self._isurl(path): try: with closing(urlopen(path)) as openedurl: From 29fe679e669167d5b6566d7b84d01912fa4b915b Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:44:20 -0400 Subject: [PATCH 05/11] delay importing platform. --- numpy/core/_internal.py | 6 ++++-- numpy/ctypeslib.py | 9 ++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 5f6fe960cbab..5401089fee6f 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -8,7 +8,6 @@ import re import sys -import platform from numpy.compat import unicode from .multiarray import dtype, array, ndarray @@ -17,7 +16,10 @@ except ImportError: ctypes = None -IS_PYPY = platform.python_implementation() == 'PyPy' +# While one could use platform.python_implementation +# importing platform is quite slow +IS_PYPY = "PyPy" in sys.version + if (sys.byteorder == 'little'): _nbo = b'<' diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index 1f842d003cee..2c629d47ac51 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -92,11 +92,11 @@ def _dummy(*args, **kwds): # Adapted from Albert Strasheim def load_library(libname, loader_path): """ - It is possible to load a library using + It is possible to load a library using >>> lib = ctypes.cdll[] # doctest: +SKIP But there are cross-platform considerations, such as library file extensions, - plus the fact Windows will just load the first library it finds with that name. + plus the fact Windows will just load the first library it finds with that name. NumPy supplies the load_library function as a convenience. Parameters @@ -110,12 +110,12 @@ def load_library(libname, loader_path): Returns ------- ctypes.cdll[libpath] : library object - A ctypes library object + A ctypes library object Raises ------ OSError - If there is no library with the expected extension, or the + If there is no library with the expected extension, or the library is defective and cannot be loaded. """ if ctypes.__version__ < '1.0.1': @@ -355,7 +355,6 @@ def _ctype_ndarray(element_type, shape): element_type.__module__ = None return element_type - def _get_scalar_type_map(): """ Return a dictionary mapping native endian scalar dtype to ctypes types From 0ae6ba7a51c57e77313fc7ca842f8abe584fbd1c Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:47:12 -0400 Subject: [PATCH 06/11] lazy import weakref --- numpy/lib/npyio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 5f049b8b072c..9d03d20b921a 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -6,7 +6,6 @@ import functools import itertools import warnings -import weakref import contextlib from operator import itemgetter, index as opindex @@ -87,6 +86,7 @@ class BagObj(object): """ def __init__(self, obj): + import weakref # Use weakref to make NpzFile objects collectable by refcount self._obj = weakref.proxy(obj) From 172347557db6096a0ac5ab312996888d7a746b29 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 01:01:13 -0400 Subject: [PATCH 07/11] Delay importing Decimal --- numpy/lib/financial.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py index 216687475c92..d2e7480656ef 100644 --- a/numpy/lib/financial.py +++ b/numpy/lib/financial.py @@ -12,7 +12,6 @@ """ from __future__ import division, absolute_import, print_function -from decimal import Decimal import functools import numpy as np @@ -621,6 +620,8 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): OpenDocument-formula-20090508.odt """ + from decimal import Decimal + when = _convert_when(when) default_type = Decimal if isinstance(pmt, Decimal) else float @@ -812,6 +813,8 @@ def mirr(values, finance_rate, reinvest_rate): Modified internal rate of return """ + from decimal import Decimal + values = np.asarray(values) n = values.size From e57cf064cc3412fc91397fbc5703fae1c1e8d13d Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 02:03:34 -0400 Subject: [PATCH 08/11] bit_generator use fastrlock instead of Threading. --- numpy/random/bit_generator.pyx | 6 +----- numpy/random/philox.pyx | 5 ----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/numpy/random/bit_generator.pyx b/numpy/random/bit_generator.pyx index 6694e5e4db02..7aabb6f3d7b4 100644 --- a/numpy/random/bit_generator.pyx +++ b/numpy/random/bit_generator.pyx @@ -43,12 +43,8 @@ except ImportError: from random import SystemRandom randbits = SystemRandom().getrandbits -try: - from threading import Lock -except ImportError: - from dummy_threading import Lock - from cpython.pycapsule cimport PyCapsule_New +from fastrlock.rlock cimport create_fastrlock as Lock import numpy as np cimport numpy as np diff --git a/numpy/random/philox.pyx b/numpy/random/philox.pyx index 8b76830172cb..ceeff5f59843 100644 --- a/numpy/random/philox.pyx +++ b/numpy/random/philox.pyx @@ -1,10 +1,5 @@ from cpython.pycapsule cimport PyCapsule_New -try: - from threading import Lock -except ImportError: - from dummy_threading import Lock - import numpy as np from .common cimport * From 4329d6acec8387225ae05e09c073174c913aed02 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 02:27:35 -0400 Subject: [PATCH 09/11] create the randbits object ourselves. --- numpy/random/bit_generator.pyx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/numpy/random/bit_generator.pyx b/numpy/random/bit_generator.pyx index 7aabb6f3d7b4..4fb3e81817cc 100644 --- a/numpy/random/bit_generator.pyx +++ b/numpy/random/bit_generator.pyx @@ -249,6 +249,11 @@ cdef class SeedlessSeedSequence(): # we must register it after the fact. ISpawnableSeedSequence.register(SeedlessSeedSequence) +# Taken from https://github.com/python/cpython/blob/3.7/Lib/secrets.py#L24 +# secret imports many things we don't need +from random import SystemRandom +_sysrand = SystemRandom() +randbits = _sysrand.getrandbits cdef class SeedSequence(): """ From df0c827fe06a67abc68ebda145aa87443a464879 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 22 Jul 2019 00:37:00 -0400 Subject: [PATCH 10/11] don't import textwrap --- numpy/core/overrides.py | 15 ++++++------ numpy/ma/core.py | 53 ++++++++++++++++++++--------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/numpy/core/overrides.py b/numpy/core/overrides.py index 04a5a995fba1..c95dca7d5742 100644 --- a/numpy/core/overrides.py +++ b/numpy/core/overrides.py @@ -2,7 +2,6 @@ import collections import functools import os -import textwrap from numpy.core._multiarray_umath import ( add_docstring, implement_array_function, _get_implementing_args) @@ -163,13 +162,13 @@ def decorator(implementation): # more interpettable name. Otherwise, the original function does not # show up at all in many cases, e.g., if it's written in C or if the # dispatcher gets an invalid keyword argument. - source = textwrap.dedent(""" - @functools.wraps(implementation) - def {name}(*args, **kwargs): - relevant_args = dispatcher(*args, **kwargs) - return implement_array_function( - implementation, {name}, relevant_args, args, kwargs) - """).format(name=implementation.__name__) + source = """ +@functools.wraps(implementation) +def {name}(*args, **kwargs): + relevant_args = dispatcher(*args, **kwargs) + return implement_array_function( + implementation, {name}, relevant_args, args, kwargs) + """.format(name=implementation.__name__) source_object = compile( source, filename='<__array_function__ internals>', mode='exec') diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 0b44d2b4998f..c0af04116b67 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -25,7 +25,6 @@ import sys import operator import warnings -import textwrap import re from functools import reduce @@ -2445,32 +2444,32 @@ def _recursive_printoption(result, mask, printopt): # For better or worse, these end in a newline _legacy_print_templates = dict( - long_std=textwrap.dedent("""\ - masked_%(name)s(data = - %(data)s, - %(nlen)s mask = - %(mask)s, - %(nlen)s fill_value = %(fill)s) - """), - long_flx=textwrap.dedent("""\ - masked_%(name)s(data = - %(data)s, - %(nlen)s mask = - %(mask)s, - %(nlen)s fill_value = %(fill)s, - %(nlen)s dtype = %(dtype)s) - """), - short_std=textwrap.dedent("""\ - masked_%(name)s(data = %(data)s, - %(nlen)s mask = %(mask)s, - %(nlen)s fill_value = %(fill)s) - """), - short_flx=textwrap.dedent("""\ - masked_%(name)s(data = %(data)s, - %(nlen)s mask = %(mask)s, - %(nlen)s fill_value = %(fill)s, - %(nlen)s dtype = %(dtype)s) - """) + long_std="""\ +masked_%(name)s(data = + %(data)s, +%(nlen)s mask = + %(mask)s, +%(nlen)s fill_value = %(fill)s) +""", + long_flx="""\ +masked_%(name)s(data = + %(data)s, +%(nlen)s mask = + %(mask)s, +%(nlen)s fill_value = %(fill)s, +%(nlen)s dtype = %(dtype)s) +""", + short_std="""\ +masked_%(name)s(data = %(data)s, +%(nlen)s mask = %(mask)s, +%(nlen)s fill_value = %(fill)s) +""", + short_flx="""\ +masked_%(name)s(data = %(data)s, +%(nlen)s mask = %(mask)s, +%(nlen)s fill_value = %(fill)s, +%(nlen)s dtype = %(dtype)s) +""" ) ############################################################################### From 2e13de7794999d946b2f85666289a3757840385b Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Tue, 23 Jul 2019 00:49:21 -0400 Subject: [PATCH 11/11] justtotest: install fastrlock in builds. --- .circleci/config.yml | 4 ++-- azure-pipelines.yml | 8 ++++---- shippable.yml | 2 ++ tools/pypy-test.sh | 2 +- tools/travis-before-install.sh | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b4ab812ff9d..4b3de4cac47d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: python3 -m venv venv ln -s $(which python3) venv/bin/python3.6 . venv/bin/activate - pip install cython sphinx==1.8.5 matplotlib ipython + pip install cython sphinx==1.8.5 matplotlib ipython fastrlock sudo apt-get update sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex @@ -30,7 +30,7 @@ jobs: command: | . venv/bin/activate pip install --upgrade pip setuptools - pip install cython + pip install cython fastrlock pip install . pip install scipy diff --git a/azure-pipelines.yml b/azure-pipelines.yml index edb577cdb63f..d666f97430d3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ jobs: apt-get -y update && \ apt-get -y install python3.6-dev python3-pip locales python3-certifi && \ locale-gen fr_FR && update-locale && \ - pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 && \ + pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 fastrlock && \ apt-get -y install gfortran-5 wget && \ target=\$(python3 tools/openblas_support.py) && \ cp -r \$target/usr/local/lib/* /usr/lib && \ @@ -85,7 +85,7 @@ jobs: displayName: 'install pre-built openblas' - script: python -m pip install --upgrade pip setuptools wheel displayName: 'Install tools' - - script: python -m pip install cython nose pytz pytest pickle5 vulture docutils sphinx==1.8.5 numpydoc + - script: python -m pip install cython fastrlock nose pytz pytest pickle5 vulture docutils sphinx==1.8.5 numpydoc displayName: 'Install dependencies; some are optional to avoid test skips' - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'" displayName: 'Check for unreachable code paths in Python modules' @@ -156,13 +156,13 @@ jobs: architecture: $(PYTHON_ARCH) - script: python -m pip install --upgrade pip setuptools wheel displayName: 'Install tools' - - script: python -m pip install cython nose pytz pytest + - script: python -m pip install cython fastrlock nose pytz pytest displayName: 'Install dependencies; some are optional to avoid test skips' - script: if [%INSTALL_PICKLE5%]==[1] python -m pip install pickle5 displayName: 'Install optional pickle5 backport (only for python3.6 and 3.7)' - powershell: | - $pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])" + $pyversion = python -c " Write-Host "Python Version: $pyversion" $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a" Write-Host "target path: $target" diff --git a/shippable.yml b/shippable.yml index cf09b791dbd6..db1941229a95 100644 --- a/shippable.yml +++ b/shippable.yml @@ -32,6 +32,8 @@ build: # version is scraped in by pip; otherwise, use the cached # wheel shippable places on Amazon S3 after we build it once - pip install cython --cache-dir=/root/.cache/pip/wheels/$SHIPPABLE_PYTHON_VERSION + # Install fastrlock for locks in random + - pip install fastrlock # install pytz for datetime testing - pip install pytz # install pytest-xdist to leverage a second core diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 038748af9402..2bc6bcce5085 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -32,7 +32,7 @@ mkdir -p pypy3 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) pypy3/bin/pypy3 -mensurepip pypy3/bin/pypy3 -m pip install --upgrade pip setuptools -pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location +pypy3/bin/pypy3 -m pip install --user fastrlock cython==0.29.0 pytest pytz --no-warn-script-location echo echo pypy3 version diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index b1c1f2ca1c37..9c2702715166 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -36,6 +36,6 @@ fi pip install --upgrade pip setuptools -pip install nose pytz cython pytest +pip install nose pytz cython pytest fastrlock if [ -n "$USE_ASV" ]; then pip install asv; fi popd