MAINT: Hide internals of np.lib to only show submodules

seberg · seberg · commit 91bfcfa8eab5 · 2021-02-19T19:20:09.000-06:00
This commit creates a new module to "conveniently" import all
symbols into the main namespace, while hiding them in `np.lib`.
diff --git a/benchmarks/asv_compare.conf.json.tpl b/benchmarks/asv_compare.conf.json.tpl
@@ -39,7 +39,7 @@
 
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
-    "pythons": ["3.7"],
+    "pythons": ["3.9"],
 
     // The matrix of dependencies to test.  Each key is the name of a
     // package (in PyPI) and the values are version numbers.  An empty
diff --git a/numpy/__init__.py b/numpy/__init__.py
@@ -153,7 +153,7 @@
     from . import lib
     # NOTE: to be revisited following future namespace cleanup.
     # See gh-14454 and gh-15672 for discussion.
-    from .lib import *
+    from .lib._lib_importer import *
 
     from . import linalg
     from . import fft
@@ -225,7 +225,7 @@
     __all__.extend(['__version__', 'show_config'])
     __all__.extend(core.__all__)
     __all__.extend(_mat.__all__)
-    __all__.extend(lib.__all__)
+    __all__.extend(lib._lib_importer.__all__)
     __all__.extend(['linalg', 'fft', 'random', 'ctypeslib', 'ma'])
 
     # These are exported by np.core, but are replaced by the builtins below
@@ -235,14 +235,6 @@
     __all__.remove('long')
     __all__.remove('unicode')
 
-    # Remove things that are in the numpy.lib but not in the numpy namespace
-    # Note that there is a test (numpy/tests/test_public_api.py:test_numpy_namespace)
-    # that prevents adding more things to the main namespace by accident.
-    # The list below will grow until the `from .lib import *` fixme above is
-    # taken care of
-    __all__.remove('Arrayterator')
-    del Arrayterator
-
     # These names were removed in NumPy 1.20.  For at least one release,
     # attempts to access these names in the numpy namespace will trigger
     # a warning, and calling the function will raise an exception.
diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py
@@ -1,5 +1,5 @@
 """
-**Note:** almost all functions in the ``numpy.lib`` namespace
+**Note:** almost all functions in the ``numpy.lib.submodule`` namespaces
 are also present in the main ``numpy`` namespace.  Please use the
 functions as ``np.<funcname>`` where possible.
 
@@ -11,51 +11,83 @@
 useful to have in the main name-space.
 
 """
-import math
-
-from numpy.version import version as __version__
-
-# Public submodules
-# Note: recfunctions and (maybe) format are public too, but not imported
-from . import mixins
-from . import scimath as emath
-
-# Private submodules
-from .type_check import *
-from .index_tricks import *
-from .function_base import *
-from .nanfunctions import *
-from .shape_base import *
-from .stride_tricks import *
-from .twodim_base import *
-from .ufunclike import *
-from .histograms import *
-
-from .polynomial import *
-from .utils import *
-from .arraysetops import *
-from .npyio import *
+# NOTE: This file is the _public_ interface for additional submodules
+# housed in `numpy.lib`. The private import mechanism lives in the
+# `_lib_importer.py` file, which defines `__all__` to include all names to
+# be added to the main namespace!
+
+from . import scimath as emath  # import explicitly to rename
+
 from .arrayterator import Arrayterator
-from .arraypad import *
-from ._version import *
 from numpy.core._multiarray_umath import tracemalloc_domain
-
-__all__ = ['emath', 'math', 'tracemalloc_domain', 'Arrayterator']
-__all__ += type_check.__all__
-__all__ += index_tricks.__all__
-__all__ += function_base.__all__
-__all__ += shape_base.__all__
-__all__ += stride_tricks.__all__
-__all__ += twodim_base.__all__
-__all__ += ufunclike.__all__
-__all__ += arraypad.__all__
-__all__ += polynomial.__all__
-__all__ += utils.__all__
-__all__ += arraysetops.__all__
-__all__ += npyio.__all__
-__all__ += nanfunctions.__all__
-__all__ += histograms.__all__
+from ._version import NumpyVersion
 
 from numpy._pytesttester import PytestTester
 test = PytestTester(__name__)
 del PytestTester
+
+from . import _lib_importer as _hidden_lib_namespace
+
+
+# Ensure that __all__ only contains those things that we want it to contain.
+# Because this is an __init__, we would otherwise include all submodules.
+#
+# NOTE: The symbols not included in `__all__` will be hidden, but do not go
+#       through `__getattr__` (see `__dir__`).
+__all__ = {'emath', 'Arrayterator', 'tracemalloc_domain', 'NumpyVersion'}
+
+__lib_submodules_ = {
+    'type_check',
+    'emath',
+    'arraysetops',
+    'utils',
+    'format',
+    'npyio',
+    'arrayterator',
+    'mixins',
+    'stride_tricks',
+    'twodim_base',
+    'histograms',
+    'index_tricks',
+    'nanfunctions',
+    'shape_base',
+}
+# Add submodules to all (and finalize all)
+__all__.update(__lib_submodules_)
+__all__ = sorted(__all__)
+
+# Add hidden submodules:
+__lib_submodules_.update({
+    'arraypad',
+    'function_base',
+    'polynomial',
+    'scimath',
+    "ufunclike",
+    })
+
+
+def __getattr__(name):
+    if name in __lib_submodules_:
+        import importlib
+        # Allow lazy import (and avoid requiring explicit import), at the
+        # time of writing `mixins` may be the only namespace using this.
+        return importlib.import_module("." + name, __name__)
+
+    try:
+        attribute = getattr(_hidden_lib_namespace, name)
+    except AttributeError:
+        pass
+    else:
+        # Adding a warning in this branch would allow us to properly deprecate
+        # all hidden attributes. Raise an error here to test NumPy itself.
+        return attribute
+
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__():
+    # Overriding __dir__ hides symbols from tab completion for example.
+    # Note that it does _not_ hide symbols from `np.lib.__dict__()`, to do
+    # that, we would have to delete them from the locals here and hide them
+    # into `__getattr__`.  But this seems sufficient, e.g. for tab completion.
+    return __all__
diff --git a/numpy/lib/_lib_importer.py b/numpy/lib/_lib_importer.py
@@ -0,0 +1,39 @@
+# This file defines which functions end up in the main namespace of NumPy.
+# Note that the use of * import with an __all__ based on the locals(), means
+# that the __all__ of the subpackages also defines what
+# There are at least two approaches how this could be modified in the future
+# conveniently:
+# * We could create an `__export_to_numpy_namespace__` list in each package
+#   and use this to define the `__all__` here (which is still copied into
+#   the main namespace.
+# * At least all functions copied to the main namespace probably already
+#   use the `@set_module` decorator. So we could just use that (or something
+#   similar) to manually add it to the main namespace.
+#   (in which case we still need to make sure the file is imported)
+
+
+import math
+
+# Public submodules
+# Note: recfunctions and (maybe) format are public too, but not imported
+from . import scimath as emath
+
+# Private submodules
+from .type_check import *
+from .index_tricks import *
+from .function_base import *
+from .nanfunctions import *
+from .shape_base import *
+from .stride_tricks import *
+from .twodim_base import *
+from .ufunclike import *
+from .histograms import *
+
+from .polynomial import *
+from .utils import *
+from .arraysetops import *
+from .npyio import *
+from .arraypad import *
+
+
+__all__ = [k for k in locals().keys() if not k.startswith("_")]
diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py
@@ -17,6 +17,10 @@
 
 _check_fill_value = np.ma.core._check_fill_value
 
+# Make "recfunctions" available in numpy.lib for convenience, we could
+# make it a lazy import instead/additionally.
+np.lib.__all__.append("recfunctions")
+np.lib.__all__.sort()
 
 __all__ = [
     'append_fields', 'apply_along_fields', 'assign_fields_by_name',
diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py
@@ -629,7 +629,7 @@ def test_check_constant_odd_pad_amount(self):
 
     def test_check_constant_pad_2d(self):
         arr = np.arange(4).reshape(2, 2)
-        test = np.lib.pad(arr, ((1, 2), (1, 3)), mode='constant',
+        test = np.pad(arr, ((1, 2), (1, 3)), mode='constant',
                           constant_values=((1, 2), (3, 4)))
         expected = np.array(
             [[3, 1, 1, 4, 4, 4],
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
@@ -19,12 +19,14 @@
     )
 import numpy.lib.function_base as nfb
 from numpy.random import rand
-from numpy.lib import (
+from numpy.lib.function_base import (
     add_newdoc_ufunc, angle, average, bartlett, blackman, corrcoef, cov,
-    delete, diff, digitize, extract, flipud, gradient, hamming, hanning,
+    delete, diff, digitize, extract, gradient, hamming, hanning,
     i0, insert, interp, kaiser, meshgrid, msort, piecewise, place, rot90,
-    select, setxor1d, sinc, trapz, trim_zeros, unwrap, unique, vectorize
+    select, sinc, trapz, trim_zeros, unwrap, vectorize,
     )
+from numpy.lib.arraysetops import setxor1d, unique
+from numpy.lib.twodim_base import flipud
 
 
 def get_mat(n):
@@ -1928,14 +1930,14 @@ def test_simple(self):
         a = [1, 2, 3]
         b = [1, 2, np.inf]
         c = [1, 2, np.nan]
-        np.lib.asarray_chkfinite(a)
-        assert_raises(ValueError, np.lib.asarray_chkfinite, b)
-        assert_raises(ValueError, np.lib.asarray_chkfinite, c)
+        np.asarray_chkfinite(a)
+        assert_raises(ValueError, np.asarray_chkfinite, b)
+        assert_raises(ValueError, np.asarray_chkfinite, c)
 
     def test_dtype_order(self):
         # Regression test for missing dtype and order arguments
         a = [1, 2, 3]
-        a = np.lib.asarray_chkfinite(a, order='F', dtype=np.float64)
+        a = np.asarray_chkfinite(a, order='F', dtype=np.float64)
         assert_(a.dtype == np.float64)
 
 
diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py
@@ -4,8 +4,8 @@
 
 from numpy.core import arange
 from numpy.testing import assert_, assert_equal, assert_raises_regex
-from numpy.lib import deprecate, deprecate_with_doc
 import numpy.lib.utils as utils
+from numpy.lib.utils import deprecate, deprecate_with_doc
 
 from io import StringIO
 
diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py
@@ -339,7 +339,7 @@ def assert_equal(actual, desired, err_msg='', verbose=True):
                          verbose)
         return
     from numpy.core import ndarray, isscalar, signbit
-    from numpy.lib import iscomplexobj, real, imag
+    from numpy.lib.type_check import iscomplexobj, real, imag
     if isinstance(actual, ndarray) or isinstance(desired, ndarray):
         return assert_array_equal(actual, desired, err_msg, verbose)
     msg = build_err_msg([actual, desired], err_msg, verbose=verbose)
@@ -542,7 +542,7 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True):
     """
     __tracebackhide__ = True  # Hide traceback for py.test
     from numpy.core import ndarray
-    from numpy.lib import iscomplexobj, real, imag
+    from numpy.lib.type_check import iscomplexobj, real, imag
 
     # Handle complex numbers: separate into real/imag to handle
     # nan/inf/negative zero correctly
