Simplify extension setup.

anntzer · anntzer · commit a93799e7fcf5 · 2018-08-28T20:35:51.000+02:00
setupext currently has a complex machinery to define extension modules.
Essentially, the problem is that numpy may not be installed at the time
the extensions are declared, so we can't call np.get_include() to get
the include paths.  So we use a DelayedExtension class and a hook
mechanism to inject the numpy include path later, once it becomes
available.

Instead, we can just declare a dummy extension (so that setuptools
doesn't elide the call to build_ext), then swap in the correct
extensions in build_ext.finalize_options(): at that point, numpy will
have been installed and we don't need any crazy machinery.
diff --git a/setup.py b/setup.py
@@ -8,7 +8,7 @@
 # and/or pip.
 from string import Template
 import sys
-from setuptools import setup
+from setuptools import setup, Extension
 from setuptools.command.test import test as TestCommand
 from setuptools.command.build_ext import build_ext as BuildExtCommand
 
@@ -108,6 +108,12 @@ def __init__(self, dist):
 
 
 class BuildExtraLibraries(BuildExtCommand):
+    def finalize_options(self):
+        self.distribution.ext_modules[:] = [
+            *filter(None, (package.get_extension()
+                           for package in good_packages))]
+        super().finalize_options()
+
     def run(self):
         for package in good_packages:
             package.do_custom_build()
@@ -128,7 +134,9 @@ def run(self):
     packages = []
     namespace_packages = []
     py_modules = []
-    ext_modules = []
+    # Dummy extension to trigger build_ext, which will swap it out with real
+    # extensions that can depend on numpy for the build.
+    ext_modules = [Extension('', [])]
     package_data = {}
     package_dir = {'': 'lib'}
     install_requires = []
@@ -192,9 +200,8 @@ def run(self):
             packages.extend(package.get_packages())
             namespace_packages.extend(package.get_namespace_packages())
             py_modules.extend(package.get_py_modules())
-            ext = package.get_extension()
-            if ext is not None:
-                ext_modules.append(ext)
+            # Extension modules only get added in build_ext, as numpy will have
+            # been installed (as setup_requires) at that point.
             data = package.get_package_data()
             for key, val in data.items():
                 package_data.setdefault(key, [])
@@ -212,11 +219,6 @@ def run(self):
         with open('lib/matplotlib/mpl-data/matplotlibrc', 'w') as fd:
             fd.write(template)
 
-        # Finalize the extension modules so they can get the Numpy include
-        # dirs
-        for mod in ext_modules:
-            mod.finalize()
-
     # Finally, pass this all along to distutils to do the heavy lifting.
     setup(
         name="matplotlib",
diff --git a/setupext.py b/setupext.py
@@ -231,7 +231,7 @@ def make_extension(name, files, *args, **kwargs):
     Any additional arguments are passed to the
     `distutils.core.Extension` constructor.
     """
-    ext = DelayedExtension(name, files, *args, **kwargs)
+    ext = Extension(name, files, *args, **kwargs)
     for dir in get_base_dirs():
         include_dir = os.path.join(dir, 'include')
         if os.path.exists(include_dir):
@@ -241,7 +241,6 @@ def make_extension(name, files, *args, **kwargs):
             if os.path.exists(lib_dir):
                 ext.library_dirs.append(lib_dir)
     ext.include_dirs.append('.')
-
     return ext
 
 
@@ -800,77 +799,9 @@ def get_namespace_packages(self):
         return ['mpl_toolkits']
 
 
-class DelayedExtension(Extension, object):
-    """
-    A distutils Extension subclass where some of its members
-    may have delayed computation until reaching the build phase.
-
-    This is so we can, for example, get the Numpy include dirs
-    after pip has installed Numpy for us if it wasn't already
-    on the system.
-    """
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._finalized = False
-        self._hooks = {}
-
-    def add_hook(self, member, func):
-        """
-        Add a hook to dynamically compute a member.
-
-        Parameters
-        ----------
-        member : string
-            The name of the member
-
-        func : callable
-            The function to call to get dynamically-computed values
-            for the member.
-        """
-        self._hooks[member] = func
-
-    def finalize(self):
-        self._finalized = True
-
-    class DelayedMember(property):
-        def __init__(self, name):
-            self._name = name
-
-        def __get__(self, obj, objtype=None):
-            result = getattr(obj, '_' + self._name, [])
-
-            if obj._finalized:
-                if self._name in obj._hooks:
-                    result = obj._hooks[self._name]() + result
-
-            return result
-
-        def __set__(self, obj, value):
-            setattr(obj, '_' + self._name, value)
-
-    include_dirs = DelayedMember('include_dirs')
-
-
 class Numpy(SetupPackage):
     name = "numpy"
 
-    @staticmethod
-    def include_dirs_hook():
-        if hasattr(builtins, '__NUMPY_SETUP__'):
-            del builtins.__NUMPY_SETUP__
-        import numpy
-        importlib.reload(numpy)
-
-        ext = Extension('test', [])
-        ext.include_dirs.append(numpy.get_include())
-        if not has_include_file(
-                ext.include_dirs, os.path.join("numpy", "arrayobject.h")):
-            warnings.warn(
-                "The C headers for numpy could not be found. "
-                "You may need to install the development package")
-
-        return [numpy.get_include()]
-
     def check(self):
         min_version = extract_versions()['__version__numpy__']
         try:
@@ -886,16 +817,14 @@ def check(self):
         return 'version %s' % numpy.__version__
 
     def add_flags(self, ext):
+        import numpy as np
+        ext.include_dirs.append(np.get_include())
         # Ensure that PY_ARRAY_UNIQUE_SYMBOL is uniquely defined for
         # each extension
         array_api_name = 'MPL_' + ext.name.replace('.', '_') + '_ARRAY_API'
-
         ext.define_macros.append(('PY_ARRAY_UNIQUE_SYMBOL', array_api_name))
-        ext.add_hook('include_dirs', self.include_dirs_hook)
-
         ext.define_macros.append(('NPY_NO_DEPRECATED_API',
                                   'NPY_1_7_API_VERSION'))
-
         # Allow NumPy's printf format specifiers in C++.
         ext.define_macros.append(('__STDC_FORMAT_MACROS', 1))
 
@@ -910,32 +839,23 @@ class LibAgg(SetupPackage):
     name = 'libagg'
 
     def check(self):
-        self.__class__.found_external = True
-        try:
-            return self._check_for_pkg_config(
-                'libagg', 'agg2/agg_basics.h', min_version='PATCH')
-        except CheckFailed as e:
-            self.__class__.found_external = False
-            return str(e) + ' Using local copy.'
+        return 'requires patched version; using local copy.'
 
     def add_flags(self, ext, add_sources=True):
-        if self.found_external:
-            pkg_config.setup_extension(ext, 'libagg')
-        else:
-            ext.include_dirs.insert(0, 'extern/agg24-svn/include')
-            if add_sources:
-                agg_sources = [
-                    'agg_bezier_arc.cpp',
-                    'agg_curves.cpp',
-                    'agg_image_filters.cpp',
-                    'agg_trans_affine.cpp',
-                    'agg_vcgen_contour.cpp',
-                    'agg_vcgen_dash.cpp',
-                    'agg_vcgen_stroke.cpp',
-                    'agg_vpgen_segmentator.cpp'
-                    ]
-                ext.sources.extend(
-                    os.path.join('extern', 'agg24-svn', 'src', x) for x in agg_sources)
+        ext.include_dirs.insert(0, 'extern/agg24-svn/include')
+        if add_sources:
+            agg_sources = [
+                'agg_bezier_arc.cpp',
+                'agg_curves.cpp',
+                'agg_image_filters.cpp',
+                'agg_trans_affine.cpp',
+                'agg_vcgen_contour.cpp',
+                'agg_vcgen_dash.cpp',
+                'agg_vcgen_stroke.cpp',
+                'agg_vpgen_segmentator.cpp'
+                ]
+            ext.sources.extend(
+                os.path.join('extern', 'agg24-svn', 'src', x) for x in agg_sources)
 
 
 class FreeType(SetupPackage):
@@ -1211,7 +1131,14 @@ def get_extension(self):
         pkg_config.setup_extension(
             ext, 'libpng', default_libraries=['png', 'z'],
             alt_exec='libpng-config --ldflags')
-        Numpy().add_flags(ext)
+        # We call this twice: once early, to check whether png exists (from
+        # _check_for_pkg_config), and once late, to build the real extension.
+        # Only at the second time is numpy guaranteed to be installed, so don't
+        # error out if it isn't the first time.
+        try:
+            Numpy().add_flags(ext)
+        except ImportError:
+            pass
         return ext
 
 
@@ -1224,7 +1151,6 @@ def check(self):
             return self._check_for_pkg_config(
                 'libqhull', 'libqhull/qhull_a.h', min_version='2015.2')
         except CheckFailed as e:
-            self.__class__.found_pkgconfig = False
             self.__class__.found_external = False
             return str(e) + ' Using local copy.'
 
