diff --git a/setup.py b/setup.py
index c4f1913dc556..f1dbab2a9e6b 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@
 import urllib.request
 from zipfile import ZipFile
 
-from setuptools import setup
+from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext as BuildExtCommand
 from setuptools.command.develop import develop as DevelopCommand
 from setuptools.command.install_lib import install_lib as InstallLibCommand
@@ -101,6 +101,11 @@ def __init__(self, dist):
 
 
 class BuildExtraLibraries(BuildExtCommand):
+    def finalize_options(self):
+        self.distribution.ext_modules[:] = filter(
+            None, (package.get_extension() for package in good_packages))
+        super().finalize_options()
+
     def build_extensions(self):
         # Remove the -Wstrict-prototypes option, it's not valid for C++.  Fixed
         # in Py3.7 as bpo-5755.
@@ -171,7 +176,9 @@ def run(self):
     packages = []
     namespace_packages = []
     py_modules = []
-    ext_modules = []
+    # Dummy extension to trigger build_ext, which will swap it out with real
+    # extensions that can depend on numpy for the build.
+    ext_modules = [Extension('', [])]
     package_data = {}
     package_dir = {'': 'lib'}
     install_requires = []
@@ -230,9 +237,8 @@ def run(self):
             packages.extend(package.get_packages())
             namespace_packages.extend(package.get_namespace_packages())
             py_modules.extend(package.get_py_modules())
-            ext = package.get_extension()
-            if ext is not None:
-                ext_modules.append(ext)
+            # Extension modules only get added in build_ext, as numpy will have
+            # been installed (as setup_requires) at that point.
             data = package.get_package_data()
             for key, val in data.items():
                 package_data.setdefault(key, [])
@@ -252,11 +258,6 @@ def run(self):
         with open('lib/matplotlib/mpl-data/matplotlibrc', 'w') as fd:
             fd.write(''.join(template_lines))
 
-        # Finalize the extension modules so they can get the Numpy include
-        # dirs
-        for mod in ext_modules:
-            mod.finalize()
-
     # Finally, pass this all along to distutils to do the heavy lifting.
     setup(
         name="matplotlib",
diff --git a/setupext.py b/setupext.py
index 7b70197cdf07..5ff2c9dc165e 100644
--- a/setupext.py
+++ b/setupext.py
@@ -319,7 +319,7 @@ def make_extension(name, files, *args, **kwargs):
     Any additional arguments are passed to the
     `distutils.core.Extension` constructor.
     """
-    ext = DelayedExtension(name, files, *args, **kwargs)
+    ext = Extension(name, files, *args, **kwargs)
     for dir in get_base_dirs():
         include_dir = os.path.join(dir, 'include')
         if os.path.exists(include_dir):
@@ -329,7 +329,6 @@ def make_extension(name, files, *args, **kwargs):
             if os.path.exists(lib_dir):
                 ext.library_dirs.append(lib_dir)
     ext.include_dirs.append('.')
-
     return ext
 
 
@@ -789,79 +788,12 @@ def get_package_data(self):
         }
 
 
-class DelayedExtension(Extension, object):
-    """
-    A distutils Extension subclass where some of its members
-    may have delayed computation until reaching the build phase.
-
-    This is so we can, for example, get the Numpy include dirs
-    after pip has installed Numpy for us if it wasn't already
-    on the system.
-    """
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._finalized = False
-        self._hooks = {}
-
-    def add_hook(self, member, func):
-        """
-        Add a hook to dynamically compute a member.
-
-        Parameters
-        ----------
-        member : string
-            The name of the member
-
-        func : callable
-            The function to call to get dynamically-computed values
-            for the member.
-        """
-        self._hooks[member] = func
-
-    def finalize(self):
-        self._finalized = True
-
-    class DelayedMember(property):
-        def __init__(self, name):
-            self._name = name
-
-        def __get__(self, obj, objtype=None):
-            result = getattr(obj, '_' + self._name, [])
-
-            if obj._finalized:
-                if self._name in obj._hooks:
-                    result = obj._hooks[self._name]() + result
-
-            return result
-
-        def __set__(self, obj, value):
-            setattr(obj, '_' + self._name, value)
-
-    include_dirs = DelayedMember('include_dirs')
-
-
 class Numpy(SetupPackage):
     name = "numpy"
 
-    @staticmethod
-    def include_dirs_hook():
-        if hasattr(builtins, '__NUMPY_SETUP__'):
-            del builtins.__NUMPY_SETUP__
-        import numpy
-        importlib.reload(numpy)
-
-        ext = Extension('test', [])
-        ext.include_dirs.append(numpy.get_include())
-        if not has_include_file(
-                ext.include_dirs, os.path.join("numpy", "arrayobject.h")):
-            _log.warning(
-                "The C headers for numpy could not be found. "
-                "You may need to install the development package")
-
-        return [numpy.get_include()]
-
     def add_flags(self, ext):
-        ext.add_hook('include_dirs', self.include_dirs_hook)
+        import numpy as np
+        ext.include_dirs.append(np.get_include())
         ext.define_macros.extend([
             # Ensure that PY_ARRAY_UNIQUE_SYMBOL is uniquely defined for each
             # extension.