WIN: Rewrite local freetype build script

Kojoley · Kojoley · commit 9b6989e8aa94 · 2017-10-11T03:25:02.000+03:00
These changes should fix some build problems on windows:
 - Mismatch Python MSVC runtime version and local freetype
 - `zlib` and `libpng` linking
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -95,17 +95,6 @@ install:
   # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
   - pip install -q pytest "pytest-cov>=2.3.1" pytest-rerunfailures pytest-timeout
 
-  # Let the install prefer the static builds of the libs
-  - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib
-  - mkdir lib || cmd /c "exit /b 0"
-  - copy /y %LIBRARY_LIB%\zlibstatic.lib lib\z.lib
-  - copy /y %LIBRARY_LIB%\libpng_static.lib lib\png.lib
-  # These z.lib / png.lib are not static versions but files which end up as
-  # dependencies to the dll file. This is fine for the conda build, but not here
-  # and for the wheels
-  - del %LIBRARY_LIB%\png.lib
-  - del %LIBRARY_LIB%\z.lib
-  - set MPLBASEDIRLIST=%CONDA_PREFIX%\Library\;.
   # enables the local freetype build
   - copy ci\travis\setup.cfg .
   # Show the installed packages + versions
@@ -146,9 +135,6 @@ after_test:
   # And now the conda build after a cleanup...
   # cleanup build files so that they don't pollute the conda build but keep the wheel in dist...
   - git clean -xdfq -e dist/
-  # cleanup the environment so that the test-environment does not leak into the conda build...
-  - set MPLBASEDIRLIST=
-  - set LIBRARY_LIB=
   - deactivate
   - path
   - where python
diff --git a/INSTALL.rst b/INSTALL.rst
@@ -332,16 +332,6 @@ without fiddling with environment variables::
   # for Python 2.7
   conda install -c conda-forge backports.functools_lru_cache
 
-  # copy the libs which have "wrong" names
-  set LIBRARY_LIB=%CONDA_DEFAULT_ENV%\Library\lib
-  mkdir lib || cmd /c "exit /b 0"
-  copy %LIBRARY_LIB%\zlibstatic.lib lib\z.lib
-  copy %LIBRARY_LIB%\libpng_static.lib lib\png.lib
-
-  # Make the header files and the rest of the static libs available during the build
-  # CONDA_DEFAULT_ENV is a env variable which is set to the currently active environment path
-  set MPLBASEDIRLIST=%CONDA_DEFAULT_ENV%\Library\;.
-
   # build the wheel
   python setup.py bdist_wheel
 
diff --git a/ci/conda_recipe/bld.bat b/ci/conda_recipe/bld.bat
@@ -1,16 +1,9 @@
-set LIBPATH=%LIBRARY_LIB%;
-set INCLUDE=%INCLUDE%;%PREFIX%\Library\include\freetype2
-
 ECHO [directories] > setup.cfg
 ECHO basedirlist = %LIBRARY_PREFIX% >> setup.cfg
 ECHO [packages] >> setup.cfg
 ECHO tests = False >> setup.cfg
 ECHO sample_data = False >> setup.cfg
 ECHO toolkits_tests = False >> setup.cfg
 
-@rem workaround for https://github.com/matplotlib/matplotlib/issues/6460
-@rem see also https://github.com/conda-forge/libpng-feedstock/pull/4
-copy /y %LIBRARY_LIB%\libpng16.lib %LIBRARY_LIB%\png.lib
-
 %PYTHON% setup.py install --single-version-externally-managed --record=record.txt
 if errorlevel 1 exit 1
diff --git a/setup.py b/setup.py
@@ -136,11 +136,11 @@ def run(self):
 
 
 class BuildExtraLibraries(BuildExtCommand):
-    def run(self):
+    def build_extensions(self):
         for package in good_packages:
-            package.do_custom_build()
+            package.do_custom_build(self)
 
-        return BuildExtCommand.run(self)
+        return BuildExtCommand.build_extensions(self)
 
 
 cmdclass = versioneer.get_cmdclass()
diff --git a/setupext.py b/setupext.py
@@ -124,6 +124,7 @@ def get_win32_compiler():
     """
     Determine the compiler being used on win32.
     """
+    # TODO: rewrite to use distutils
     # Used to determine mingw32 or msvc
     # This is pretty bad logic, someone know a better way?
     for v in sys.argv:
@@ -151,7 +152,8 @@ def has_include_file(include_dirs, filename):
     directories in `include_dirs`.
     """
     if sys.platform == 'win32':
-        include_dirs += os.environ.get('INCLUDE', '.').split(';')
+        include_dirs = list(include_dirs)
+        include_dirs += os.environ.get('INCLUDE', '.').split(os.pathsep)
     for dir in include_dirs:
         if os.path.exists(os.path.join(dir, filename)):
             return True
@@ -180,10 +182,14 @@ def get_base_dirs():
         return os.environ.get('MPLBASEDIRLIST').split(os.pathsep)
 
     win_bases = ['win32_static', ]
-    # on conda windows, we also add the <installdir>\Library of the local interpreter,
+    # on conda windows, we also add the <conda_env_dir>\Library,
     # as conda installs libs/includes there
-    if os.getenv('CONDA_DEFAULT_ENV'):
-        win_bases.append(os.path.join(os.getenv('CONDA_DEFAULT_ENV'), "Library"))
+    # env var names mess: https://github.com/conda/conda/issues/2312
+    conda_env_path = os.getenv('CONDA_PREFIX')  # conda >= 4.1
+    if not conda_env_path:
+        conda_env_path = os.getenv('CONDA_DEFAULT_ENV')  # conda < 4.1
+    if conda_env_path and os.path.isdir(conda_env_path):
+        win_bases.append(os.path.join(conda_env_path, "Library"))
 
     basedir_map = {
         'win32': win_bases,
@@ -541,11 +547,16 @@ def _check_for_pkg_config(self, package, include_file, min_version=None,
 
         return 'version %s' % version
 
-    def do_custom_build(self):
+    def do_custom_build(self, cmd):
         """
         If a package needs to do extra custom things, such as building a
         third-party library, before building an extension, it should
         override this method.
+
+        Parameters
+        ----------
+        cmd : distutils.command.build_ext
+            The 'build_ext' instance
         """
         pass
 
@@ -1088,19 +1099,24 @@ def version_from_header(self):
                                 patch = value
                 return '.'.join([major, minor, patch])
 
+    def get_local_freetype_path(self, version=LOCAL_FREETYPE_VERSION):
+        return os.path.join('build', 'freetype-{0}'.format(version))
+
+    def get_local_freetype_lib_path(self, version=LOCAL_FREETYPE_VERSION):
+        if sys.platform == 'win32':
+            libfreetype = 'libfreetype.lib'
+        else:
+            libfreetype = 'libfreetype.a'
+
+        src_path = self.get_local_freetype_path(version)
+        return os.path.join(src_path, 'objs', '.libs', libfreetype)
+
     def add_flags(self, ext):
         if options.get('local_freetype'):
-            src_path = os.path.join(
-                'build', 'freetype-{0}'.format(LOCAL_FREETYPE_VERSION))
-            # Statically link to the locally-built freetype.
-            # This is certainly broken on Windows.
+            src_path = self.get_local_freetype_path()
+            # link to the locally-built freetype.
             ext.include_dirs.insert(0, os.path.join(src_path, 'include'))
-            if sys.platform == 'win32':
-                libfreetype = 'libfreetype.lib'
-            else:
-                libfreetype = 'libfreetype.a'
-            ext.extra_objects.insert(
-                0, os.path.join(src_path, 'objs', '.libs', libfreetype))
+            ext.extra_objects.insert(0, self.get_local_freetype_lib_path())
             ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'local'))
         else:
             pkg_config.setup_extension(
@@ -1111,26 +1127,21 @@ def add_flags(self, ext):
                     'lib/freetype2/include/freetype2'],
                 default_library_dirs=[
                     'freetype2/lib'],
-                default_libraries=['freetype', 'z'])
+                default_libraries=['freetype'])
             ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'system'))
 
-    def do_custom_build(self):
-        # We're using a system freetype
+    def do_custom_build(self, cmd):
         if not options.get('local_freetype'):
+            # We're using a system freetype
             return
 
-        src_path = os.path.join(
-            'build', 'freetype-{0}'.format(LOCAL_FREETYPE_VERSION))
-
-        # We've already built freetype
-        if sys.platform == 'win32':
-            libfreetype = 'libfreetype.lib'
-        else:
-            libfreetype = 'libfreetype.a'
-
-        if os.path.isfile(os.path.join(src_path, 'objs', '.libs', libfreetype)):
+        libdir = os.path.dirname(self.get_local_freetype_lib_path())
+        if os.path.isfile(self.get_local_freetype_lib_path()):
+            # We've already built freetype
+            # TODO: This will prevent x86/x64 debug/release switch
             return
 
+        src_path = self.get_local_freetype_path()
         tarball = 'freetype-{0}.tar.gz'.format(LOCAL_FREETYPE_VERSION)
         tarball_path = os.path.join('build', tarball)
         try:
@@ -1217,40 +1228,72 @@ def do_custom_build(self):
             subprocess.check_call(
                 [cflags + 'make'], shell=True, cwd=src_path)
         else:
-            # compilation on windows
-            FREETYPE_BUILD_CMD = """\
-call "%ProgramFiles%\\Microsoft SDKs\\Windows\\v7.0\\Bin\\SetEnv.Cmd" /Release /{xXX} /xp
-call "{vcvarsall}" {xXX}
-set MSBUILD=C:\\Windows\\Microsoft.NET\\Framework\\v4.0.30319\\MSBuild.exe
-rd /S /Q %FREETYPE%\\objs
-%MSBUILD% %FREETYPE%\\builds\\windows\\{vc20xx}\\freetype.sln /t:Clean;Build /p:Configuration="{config}";Platform={WinXX}
-echo Build completed, moving result"
-:: move to the "normal" path for the unix builds...
-mkdir %FREETYPE%\\objs\\.libs
-:: REMINDER: fix when changing the version
-copy %FREETYPE%\\objs\\{vc20xx}\\{xXX}\\freetype261.lib %FREETYPE%\\objs\\.libs\\libfreetype.lib
-if errorlevel 1 (
-  rem This is a py27 version, which has a different location for the lib file :-/
-  copy %FREETYPE%\\objs\\win32\\{vc20xx}\\freetype261.lib %FREETYPE%\\objs\\.libs\\libfreetype.lib
-)
-"""
-            from setup_external_compile import fixproj, prepare_build_cmd, VS2010, X64, tar_extract
-            # Note: freetype has no build profile for 2014, so we don't bother...
-            vc = 'vc2010' if VS2010 else 'vc2008'
-            WinXX = 'x64' if X64 else 'Win32'
+            assert cmd.compiler.compiler_type == 'msvc', (
+                'Support for other compilers is not implemented')
+
+            #from distutils.spawn import find_executable
+            from distutils.msvccompiler import get_build_version
+            from setup_external_compile import fixproj, X64, tar_extract
+
+            if not cmd.compiler.initialized:
+                cmd.compiler.initialize()
+
+            vc = {9: 'vc2005', 11: 'vc2010'}[min(int(get_build_version()), 11)]
+            ftproj = os.path.join(src_path, 'builds', 'windows', vc, 'freetype')
+
+            vc_config = 'Debug' if cmd.debug else 'Release'
+            #vc_config += '|x64' if X64 else '|Win32'
+            vc_platform = 'x64' if X64 else 'x86'
+
             tar_extract(tarball_path, "build")
-            # This is only false for py2.7, even on py3.5...
-            if not VS2010:
-                fixproj(os.path.join(src_path, 'builds', 'windows', vc, 'freetype.sln'), WinXX)
-                fixproj(os.path.join(src_path, 'builds', 'windows', vc, 'freetype.vcproj'), WinXX)
 
-            cmdfile = os.path.join("build", 'build_freetype.cmd')
-            with open(cmdfile, 'w') as cmd:
-                cmd.write(prepare_build_cmd(FREETYPE_BUILD_CMD, vc20xx=vc, WinXX=WinXX,
-                                            config='Release' if VS2010 else 'LIB Release'))
+            if not cmd.compiler.initialized:
+                cmd.compiler.initialize()
 
-            os.environ['FREETYPE'] = src_path
-            subprocess.check_call([cmdfile], shell=True)
+            def spawn(command):
+                command = list(command)
+                if hasattr(cmd.compiler, 'find_exe'):
+                    executable = cmd.compiler.find_exe(command[0])
+                    if not executable:
+                        raise ValueError("Couldn't find %s" % executable)
+                    return cmd.compiler.spawn([executable] + command[1:])
+                else:
+                    return cmd.compiler.spawn(command)
+
+            if get_build_version() < 11.0:
+                vc_config = 'LIB ' + vc_config
+                # monkey-patch project files, because
+                # vc2005 and vc2008 have only Win32 targets
+                # TODO: fix `win32` output dir to `x64`?
+                fixproj(ftproj + '.sln', vc_platform)
+                fixproj(ftproj + '.vcproj', vc_platform)
+                spawn(['vcbuild', ftproj + '.sln', '/M', '/time',
+                       '/platform:{}'.format(vc_platform),
+                       '{config}|{platform}'.format(config=vc_config,
+                           platform='x64' if X64 else 'Win32')
+                      ])
+                postfix = '_D' if cmd.debug else ''
+                builddir = os.path.join(src_path, 'objs', 'win32', vc)
+            else:
+                spawn(['msbuild', ftproj + '.sln', '/m', '/t:Clean;Build',
+                       '/toolsversion:{}'.format(get_build_version()),
+                       ('/p:Configuration="{config}"'
+                        ';Platform={platform}'
+                        ';VisualStudioVersion={version}'
+                        ';ToolsVersion={version}'
+                        ';PlatformToolset=v{toolset:d}'
+                       ).format(config=vc_config, platform=vc_platform,
+                                version=get_build_version(),
+                                toolset=int(get_build_version() * 10)),
+                      ])
+                postfix = 'd' if cmd.debug else ''
+                builddir = os.path.join(src_path, 'objs', vc, vc_platform)
+
+            verstr = LOCAL_FREETYPE_VERSION.replace('.', '')
+            buildname = 'freetype{0}{1}.lib'.format(verstr, postfix)
+            buildpath = os.path.join(builddir, buildname)
+            assert os.path.isfile(buildpath)
+            os.renames(buildpath, self.get_local_freetype_lib_path())
 
 
 class FT2Font(SetupPackage):
@@ -1267,6 +1310,23 @@ def get_extension(self):
         Numpy().add_flags(ext)
         return ext
 
+    def do_custom_build(self, cmd):
+        if cmd.compiler.compiler_type != 'msvc':
+            return
+
+        if not cmd.compiler.initialized:
+            cmd.compiler.initialize()
+
+        # TODO: What about compiler pathes?
+        if not has_include_file(cmd.compiler.include_dirs, 'stdint.h'):
+            if os.getenv('CONDA_DEFAULT_ENV'):
+                # TODO: spawn conda install msinttypes?
+                pass
+            else:
+                # TODO: download from https://code.google.com/p/msinttypes/
+                #cmd.compiler.add_include_dir(...)
+                pass
+
 
 class Png(SetupPackage):
     name = "png"
@@ -1300,13 +1360,24 @@ def check(self):
             raise
 
     def get_extension(self):
+        # TODO: implement a dynamic/static linking switch
+        zlib = 'z' if sys.platform != 'win32' else 'zlibstatic'
+        if sys.platform != 'win32':
+            png = 'png'
+            zlib = 'z'
+        elif os.getenv('CONDA_DEFAULT_ENV'):
+            # libpng in conda copies versioned lib to unversioned
+            png = 'libpng_static'
+        else:
+            # TODO: there is no good solution with our build system
+            png = 'libpng16_static'
         sources = [
             'src/_png.cpp',
             'src/mplutils.cpp'
             ]
         ext = make_extension('matplotlib._png', sources)
         pkg_config.setup_extension(
-            ext, 'libpng', default_libraries=['png', 'z'],
+            ext, 'libpng', default_libraries=[png, zlib],
             alt_exec='libpng-config --ldflags')
         Numpy().add_flags(ext)
         return ext
