From a4c56f209f6422e36e889bc518e832aee047d764 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Tue, 10 Oct 2017 19:12:32 +0300 Subject: [PATCH 1/3] BLD: Rewrite local freetype build script These changes should fix some build problems on windows: - Mismatch Python MSVC runtime version and local freetype - `zlib` and `libpng` linking --- .appveyor.yml | 14 --- INSTALL.rst | 10 --- ci/conda_recipe/bld.bat | 7 -- setup.py | 6 +- setupext.py | 191 +++++++++++++++++++++++++++------------- 5 files changed, 134 insertions(+), 94 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 2340e11f4bab..86f7a5654656 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -99,17 +99,6 @@ install: curl -sL https://github.com/python/cpython/pull/1224.patch | patch -fsup 1 -d %CONDA_PREFIX% ) || ( set errorlevel= ) - # Let the install prefer the static builds of the libs - - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib - - mkdir lib || cmd /c "exit /b 0" - - copy /y %LIBRARY_LIB%\zlibstatic.lib lib\z.lib - - copy /y %LIBRARY_LIB%\libpng_static.lib lib\png.lib - # These z.lib / png.lib are not static versions but files which end up as - # dependencies to the dll file. This is fine for the conda build, but not here - # and for the wheels - - del %LIBRARY_LIB%\png.lib - - del %LIBRARY_LIB%\z.lib - - set MPLBASEDIRLIST=%CONDA_PREFIX%\Library\;. # enables the local freetype build - copy ci\travis\setup.cfg . # Show the installed packages + versions @@ -150,9 +139,6 @@ after_test: # And now the conda build after a cleanup... # cleanup build files so that they don't pollute the conda build but keep the wheel in dist... - git clean -xdfq -e dist/ - # cleanup the environment so that the test-environment does not leak into the conda build... - - set MPLBASEDIRLIST= - - set LIBRARY_LIB= - deactivate - path - where python diff --git a/INSTALL.rst b/INSTALL.rst index 354a34eb89c5..872de4e224b0 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -332,16 +332,6 @@ without fiddling with environment variables:: # for Python 2.7 conda install -c conda-forge backports.functools_lru_cache - # copy the libs which have "wrong" names - set LIBRARY_LIB=%CONDA_DEFAULT_ENV%\Library\lib - mkdir lib || cmd /c "exit /b 0" - copy %LIBRARY_LIB%\zlibstatic.lib lib\z.lib - copy %LIBRARY_LIB%\libpng_static.lib lib\png.lib - - # Make the header files and the rest of the static libs available during the build - # CONDA_DEFAULT_ENV is a env variable which is set to the currently active environment path - set MPLBASEDIRLIST=%CONDA_DEFAULT_ENV%\Library\;. - # build the wheel python setup.py bdist_wheel diff --git a/ci/conda_recipe/bld.bat b/ci/conda_recipe/bld.bat index a7810d418d2f..550b25cb4d3b 100644 --- a/ci/conda_recipe/bld.bat +++ b/ci/conda_recipe/bld.bat @@ -1,6 +1,3 @@ -set LIBPATH=%LIBRARY_LIB%; -set INCLUDE=%INCLUDE%;%PREFIX%\Library\include\freetype2 - ECHO [directories] > setup.cfg ECHO basedirlist = %LIBRARY_PREFIX% >> setup.cfg ECHO [packages] >> setup.cfg @@ -8,9 +5,5 @@ ECHO tests = False >> setup.cfg ECHO sample_data = False >> setup.cfg ECHO toolkits_tests = False >> setup.cfg -@rem workaround for https://github.com/matplotlib/matplotlib/issues/6460 -@rem see also https://github.com/conda-forge/libpng-feedstock/pull/4 -copy /y %LIBRARY_LIB%\libpng16.lib %LIBRARY_LIB%\png.lib - %PYTHON% setup.py install --single-version-externally-managed --record=record.txt if errorlevel 1 exit 1 diff --git a/setup.py b/setup.py index 9a60d269cf2c..4fb5bbbda321 100644 --- a/setup.py +++ b/setup.py @@ -136,11 +136,11 @@ def run(self): class BuildExtraLibraries(BuildExtCommand): - def run(self): + def build_extensions(self): for package in good_packages: - package.do_custom_build() + package.do_custom_build(self) - return BuildExtCommand.run(self) + return BuildExtCommand.build_extensions(self) cmdclass = versioneer.get_cmdclass() diff --git a/setupext.py b/setupext.py index c56ee47cd30e..707f10844fb9 100644 --- a/setupext.py +++ b/setupext.py @@ -124,6 +124,7 @@ def get_win32_compiler(): """ Determine the compiler being used on win32. """ + # TODO: rewrite to use distutils # Used to determine mingw32 or msvc # This is pretty bad logic, someone know a better way? for v in sys.argv: @@ -151,7 +152,8 @@ def has_include_file(include_dirs, filename): directories in `include_dirs`. """ if sys.platform == 'win32': - include_dirs += os.environ.get('INCLUDE', '.').split(';') + include_dirs = list(include_dirs) + include_dirs += os.environ.get('INCLUDE', '.').split(os.pathsep) for dir in include_dirs: if os.path.exists(os.path.join(dir, filename)): return True @@ -180,10 +182,14 @@ def get_base_dirs(): return os.environ.get('MPLBASEDIRLIST').split(os.pathsep) win_bases = ['win32_static', ] - # on conda windows, we also add the \Library of the local interpreter, + # on conda windows, we also add the \Library, # as conda installs libs/includes there - if os.getenv('CONDA_DEFAULT_ENV'): - win_bases.append(os.path.join(os.getenv('CONDA_DEFAULT_ENV'), "Library")) + # env var names mess: https://github.com/conda/conda/issues/2312 + conda_env_path = os.getenv('CONDA_PREFIX') # conda >= 4.1 + if not conda_env_path: + conda_env_path = os.getenv('CONDA_DEFAULT_ENV') # conda < 4.1 + if conda_env_path and os.path.isdir(conda_env_path): + win_bases.append(os.path.join(conda_env_path, "Library")) basedir_map = { 'win32': win_bases, @@ -541,11 +547,16 @@ def _check_for_pkg_config(self, package, include_file, min_version=None, return 'version %s' % version - def do_custom_build(self): + def do_custom_build(self, cmd): """ If a package needs to do extra custom things, such as building a third-party library, before building an extension, it should override this method. + + Parameters + ---------- + cmd : distutils.command.build_ext + The 'build_ext' instance """ pass @@ -1094,19 +1105,24 @@ def version_from_header(self): patch = value return '.'.join([major, minor, patch]) + def get_local_freetype_path(self, version=LOCAL_FREETYPE_VERSION): + return os.path.join('build', 'freetype-{0}'.format(version)) + + def get_local_freetype_lib_path(self, version=LOCAL_FREETYPE_VERSION): + if sys.platform == 'win32': + libfreetype = 'libfreetype.lib' + else: + libfreetype = 'libfreetype.a' + + src_path = self.get_local_freetype_path(version) + return os.path.join(src_path, 'objs', '.libs', libfreetype) + def add_flags(self, ext): if options.get('local_freetype'): - src_path = os.path.join( - 'build', 'freetype-{0}'.format(LOCAL_FREETYPE_VERSION)) - # Statically link to the locally-built freetype. - # This is certainly broken on Windows. + src_path = self.get_local_freetype_path() + # link to the locally-built freetype. ext.include_dirs.insert(0, os.path.join(src_path, 'include')) - if sys.platform == 'win32': - libfreetype = 'libfreetype.lib' - else: - libfreetype = 'libfreetype.a' - ext.extra_objects.insert( - 0, os.path.join(src_path, 'objs', '.libs', libfreetype)) + ext.extra_objects.insert(0, self.get_local_freetype_lib_path()) ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'local')) else: pkg_config.setup_extension( @@ -1117,26 +1133,21 @@ def add_flags(self, ext): 'lib/freetype2/include/freetype2'], default_library_dirs=[ 'freetype2/lib'], - default_libraries=['freetype', 'z']) + default_libraries=['freetype']) ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'system')) - def do_custom_build(self): - # We're using a system freetype + def do_custom_build(self, cmd): if not options.get('local_freetype'): + # We're using a system freetype return - src_path = os.path.join( - 'build', 'freetype-{0}'.format(LOCAL_FREETYPE_VERSION)) - - # We've already built freetype - if sys.platform == 'win32': - libfreetype = 'libfreetype.lib' - else: - libfreetype = 'libfreetype.a' - - if os.path.isfile(os.path.join(src_path, 'objs', '.libs', libfreetype)): + libdir = os.path.dirname(self.get_local_freetype_lib_path()) + if os.path.isfile(self.get_local_freetype_lib_path()): + # We've already built freetype + # TODO: This will prevent x86/x64 debug/release switch return + src_path = self.get_local_freetype_path() tarball = 'freetype-{0}.tar.gz'.format(LOCAL_FREETYPE_VERSION) tarball_path = os.path.join('build', tarball) try: @@ -1226,40 +1237,72 @@ def do_custom_build(self): subprocess.check_call( [cflags + 'make'], shell=True, cwd=src_path) else: - # compilation on windows - FREETYPE_BUILD_CMD = """\ -call "%ProgramFiles%\\Microsoft SDKs\\Windows\\v7.0\\Bin\\SetEnv.Cmd" /Release /{xXX} /xp -call "{vcvarsall}" {xXX} -set MSBUILD=C:\\Windows\\Microsoft.NET\\Framework\\v4.0.30319\\MSBuild.exe -rd /S /Q %FREETYPE%\\objs -%MSBUILD% %FREETYPE%\\builds\\windows\\{vc20xx}\\freetype.sln /t:Clean;Build /p:Configuration="{config}";Platform={WinXX} -echo Build completed, moving result" -:: move to the "normal" path for the unix builds... -mkdir %FREETYPE%\\objs\\.libs -:: REMINDER: fix when changing the version -copy %FREETYPE%\\objs\\{vc20xx}\\{xXX}\\freetype261.lib %FREETYPE%\\objs\\.libs\\libfreetype.lib -if errorlevel 1 ( - rem This is a py27 version, which has a different location for the lib file :-/ - copy %FREETYPE%\\objs\\win32\\{vc20xx}\\freetype261.lib %FREETYPE%\\objs\\.libs\\libfreetype.lib -) -""" - from setup_external_compile import fixproj, prepare_build_cmd, VS2010, X64, tar_extract - # Note: freetype has no build profile for 2014, so we don't bother... - vc = 'vc2010' if VS2010 else 'vc2008' - WinXX = 'x64' if X64 else 'Win32' + assert cmd.compiler.compiler_type == 'msvc', ( + 'Support for other compilers is not implemented') + + from distutils.msvccompiler import get_build_version + from setup_external_compile import fixproj, X64, tar_extract + + if not cmd.compiler.initialized: + cmd.compiler.initialize() + + # NOTE: need change on upgrade of local freetype to newer versions + vc = {9: 'vc2008', 10: 'vc2010'}[min(int(get_build_version()), 10)] + ftproj = os.path.join(src_path, 'builds', 'windows', vc, 'freetype') + + vc_config = 'Debug' if cmd.debug else 'Release' + tar_extract(tarball_path, "build") - # This is only false for py2.7, even on py3.5... - if not VS2010: - fixproj(os.path.join(src_path, 'builds', 'windows', vc, 'freetype.sln'), WinXX) - fixproj(os.path.join(src_path, 'builds', 'windows', vc, 'freetype.vcproj'), WinXX) - cmdfile = os.path.join("build", 'build_freetype.cmd') - with open(cmdfile, 'w') as cmd: - cmd.write(prepare_build_cmd(FREETYPE_BUILD_CMD, vc20xx=vc, WinXX=WinXX, - config='Release' if VS2010 else 'LIB Release')) + if not cmd.compiler.initialized: + cmd.compiler.initialize() - os.environ['FREETYPE'] = src_path - subprocess.check_call([cmdfile], shell=True) + def spawn(command): + command = list(command) + if hasattr(cmd.compiler, 'find_exe'): + executable = cmd.compiler.find_exe(command[0]) + if not executable: + raise ValueError("Couldn't find %s" % command[0]) + return cmd.compiler.spawn([executable] + command[1:]) + else: + return cmd.compiler.spawn(command) + + if get_build_version() < 11.0: + vc_platform = 'x64' if X64 else 'Win32' + vc_config = 'LIB ' + vc_config + # monkey-patch project files, because + # vc2005 and vc2008 have only Win32 targets + # TODO: fix `win32` output dir to `x64`? + fixproj(ftproj + '.sln', vc_platform) + fixproj(ftproj + '.vcproj', vc_platform) + spawn(['vcbuild', ftproj + '.sln', '/M', '/time', + '/platform:{}'.format(vc_platform), + '{config}|{platform}'.format(config=vc_config, + platform=vc_platform) + ]) + postfix = '_D' if cmd.debug else '' + builddir = os.path.join(src_path, 'objs', 'win32', vc) + else: + vc_platform = 'x64' if X64 else 'x86' + spawn(['msbuild', ftproj + '.sln', '/m', '/t:Clean;Build', + '/toolsversion:{}'.format(get_build_version()), + ('/p:Configuration={config}' + ';Platform={platform}' + ';VisualStudioVersion={version}' + ';ToolsVersion={version}' + ';PlatformToolset=v{toolset:d}' + ).format(config=vc_config, platform=vc_platform, + version=get_build_version(), + toolset=int(get_build_version() * 10)), + ]) + postfix = 'd' if cmd.debug else '' + builddir = os.path.join(src_path, 'objs', vc, vc_platform) + + verstr = LOCAL_FREETYPE_VERSION.replace('.', '') + buildname = 'freetype{0}{1}.lib'.format(verstr, postfix) + buildpath = os.path.join(builddir, buildname) + assert os.path.isfile(buildpath) + os.renames(buildpath, self.get_local_freetype_lib_path()) class FT2Font(SetupPackage): @@ -1276,6 +1319,23 @@ def get_extension(self): Numpy().add_flags(ext) return ext + def do_custom_build(self, cmd): + if cmd.compiler.compiler_type != 'msvc': + return + + if not cmd.compiler.initialized: + cmd.compiler.initialize() + + # TODO: What about compiler pathes? + if not has_include_file(cmd.compiler.include_dirs, 'stdint.h'): + if os.getenv('CONDA_DEFAULT_ENV'): + # TODO: spawn conda install msinttypes? + pass + else: + # TODO: download from https://code.google.com/p/msinttypes/ + #cmd.compiler.add_include_dir(...) + pass + class Png(SetupPackage): name = "png" @@ -1309,13 +1369,24 @@ def check(self): raise def get_extension(self): + # TODO: implement a dynamic/static linking switch + zlib = 'z' if sys.platform != 'win32' else 'zlibstatic' + if sys.platform != 'win32': + png = 'png' + zlib = 'z' + elif os.getenv('CONDA_DEFAULT_ENV'): + # libpng in conda copies versioned lib to unversioned + png = 'libpng_static' + else: + # TODO: there is no good solution with our build system + png = 'libpng16_static' sources = [ 'src/_png.cpp', 'src/mplutils.cpp' ] ext = make_extension('matplotlib._png', sources) pkg_config.setup_extension( - ext, 'libpng', default_libraries=['png', 'z'], + ext, 'libpng', default_libraries=[png, zlib], alt_exec='libpng-config --ldflags') Numpy().add_flags(ext) return ext From ae29b0d506e07cf2fb6d91a78f3e586651781f83 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Thu, 12 Oct 2017 17:32:43 +0300 Subject: [PATCH 2/3] BLD: Removed `build_alllocal.cmd` as not needed --- build_alllocal.cmd | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 build_alllocal.cmd diff --git a/build_alllocal.cmd b/build_alllocal.cmd deleted file mode 100644 index 7a357302c4ae..000000000000 --- a/build_alllocal.cmd +++ /dev/null @@ -1,36 +0,0 @@ -:: This assumes you have installed all the dependencies via conda packages: -:: # create a new environment with the required packages -:: conda create -n "matplotlib_build" python=3.4 numpy python-dateutil pyparsing pytz tornado "cycler>=0.10" tk libpng zlib freetype -:: activate matplotlib_build -:: if you want qt backend, you also have to install pyqt -:: conda install pyqt -:: # this package is only available in the conda-forge channel -:: conda install -c conda-forge msinttypes -:: if you build on py2.7: -:: conda install -c conda-forge backports.functools_lru_cache - -set TARGET=bdist_wheel -IF [%1]==[] ( - echo Using default target: %TARGET% -) else ( - set TARGET=%1 - echo Using user supplied target: %TARGET% -) - -IF NOT DEFINED CONDA_PREFIX ( - echo No Conda env activated: you need to create a conda env with the right packages and activate it! - GOTO:eof -) - -:: copy the libs which have "wrong" names -set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib -mkdir lib || cmd /c "exit /b 0" -copy %LIBRARY_LIB%\zlibstatic.lib lib\z.lib -copy %LIBRARY_LIB%\libpng_static.lib lib\png.lib - -:: Make the header files and the rest of the static libs available during the build -:: CONDA_PREFIX is a env variable which is set to the currently active environment path -set MPLBASEDIRLIST=%CONDA_PREFIX%\Library\;. - -:: build the target -python setup.py %TARGET% From ea1c43a1bf4474dd942447ffd4e3fe3c247998f4 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Tue, 17 Oct 2017 19:26:45 +0300 Subject: [PATCH 3/3] BLD: Detect /MT flag and build corresponding freetype --- setupext.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setupext.py b/setupext.py index 707f10844fb9..a09c5be6bbe1 100644 --- a/setupext.py +++ b/setupext.py @@ -1267,6 +1267,16 @@ def spawn(command): else: return cmd.compiler.spawn(command) + if cmd.debug: + mt = '/MTd' in cmd.compiler.compile_options_debug + else: + mt = '/MT' in cmd.compiler.compile_options + + if mt: + vc_config += ' Multithreaded' + + postfix = 'MT' if mt else '' + if get_build_version() < 11.0: vc_platform = 'x64' if X64 else 'Win32' vc_config = 'LIB ' + vc_config @@ -1280,7 +1290,7 @@ def spawn(command): '{config}|{platform}'.format(config=vc_config, platform=vc_platform) ]) - postfix = '_D' if cmd.debug else '' + postfix += '_D' if cmd.debug else '' builddir = os.path.join(src_path, 'objs', 'win32', vc) else: vc_platform = 'x64' if X64 else 'x86' @@ -1295,7 +1305,7 @@ def spawn(command): version=get_build_version(), toolset=int(get_build_version() * 10)), ]) - postfix = 'd' if cmd.debug else '' + postfix += 'd' if cmd.debug else '' builddir = os.path.join(src_path, 'objs', vc, vc_platform) verstr = LOCAL_FREETYPE_VERSION.replace('.', '')