diff --git a/.appveyor.yml b/.appveyor.yml index aa747afd2d03..9ddf5068592f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -68,7 +68,7 @@ install: - activate test-environment - echo %PYTHON_VERSION% %TARGET_ARCH% # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124 - - pip install -q "pytest>=3.4,!=4.6.0" "pytest-cov>=2.3.1" pytest-rerunfailures pytest-timeout pytest-xdist "pandas!=0.25.0" + - pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis36.txt # Apply patch to `subprocess` on Python versions > 2 and < 3.6.3 # https://github.com/matplotlib/matplotlib/issues/9176 @@ -76,16 +76,6 @@ install: curl -sL https://github.com/python/cpython/pull/1224.patch | patch -fsup 1 -d %CONDA_PREFIX% ) || cmd /c "exit /b 0" - # Let the install prefer the static builds of the libs - - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib - - mkdir lib || cmd /c "exit /b 0" - - copy /y %LIBRARY_LIB%\zlibstatic.lib lib\zlib.lib - - copy /y %LIBRARY_LIB%\libpng16_static.lib lib\libpng16.lib - # These z.lib / png.lib are not static versions but files which end up as - # dependencies to the dll file. This is fine for the conda build, but not here - # and for the wheels - - del %LIBRARY_LIB%\png.lib - - del %LIBRARY_LIB%\z.lib # enables the local freetype build - set MPLLOCALFREETYPE=1 # Show the installed packages + versions @@ -97,9 +87,9 @@ test_script: - pip install -ve . # these should show no z, png, or freetype dll... - set "DUMPBIN=%VS140COMNTOOLS%\..\..\VC\bin\dumpbin.exe" - - '"%DUMPBIN%" /DEPENDENTS lib\matplotlib\ft2font*.pyd | findstr freetype.*.dll && exit /b 1 || exit /b 0' - - '"%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr z.*.dll && exit /b 1 || exit /b 0' - - '"%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr png.*.dll && exit /b 1 || exit /b 0' + - 'if x%MPLSTATICBUILD% == xTrue "%DUMPBIN%" /DEPENDENTS lib\matplotlib\ft2font*.pyd | findstr freetype.*.dll && exit /b 1 || exit /b 0' + - 'if x%MPLSTATICBUILD% == xTrue "%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr z.*.dll && exit /b 1 || exit /b 0' + - 'if x%MPLSTATICBUILD% == xTrue "%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr png.*.dll && exit /b 1 || exit /b 0' # this are optional dependencies so that we don't skip so many tests... - if x%TEST_ALL% == xyes conda install -q ffmpeg inkscape miktex pillow @@ -116,6 +106,7 @@ test_script: after_test: # After the tests were a success, build wheels with the static libs # Hide the output, the copied files really clutter the build log... + - set MPLSTATICBUILD=True - 'python setup.py bdist_wheel > NUL:' - dir dist\ - echo finished... diff --git a/.flake8 b/.flake8 index 88083c30645c..2aad08357dc9 100644 --- a/.flake8 +++ b/.flake8 @@ -28,6 +28,7 @@ exclude = per-file-ignores = setup.py: E402 + setupext.py: E501 tools/subset.py: E221, E251, E261, E302, E501 diff --git a/INSTALL.rst b/INSTALL.rst index f256b0923c93..60737bb8d511 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -307,11 +307,10 @@ the list of conda packages. conda create -n "matplotlib_build" python=3.7 numpy python-dateutil pyparsing tornado cycler tk libpng zlib freetype msinttypes conda activate matplotlib_build + # force the build against static libpng and zlib libraries + set MPLSTATICBUILD=True + python setup.py bdist_wheel -For building, call the script ``build_alllocal.cmd`` in the root folder of the -repository:: - - build_alllocal.cmd Conda packages ^^^^^^^^^^^^^^ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 795101bee9cf..1b71826d38f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,9 +5,9 @@ jobs: -- job: 'Test' +- job: 'Linux_Test' pool: - vmImage: 'Ubuntu 16.04' + vmImage: ubuntu-16.04 strategy: matrix: Python36: @@ -17,31 +17,43 @@ jobs: maxParallel: 4 steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' + - template: ci/azure-pipelines-steps.yml + parameters: + platform: ubuntu + installer: apt - - script: | - python -m pip install --upgrade pip - pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis36.txt - displayName: 'Install dependencies' - - - script: | - export MPLLOCALFREETYPE=1 - pip install -ve . - displayName: "Install self" +- job: 'Windows_Test' + pool: + vmImage: vs2017-win2016 + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + PythonPreview: + python.version: 'Pre' + maxParallel: 4 - - script: env - displayName: 'print env' + steps: + - template: ci/azure-pipelines-steps.yml + parameters: + platform: windows + installer: nuget - - script: | - env - PYTHONFAULTHANDLER=1 pytest --junitxml=junit/test-results.xml -raR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n 2 --log-level=DEBUG - displayName: 'pytest' +- job: 'macOS_Test' + pool: + vmImage: xcode9-macos10.13 + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + maxParallel: 4 - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() + steps: + - template: ci/azure-pipelines-steps.yml + parameters: + platform: macos + installer: brew diff --git a/build_alllocal.cmd b/build_alllocal.cmd deleted file mode 100644 index 08845a80fa16..000000000000 --- a/build_alllocal.cmd +++ /dev/null @@ -1,27 +0,0 @@ -:: This assumes you have installed all the dependencies via conda packages: -:: # create a new environment with the required packages -:: # if you want a qt backend, add "pyqt" to the list of conda packages -:: conda create -n "matplotlib_build" python=3.7 numpy python-dateutil pyparsing tornado cycler tk libpng zlib freetype msinttypes -:: conda activate matplotlib_build - -set TARGET=bdist_wheel -IF [%1]==[] ( - echo Using default target: %TARGET% -) else ( - set TARGET=%1 - echo Using user supplied target: %TARGET% -) - -IF NOT DEFINED CONDA_PREFIX ( - echo No Conda env activated: you need to create a conda env with the right packages and activate it! - GOTO:eof -) - -:: copy the libs which have "wrong" names -set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib -mkdir lib || cmd /c "exit /b 0" -copy %LIBRARY_LIB%\zlibstatic.lib lib\zlib.lib -copy %LIBRARY_LIB%\libpng16_static.lib lib\libpng16.lib - -:: build the target -python setup.py %TARGET% diff --git a/ci/azure-pipelines-steps.yml b/ci/azure-pipelines-steps.yml new file mode 100644 index 000000000000..f8b2a58fe3eb --- /dev/null +++ b/ci/azure-pipelines-steps.yml @@ -0,0 +1,92 @@ +parameters: + platform: none + installer: none + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + displayName: 'Use Python $(python.version)' + condition: and(succeeded(), ne(variables['python.version'], 'Pre')) + +- task: stevedower.python.InstallPython.InstallPython@1 + displayName: 'Use prerelease Python' + inputs: + prerelease: true + condition: and(succeeded(), eq(variables['python.version'], 'Pre')) + +- ${{ if eq(parameters.installer, 'nuget') }}: + - task: NuGetToolInstaller@0 + displayName: 'Use latest available Nuget' + + - script: | + nuget install libpng-msvc14-x64 -ExcludeVersion -OutputDirectory "$(build.BinariesDirectory)" + nuget install zlib-msvc14-x64 -ExcludeVersion -OutputDirectory "$(build.BinariesDirectory)" + echo ##vso[task.prependpath]$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\bin_release + echo ##vso[task.prependpath]$(build.BinariesDirectory)\zlib-msvc14-x64\build\native\bin_release + echo ##vso[task.setvariable variable=CL]/I$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\include /I$(build.BinariesDirectory)\zlib-msvc14-x64\build\native\include + echo ##vso[task.setvariable variable=LINK]/LIBPATH:$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\lib_release /LIBPATH:$(build.BinariesDirectory)\zlib-msvc14-x64\build\native\lib_release + + displayName: 'Install dependencies with nuget' + +- ${{ if eq(parameters.installer, 'brew') }}: + - script: | + brew install pkg-config ffmpeg imagemagick mplayer ccache + displayName: 'Install dependencies with brew' + +- ${{ if eq(parameters.installer, 'apt') }}: + - script: | + sudo apt-add-repository ppa:jonathonf/ffmpeg-3 + sudo apt-get update + sudo apt-get install \ + cm-super \ + dvipng \ + ffmpeg \ + gdb \ + gir1.2-gtk-3.0 \ + graphviz \ + inkscape \ + libcairo2 \ + libgeos-dev \ + libgirepository-1.0.1 \ + lmodern \ + otf-freefont \ + pgf \ + texlive-fonts-recommended \ + texlive-latex-base \ + texlive-latex-extra \ + texlive-latex-recommended \ + texlive-xetex texlive-luatex + displayName: 'Install dependencies with apt' + +- script: | + + python -m pip install --upgrade pip + pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis36.txt + + displayName: 'Install dependencies with pip' + +- script: | + + pip install -ve . + + displayName: "Install self" + env: + MPLLOCALFREETYPE: 1 + +- script: env + displayName: 'print env' + +- script: | + env + pytest --junitxml=junit/test-results.xml -raR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n 2 + displayName: 'pytest' + env: + PYTHONFAULTHANDLER: 1 + +- task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-results.xml' + testRunTitle: 'Python $(python.version)' + condition: succeededOrFailed() diff --git a/setup.cfg.template b/setup.cfg.template index 45f448af4d93..ab2fa6ff0531 100644 --- a/setup.cfg.template +++ b/setup.cfg.template @@ -61,3 +61,8 @@ # modules. The default is determined by fallback. # #backend = Agg + +[build] +# Build options +# If we should try to use static libraries on Windows +#staticbuild = True diff --git a/setupext.py b/setupext.py index ea3b65ec46a3..74e0ffb96461 100644 --- a/setupext.py +++ b/setupext.py @@ -1,5 +1,5 @@ import configparser -from distutils import sysconfig +from distutils import ccompiler, sysconfig from distutils.core import Extension import functools import glob @@ -151,6 +151,7 @@ def write_cache(local_fn, data): # matplotlib build options, which can be altered using setup.cfg options = { 'backend': None, + 'staticbuild': False, } @@ -164,12 +165,18 @@ def write_cache(local_fn, data): if config.has_option('test', 'local_freetype'): options['local_freetype'] = config.getboolean("test", "local_freetype") + + if config.has_option('build', 'staticbuild'): + options['staticbuild'] = config.getboolean("build", "staticbuild") else: config = None lft = bool(os.environ.get('MPLLOCALFREETYPE', False)) options['local_freetype'] = lft or options.get('local_freetype', False) +staticbuild = bool(os.environ.get('MPLSTATICBUILD', os.name == 'nt')) +options['staticbuild'] = staticbuild or options.get('staticbuild', False) + if '-q' in sys.argv or '--quiet' in sys.argv: def print_raw(*args, **kwargs): pass # Suppress our own output. @@ -195,6 +202,23 @@ def get_buffer_hash(fd): return hasher.hexdigest() +def deplib(libname): + if sys.platform != 'win32': + return libname + + known_libs = { + # TODO: support versioned libpng on build system rewrite + 'libpng16': ('libpng16', '_static'), + 'z': ('zlib', 'static'), + } + + libname, static_postfix = known_libs[libname] + if options['staticbuild']: + libname += static_postfix + + return libname + + @functools.lru_cache(1) # We only need to compute this once. def get_pkg_config(): """ @@ -501,7 +525,7 @@ def add_flags(self, ext): ext, 'freetype2', atleast_version='9.11.3', alt_exec=['freetype-config'], - default_libraries=['freetype', 'z']) + default_libraries=['freetype', deplib('z')]) ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'system')) def do_custom_build(self): @@ -569,30 +593,35 @@ def do_custom_build(self): subprocess.check_call(["make"], env=env, cwd=src_path) else: # compilation on windows - shutil.rmtree(pathlib.Path(src_path, "objs"), ignore_errors=True) - import distutils.msvc9compiler as msvc - # FreeType has no build profile for 2014, so we don't bother. + shutil.rmtree(str(pathlib.Path(src_path, "objs")), + ignore_errors=True) + msbuild_platform = ( + 'x64' if platform.architecture()[0] == '64bit' else 'Win32') + base_path = pathlib.Path("build/freetype-2.6.1/builds/windows") vc = 'vc2010' - WinXX = 'x64' if platform.architecture()[0] == '64bit' else 'Win32' - xXX = 'x64' if platform.architecture()[0] == '64bit' else 'x86' - vcvarsall = msvc.find_vcvarsall(10.0) - if vcvarsall is None: - raise RuntimeError('Microsoft VS 2010 required') - cmdfile = pathlib.Path("build/build_freetype.cmd") - cmdfile.write_text(fr""" -call "%ProgramFiles%\Microsoft SDKs\Windows\v7.0\Bin\SetEnv.Cmd" ^ - /Release /{xXX} /xp -call "{vcvarsall}" {xXX} -set MSBUILD=C:\Windows\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe -%MSBUILD% "builds\windows\{vc}\freetype.sln" ^ - /t:Clean;Build /p:Configuration="Release";Platform={WinXX} + sln_path = ( + base_path / vc / "freetype.sln" + ) + # https://developercommunity.visualstudio.com/comments/190992/view.html + (sln_path.parent / "Directory.Build.props").write_text(""" + + + + $([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0')) + + """) - subprocess.check_call([str(cmdfile.resolve())], - shell=True, cwd=src_path) + cc = ccompiler.new_compiler() + cc.initialize() # Get devenv & msbuild in the %PATH% of cc.spawn. + cc.spawn(["devenv", str(sln_path), "/upgrade"]) + cc.spawn(["msbuild", str(sln_path), + "/t:Clean;Build", + f"/p:Configuration=Release;Platform={msbuild_platform}"]) # Move to the corresponding Unix build path. (src_path / "objs" / ".libs").mkdir() # Be robust against change of FreeType version. - lib_path, = (src_path / "objs" / vc / xXX).glob("freetype*.lib") + lib_path, = (src_path / "objs" / vc / msbuild_platform).glob( + "freetype*.lib") shutil.copy2(lib_path, src_path / "objs/.libs/libfreetype.lib") @@ -630,8 +659,7 @@ def get_extension(self): default_libraries=( ['png', 'z'] if os.name == 'posix' else # libpng upstream names their lib libpng16.lib, not png.lib. - # zlib upstream names their lib zlib.lib, not z.lib. - ['libpng16', 'zlib'] if os.name == 'nt' else + [deplib('libpng16'), deplib('z')] if os.name == 'nt' else [] )) add_numpy_flags(ext)