From 0ff669fcd884ba07fcb29806cdce4d193b5b05ae Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 12 Dec 2018 16:16:24 -0800 Subject: [PATCH 1/5] CI: set up OSX and windows azure pipelines Adds macOS and Windows builds to Azure Pipelines Correct VM image name Fix job names and remove extension reference Add dependency installs for macOS and Ubuntu Attempt to use libpng from nuget for Windows build Ensures nuget is available on PATH --- azure-pipelines.yml | 64 +++++++++++++++----------- ci/azure-pipelines-steps.yml | 87 ++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 26 deletions(-) create mode 100644 ci/azure-pipelines-steps.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 795101bee9cf..1b71826d38f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,9 +5,9 @@ jobs: -- job: 'Test' +- job: 'Linux_Test' pool: - vmImage: 'Ubuntu 16.04' + vmImage: ubuntu-16.04 strategy: matrix: Python36: @@ -17,31 +17,43 @@ jobs: maxParallel: 4 steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' + - template: ci/azure-pipelines-steps.yml + parameters: + platform: ubuntu + installer: apt - - script: | - python -m pip install --upgrade pip - pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis36.txt - displayName: 'Install dependencies' - - - script: | - export MPLLOCALFREETYPE=1 - pip install -ve . - displayName: "Install self" +- job: 'Windows_Test' + pool: + vmImage: vs2017-win2016 + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + PythonPreview: + python.version: 'Pre' + maxParallel: 4 - - script: env - displayName: 'print env' + steps: + - template: ci/azure-pipelines-steps.yml + parameters: + platform: windows + installer: nuget - - script: | - env - PYTHONFAULTHANDLER=1 pytest --junitxml=junit/test-results.xml -raR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n 2 --log-level=DEBUG - displayName: 'pytest' +- job: 'macOS_Test' + pool: + vmImage: xcode9-macos10.13 + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + maxParallel: 4 - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() + steps: + - template: ci/azure-pipelines-steps.yml + parameters: + platform: macos + installer: brew diff --git a/ci/azure-pipelines-steps.yml b/ci/azure-pipelines-steps.yml new file mode 100644 index 000000000000..3d47bb350418 --- /dev/null +++ b/ci/azure-pipelines-steps.yml @@ -0,0 +1,87 @@ +parameters: + platform: none + installer: none + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + displayName: 'Use Python $(python.version)' + condition: and(succeeded(), ne(variables['python.version'], 'Pre')) + +- task: stevedower.python.InstallPython.InstallPython@1 + displayName: 'Use prerelease Python' + inputs: + prerelease: true + condition: and(succeeded(), eq(variables['python.version'], 'Pre')) + +- ${{ if eq(parameters.installer, 'nuget') }}: + - task: NuGetToolInstaller@0 + displayName: 'Use latest available Nuget' + + - script: | + nuget install libpng-msvc14-x64 -ExcludeVersion -OutputDirectory "$(build.BinariesDirectory)" + echo ##vso[task.setvariable variable=MPLBASEDIRLIST]win32_static;$(build.BinariesDirectory)\libpng-msvc14-x64\build\native + echo ##vso[task.setvariable variable=LIBDIR]%LIBDIR%;$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\lib_release + echo ##vso[task.prependpath]$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\bin_release + displayName: 'Install dependencies with nuget' + +- ${{ if eq(parameters.installer, 'brew') }}: + - script: | + brew install pkg-config ffmpeg imagemagick mplayer ccache + displayName: 'Install dependencies with brew' + +- ${{ if eq(parameters.installer, 'apt') }}: + - script: | + sudo apt-add-repository ppa:jonathonf/ffmpeg-3 + sudo apt-get update + sudo apt-get install \ + cm-super \ + dvipng \ + ffmpeg \ + gdb \ + gir1.2-gtk-3.0 \ + graphviz \ + inkscape \ + libcairo2 \ + libgeos-dev \ + libgirepository-1.0.1 \ + lmodern \ + otf-freefont \ + pgf \ + texlive-fonts-recommended \ + texlive-latex-base \ + texlive-latex-extra \ + texlive-latex-recommended \ + texlive-xetex texlive-luatex + displayName: 'Install dependencies with apt' + +- script: | + + python -m pip install --upgrade pip + pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis36.txt + + displayName: 'Install dependencies with pip' + +- script: | + + pip install -ve . + + displayName: "Install self" + env: + MPLLOCALFREETYPE: 1 + +- script: env + displayName: 'print env' + +- script: | + env + PYTHONFAULTHANDLER=1 pytest --junitxml=junit/test-results.xml -raR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n 2 + displayName: 'pytest' + +- task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-results.xml' + testRunTitle: 'Python $(python.version)' + condition: succeededOrFailed() From 6c7e672fcd92b2df3a07fc7ff2dde03a16be37bb Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Wed, 2 Jan 2019 14:49:50 +0100 Subject: [PATCH 2/5] CI: Fix FreeType build. --- .flake8 | 1 + ci/azure-pipelines-steps.yml | 4 +-- setupext.py | 48 +++++++++++++++++++----------------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/.flake8 b/.flake8 index 88083c30645c..2aad08357dc9 100644 --- a/.flake8 +++ b/.flake8 @@ -28,6 +28,7 @@ exclude = per-file-ignores = setup.py: E402 + setupext.py: E501 tools/subset.py: E221, E251, E261, E302, E501 diff --git a/ci/azure-pipelines-steps.yml b/ci/azure-pipelines-steps.yml index 3d47bb350418..c68fed6d9abc 100644 --- a/ci/azure-pipelines-steps.yml +++ b/ci/azure-pipelines-steps.yml @@ -22,8 +22,8 @@ steps: - script: | nuget install libpng-msvc14-x64 -ExcludeVersion -OutputDirectory "$(build.BinariesDirectory)" - echo ##vso[task.setvariable variable=MPLBASEDIRLIST]win32_static;$(build.BinariesDirectory)\libpng-msvc14-x64\build\native - echo ##vso[task.setvariable variable=LIBDIR]%LIBDIR%;$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\lib_release + echo ##vso[task.setvariable variable=CL]/I$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\include + echo ##vso[task.setvariable variable=LINK]/LIBPATH:$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\lib_release echo ##vso[task.prependpath]$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\bin_release displayName: 'Install dependencies with nuget' diff --git a/setupext.py b/setupext.py index ea3b65ec46a3..d60dfdc01b5f 100644 --- a/setupext.py +++ b/setupext.py @@ -1,5 +1,5 @@ import configparser -from distutils import sysconfig +from distutils import ccompiler, sysconfig from distutils.core import Extension import functools import glob @@ -569,30 +569,35 @@ def do_custom_build(self): subprocess.check_call(["make"], env=env, cwd=src_path) else: # compilation on windows - shutil.rmtree(pathlib.Path(src_path, "objs"), ignore_errors=True) - import distutils.msvc9compiler as msvc - # FreeType has no build profile for 2014, so we don't bother. + shutil.rmtree(str(pathlib.Path(src_path, "objs")), + ignore_errors=True) + msbuild_platform = ( + 'x64' if platform.architecture()[0] == '64bit' else 'Win32') + base_path = pathlib.Path("build/freetype-2.6.1/builds/windows") vc = 'vc2010' - WinXX = 'x64' if platform.architecture()[0] == '64bit' else 'Win32' - xXX = 'x64' if platform.architecture()[0] == '64bit' else 'x86' - vcvarsall = msvc.find_vcvarsall(10.0) - if vcvarsall is None: - raise RuntimeError('Microsoft VS 2010 required') - cmdfile = pathlib.Path("build/build_freetype.cmd") - cmdfile.write_text(fr""" -call "%ProgramFiles%\Microsoft SDKs\Windows\v7.0\Bin\SetEnv.Cmd" ^ - /Release /{xXX} /xp -call "{vcvarsall}" {xXX} -set MSBUILD=C:\Windows\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe -%MSBUILD% "builds\windows\{vc}\freetype.sln" ^ - /t:Clean;Build /p:Configuration="Release";Platform={WinXX} + sln_path = ( + base_path / vc / "freetype.sln" + ) + # https://developercommunity.visualstudio.com/comments/190992/view.html + (sln_path.parent / "Directory.Build.props").write_text(""" + + + + $([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0')) + + """) - subprocess.check_call([str(cmdfile.resolve())], - shell=True, cwd=src_path) + cc = ccompiler.new_compiler() + cc.initialize() # Get devenv & msbuild in the %PATH% of cc.spawn. + cc.spawn(["devenv", str(sln_path), "/upgrade"]) + cc.spawn(["msbuild", str(sln_path), + "/t:Clean;Build", + f"/p:Configuration=Release;Platform={msbuild_platform}"]) # Move to the corresponding Unix build path. (src_path / "objs" / ".libs").mkdir() # Be robust against change of FreeType version. - lib_path, = (src_path / "objs" / vc / xXX).glob("freetype*.lib") + lib_path, = (src_path / "objs" / vc / msbuild_platform).glob( + "freetype*.lib") shutil.copy2(lib_path, src_path / "objs/.libs/libfreetype.lib") @@ -630,8 +635,7 @@ def get_extension(self): default_libraries=( ['png', 'z'] if os.name == 'posix' else # libpng upstream names their lib libpng16.lib, not png.lib. - # zlib upstream names their lib zlib.lib, not z.lib. - ['libpng16', 'zlib'] if os.name == 'nt' else + ['libpng16'] if os.name == 'nt' else [] )) add_numpy_flags(ext) From 627624c7faf85c4f65f8a2308ab37e6d7f699175 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 2 Jan 2019 23:12:17 -0500 Subject: [PATCH 3/5] CI: use requirements files on appveyor too This should make sure we have consistent dependencies installed across all of the CI services. --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index aa747afd2d03..18ad262ebe1e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -68,7 +68,7 @@ install: - activate test-environment - echo %PYTHON_VERSION% %TARGET_ARCH% # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124 - - pip install -q "pytest>=3.4,!=4.6.0" "pytest-cov>=2.3.1" pytest-rerunfailures pytest-timeout pytest-xdist "pandas!=0.25.0" + - pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis36.txt # Apply patch to `subprocess` on Python versions > 2 and < 3.6.3 # https://github.com/matplotlib/matplotlib/issues/9176 From 44ed274763736f169725f3490fe1a6887144ba57 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Sun, 5 Nov 2017 18:29:17 +0300 Subject: [PATCH 4/5] BLD: Switch to static linking by default on windows --- .appveyor.yml | 17 ++++------------- INSTALL.rst | 7 +++---- build_alllocal.cmd | 27 --------------------------- ci/azure-pipelines-steps.yml | 7 +++++-- setup.cfg.template | 5 +++++ setupext.py | 28 ++++++++++++++++++++++++++-- 6 files changed, 43 insertions(+), 48 deletions(-) delete mode 100644 build_alllocal.cmd diff --git a/.appveyor.yml b/.appveyor.yml index 18ad262ebe1e..9ddf5068592f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -76,16 +76,6 @@ install: curl -sL https://github.com/python/cpython/pull/1224.patch | patch -fsup 1 -d %CONDA_PREFIX% ) || cmd /c "exit /b 0" - # Let the install prefer the static builds of the libs - - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib - - mkdir lib || cmd /c "exit /b 0" - - copy /y %LIBRARY_LIB%\zlibstatic.lib lib\zlib.lib - - copy /y %LIBRARY_LIB%\libpng16_static.lib lib\libpng16.lib - # These z.lib / png.lib are not static versions but files which end up as - # dependencies to the dll file. This is fine for the conda build, but not here - # and for the wheels - - del %LIBRARY_LIB%\png.lib - - del %LIBRARY_LIB%\z.lib # enables the local freetype build - set MPLLOCALFREETYPE=1 # Show the installed packages + versions @@ -97,9 +87,9 @@ test_script: - pip install -ve . # these should show no z, png, or freetype dll... - set "DUMPBIN=%VS140COMNTOOLS%\..\..\VC\bin\dumpbin.exe" - - '"%DUMPBIN%" /DEPENDENTS lib\matplotlib\ft2font*.pyd | findstr freetype.*.dll && exit /b 1 || exit /b 0' - - '"%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr z.*.dll && exit /b 1 || exit /b 0' - - '"%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr png.*.dll && exit /b 1 || exit /b 0' + - 'if x%MPLSTATICBUILD% == xTrue "%DUMPBIN%" /DEPENDENTS lib\matplotlib\ft2font*.pyd | findstr freetype.*.dll && exit /b 1 || exit /b 0' + - 'if x%MPLSTATICBUILD% == xTrue "%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr z.*.dll && exit /b 1 || exit /b 0' + - 'if x%MPLSTATICBUILD% == xTrue "%DUMPBIN%" /DEPENDENTS lib\matplotlib\_png*.pyd | findstr png.*.dll && exit /b 1 || exit /b 0' # this are optional dependencies so that we don't skip so many tests... - if x%TEST_ALL% == xyes conda install -q ffmpeg inkscape miktex pillow @@ -116,6 +106,7 @@ test_script: after_test: # After the tests were a success, build wheels with the static libs # Hide the output, the copied files really clutter the build log... + - set MPLSTATICBUILD=True - 'python setup.py bdist_wheel > NUL:' - dir dist\ - echo finished... diff --git a/INSTALL.rst b/INSTALL.rst index f256b0923c93..60737bb8d511 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -307,11 +307,10 @@ the list of conda packages. conda create -n "matplotlib_build" python=3.7 numpy python-dateutil pyparsing tornado cycler tk libpng zlib freetype msinttypes conda activate matplotlib_build + # force the build against static libpng and zlib libraries + set MPLSTATICBUILD=True + python setup.py bdist_wheel -For building, call the script ``build_alllocal.cmd`` in the root folder of the -repository:: - - build_alllocal.cmd Conda packages ^^^^^^^^^^^^^^ diff --git a/build_alllocal.cmd b/build_alllocal.cmd deleted file mode 100644 index 08845a80fa16..000000000000 --- a/build_alllocal.cmd +++ /dev/null @@ -1,27 +0,0 @@ -:: This assumes you have installed all the dependencies via conda packages: -:: # create a new environment with the required packages -:: # if you want a qt backend, add "pyqt" to the list of conda packages -:: conda create -n "matplotlib_build" python=3.7 numpy python-dateutil pyparsing tornado cycler tk libpng zlib freetype msinttypes -:: conda activate matplotlib_build - -set TARGET=bdist_wheel -IF [%1]==[] ( - echo Using default target: %TARGET% -) else ( - set TARGET=%1 - echo Using user supplied target: %TARGET% -) - -IF NOT DEFINED CONDA_PREFIX ( - echo No Conda env activated: you need to create a conda env with the right packages and activate it! - GOTO:eof -) - -:: copy the libs which have "wrong" names -set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib -mkdir lib || cmd /c "exit /b 0" -copy %LIBRARY_LIB%\zlibstatic.lib lib\zlib.lib -copy %LIBRARY_LIB%\libpng16_static.lib lib\libpng16.lib - -:: build the target -python setup.py %TARGET% diff --git a/ci/azure-pipelines-steps.yml b/ci/azure-pipelines-steps.yml index c68fed6d9abc..da45f9c94314 100644 --- a/ci/azure-pipelines-steps.yml +++ b/ci/azure-pipelines-steps.yml @@ -22,9 +22,12 @@ steps: - script: | nuget install libpng-msvc14-x64 -ExcludeVersion -OutputDirectory "$(build.BinariesDirectory)" - echo ##vso[task.setvariable variable=CL]/I$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\include - echo ##vso[task.setvariable variable=LINK]/LIBPATH:$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\lib_release + nuget install zlib-msvc14-x64 -ExcludeVersion -OutputDirectory "$(build.BinariesDirectory)" echo ##vso[task.prependpath]$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\bin_release + echo ##vso[task.prependpath]$(build.BinariesDirectory)\zlib-msvc14-x64\build\native\bin_release + echo ##vso[task.setvariable variable=CL]/I$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\include /I$(build.BinariesDirectory)\zlib-msvc14-x64\build\native\include + echo ##vso[task.setvariable variable=LINK]/LIBPATH:$(build.BinariesDirectory)\libpng-msvc14-x64\build\native\lib_release /LIBPATH:$(build.BinariesDirectory)\zlib-msvc14-x64\build\native\lib_release + displayName: 'Install dependencies with nuget' - ${{ if eq(parameters.installer, 'brew') }}: diff --git a/setup.cfg.template b/setup.cfg.template index 45f448af4d93..ab2fa6ff0531 100644 --- a/setup.cfg.template +++ b/setup.cfg.template @@ -61,3 +61,8 @@ # modules. The default is determined by fallback. # #backend = Agg + +[build] +# Build options +# If we should try to use static libraries on Windows +#staticbuild = True diff --git a/setupext.py b/setupext.py index d60dfdc01b5f..74e0ffb96461 100644 --- a/setupext.py +++ b/setupext.py @@ -151,6 +151,7 @@ def write_cache(local_fn, data): # matplotlib build options, which can be altered using setup.cfg options = { 'backend': None, + 'staticbuild': False, } @@ -164,12 +165,18 @@ def write_cache(local_fn, data): if config.has_option('test', 'local_freetype'): options['local_freetype'] = config.getboolean("test", "local_freetype") + + if config.has_option('build', 'staticbuild'): + options['staticbuild'] = config.getboolean("build", "staticbuild") else: config = None lft = bool(os.environ.get('MPLLOCALFREETYPE', False)) options['local_freetype'] = lft or options.get('local_freetype', False) +staticbuild = bool(os.environ.get('MPLSTATICBUILD', os.name == 'nt')) +options['staticbuild'] = staticbuild or options.get('staticbuild', False) + if '-q' in sys.argv or '--quiet' in sys.argv: def print_raw(*args, **kwargs): pass # Suppress our own output. @@ -195,6 +202,23 @@ def get_buffer_hash(fd): return hasher.hexdigest() +def deplib(libname): + if sys.platform != 'win32': + return libname + + known_libs = { + # TODO: support versioned libpng on build system rewrite + 'libpng16': ('libpng16', '_static'), + 'z': ('zlib', 'static'), + } + + libname, static_postfix = known_libs[libname] + if options['staticbuild']: + libname += static_postfix + + return libname + + @functools.lru_cache(1) # We only need to compute this once. def get_pkg_config(): """ @@ -501,7 +525,7 @@ def add_flags(self, ext): ext, 'freetype2', atleast_version='9.11.3', alt_exec=['freetype-config'], - default_libraries=['freetype', 'z']) + default_libraries=['freetype', deplib('z')]) ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'system')) def do_custom_build(self): @@ -635,7 +659,7 @@ def get_extension(self): default_libraries=( ['png', 'z'] if os.name == 'posix' else # libpng upstream names their lib libpng16.lib, not png.lib. - ['libpng16'] if os.name == 'nt' else + [deplib('libpng16'), deplib('z')] if os.name == 'nt' else [] )) add_numpy_flags(ext) From 261235e3821aa1692dd569e0b0a1967b8399f70b Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Mon, 26 Aug 2019 20:37:04 -0400 Subject: [PATCH 5/5] CI: fix env on windows --- ci/azure-pipelines-steps.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/azure-pipelines-steps.yml b/ci/azure-pipelines-steps.yml index da45f9c94314..f8b2a58fe3eb 100644 --- a/ci/azure-pipelines-steps.yml +++ b/ci/azure-pipelines-steps.yml @@ -80,8 +80,10 @@ steps: - script: | env - PYTHONFAULTHANDLER=1 pytest --junitxml=junit/test-results.xml -raR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n 2 + pytest --junitxml=junit/test-results.xml -raR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n 2 displayName: 'pytest' + env: + PYTHONFAULTHANDLER: 1 - task: PublishTestResults@2 inputs: