matplotlib · SidharthBansal · Jun 25, 2020 · Jun 25, 2020 · Jun 25, 2020 · Jul 1, 2020
diff --git a/.flake8 b/.flake8
@@ -64,6 +64,7 @@ per-file-ignores =
     lib/matplotlib/pylab.py: F401, F403
     lib/matplotlib/pyplot.py: F401, F811
     lib/matplotlib/style/__init__.py: F401
+    lib/matplotlib/tests/__init__.py: F401
     lib/matplotlib/testing/conftest.py: F401
     lib/matplotlib/tests/conftest.py: F401
     lib/matplotlib/tests/test_backend_qt.py: F401
@@ -81,6 +82,7 @@ per-file-ignores =
     lib/mpl_toolkits/axisartist/axes_rgb.py: F401
     lib/mpl_toolkits/axisartist/axislines.py: F401
     lib/mpl_toolkits/mplot3d/__init__.py: F401
+    lib/mpl_toolkits/tests/__init__.py: F401
     lib/mpl_toolkits/tests/conftest.py: F401
     lib/pylab.py: F401, F403
 

diff --git a/.gitignore b/.gitignore
@@ -105,3 +105,12 @@ lib/z.lib
 #########################
 lib/matplotlib/backends/web_backend/node_modules/
 lib/matplotlib/backends/web_backend/package-lock.json
+
+# Matplotlib tempenv #
+######################
+temp/testenv
+
+# Baseline Images #
+###################
+lib/matplotlib/tests/baseline_images
+lib/mpl_toolkits/tests/baseline_images/
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -21,3 +21,4 @@ recursive-include tutorials *
 include versioneer.py
 include lib/matplotlib/_version.py
 include tests.py
+prune sub-wheels
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -85,6 +85,7 @@ steps:
 
 - bash: |
     python -m pip install --upgrade pip
+    python -m pip install --upgrade setuptools
     python -m pip install -r requirements/testing/travis_all.txt -r requirements/testing/travis_extra.txt ||
       [[ "$PYTHON_VERSION" = 'Pre' ]]
   displayName: 'Install dependencies with pip'

diff --git a/baseline_image_modifications/PR1.json b/baseline_image_modifications/PR1.json
@@ -0,0 +1,15 @@
+{
+  "files_added": [{"file" : "test_arrow_patches.py",
+                  "test" : "test_boxarrow",
+                  "img_names":  ["a.png", "b.png"]}],
+  "files_deleted": [{"file" : "test_arrow_patches.py",
+                  "test" : "test_boxarrow",
+                  "img_names":  ["a.png", "b.png"]}],
+  "files_modified": [{"prev_file" : "test_arrow_patches.py",
+                  "prev_test" : "test_boxarrow",
+                  "prev_img_names":  ["a.png", "b.png"],
+                  "new_file" : "test_arrow_patches.py",
+                  "new_test" : "test_boxarrow",
+                  "new_img_names":  ["a.png", "b.png"]
+                  }]
+}
diff --git a/doc/api/next_api_changes/behavior/baseline-images.rst b/doc/api/next_api_changes/behavior/baseline-images.rst
@@ -0,0 +1,25 @@
+Baseline image generation
+-------------------------
+
+Earlier the baseline images were present in the ``baseline_images`` folder. Now they are present in the
+:file:`sub-wheels/matpotlib-baseline-images/lib/matplotlib_baseline_images`.
+
+In order to run local image tests, the developer should checkout the ``master`` branch and run the tests once with the
+``generate_images`` flag in order to get a set of baseline images that are consistent with the code in ``master``. The
+command will be ::
+
+    python3 -mpytest --generate_images
+
+Then subsequent tests from their branches will test against these baseline images.
+
+Alternately, the developer could download the ``matplotlib-baseline-images`` package and compare against these images.
+However, the developer will have to be certain to have development environment that has the same free type and other
+dependencies as the testing environment on the the machine that made ``matplotlib-baseline-images``. Developer can
+install the ``matplotlib-baseline-images`` package by running the command ::
+
+    python3 -mpip install  -ve  sub-wheels/matplotlib-baseline-images
+
+or can install the package from PyP by the command ::
+
+    python3 -mpip install matplotlib-baseline-images
+
diff --git a/doc/devel/contributing.rst b/doc/devel/contributing.rst
@@ -590,3 +590,69 @@ example code you can load it into a file handle with::
 
     import matplotlib.cbook as cbook
     fh = cbook.get_sample_data('mydata.dat')
+
+
+Baseline image generation
+-------------------------
+The idea is to eventually remove all of the baseline images from the matplotlib repository as the baseline images are
+problematic. This is due to the reason that they cause the repo size to grow rather quickly. they also force the
+developers to pin to a somewhat old version of freetype, because nearly every release of freetype causes tiny
+rasterization changes that would entail regenerating all baseline images (and thus cause even more repo size growth).
+After removal of the baseline images, the developer have two options to get the baselines from.
+In order to run local image tests, the developer should checkout the ``master`` branch and run the tests once with the
+``generate_images`` flag in order to get a set of baseline images that are consistent with the code in ``master``.  Then
+subsequent tests from their branches will test against these baseline images.
+
+Alternately, the developer could download the ``matplotlib-baseline-images`` package and
+compare against these images.  However, the developer will have to be certain to have
+development environment that has the same free type and other dependencies as the testing
+environment on the the machine that made ``matplotlib-baseline-images``.
+
+These methods are described below. Later on, developer can run ::
+
+    python3 -pytest
+
+
+How to install matplotlib-baseline-images package?
+--------------------------------------------------
+Developer can install the ``matplotlib-baseline-images`` package by running the command ::
+
+    python3 -mpip install  -ve  sub-wheels/matplotlib-baseline-images
+
+or can install the package from PyP by the command ::
+
+    python3 -mpip install matplotlib-baseline-images
+
+Canonical images will be tied to a given release.
+
+
+Baseline image generation at the fresh install of matplotlib through command line
+----------------------------------------------------------------------------------
+The baseline images can be generated in the ``baseline_images`` directory by using ::
+
+    python3 -mpytest --generate_images
+
+For a given developer the baseline images don't change due to local changes (even if there may be some variation
+between developers respective baselines due to different freetype / latex etc.
+
+
+How to update an existing baseline image?
+---------------------------------------------
+Tests generating the baseline images may change over time. So, the baseline images needs to be regenerated by the
+developer after fetching the changes through ``git pull`` or ``git checkout -b featured_branch``. Missing baseline
+images can be generated in the ``baseline_images`` directory with the following command::
+
+    python3 -mpytest --generate_images
+
+Later on, developer can run the test suite with ::
+
+    python3 -pytest
+
+
+Add new tests that require new images
+-------------------------------------
+Tests which require new expected image can be inserted in the test files without adding the baseline or expected images.
+The baseline or expected images will be first created by::
+
+    python3 -mpytest --generate_images
+
diff --git a/doc/devel/testing.rst b/doc/devel/testing.rst
@@ -144,13 +144,14 @@ tests it::
        fig, ax = plt.subplots()
        ax.plot(range(10), linestyle=(0, (3, 3)), lw=5)
 
-The first time this test is run, there will be no baseline image to compare
-against, so the test will fail.  Copy the output images (in this case
-:file:`result_images/test_lines/test_line_dashes.png`) to the correct
-subdirectory of :file:`baseline_images` tree in the source directory (in this
-case :file:`lib/matplotlib/tests/baseline_images/test_lines`).  Put this new
-file under source code revision control (with ``git add``).  When rerunning
-the tests, they should now pass.
+
+What should a developer do if they want to change any baseline images?
+----------------------------------------------------------------------
+In the previous flow the committed baseline images were changed when the tests generating them changed. Now, there is
+no need to change the baseline images as they will be generated by running the tests by other developers. Instead,
+change the name of the baseline images. It can be done by changing the id appended to the name of the test to the next
+corresponding id if the id field is already present in the image name. Otherwise, add an id equals to ``1`` to the
+image names.
 
 Baseline images take a lot of space in the Matplotlib repository.
 An alternative approach for image comparison tests is to use the

diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py
@@ -17,6 +17,7 @@ def pytest_configure(config):
         ("markers", "baseline_images: Compare output against references."),
         ("markers", "pytz: Tests that require pytz to be installed."),
         ("markers", "network: Tests that reach out to the network."),
+        ("markers", "generate_images: Flag to generate baseline images."),
         ("filterwarnings", "error"),
     ]:
         config.addinivalue_line(key, value)

diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py
@@ -17,7 +17,7 @@
 from matplotlib import ft2font
 from matplotlib import pyplot as plt
 from matplotlib import ticker
-
+from matplotlib.testing.compare import get_cache_dir
 from .compare import comparable_formats, compare_images, make_test_filename
 from .exceptions import ImageComparisonFailure
 
@@ -199,11 +199,29 @@ def copy_baseline(self, baseline, extension):
                 f"{orig_expected_path}") from err
         return expected_fname
 
-    def compare(self, idx, baseline, extension, *, _lock=False):
+    def create_baseline(self, baseline, extension):
+        src_path = self.result_dir / baseline
+        orig_src_path = src_path.with_suffix(f'.{extension}')
+        if extension == 'eps' and not orig_src_path.exists():
+            orig_src_path = orig_src_path.with_suffix('.pdf')
+        dest_path = Path(self.baseline_dir / orig_src_path.name)
+        try:
+            if dest_path.exists():
+                return dest_path
+            Path(dest_path).parent.mkdir(parents=True, exist_ok=True)
+            try:
+                os.symlink(orig_src_path, dest_path)
+            except OSError:  # On Windows, symlink *may* be unavailable.
+                shutil.copyfile(orig_src_path, dest_path)
+        except OSError as err:
+            raise ValueError("Failed to put the image in the right place")
+        return dest_path
+
+    def compare(self, idx, baseline, extension, *, _lock=False,
+                generating=False):
         __tracebackhide__ = True
         fignum = plt.get_fignums()[idx]
         fig = plt.figure(fignum)
-
         if self.remove_text:
             remove_ticks_and_titles(fig)
 
@@ -218,8 +236,12 @@ def compare(self, idx, baseline, extension, *, _lock=False):
                 if _lock else contextlib.nullcontext())
         with lock:
             fig.savefig(actual_path, **kwargs)
-            expected_path = self.copy_baseline(baseline, extension)
-            _raise_on_image_difference(expected_path, actual_path, self.tol)
+            if generating:
+                self.create_baseline(baseline, extension)
+            else:
+                expected_path = self.copy_baseline(baseline, extension)
+                _raise_on_image_difference(expected_path, actual_path,
+                                           self.tol)
 
 
 def _pytest_image_comparison(baseline_images, extensions, tol,
@@ -244,12 +266,14 @@ def decorator(func):
         @pytest.mark.style(style)
         @_checked_on_freetype_version(freetype_version)
         @functools.wraps(func)
+        @pytest.mark.generate_images
         def wrapper(*args, extension, request, **kwargs):
             __tracebackhide__ = True
             if 'extension' in old_sig.parameters:
                 kwargs['extension'] = extension
             if 'request' in old_sig.parameters:
                 kwargs['request'] = request
+            generate_images = request.config.getoption("--generate_images")
 
             img = _ImageComparisonBase(func, tol=tol, remove_text=remove_text,
                                        savefig_kwargs=savefig_kwargs)
@@ -275,7 +299,8 @@ def wrapper(*args, extension, request, **kwargs):
                 "Test generated {} images but there are {} baseline images"
                 .format(len(plt.get_fignums()), len(our_baseline_images)))
             for idx, baseline in enumerate(our_baseline_images):
-                img.compare(idx, baseline, extension, _lock=needs_lock)
+                img.compare(idx, baseline, extension, _lock=needs_lock,
+                            generating=generate_images)
 
         parameters = list(old_sig.parameters.values())
         if 'extension' not in old_sig.parameters:
@@ -483,7 +508,41 @@ def _image_directories(func):
     doesn't exist.
     """
     module_path = Path(sys.modules[func.__module__].__file__)
-    baseline_dir = module_path.parent / "baseline_images" / module_path.stem
+    cache_dir = Path(get_cache_dir()) / "baseline_images" / module_path.stem
+    if func.__module__.startswith("matplotlib."):
+        try:
+            import matplotlib_baseline_images
+            baseline_dir = (Path(matplotlib_baseline_images.__file__).parent /
+                            module_path.stem)
+        except ImportError:
+            baseline_dir = (module_path.parent /
+                            "baseline_images" /
+                            module_path.stem)
+
+    elif func.__module__.startswith("mpl_toolkits."):
+        try:
+            import mpl_toolkits_baseline_images
+            baseline_file = mpl_toolkits_baseline_images.__file__
+            baseline_dir = (Path(baseline_file).parent /
+                            module_path.stem)
+        except ImportError:
+            baseline_dir = (module_path.parent /
+                            "baseline_images" /
+                            module_path.stem)
+    extensions = ['**/*.svg', '**/*.eps', '**/*.png', '**/*.pdf']
+    for ext in extensions:
+        for baseline_img in Path(baseline_dir).glob(ext):
+            cache_dir_img = (cache_dir) / \
+                            (baseline_img).relative_to(baseline_dir)
+            if not cache_dir_img.exists():
+                cache_dir_img.parent.mkdir(parents=True, exist_ok=True)
+                try:
+                    shutil.copyfile(baseline_img, cache_dir_img)
+                except shutil.SameFileError:
+                    if not cache_dir_img.exists():
+                        raise
+                except FileNotFoundError:
+                    raise
     result_dir = Path().resolve() / "result_images" / module_path.stem
     result_dir.mkdir(parents=True, exist_ok=True)
-    return baseline_dir, result_dir
+    return cache_dir, result_dir
diff --git a/lib/matplotlib/testing/generate_images_plugin.py b/lib/matplotlib/testing/generate_images_plugin.py
@@ -0,0 +1,23 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    try:
+        parser.addoption(
+            "--generate_images",
+             action="store_true",
+             default=False,
+             help="run matplotlib baseline image generation tests"
+    )
+    except:
+        pass
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--generate_images"):
+        skip_non_baseline_img_gen_tests \
+            = pytest.mark.skip(reason="No need to run non "
+                                      "image generation tests")
+        for item in items:
+            if "generate_images" not in item.keywords:
+                item.add_marker(skip_non_baseline_img_gen_tests)
diff --git a/lib/matplotlib/tests/__init__.py b/lib/matplotlib/tests/__init__.py
@@ -1,10 +0,0 @@
-from pathlib import Path
-
-
-# Check that the test directories exist.
-if not (Path(__file__).parent / 'baseline_images').exists():
-    raise IOError(
-        'The baseline image directory does not exist. '
-        'This is most likely because the test data is not installed. '
-        'You may need to install matplotlib from source to get the '
-        'test data.')

diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf
diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png