From e3e2be95ce8ab60c5c00944ce1dc593e4ac9487c Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sun, 6 Dec 2015 22:28:30 +0100 Subject: [PATCH 1/7] Reset the available animation movie writer on rcParam change If one of the rcParams for a path to a program, which was called by a movie writer, is changed, the the available movie writer in the registry should be reevaluated if they are (still/became) available. This also fixes the problem that you have to set the path to a movie writer before importing mpl.animation, as before the state was fixed on import time. --- lib/matplotlib/animation.py | 23 +++++++++++++++++++++++ lib/matplotlib/rcsetup.py | 23 +++++++++++++++++++---- lib/matplotlib/tests/test_animation.py | 26 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/lib/matplotlib/animation.py b/lib/matplotlib/animation.py index 57e054171196..b337f7a65928 100644 --- a/lib/matplotlib/animation.py +++ b/lib/matplotlib/animation.py @@ -64,6 +64,12 @@ class MovieWriterRegistry(object): def __init__(self): self.avail = dict() + self._registered = dict() + self._dirty = False + + def set_dirty(self): + """Sets a flag to re-setup the writers""" + self._dirty = True # Returns a decorator that can be used on classes to register them under # a name. As in: @@ -72,19 +78,36 @@ def __init__(self): # pass def register(self, name): def wrapper(writerClass): + self._registered[name] = writerClass if writerClass.isAvailable(): self.avail[name] = writerClass return writerClass return wrapper + def ensure_not_dirty(self): + """If dirty, reasks the writers if they are available""" + if self._dirty: + self.reset_available_writers() + + def reset_available_writers(self): + """Reset the available state of all registered writers""" + self.avail = {} + for name, writerClass in self._registered.items(): + if writerClass.isAvailable(): + self.avail[name] = writerClass + self._dirty = False + def list(self): ''' Get a list of available MovieWriters.''' + self.ensure_not_dirty() return list(self.avail.keys()) def is_available(self, name): + self.ensure_not_dirty() return name in self.avail def __getitem__(self, name): + self.ensure_not_dirty() if not self.avail: raise RuntimeError("No MovieWriters available!") return self.avail[name] diff --git a/lib/matplotlib/rcsetup.py b/lib/matplotlib/rcsetup.py index f8c239867087..18cf34ce3893 100644 --- a/lib/matplotlib/rcsetup.py +++ b/lib/matplotlib/rcsetup.py @@ -802,6 +802,21 @@ def validate_hist_bins(s): raise ValueError("'hist.bins' must be 'auto', an int or " + "a sequence of floats") +def validate_animation_writer_path(p): + # Make sure it's a string and then figure out if the animations + # are already loaded and reset the writers (which will validate + # the path on next call) + if not isinstance(p, six.text_type): + raise ValueError("path must be a (unicode) string") + from sys import modules + # set dirty, so that the next call to the registry will re-evaluate + # the state. + # only set dirty if already loaded. If not loaded, the load will + # trigger the checks. + if "matplotlib.animation" in modules: + modules["matplotlib.animation"].writers.set_dirty() + return p + # a map from key -> value, converter defaultParams = { @@ -1222,20 +1237,20 @@ def validate_hist_bins(s): # Controls image format when frames are written to disk 'animation.frame_format': ['png', validate_movie_frame_fmt], # Path to FFMPEG binary. If just binary name, subprocess uses $PATH. - 'animation.ffmpeg_path': ['ffmpeg', six.text_type], + 'animation.ffmpeg_path': ['ffmpeg', validate_animation_writer_path], # Additional arguments for ffmpeg movie writer (using pipes) 'animation.ffmpeg_args': [[], validate_stringlist], # Path to AVConv binary. If just binary name, subprocess uses $PATH. - 'animation.avconv_path': ['avconv', six.text_type], + 'animation.avconv_path': ['avconv', validate_animation_writer_path], # Additional arguments for avconv movie writer (using pipes) 'animation.avconv_args': [[], validate_stringlist], # Path to MENCODER binary. If just binary name, subprocess uses $PATH. - 'animation.mencoder_path': ['mencoder', six.text_type], + 'animation.mencoder_path': ['mencoder', validate_animation_writer_path], # Additional arguments for mencoder movie writer (using pipes) 'animation.mencoder_args': [[], validate_stringlist], # Path to convert binary. If just binary name, subprocess uses $PATH - 'animation.convert_path': ['convert', six.text_type], + 'animation.convert_path': ['convert', validate_animation_writer_path], # Additional arguments for mencoder movie writer (using pipes) 'animation.convert_args': [[], validate_stringlist], diff --git a/lib/matplotlib/tests/test_animation.py b/lib/matplotlib/tests/test_animation.py index 2a62cfb7a2ac..ec2bc3a1ed52 100644 --- a/lib/matplotlib/tests/test_animation.py +++ b/lib/matplotlib/tests/test_animation.py @@ -4,10 +4,13 @@ from matplotlib.externals import six import os +import sys import tempfile import numpy as np from numpy.testing import assert_equal from nose import with_setup +from nose.tools import assert_false, assert_true +import matplotlib as mpl from matplotlib import pyplot as plt from matplotlib import animation from matplotlib.testing.noseclasses import KnownFailureTest @@ -163,6 +166,29 @@ def animate(i): frames=iter(range(5))) +def test_movie_writer_registry(): + ffmpeg_path = mpl.rcParams['animation.ffmpeg_path'] + # Not sure about the first state as there could be some writer + # which set rcparams + #assert_false(animation.writers._dirty) + assert_true(len(animation.writers._registered) > 0) + animation.writers.list() # resets dirty state + assert_false(animation.writers._dirty) + mpl.rcParams['animation.ffmpeg_path'] = u"not_available_ever_xxxx" + assert_true(animation.writers._dirty) + animation.writers.list() # resets + assert_false(animation.writers._dirty) + assert_false(animation.writers.is_available("ffmpeg")) + # something which is garanteered to be available in path + bin = u"python" + mpl.rcParams['animation.ffmpeg_path'] = bin + assert_true(animation.writers._dirty) + animation.writers.list() # resets + assert_false(animation.writers._dirty) + assert_true(animation.writers.is_available("ffmpeg")) + mpl.rcParams['animation.ffmpeg_path'] = ffmpeg_path + + if __name__ == "__main__": import nose nose.runmodule(argv=['-s', '--with-doctest'], exit=False) From 6a01829ac023c842b215c05da86a062f6c95d9b2 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Wed, 9 Dec 2015 11:06:17 +0100 Subject: [PATCH 2/7] TST: Use a better command for annimation tests python is blocking, leading to timeouts... --- lib/matplotlib/tests/test_animation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/matplotlib/tests/test_animation.py b/lib/matplotlib/tests/test_animation.py index ec2bc3a1ed52..7ec385d12bd6 100644 --- a/lib/matplotlib/tests/test_animation.py +++ b/lib/matplotlib/tests/test_animation.py @@ -180,7 +180,8 @@ def test_movie_writer_registry(): assert_false(animation.writers._dirty) assert_false(animation.writers.is_available("ffmpeg")) # something which is garanteered to be available in path - bin = u"python" + # and exits immediately + bin = u"true" if sys.platform != 'win32' else u"where" mpl.rcParams['animation.ffmpeg_path'] = bin assert_true(animation.writers._dirty) animation.writers.list() # resets From 6c6cf91fa72398d918ba752bff91a4ba77c01bec Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sun, 7 Feb 2016 10:42:15 +0100 Subject: [PATCH 3/7] Fix: rerun check for imagemagick in isAvailable On windows the check ensures that the imagemagick path for 'convert' was set to '' if it was just 'convert' to prevent clashes with the windows built-in command 'convert' (doing filesystem conversion). Unfortunately, the command only set this values in the current rcParams and rcParamsDefault, but a use of `matplotlib.style.use("classic")` after importing mpl.animation would again overwrite it with `convert`. In this case, the image comparison decorator sets the style to classic and appveyor went BOOM... --- lib/matplotlib/animation.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/matplotlib/animation.py b/lib/matplotlib/animation.py index b337f7a65928..05903da0c6eb 100644 --- a/lib/matplotlib/animation.py +++ b/lib/matplotlib/animation.py @@ -338,10 +338,11 @@ def isAvailable(cls): Check to see if a MovieWriter subclass is actually available by running the commandline tool. ''' - if not cls.bin_path(): + bin_path = cls.bin_path() + if not bin_path: return False try: - p = subprocess.Popen(cls.bin_path(), + p = subprocess.Popen(bin_path, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -642,12 +643,22 @@ def _init_from_registry(cls): binpath = '' rcParams[cls.exec_key] = rcParamsDefault[cls.exec_key] = binpath + @classmethod + def isAvailable(cls): + ''' + Check to see if a MovieWriter subclass is actually available by + running the commandline tool. + ''' + bin_path = cls.bin_path() + if bin_path == "convert": + cls._init_from_registry() + return super(ImageMagickBase, cls).isAvailable() ImageMagickBase._init_from_registry() @writers.register('imagemagick') -class ImageMagickWriter(MovieWriter, ImageMagickBase): +class ImageMagickWriter(ImageMagickBase, MovieWriter, ): def _args(self): return ([self.bin_path(), '-size', '%ix%i' % self.frame_size, '-depth', '8', @@ -657,7 +668,7 @@ def _args(self): @writers.register('imagemagick_file') -class ImageMagickFileWriter(FileMovieWriter, ImageMagickBase): +class ImageMagickFileWriter(ImageMagickBase, FileMovieWriter): supported_formats = ['png', 'jpeg', 'ppm', 'tiff', 'sgi', 'bmp', 'pbm', 'raw', 'rgba'] From 96b439a167d53ccbe23574aada1d16417cf69575 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sun, 7 Feb 2016 17:37:22 +0100 Subject: [PATCH 4/7] fix spelling --- lib/matplotlib/tests/test_animation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/matplotlib/tests/test_animation.py b/lib/matplotlib/tests/test_animation.py index 7ec385d12bd6..ea395f7e9cbb 100644 --- a/lib/matplotlib/tests/test_animation.py +++ b/lib/matplotlib/tests/test_animation.py @@ -179,7 +179,7 @@ def test_movie_writer_registry(): animation.writers.list() # resets assert_false(animation.writers._dirty) assert_false(animation.writers.is_available("ffmpeg")) - # something which is garanteered to be available in path + # something which is guaranteed to be available in path # and exits immediately bin = u"true" if sys.platform != 'win32' else u"where" mpl.rcParams['animation.ffmpeg_path'] = bin From 9b61b341415825f4996ec671f3a157b8be9687ec Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sun, 7 Feb 2016 18:57:56 +0100 Subject: [PATCH 5/7] FIX: add debugging output when the MovieWriter fails This pulls out the stderr and stdout from the internal POpen buffers (tested on py3.5 and not shown if any error occures while pulling out the messages...). It also changes the exception message to mention mpl.verbose.set_level("helpful") because adding a commandline switch is probably not helpful in todays world where everybody uses the Jupyter notebook and can't add to the kernel commandline... It would be nice if this verbose level could be set by nose / tests.py, but I have found no way yet :-( --- lib/matplotlib/animation.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/matplotlib/animation.py b/lib/matplotlib/animation.py index 05903da0c6eb..96c6d7199496 100644 --- a/lib/matplotlib/animation.py +++ b/lib/matplotlib/animation.py @@ -456,9 +456,16 @@ def finish(self): # Check error code for creating file here, since we just run # the process here, rather than having an open pipe. if self._proc.returncode: + try: + stdout = [s.decode() for s in self._proc._stdout_buff] + stderr = [s.decode() for s in self._proc._stderr_buff] + verbose.report("MovieWriter.finish: stdout: %s" % stdout, level='helpful') + verbose.report("MovieWriter.finish: stderr: %s" % stderr, level='helpful') + except Exception as e: + pass raise RuntimeError('Error creating movie, return code: ' + str(self._proc.returncode) - + ' Try running with --verbose-debug') + + ' Try setting mpl.verbose.set_level("helpful")') def cleanup(self): MovieWriter.cleanup(self) From 33f564ac275d997458aab4e79a96ccdb9e547f97 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sun, 7 Feb 2016 21:26:32 +0100 Subject: [PATCH 6/7] Pep8 fixes --- lib/matplotlib/animation.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/matplotlib/animation.py b/lib/matplotlib/animation.py index 96c6d7199496..c932580298a3 100644 --- a/lib/matplotlib/animation.py +++ b/lib/matplotlib/animation.py @@ -459,13 +459,16 @@ def finish(self): try: stdout = [s.decode() for s in self._proc._stdout_buff] stderr = [s.decode() for s in self._proc._stderr_buff] - verbose.report("MovieWriter.finish: stdout: %s" % stdout, level='helpful') - verbose.report("MovieWriter.finish: stderr: %s" % stderr, level='helpful') + verbose.report("MovieWriter.finish: stdout: %s" % stdout, + level='helpful') + verbose.report("MovieWriter.finish: stderr: %s" % stderr, + level='helpful') except Exception as e: pass - raise RuntimeError('Error creating movie, return code: ' - + str(self._proc.returncode) - + ' Try setting mpl.verbose.set_level("helpful")') + msg = ('Error creating movie, return code: ' + + str(self._proc.returncode) + + ' Try setting mpl.verbose.set_level("helpful")') + raise RuntimeError(msg) def cleanup(self): MovieWriter.cleanup(self) From e7838671ea8f26065ca0927235dd8a3b319d94b2 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sun, 7 Feb 2016 21:32:40 +0100 Subject: [PATCH 7/7] Add comment to explain the fix in ImageMagickBase --- lib/matplotlib/animation.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/animation.py b/lib/matplotlib/animation.py index c932580298a3..d70c218c525c 100644 --- a/lib/matplotlib/animation.py +++ b/lib/matplotlib/animation.py @@ -656,7 +656,9 @@ def _init_from_registry(cls): @classmethod def isAvailable(cls): ''' - Check to see if a MovieWriter subclass is actually available by + Check to see if a ImageMagickWriter is actually available + + Done by first checking the windows registry (if applicable) and then running the commandline tool. ''' bin_path = cls.bin_path() @@ -667,8 +669,12 @@ def isAvailable(cls): ImageMagickBase._init_from_registry() +# Note: the base classes need to be in that order to get +# isAvailable() from ImageMagickBase called and not the +# one from MovieWriter. The latter is then called by the +# former. @writers.register('imagemagick') -class ImageMagickWriter(ImageMagickBase, MovieWriter, ): +class ImageMagickWriter(ImageMagickBase, MovieWriter): def _args(self): return ([self.bin_path(), '-size', '%ix%i' % self.frame_size, '-depth', '8', @@ -677,6 +683,10 @@ def _args(self): + self.output_args) +# Note: the base classes need to be in that order to get +# isAvailable() from ImageMagickBase called and not the +# one from MovieWriter. The latter is then called by the +# former. @writers.register('imagemagick_file') class ImageMagickFileWriter(ImageMagickBase, FileMovieWriter): supported_formats = ['png', 'jpeg', 'ppm', 'tiff', 'sgi', 'bmp',