From 349762b8d2fba9638746e6ed1c8d237a925561ec Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 21 Oct 2015 14:31:34 -0400 Subject: [PATCH 1/3] Reduce number of font files opened This should hopefully address the long-reported "Too many open files" error message (Fix #3315). To reproduce: On a Mac or Windows box with starvation for file handles (Linux has a much higher file handle limit by default), build the docs, then immediately build again. This will trigger the caching bug. The font cache in the mathtext renderer was broken. It was caching a font file once for every *combination* of font properties, including things like size. Therefore, in a complex math expression containing many different sizes of the same font, the font file was opened once for each of those sizes. Font files are opened and kept open (rather than opened, read, and closed) so that FreeType only needs to load the actual glyphs that are used, rather than the entire font. In an era of cheap memory and fast disk, it probably doesn't matter for our current fonts, but once #5214 is merged, we will have larger font files with many more glyphs and this loading time will matter more. The solution here is to do all font file loading in one place and to use `lru_cache` (available since Python 3.2) to do the caching, and to use only the file name and hinting parameters as a cache key. For earlier versions of Python, the functools32 backport package is required. (Or we can discuss whether we want to vendor it). --- lib/matplotlib/backends/backend_agg.py | 22 +++++++--------------- lib/matplotlib/backends/backend_pdf.py | 20 ++++++-------------- lib/matplotlib/backends/backend_pgf.py | 3 +-- lib/matplotlib/backends/backend_ps.py | 18 +++++------------- lib/matplotlib/backends/backend_svg.py | 17 +++++------------ lib/matplotlib/font_manager.py | 13 +++++++++++-- lib/matplotlib/mathtext.py | 9 ++++----- lib/matplotlib/tests/__init__.py | 6 ------ lib/matplotlib/textpath.py | 8 ++++---- lib/mpl_toolkits/tests/__init__.py | 6 ------ setup.py | 1 + setupext.py | 23 +++++++++++++++++++++++ 12 files changed, 67 insertions(+), 79 deletions(-) diff --git a/lib/matplotlib/backends/backend_agg.py b/lib/matplotlib/backends/backend_agg.py index f3b23ca95531..5c4c4d96c6c4 100644 --- a/lib/matplotlib/backends/backend_agg.py +++ b/lib/matplotlib/backends/backend_agg.py @@ -32,8 +32,8 @@ FigureManagerBase, FigureCanvasBase from matplotlib.cbook import is_string_like, maxdict, restrict_dict from matplotlib.figure import Figure -from matplotlib.font_manager import findfont -from matplotlib.ft2font import FT2Font, LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING, \ +from matplotlib.font_manager import findfont, get_font +from matplotlib.ft2font import LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING, \ LOAD_DEFAULT, LOAD_NO_AUTOHINT from matplotlib.mathtext import MathTextParser from matplotlib.path import Path @@ -81,7 +81,6 @@ class RendererAgg(RendererBase): # renderer at a time lock = threading.RLock() - _fontd = maxdict(50) def __init__(self, width, height, dpi): if __debug__: verbose.report('RendererAgg.__init__', 'debug-annoying') RendererBase.__init__(self) @@ -191,6 +190,7 @@ def draw_text(self, gc, x, y, s, prop, angle, ismath=False, mtext=None): flags = get_hinting_flag() font = self._get_agg_font(prop) + if font is None: return None if len(s) == 1 and ord(s) > 127: font.load_char(ord(s), flags=flags) @@ -272,18 +272,10 @@ def _get_agg_font(self, prop): if __debug__: verbose.report('RendererAgg._get_agg_font', 'debug-annoying') - key = hash(prop) - font = RendererAgg._fontd.get(key) - - if font is None: - fname = findfont(prop) - font = RendererAgg._fontd.get(fname) - if font is None: - font = FT2Font( - fname, - hinting_factor=rcParams['text.hinting_factor']) - RendererAgg._fontd[fname] = font - RendererAgg._fontd[key] = font + fname = findfont(prop) + font = get_font( + fname, + hinting_factor=rcParams['text.hinting_factor']) font.clear() size = prop.get_size_in_points() diff --git a/lib/matplotlib/backends/backend_pdf.py b/lib/matplotlib/backends/backend_pdf.py index 0ef7c1c49634..0dad65eec996 100644 --- a/lib/matplotlib/backends/backend_pdf.py +++ b/lib/matplotlib/backends/backend_pdf.py @@ -35,11 +35,11 @@ from matplotlib.cbook import Bunch, is_string_like, \ get_realpath_and_stat, is_writable_file_like, maxdict from matplotlib.figure import Figure -from matplotlib.font_manager import findfont, is_opentype_cff_font +from matplotlib.font_manager import findfont, is_opentype_cff_font, get_font from matplotlib.afm import AFM import matplotlib.type1font as type1font import matplotlib.dviread as dviread -from matplotlib.ft2font import FT2Font, FIXED_WIDTH, ITALIC, LOAD_NO_SCALE, \ +from matplotlib.ft2font import FIXED_WIDTH, ITALIC, LOAD_NO_SCALE, \ LOAD_NO_HINTING, KERNING_UNFITTED from matplotlib.mathtext import MathTextParser from matplotlib.transforms import Affine2D, BboxBase @@ -757,7 +757,7 @@ def createType1Descriptor(self, t1font, fontfile): if 0: flags |= 1 << 18 - ft2font = FT2Font(fontfile) + ft2font = get_font(fontfile) descriptor = { 'Type': Name('FontDescriptor'), @@ -817,7 +817,7 @@ def _get_xobject_symbol_name(self, filename, symbol_name): def embedTTF(self, filename, characters): """Embed the TTF font from the named file into the document.""" - font = FT2Font(filename) + font = get_font(filename) fonttype = rcParams['pdf.fonttype'] def cvt(length, upe=font.units_per_EM, nearest=True): @@ -1526,7 +1526,6 @@ def writeTrailer(self): class RendererPdf(RendererBase): - truetype_font_cache = maxdict(50) afm_font_cache = maxdict(50) def __init__(self, file, image_dpi): @@ -2126,15 +2125,8 @@ def _get_font_afm(self, prop): return font def _get_font_ttf(self, prop): - key = hash(prop) - font = self.truetype_font_cache.get(key) - if font is None: - filename = findfont(prop) - font = self.truetype_font_cache.get(filename) - if font is None: - font = FT2Font(filename) - self.truetype_font_cache[filename] = font - self.truetype_font_cache[key] = font + filename = findfont(prop) + font = get_font(filename) font.clear() font.set_size(prop.get_size_in_points(), 72) return font diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py index ce00652e1388..4f00ea0b43ab 100644 --- a/lib/matplotlib/backends/backend_pgf.py +++ b/lib/matplotlib/backends/backend_pgf.py @@ -36,10 +36,9 @@ system_fonts = [] if sys.platform.startswith('win'): from matplotlib import font_manager - from matplotlib.ft2font import FT2Font for f in font_manager.win32InstalledFonts(): try: - system_fonts.append(FT2Font(str(f)).family_name) + system_fonts.append(font_manager.get_font(str(f)).family_name) except: pass # unknown error, skip this font else: diff --git a/lib/matplotlib/backends/backend_ps.py b/lib/matplotlib/backends/backend_ps.py index 3f1145a5b14e..bb6f83afc667 100644 --- a/lib/matplotlib/backends/backend_ps.py +++ b/lib/matplotlib/backends/backend_ps.py @@ -28,8 +28,8 @@ def _fn_name(): return sys._getframe(1).f_code.co_name is_writable_file_like, maxdict, file_requires_unicode from matplotlib.figure import Figure -from matplotlib.font_manager import findfont, is_opentype_cff_font -from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING +from matplotlib.font_manager import findfont, is_opentype_cff_font, get_font +from matplotlib.ft2font import KERNING_DEFAULT, LOAD_NO_HINTING from matplotlib.ttconv import convert_ttf_to_ps from matplotlib.mathtext import MathTextParser from matplotlib._mathtext_data import uni2type1 @@ -199,7 +199,6 @@ class RendererPS(RendererBase): context instance that controls the colors/styles. """ - fontd = maxdict(50) afmfontd = maxdict(50) def __init__(self, width, height, pswriter, imagedpi=72): @@ -393,15 +392,8 @@ def _get_font_afm(self, prop): return font def _get_font_ttf(self, prop): - key = hash(prop) - font = self.fontd.get(key) - if font is None: - fname = findfont(prop) - font = self.fontd.get(fname) - if font is None: - font = FT2Font(fname) - self.fontd[fname] = font - self.fontd[key] = font + fname = findfont(prop) + font = get_font(fname) font.clear() size = prop.get_size_in_points() font.set_size(size, 72.0) @@ -1145,7 +1137,7 @@ def print_figure_impl(): if not rcParams['ps.useafm']: for font_filename, chars in six.itervalues(ps_renderer.used_characters): if len(chars): - font = FT2Font(font_filename) + font = get_font(font_filename) cmap = font.get_charmap() glyph_ids = [] for c in chars: diff --git a/lib/matplotlib/backends/backend_svg.py b/lib/matplotlib/backends/backend_svg.py index 4e137936b96d..9c59c8bbbfee 100644 --- a/lib/matplotlib/backends/backend_svg.py +++ b/lib/matplotlib/backends/backend_svg.py @@ -19,8 +19,8 @@ from matplotlib.cbook import is_string_like, is_writable_file_like, maxdict from matplotlib.colors import rgb2hex from matplotlib.figure import Figure -from matplotlib.font_manager import findfont, FontProperties -from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING +from matplotlib.font_manager import findfont, FontProperties, get_font +from matplotlib.ft2font import KERNING_DEFAULT, LOAD_NO_HINTING from matplotlib.mathtext import MathTextParser from matplotlib.path import Path from matplotlib import _path @@ -326,15 +326,8 @@ def _make_flip_transform(self, transform): .translate(0.0, self.height)) def _get_font(self, prop): - key = hash(prop) - font = self.fontd.get(key) - if font is None: - fname = findfont(prop) - font = self.fontd.get(fname) - if font is None: - font = FT2Font(fname) - self.fontd[fname] = font - self.fontd[key] = font + fname = findfont(prop) + font = get_font(fname) font.clear() size = prop.get_size_in_points() font.set_size(size, 72.0) @@ -495,7 +488,7 @@ def _write_svgfonts(self): writer = self.writer writer.start('defs') for font_fname, chars in six.iteritems(self._fonts): - font = FT2Font(font_fname) + font = get_font(font_fname) font.set_size(72, 72) sfnt = font.get_sfnt() writer.start('font', id=sfnt[(1, 0, 0, 4)]) diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py index 7bba3b8ae3ae..1dfae5e10c97 100644 --- a/lib/matplotlib/font_manager.py +++ b/lib/matplotlib/font_manager.py @@ -63,6 +63,12 @@ from matplotlib.fontconfig_pattern import \ parse_fontconfig_pattern, generate_fontconfig_pattern +try: + from functools import lru_cache +except ImportError: + from functools32 import lru_cache + + USE_FONTCONFIG = False verbose = matplotlib.verbose @@ -733,7 +739,7 @@ def get_name(self): Return the name of the font that best matches the font properties. """ - return ft2font.FT2Font(findfont(self)).family_name + return get_font(findfont(self)).family_name def get_style(self): """ @@ -1336,7 +1342,6 @@ def findfont(self, prop, fontext='ttf', directory=None, _lookup_cache[fontext].set(prop, result) return result - _is_opentype_cff_font_cache = {} def is_opentype_cff_font(filename): """ @@ -1357,6 +1362,10 @@ def is_opentype_cff_font(filename): fontManager = None _fmcache = None + +get_font = lru_cache(64)(ft2font.FT2Font) + + # The experimental fontconfig-based backend. if USE_FONTCONFIG and sys.platform != 'win32': import re diff --git a/lib/matplotlib/mathtext.py b/lib/matplotlib/mathtext.py index 90e45a3c4d75..7971366a0dd0 100644 --- a/lib/matplotlib/mathtext.py +++ b/lib/matplotlib/mathtext.py @@ -50,8 +50,8 @@ from matplotlib.afm import AFM from matplotlib.cbook import Bunch, get_realpath_and_stat, \ is_string_like, maxdict -from matplotlib.ft2font import FT2Font, FT2Image, KERNING_DEFAULT, LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING -from matplotlib.font_manager import findfont, FontProperties +from matplotlib.ft2font import FT2Image, KERNING_DEFAULT, LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING +from matplotlib.font_manager import findfont, FontProperties, get_font from matplotlib._mathtext_data import latex_to_bakoma, \ latex_to_standard, tex2uni, latex_to_cmex, stix_virtual_fonts from matplotlib import get_data_path, rcParams @@ -563,7 +563,7 @@ def __init__(self, default_font_prop, mathtext_backend): self._fonts = {} filename = findfont(default_font_prop) - default_font = self.CachedFont(FT2Font(filename)) + default_font = self.CachedFont(get_font(filename)) self._fonts['default'] = default_font self._fonts['regular'] = default_font @@ -576,10 +576,9 @@ def _get_font(self, font): basename = self.fontmap[font] else: basename = font - cached_font = self._fonts.get(basename) if cached_font is None and os.path.exists(basename): - font = FT2Font(basename) + font = get_font(basename) cached_font = self.CachedFont(font) self._fonts[basename] = cached_font self._fonts[font.postscript_name] = cached_font diff --git a/lib/matplotlib/tests/__init__.py b/lib/matplotlib/tests/__init__.py index 20a4ae3c6087..9b06bd1cbc91 100644 --- a/lib/matplotlib/tests/__init__.py +++ b/lib/matplotlib/tests/__init__.py @@ -49,12 +49,6 @@ def setup(): rcParams['text.hinting'] = False rcParams['text.hinting_factor'] = 8 - # Clear the font caches. Otherwise, the hinting mode can travel - # from one test to another. - backend_agg.RendererAgg._fontd.clear() - backend_pdf.RendererPdf.truetype_font_cache.clear() - backend_svg.RendererSVG.fontd.clear() - def assert_str_equal(reference_str, test_str, format_str=('String {str1} and {str2} do not ' diff --git a/lib/matplotlib/textpath.py b/lib/matplotlib/textpath.py index d46dd27898b1..e0f7fbe8d400 100644 --- a/lib/matplotlib/textpath.py +++ b/lib/matplotlib/textpath.py @@ -13,11 +13,11 @@ from matplotlib.path import Path from matplotlib import rcParams import matplotlib.font_manager as font_manager -from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING +from matplotlib.ft2font import KERNING_DEFAULT, LOAD_NO_HINTING from matplotlib.ft2font import LOAD_TARGET_LIGHT from matplotlib.mathtext import MathTextParser import matplotlib.dviread as dviread -from matplotlib.font_manager import FontProperties +from matplotlib.font_manager import FontProperties, get_font from matplotlib.transforms import Affine2D from matplotlib.externals.six.moves.urllib.parse import quote as urllib_quote @@ -54,7 +54,7 @@ def _get_font(self, prop): find a ttf font. """ fname = font_manager.findfont(prop) - font = FT2Font(fname) + font = get_font(fname) font.set_size(self.FONT_SCALE, self.DPI) return font @@ -334,7 +334,7 @@ def get_glyphs_tex(self, prop, s, glyph_map=None, font_bunch = self.tex_font_map[dvifont.texname] if font_and_encoding is None: - font = FT2Font(font_bunch.filename) + font = get_font(font_bunch.filename) for charmap_name, charmap_code in [("ADOBE_CUSTOM", 1094992451), diff --git a/lib/mpl_toolkits/tests/__init__.py b/lib/mpl_toolkits/tests/__init__.py index 20a4ae3c6087..9b06bd1cbc91 100644 --- a/lib/mpl_toolkits/tests/__init__.py +++ b/lib/mpl_toolkits/tests/__init__.py @@ -49,12 +49,6 @@ def setup(): rcParams['text.hinting'] = False rcParams['text.hinting_factor'] = 8 - # Clear the font caches. Otherwise, the hinting mode can travel - # from one test to another. - backend_agg.RendererAgg._fontd.clear() - backend_pdf.RendererPdf.truetype_font_cache.clear() - backend_svg.RendererSVG.fontd.clear() - def assert_str_equal(reference_str, test_str, format_str=('String {str1} and {str2} do not ' diff --git a/setup.py b/setup.py index f3ecd708f54f..b3fb0413699e 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ 'Required dependencies and extensions', setupext.Numpy(), setupext.Dateutil(), + setupext.FuncTools32(), setupext.Pytz(), setupext.Cycler(), setupext.Tornado(), diff --git a/setupext.py b/setupext.py index 0a9660b6f7c8..6ef9480c943d 100755 --- a/setupext.py +++ b/setupext.py @@ -1221,6 +1221,29 @@ def get_install_requires(self): return [dateutil] +class FuncTools32(SetupPackage): + name = "functools32" + + def check(self): + if sys.version_info[:2] < (3, 2): + try: + import functools32 + except ImportError: + return ( + "functools32 was not found. It is required for for" + "python versions prior to 3.2") + + return "using functools32 version %s" % functools32.__version__ + else: + return "Not required" + + def get_install_requires(self): + if sys.version_info[:2] < (3, 2): + return ['functools32'] + else: + return [] + + class Tornado(OptionalPackage): name = "tornado" From 082a3a5890ba37b10f191ba8aba25a8aed723677 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 21 Oct 2015 14:38:56 -0400 Subject: [PATCH 2/3] Add INSTALL note about functools32 --- INSTALL | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/INSTALL b/INSTALL index a81ae7e1a104..02e5946e89d5 100644 --- a/INSTALL +++ b/INSTALL @@ -214,6 +214,10 @@ libpng 1.2 (or later) ``cycler`` 0.9 or later Composable cycle class used for constructing style-cycles +`functools32` + Required for compatibility if running on versions of Python before + Python 3.2. + Optional GUI framework ^^^^^^^^^^^^^^^^^^^^^^ From 5e93dfc265e041aa6855f2eb63946dfc0df4ffad Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 21 Oct 2015 15:54:07 -0400 Subject: [PATCH 3/3] functools32 has no version --- setupext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setupext.py b/setupext.py index 6ef9480c943d..1c70adde064e 100755 --- a/setupext.py +++ b/setupext.py @@ -1233,7 +1233,7 @@ def check(self): "functools32 was not found. It is required for for" "python versions prior to 3.2") - return "using functools32 version %s" % functools32.__version__ + return "using functools32" else: return "Not required"