diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py
index 917810b31518..1a7fcf429fa1 100644
--- a/lib/matplotlib/backends/backend_pgf.py
+++ b/lib/matplotlib/backends/backend_pgf.py
@@ -18,8 +18,6 @@
 from matplotlib.text import Text
 from matplotlib.path import Path
 from matplotlib import _png, rcParams
-from matplotlib import font_manager
-from matplotlib.ft2font import FT2Font
 from matplotlib.cbook import is_string_like, is_writable_file_like
 from matplotlib.compat import subprocess
 from matplotlib.compat.subprocess import check_output
@@ -29,14 +27,23 @@
 
 # create a list of system fonts, all of these should work with xe/lua-latex
 system_fonts = []
-for f in font_manager.findSystemFonts():
+if sys.platform == 'win32':
+    from matplotlib import font_manager
+    from matplotlib.ft2font import FT2Font
+    for f in font_manager.win32InstalledFonts():
+        try:
+            system_fonts.append(FT2Font(str(f)).family_name)
+        except:
+            pass  # unknown error, skip this font
+else:
+    # assuming fontconfig is installed and the command 'fc-list' exists
     try:
-        system_fonts.append(FT2Font(str(f)).family_name)
-    except RuntimeError:
-        pass  # some fonts on osx are known to fail, print?
+        # search for scalable (non-bitmap) fonts
+        fc_list = check_output(["fc-list", ":outline,scalable", "family"])
+        system_fonts = [f.split(",")[0] for f in fc_list.splitlines()]
+        system_fonts = list(set(system_fonts))
     except:
-        pass  # unknown error, skip this font
-
+        pass
 
 def get_texcommand():
     """Get chosen TeX system from rc."""
diff --git a/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf b/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf
index 86dfbbc527cf..8f510ea867d9 100644
Binary files a/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf and b/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf differ
diff --git a/lib/matplotlib/tests/test_backend_pgf.py b/lib/matplotlib/tests/test_backend_pgf.py
index 2a960e493201..0ff6c27be407 100644
--- a/lib/matplotlib/tests/test_backend_pgf.py
+++ b/lib/matplotlib/tests/test_backend_pgf.py
@@ -138,6 +138,10 @@ def test_pathclip():
     if not check_for('xelatex'):
         raise SkipTest('xelatex + pgf is required')
 
+    rc_xelatex = {'font.family': 'serif',
+                   'pgf.rcfonts': False,}
+    mpl.rcParams.update(rc_xelatex)
+
     plt.figure()
     plt.plot([0., 1e100], [0., 1e100])
     plt.xlim(0, 1)
@@ -152,6 +156,10 @@ def test_mixedmode():
     if not check_for('xelatex'):
         raise SkipTest('xelatex + pgf is required')
 
+    rc_xelatex = {'font.family': 'serif',
+                   'pgf.rcfonts': False,}
+    mpl.rcParams.update(rc_xelatex)
+
     Y, X = np.ogrid[-1:1:40j, -1:1:40j]
     plt.figure()
     plt.pcolor(X**2 + Y**2).set_rasterized(True)