diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py
index 4c634938706a..87fd7adceac7 100644
--- a/lib/matplotlib/backends/backend_pgf.py
+++ b/lib/matplotlib/backends/backend_pgf.py
@@ -22,8 +22,6 @@
 from matplotlib.text import Text
 from matplotlib.path import Path
 from matplotlib import _png, rcParams
-from matplotlib import font_manager
-from matplotlib.ft2font import FT2Font
 from matplotlib.cbook import is_string_like, is_writable_file_like
 from matplotlib.compat import subprocess
 from matplotlib.compat.subprocess import check_output
@@ -33,14 +31,24 @@
 
 # create a list of system fonts, all of these should work with xe/lua-latex
 system_fonts = []
-for f in font_manager.findSystemFonts():
+if sys.platform.startswith('win'):
+    from matplotlib import font_manager
+    from matplotlib.ft2font import FT2Font
+    for f in font_manager.win32InstalledFonts():
+        try:
+            system_fonts.append(FT2Font(str(f)).family_name)
+        except:
+            pass # unknown error, skip this font
+else:
+    # assuming fontconfig is installed and the command 'fc-list' exists
     try:
-        system_fonts.append(FT2Font(f).family_name)
-    except RuntimeError:
-        pass  # some fonts on osx are known to fail, print?
+        # list scalable (non-bitmap) fonts
+        fc_list = check_output(['fc-list', ':outline,scalable', 'family'])
+        fc_list = fc_list.decode('utf8')
+        system_fonts = [f.split(',')[0] for f in fc_list.splitlines()]
+        system_fonts = list(set(system_fonts))
     except:
-        pass  # unknown error, skip this font
-
+        warnings.warn('error getting fonts from fc-list', UserWarning)
 
 def get_texcommand():
     """Get chosen TeX system from rc."""
diff --git a/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_bbox_inches.pdf b/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_bbox_inches.pdf
index dcc0f56c998b..dcd7455e718b 100644
Binary files a/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_bbox_inches.pdf and b/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_bbox_inches.pdf differ
diff --git a/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf b/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf
index 86dfbbc527cf..8f510ea867d9 100644
Binary files a/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf and b/lib/matplotlib/tests/baseline_images/test_backend_pgf/pgf_mixedmode.pdf differ
diff --git a/lib/matplotlib/tests/test_backend_pgf.py b/lib/matplotlib/tests/test_backend_pgf.py
index 2703c4ca5809..2fbbea02d036 100644
--- a/lib/matplotlib/tests/test_backend_pgf.py
+++ b/lib/matplotlib/tests/test_backend_pgf.py
@@ -148,6 +148,10 @@ def test_pathclip():
     if not check_for('xelatex'):
         raise SkipTest('xelatex + pgf is required')
 
+    rc_xelatex = {'font.family': 'serif',
+                  'pgf.rcfonts': False}
+    mpl.rcParams.update(rc_xelatex)
+
     plt.figure()
     plt.plot([0., 1e100], [0., 1e100])
     plt.xlim(0, 1)
@@ -162,6 +166,10 @@ def test_mixedmode():
     if not check_for('xelatex'):
         raise SkipTest('xelatex + pgf is required')
 
+    rc_xelatex = {'font.family': 'serif',
+                  'pgf.rcfonts': False}
+    mpl.rcParams.update(rc_xelatex)
+
     Y, X = np.ogrid[-1:1:40j, -1:1:40j]
     plt.figure()
     plt.pcolor(X**2 + Y**2).set_rasterized(True)
@@ -174,6 +182,10 @@ def test_bbox_inches():
     if not check_for('xelatex'):
         raise SkipTest('xelatex + pgf is required')
 
+    rc_xelatex = {'font.family': 'serif',
+                  'pgf.rcfonts': False}
+    mpl.rcParams.update(rc_xelatex)
+
     Y, X = np.ogrid[-1:1:40j, -1:1:40j]
     fig = plt.figure()
     ax1 = fig.add_subplot(121)