diff --git a/lib/matplotlib/backends/qt_compat.py b/lib/matplotlib/backends/qt_compat.py
index f237d449eb20..c564fd0c6c02 100644
--- a/lib/matplotlib/backends/qt_compat.py
+++ b/lib/matplotlib/backends/qt_compat.py
@@ -6,6 +6,7 @@
 from matplotlib.externals import six
 
 import os
+import sys
 from matplotlib import rcParams, verbose
 
 # Available APIs.
@@ -26,8 +27,18 @@
 
 if rcParams['backend'] == 'Qt5Agg':
     QT_RC_MAJOR_VERSION = 5
-else:
+elif rcParams['backend'] == 'Qt4Agg':
     QT_RC_MAJOR_VERSION = 4
+else:
+    # A different backend was specified, but we still got here because a Qt
+    # related file was imported. This is allowed, so lets try and guess
+    # what we should be using.
+    if "PyQt4" in sys.modules or "PySide" in sys.modules:
+        # PyQt4 or PySide is actually used.
+        QT_RC_MAJOR_VERSION = 4
+    else:
+        # This is a fallback: PyQt5
+        QT_RC_MAJOR_VERSION = 5
 
 QT_API = None
 
@@ -47,8 +58,18 @@
     # No ETS environment or incompatible so use rcParams.
     if rcParams['backend'] == 'Qt5Agg':
         QT_API = rcParams['backend.qt5']
-    else:
+    elif rcParams['backend'] == 'Qt4Agg':
         QT_API = rcParams['backend.qt4']
+    else:
+        # A different backend was specified, but we still got here because a Qt
+        # related file was imported. This is allowed, so lets try and guess
+        # what we should be using.
+        if "PyQt4" in sys.modules or "PySide" in sys.modules:
+            # PyQt4 or PySide is actually used.
+            QT_API = rcParams['backend.qt4']
+        else:
+            # This is a fallback: PyQt5
+            QT_API = rcParams['backend.qt5']
 
 # We will define an appropriate wrapper for the differing versions
 # of file dialog.