diff --git a/lib/matplotlib/_pylab_helpers.py b/lib/matplotlib/_pylab_helpers.py
index 4c95dc658bd1..907e69957ca2 100644
--- a/lib/matplotlib/_pylab_helpers.py
+++ b/lib/matplotlib/_pylab_helpers.py
@@ -57,6 +57,8 @@ def destroy(cls, num):
             manager = num
             if cls.figs.get(manager.num) is manager:
                 cls.figs.pop(manager.num)
+            else:
+                return
         else:
             try:
                 manager = cls.figs.pop(num)