diff --git a/lib/matplotlib/axis.py b/lib/matplotlib/axis.py
index 0710eddf1154..277869e1123e 100644
--- a/lib/matplotlib/axis.py
+++ b/lib/matplotlib/axis.py
@@ -235,12 +235,14 @@ def get_loc(self):
     @allow_rasterization
     def draw(self, renderer):
         if not self.get_visible():
+            self.stale = False
             return
-        renderer.open_group(self.__name__)
+
         midPoint = mtransforms.interval_contains(self.get_view_interval(),
                                                  self.get_loc())
 
         if midPoint:
+            renderer.open_group(self.__name__)
             if self.gridOn:
                 self.gridline.draw(renderer)
             if self.tick1On:
@@ -248,12 +250,12 @@ def draw(self, renderer):
             if self.tick2On:
                 self.tick2line.draw(renderer)
 
-        if self.label1On:
-            self.label1.draw(renderer)
-        if self.label2On:
-            self.label2.draw(renderer)
+            if self.label1On:
+                self.label1.draw(renderer)
+            if self.label2On:
+                self.label2.draw(renderer)
+            renderer.close_group(self.__name__)
 
-        renderer.close_group(self.__name__)
         self.stale = False
 
     def set_label1(self, s):