diff --git a/lib/matplotlib/colorbar.py b/lib/matplotlib/colorbar.py
index 0d20a59d67c3..39f478944ecc 100644
--- a/lib/matplotlib/colorbar.py
+++ b/lib/matplotlib/colorbar.py
@@ -227,9 +227,9 @@ def __init__(self, parent, userax=True):
             True if the user passed `.Figure.colorbar` the axes manually.
         """
 
+        fig = parent.figure
         if userax:
             # copy position:
-            fig = parent.figure
             outer_ax = fig.add_axes(parent.get_position())
             # copy the locator if one exists:
             outer_ax._axes_locator = parent._axes_locator
@@ -239,28 +239,48 @@ def __init__(self, parent, userax=True):
                 parent._axes.add_child_axes(outer_ax)
                 outer_ax._axes.child_axes.remove(parent)
             else:
-                parent.remove()
+                try:
+                    parent.remove()
+                except ValueError:
+                    pass  # Already removed
         else:
             outer_ax = parent
 
+        # swap axes in the stack if its in there:
+        if outer_ax in fig._localaxes:
+            fig._localaxes.remove(outer_ax)
+            fig._axstack.remove(outer_ax)
+            fig._localaxes.add(self)
+            fig._axstack.add(self)
         inner_ax = outer_ax.inset_axes([0, 0, 1, 1])
         self.__dict__.update(inner_ax.__dict__)
 
         self.outer_ax = outer_ax
         self.inner_ax = inner_ax
+
         self.outer_ax.xaxis.set_visible(False)
         self.outer_ax.yaxis.set_visible(False)
         self.outer_ax.set_facecolor('none')
         self.outer_ax.tick_params = self.inner_ax.tick_params
         self.outer_ax.set_xticks = self.inner_ax.set_xticks
         self.outer_ax.set_yticks = self.inner_ax.set_yticks
-        for attr in ["get_position", "set_position", "set_aspect"]:
+        for attr in ["get_position", "set_aspect",
+                     "_remove_method", "_set_position",
+                     "set_position", "cla", "draw"]:
             setattr(self, attr, getattr(self.outer_ax, attr))
         self._colorbar_info = None  # used for mpl-created axes
+        if hasattr(self.outer_ax, "get_subplotspec"):
+            attr = "get_subplotspec"
+            setattr(self, attr, getattr(self.outer_ax, attr))
+
         if userax:
             self._colorbar_info = 'user'
             # point the parent's methods all at this axes...
+            origdict = parent.__dict__
             parent.__dict__ = self.__dict__
+            for key in origdict.keys():
+                if key not in parent.__dict__:
+                    parent.__dict__[key] = origdict[key]
 
     def _set_inner_bounds(self, bounds):
         """
@@ -949,8 +969,7 @@ def remove(self):
         If the colorbar was created with ``use_gridspec=True`` the previous
         gridspec is restored.
         """
-        self.ax.inner_ax.remove()
-        self.ax.outer_ax.remove()
+        self.ax.remove()
 
         self.mappable.callbacksSM.disconnect(self.mappable.colorbar_cid)
         self.mappable.colorbar = None
diff --git a/lib/matplotlib/figure.py b/lib/matplotlib/figure.py
index dee6024f528e..ba902506a9af 100644
--- a/lib/matplotlib/figure.py
+++ b/lib/matplotlib/figure.py
@@ -1200,7 +1200,7 @@ def subplots_adjust(self, left=None, bottom=None, right=None, top=None,
                 "disabling constrained_layout.")
         self.subplotpars.update(left, bottom, right, top, wspace, hspace)
         for ax in self.axes:
-            if isinstance(ax, SubplotBase):
+            if hasattr(ax, 'get_subplotspec'):
                 ax._set_position(ax.get_subplotspec().get_position(self))
         self.stale = True
 
diff --git a/lib/mpl_toolkits/axes_grid1/axes_grid.py b/lib/mpl_toolkits/axes_grid1/axes_grid.py
index 7720b7bbd6dc..58b111865aa0 100644
--- a/lib/mpl_toolkits/axes_grid1/axes_grid.py
+++ b/lib/mpl_toolkits/axes_grid1/axes_grid.py
@@ -3,7 +3,6 @@
 
 import numpy as np
 
-import matplotlib as mpl
 from matplotlib import _api
 from matplotlib.gridspec import SubplotSpec
 
@@ -29,28 +28,19 @@ def colorbar(self, mappable, *, ticks=None, **kwargs):
         orientation = (
             "horizontal" if self.orientation in ["top", "bottom"] else
             "vertical")
-        kwargs['userax'] = False
-        cb = mpl.colorbar.Colorbar(
-            self, mappable, orientation=orientation, ticks=ticks, **kwargs)
-        self._config_axes()
+        cb = self.figure.colorbar(mappable, cax=self, orientation=orientation,
+                                  ticks=ticks, **kwargs)
         return cb
 
-    def _config_axes(self):
-        """Make an axes patch and outline."""
-        ax = self
-        ax.set_navigate(False)
-        ax.axis[:].toggle(all=False)
-        b = self._default_label_on
-        ax.axis[self.orientation].toggle(all=b)
-
     def toggle_label(self, b):
         self._default_label_on = b
         axis = self.axis[self.orientation]
         axis.toggle(ticklabels=b, label=b)
 
     def cla(self):
+        orientation = self.orientation
         super().cla()
-        self._config_axes()
+        self.orientation = orientation
 
 
 class CbarAxes(CbarAxesBase, Axes):