diff --git a/lib/matplotlib/axes/_base.py b/lib/matplotlib/axes/_base.py
index 0078b799b159..792779a80b85 100644
--- a/lib/matplotlib/axes/_base.py
+++ b/lib/matplotlib/axes/_base.py
@@ -4693,14 +4693,25 @@ def label_outer(self, remove_inner_ticks=False):
         self._label_outer_yaxis(skip_non_rectangular_axes=False,
                                 remove_inner_ticks=remove_inner_ticks)
 
+    def _get_subplotspec_with_optional_colorbar(self):
+        """
+        Return the subplotspec for this Axes, except that if this Axes has been
+        moved to a subgridspec to make room for a colorbar, then return the
+        subplotspec that encloses both this Axes and the colorbar Axes.
+        """
+        ss = self.get_subplotspec()
+        if any(cax.get_subplotspec() for cax in self._colorbars):
+            ss = ss.get_gridspec()._subplot_spec
+        return ss
+
     def _label_outer_xaxis(self, *, skip_non_rectangular_axes,
                            remove_inner_ticks=False):
         # see documentation in label_outer.
         if skip_non_rectangular_axes and not isinstance(self.patch,
                                                         mpl.patches.Rectangle):
             return
-        ss = self.get_subplotspec()
-        if not ss:
+        ss = self._get_subplotspec_with_optional_colorbar()
+        if ss is None:
             return
         label_position = self.xaxis.get_label_position()
         if not ss.is_first_row():  # Remove top label/ticklabels/offsettext.
@@ -4726,8 +4737,8 @@ def _label_outer_yaxis(self, *, skip_non_rectangular_axes,
         if skip_non_rectangular_axes and not isinstance(self.patch,
                                                         mpl.patches.Rectangle):
             return
-        ss = self.get_subplotspec()
-        if not ss:
+        ss = self._get_subplotspec_with_optional_colorbar()
+        if ss is None:
             return
         label_position = self.yaxis.get_label_position()
         if not ss.is_first_col():  # Remove left label/ticklabels/offsettext.
diff --git a/lib/matplotlib/colorbar.py b/lib/matplotlib/colorbar.py
index 624e2250303c..1cf3b77f01c2 100644
--- a/lib/matplotlib/colorbar.py
+++ b/lib/matplotlib/colorbar.py
@@ -1453,8 +1453,7 @@ def make_axes(parents, location=None, orientation=None, fraction=0.15,
 
     cax = fig.add_axes(pbcb, label="<colorbar>")
     for a in parents:
-        # tell the parent it has a colorbar
-        a._colorbars += [cax]
+        a._colorbars.append(cax)  # tell the parent it has a colorbar
     cax._colorbar_info = dict(
         parents=parents,
         location=location,
@@ -1547,6 +1546,7 @@ def make_axes_gridspec(parent, *, location=None, orientation=None,
 
     fig = parent.get_figure()
     cax = fig.add_subplot(ss_cb, label="<colorbar>")
+    parent._colorbars.append(cax)  # tell the parent it has a colorbar
     cax.set_anchor(anchor)
     cax.set_box_aspect(aspect)
     cax.set_aspect('auto')
diff --git a/lib/matplotlib/tests/test_subplots.py b/lib/matplotlib/tests/test_subplots.py
index 704ff6e1bd3f..aa5debd0b787 100644
--- a/lib/matplotlib/tests/test_subplots.py
+++ b/lib/matplotlib/tests/test_subplots.py
@@ -4,6 +4,7 @@
 import numpy as np
 import pytest
 
+import matplotlib as mpl
 from matplotlib.axes import Axes, SubplotBase
 import matplotlib.pyplot as plt
 from matplotlib.testing.decorators import check_figures_equal, image_comparison
@@ -111,10 +112,15 @@ def test_shared():
 
 
 @pytest.mark.parametrize('remove_ticks', [True, False])
-def test_label_outer(remove_ticks):
-    f, axs = plt.subplots(2, 2, sharex=True, sharey=True)
+@pytest.mark.parametrize('layout_engine', ['none', 'tight', 'constrained'])
+@pytest.mark.parametrize('with_colorbar', [True, False])
+def test_label_outer(remove_ticks, layout_engine, with_colorbar):
+    fig = plt.figure(layout=layout_engine)
+    axs = fig.subplots(2, 2, sharex=True, sharey=True)
     for ax in axs.flat:
         ax.set(xlabel="foo", ylabel="bar")
+        if with_colorbar:
+            fig.colorbar(mpl.cm.ScalarMappable(), ax=ax)
         ax.label_outer(remove_inner_ticks=remove_ticks)
     check_ticklabel_visible(
         axs.flat, [False, False, True, True], [True, False, True, False])