diff --git a/lib/matplotlib/cm.py b/lib/matplotlib/cm.py
index 6d05be943b33..d5e71a8ae428 100644
--- a/lib/matplotlib/cm.py
+++ b/lib/matplotlib/cm.py
@@ -364,7 +364,8 @@ def set_array(self, A):
         Parameters
         ----------
         A : array-like or None
-            The values that are mapped to colors.
+            The values that are mapped to colors. If None, colormapping is
+            disabled.
 
             The base class `.ScalarMappable` does not make any assumptions on
             the dimensionality and shape of the value array *A*.
@@ -386,6 +387,9 @@ def get_array(self):
 
         The base class `.ScalarMappable` does not make any assumptions on
         the dimensionality and shape of the array.
+
+        Subclasses may decide per instance if they support colormapping.
+        Return None here in case they don't.
         """
         return self._A
 
diff --git a/lib/matplotlib/colorbar.py b/lib/matplotlib/colorbar.py
index 2e0aa0ad3c93..91ea85f80d50 100644
--- a/lib/matplotlib/colorbar.py
+++ b/lib/matplotlib/colorbar.py
@@ -380,12 +380,18 @@ def __init__(self, ax, mappable=None, *, cmap=None,
                  ):
 
         if mappable is None:
+            # Special case for a standalone colormap
             mappable = cm.ScalarMappable(norm=norm, cmap=cmap)
-
-        # Ensure the given mappable's norm has appropriate vmin and vmax
-        # set even if mappable.draw has not yet been called.
-        if mappable.get_array() is not None:
-            mappable.autoscale_None()
+        else:
+            if mappable.get_array() is None:
+                _api.deprecated("3.5",
+                                message="The given mappable instance "
+                                f"{mappable!r} does not support colormapping. "
+                                "Possibly it has explicitly specified colors.")
+            else:
+                # Ensure the given mappable's norm has appropriate vmin and
+                # vmax set even if mappable.draw has not yet been called.
+                mappable.autoscale_None()
 
         self.mappable = mappable
         cmap = mappable.cmap