diff --git a/lib/matplotlib/artist.py b/lib/matplotlib/artist.py
index 5b6fadfd364f..6e917851f91d 100644
--- a/lib/matplotlib/artist.py
+++ b/lib/matplotlib/artist.py
@@ -225,6 +225,7 @@ def __init__(self):
         self._path_effects = mpl.rcParams['path.effects']
         self._sticky_edges = _XYPair([], [])
         self._in_layout = True
+        self._in_autoscale = False
 
     def __getstate__(self):
         d = self.__dict__.copy()
@@ -903,6 +904,14 @@ def get_in_layout(self):
         """
         return self._in_layout
 
+    def _get_in_autoscale(self):
+        """
+        Return whether the artist is included in autoscaling calculations.
+
+        E.g. `.axes.Axes.autoscale_view()`.
+        """
+        return self._in_autoscale
+
     def _fully_clipped_to_axes(self):
         """
         Return a boolean flag, ``True`` if the artist is clipped to the Axes
@@ -1132,6 +1141,17 @@ def set_in_layout(self, in_layout):
         """
         self._in_layout = in_layout
 
+    def _set_in_autoscale(self, b):
+        """
+        Set if artist is to be included in autoscaling calculations,
+        E.g. `.axes.Axes.autoscale_view()`.
+
+        Parameters
+        ----------
+        b : bool
+        """
+        self._in_autoscale = b
+
     def get_label(self):
         """Return the label used for this artist in the legend."""
         return self._label
diff --git a/lib/matplotlib/axes/_base.py b/lib/matplotlib/axes/_base.py
index f89c231815dc..179774e2b7a2 100644
--- a/lib/matplotlib/axes/_base.py
+++ b/lib/matplotlib/axes/_base.py
@@ -2409,6 +2409,7 @@ def add_image(self, image):
         self._children.append(image)
         image._remove_method = self._children.remove
         self.stale = True
+        image._set_in_autoscale(True)
         return image
 
     def _update_image_limits(self, image):
@@ -2430,6 +2431,7 @@ def add_line(self, line):
         self._children.append(line)
         line._remove_method = self._children.remove
         self.stale = True
+        line._set_in_autoscale(True)
         return line
 
     def _add_text(self, txt):
@@ -2502,6 +2504,7 @@ def add_patch(self, p):
         self._update_patch_limits(p)
         self._children.append(p)
         p._remove_method = self._children.remove
+        p._set_in_autoscale(True)
         return p
 
     def _update_patch_limits(self, patch):
@@ -2599,6 +2602,8 @@ def relim(self, visible_only=False):
 
         for artist in self._children:
             if not visible_only or artist.get_visible():
+                if not artist._get_in_autoscale():
+                    continue
                 if isinstance(artist, mlines.Line2D):
                     self._update_line_limits(artist)
                 elif isinstance(artist, mpatches.Patch):