diff --git a/lib/matplotlib/artist.py b/lib/matplotlib/artist.py
index e6d428fba997..e78b59883417 100644
--- a/lib/matplotlib/artist.py
+++ b/lib/matplotlib/artist.py
@@ -103,7 +103,7 @@ def __init__(self):
         self._contains = None
         self._rasterized = None
         self._agg_filter = None
-
+        self._mouseover = False
         self.eventson = False  # fire events only if eventson
         self._oid = 0  # an observer id
         self._propobservers = {}  # a dict from oids to funcs
@@ -147,6 +147,23 @@ def remove(self):
         # callback has one parameter, which is the child to be removed.
         if self._remove_method is not None:
             self._remove_method(self)
+            # clear stale callback
+            self.stale_callback = None
+            _ax_flag = False
+            if hasattr(self, 'axes') and self.axes:
+                # remove from the mouse hit list
+                self.axes.mouseover_set.discard(self)
+                # mark the axes as stale
+                self.axes.stale = True
+                # decouple the artist from the axes
+                self.axes = None
+                _ax_flag = True
+
+            if self.figure:
+                self.figure = None
+                if not _ax_flag:
+                    self.figure = True
+
         else:
             raise NotImplementedError('cannot remove artist')
         # TODO: the fix for the collections relim problem is to move the
@@ -214,7 +231,9 @@ def axes(self):
 
     @axes.setter
     def axes(self, new_axes):
-        if self._axes is not None and new_axes != self._axes:
+
+        if (new_axes is not None and
+                (self._axes is not None and new_axes != self._axes)):
             raise ValueError("Can not reset the axes.  You are "
                              "probably trying to re-use an artist "
                              "in more than one Axes which is not "
@@ -961,6 +980,21 @@ def format_cursor_data(self, data):
             data = [data]
         return ', '.join('{:0.3g}'.format(item) for item in data)
 
+    @property
+    def mouseover(self):
+        return self._mouseover
+
+    @mouseover.setter
+    def mouseover(self, val):
+        val = bool(val)
+        self._mouseover = val
+        ax = self.axes
+        if ax:
+            if val:
+                ax.mouseover_set.add(self)
+            else:
+                ax.mouseover_set.discard(self)
+
 
 class ArtistInspector(object):
     """
diff --git a/lib/matplotlib/axes/_base.py b/lib/matplotlib/axes/_base.py
index 0b3f8b20a738..9d6d892d5373 100644
--- a/lib/matplotlib/axes/_base.py
+++ b/lib/matplotlib/axes/_base.py
@@ -786,6 +786,8 @@ def _set_artist_props(self, a):
             a.set_transform(self.transData)
 
         a.axes = self
+        if a.mouseover:
+            self.mouseover_set.add(a)
 
     def _gen_axes_patch(self):
         """
@@ -916,6 +918,7 @@ def cla(self):
         self.tables = []
         self.artists = []
         self.images = []
+        self.mouseover_set = set()
         self._current_image = None  # strictly for pyplot via _sci, _gci
         self.legend_ = None
         self.collections = []  # collection.Collection instances
diff --git a/lib/matplotlib/backend_bases.py b/lib/matplotlib/backend_bases.py
index 675cbcaa777c..cc05a59eca16 100644
--- a/lib/matplotlib/backend_bases.py
+++ b/lib/matplotlib/backend_bases.py
@@ -2815,9 +2815,11 @@ def mouse_move(self, event):
             except (ValueError, OverflowError):
                 pass
             else:
-                artists = event.inaxes.hitlist(event)
+                artists = [a for a in event.inaxes.mouseover_set
+                           if a.contains(event)]
 
                 if artists:
+
                     a = max(enumerate(artists), key=lambda x: x[1].zorder)[1]
                     if a is not event.inaxes.patch:
                         data = a.get_cursor_data(event)
diff --git a/lib/matplotlib/image.py b/lib/matplotlib/image.py
index cef36bdca578..7fcf0f6ded80 100644
--- a/lib/matplotlib/image.py
+++ b/lib/matplotlib/image.py
@@ -99,7 +99,7 @@ def __init__(self, ax,
         """
         martist.Artist.__init__(self)
         cm.ScalarMappable.__init__(self, norm, cmap)
-
+        self._mouseover = True
         if origin is None:
             origin = rcParams['image.origin']
         self.origin = origin
diff --git a/lib/matplotlib/tests/test_artist.py b/lib/matplotlib/tests/test_artist.py
index 9c92beb02e46..967a4c014e19 100644
--- a/lib/matplotlib/tests/test_artist.py
+++ b/lib/matplotlib/tests/test_artist.py
@@ -14,6 +14,8 @@
 import matplotlib.collections as mcollections
 from matplotlib.testing.decorators import image_comparison, cleanup
 
+from nose.tools import (assert_true, assert_false)
+
 
 @cleanup
 def test_patch_transform_of_none():
@@ -144,6 +146,30 @@ def test_cull_markers():
     assert len(svg.getvalue()) < 20000
 
 
+@cleanup
+def test_remove():
+    fig, ax = plt.subplots()
+    im = ax.imshow(np.arange(36).reshape(6, 6))
+
+    assert_true(fig.stale)
+    assert_true(ax.stale)
+
+    fig.canvas.draw()
+    assert_false(fig.stale)
+    assert_false(ax.stale)
+
+    assert_true(im in ax.mouseover_set)
+    assert_true(im.axes is ax)
+
+    im.remove()
+
+    assert_true(im.axes is None)
+    assert_true(im.figure is None)
+    assert_true(im not in ax.mouseover_set)
+    assert_true(fig.stale)
+    assert_true(ax.stale)
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule(argv=['-s', '--with-doctest'], exit=False)