diff --git a/doc/users/whats_new.rst b/doc/users/whats_new.rst
index 1740f66c9ff2..0b60fb54fa0e 100644
--- a/doc/users/whats_new.rst
+++ b/doc/users/whats_new.rst
@@ -19,6 +19,15 @@ New in matplotlib 2.0
 
    matplotlib 2.0 supports Python 2.7, and 3.4+
 
+The behavior of AxesStack re-add key changes
+--------------------------------------------
+
+The AxesStack does not allow users to re-add the same key. If the user tries
+to do so, a KeyError will be raised.
+If the user really wants to re-add the same key, then the old one should
+be removed first by using :func:`AxesStack.remove` and re-add the new key.
+
+:func:`~figure.AxesStack.add`
 
 
 Default style changes
diff --git a/lib/matplotlib/figure.py b/lib/matplotlib/figure.py
index 287dc64dc9af..c947631b75c4 100644
--- a/lib/matplotlib/figure.py
+++ b/lib/matplotlib/figure.py
@@ -125,12 +125,10 @@ def add(self, key, a):
         except TypeError:
             raise ValueError("first argument, %s, is not a valid key" % key)
 
-        a_existing = self.get(key)
+        a_existing = dict(self._elements).get(key)
         if a_existing is not None:
-            Stack.remove(self, (key, a_existing))
-            warnings.warn(
-                "key %s already existed; Axes is being replaced" % key)
-            # I don't think the above should ever happen.
+            raise KeyError('Key %s already exists in the AxesStack' % key)
+            # This only happens if user call this function directly
 
         if a in self:
             return None
@@ -362,7 +360,7 @@ def _repr_html_(self):
         # We can't use "isinstance" here, because then we'd end up importing
         # webagg unconditiionally.
         if (self.canvas is not None and
-            'WebAgg' in self.canvas.__class__.__name__):
+                'WebAgg' in self.canvas.__class__.__name__):
             from matplotlib.backends import backend_webagg
             return backend_webagg.ipython_inline_display(self)
 
@@ -816,7 +814,7 @@ def _make_key(self, *args, **kwargs):
         'make a hashable key out of args and kwargs'
 
         def fixitems(items):
-            #items may have arrays and lists in them, so convert them
+            # items may have arrays and lists in them, so convert them
             # to tuples for the key
             ret = []
             for k, v in items:
@@ -1745,7 +1743,7 @@ def subplots_adjust(self, *args, **kwargs):
             if not isinstance(ax, SubplotBase):
                 # Check if sharing a subplots axis
                 if (ax._sharex is not None and
-                    isinstance(ax._sharex, SubplotBase)):
+                        isinstance(ax._sharex, SubplotBase)):
                     ax._sharex.update_params()
                     ax.set_position(ax._sharex.figbox)
                 elif (ax._sharey is not None and
@@ -1921,8 +1919,8 @@ def figaspect(arg):
     # could become rc parameters, for now they're hardwired.
     figsize_min = np.array((4.0, 2.0))  # min length for width/height
     figsize_max = np.array((16.0, 16.0))  # max length for width/height
-    #figsize_min = rcParams['figure.figsize_min']
-    #figsize_max = rcParams['figure.figsize_max']
+    # figsize_min = rcParams['figure.figsize_min']
+    # figsize_max = rcParams['figure.figsize_max']
 
     # Extract the aspect ratio of the array
     if isarray:
diff --git a/lib/matplotlib/tests/test_figure.py b/lib/matplotlib/tests/test_figure.py
index 651a3b78f4c6..4854a045ca2a 100644
--- a/lib/matplotlib/tests/test_figure.py
+++ b/lib/matplotlib/tests/test_figure.py
@@ -4,12 +4,13 @@
 import six
 from six.moves import xrange
 
-from nose.tools import assert_equal, assert_true
-from matplotlib import rcParams
+from nose.tools import assert_equal, assert_true, assert_raises
+from matplotlib import rcParams, figure
 from matplotlib.testing.decorators import image_comparison, cleanup
 from matplotlib.axes import Axes
 import matplotlib.pyplot as plt
 import numpy as np
+import warnings
 
 
 @cleanup
@@ -216,6 +217,17 @@ def test_figaspect():
     assert h / w == 1
 
 
+@cleanup
+def test_stack_remove():
+    a = figure.AxesStack()
+    key = '123'
+    axes = plt.figure().add_subplot(111)
+    a.add(key, axes)
+    # Verify some things
+    assert_raises(KeyError, a.add, key, plt.figure().add_subplot(111))
+    assert_equal(a._elements, [(key, (1, axes))])
+
+
 if __name__ == "__main__":
     import nose
     nose.runmodule(argv=['-s', '--with-doctest'], exit=False)