diff --git a/lib/matplotlib/container.py b/lib/matplotlib/container.py
index 62fe9a35e666..9b76fbe4e19f 100644
--- a/lib/matplotlib/container.py
+++ b/lib/matplotlib/container.py
@@ -24,6 +24,7 @@ def __init__(self, kl, label=None):
     def remove(self):
         for c in cbook.flatten(
                 self, scalarp=lambda x: isinstance(x, Artist)):
+        self.set_label("")
             if c is not None:
                 c.remove()
         if self._remove_method: