diff --git a/doc/api/next_api_changes/deprecations/20839-EP.rst b/doc/api/next_api_changes/deprecations/20839-EP.rst
new file mode 100644
index 000000000000..1388abca194f
--- /dev/null
+++ b/doc/api/next_api_changes/deprecations/20839-EP.rst
@@ -0,0 +1,4 @@
+Selector widget state internals
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The *state_modifier_keys* attribute have been privatized and the modifier keys
+needs to be set when creating the widget.
diff --git a/doc/users/next_whats_new/rectangle_patch_rotation.rst b/doc/users/next_whats_new/rectangle_patch_rotation.rst
new file mode 100644
index 000000000000..ba3c1b336604
--- /dev/null
+++ b/doc/users/next_whats_new/rectangle_patch_rotation.rst
@@ -0,0 +1,5 @@
+Rectangle patch rotation point
+------------------------------
+
+The rotation point of the `~matplotlib.patches.Rectangle` can now be set to 'xy',
+'center' or a 2-tuple of numbers.
diff --git a/doc/users/next_whats_new/selector_improvement.rst b/doc/users/next_whats_new/selector_improvement.rst
new file mode 100644
index 000000000000..f19282335705
--- /dev/null
+++ b/doc/users/next_whats_new/selector_improvement.rst
@@ -0,0 +1,40 @@
+Selectors improvement: rotation, aspect ratio correction and add/remove state
+-----------------------------------------------------------------------------
+
+The `~matplotlib.widgets.RectangleSelector` and
+`~matplotlib.widgets.EllipseSelector` can now be rotated interactively between
+-45° and 45°. The range limits are currently dictated by the implementation.
+The rotation is enabled or disabled by striking the *r* key
+('r' is the default key mapped to 'rotate' in *state_modifier_keys*) or by calling
+``selector.add_state('rotate')``.
+
+The aspect ratio of the axes can now be taken into account when using the
+"square" state. This is enabled by specifying ``use_data_coordinates='True'`` when
+the selector is initialized.
+
+In addition to changing selector state interactively using the modifier keys
+defined in *state_modifier_keys*, the selector state can now be changed
+programmatically using the *add_state* and *remove_state* methods.
+
+
+.. code-block:: python
+
+    import matplotlib.pyplot as plt
+    from matplotlib.widgets import RectangleSelector
+    import numpy as np
+
+    values = np.arange(0, 100)
+
+    fig = plt.figure()
+    ax = fig.add_subplot()
+    ax.plot(values, values)
+
+    selector = RectangleSelector(ax, print, interactive=True,
+                                 drag_from_anywhere=True,
+                                 use_data_coordinates=True)
+    selector.add_state('rotate') # alternatively press 'r' key
+    # rotate the selector interactively
+
+    selector.remove_state('rotate') # alternatively press 'r' key
+
+    selector.add_state('square')
diff --git a/lib/matplotlib/axes/_axes.py b/lib/matplotlib/axes/_axes.py
index 6f6db320fea7..53ab692d63d5 100644
--- a/lib/matplotlib/axes/_axes.py
+++ b/lib/matplotlib/axes/_axes.py
@@ -8094,3 +8094,13 @@ def violin(self, vpstats, positions=None, vert=True, widths=0.5,
     tricontourf = mtri.tricontourf
     tripcolor = mtri.tripcolor
     triplot = mtri.triplot
+
+    def _get_aspect_ratio(self):
+        """
+        Convenience method to calculate the aspect ratio of the axes in
+        the display coordinate system.
+        """
+        figure_size = self.get_figure().get_size_inches()
+        ll, ur = self.get_position() * figure_size
+        width, height = ur - ll
+        return height / (width * self.get_data_ratio())
diff --git a/lib/matplotlib/patches.py b/lib/matplotlib/patches.py
index c13bb24cc201..b83714bdcd7b 100644
--- a/lib/matplotlib/patches.py
+++ b/lib/matplotlib/patches.py
@@ -706,7 +706,8 @@ def __str__(self):
         return fmt % pars
 
     @docstring.dedent_interpd
-    def __init__(self, xy, width, height, angle=0.0, **kwargs):
+    def __init__(self, xy, width, height, angle=0.0, *,
+                 rotation_point='xy', **kwargs):
         """
         Parameters
         ----------
@@ -717,7 +718,11 @@ def __init__(self, xy, width, height, angle=0.0, **kwargs):
         height : float
             Rectangle height.
         angle : float, default: 0
-            Rotation in degrees anti-clockwise about *xy*.
+            Rotation in degrees anti-clockwise about the rotation point.
+        rotation_point : {'xy', 'center', (number, number)}, default: 'xy'
+            If ``'xy'``, rotate around the anchor point. If ``'center'`` rotate
+            around the center. If 2-tuple of number, rotate around this
+            coordinate.
 
         Other Parameters
         ----------------
@@ -730,6 +735,14 @@ def __init__(self, xy, width, height, angle=0.0, **kwargs):
         self._width = width
         self._height = height
         self.angle = float(angle)
+        self.rotation_point = rotation_point
+        # Required for RectangleSelector with axes aspect ratio != 1
+        # The patch is defined in data coordinates and when changing the
+        # selector with square modifier and not in data coordinates, we need
+        # to correct for the aspect ratio difference between the data and
+        # display coordinate systems. Its value is typically provide by
+        # Axes._get_aspect_ratio()
+        self._aspect_ratio_correction = 1.0
         self._convert_units()  # Validate the inputs.
 
     def get_path(self):
@@ -750,9 +763,36 @@ def get_patch_transform(self):
         # important to call the accessor method and not directly access the
         # transformation member variable.
         bbox = self.get_bbox()
-        return (transforms.BboxTransformTo(bbox)
-                + transforms.Affine2D().rotate_deg_around(
-                    bbox.x0, bbox.y0, self.angle))
+        if self.rotation_point == 'center':
+            width, height = bbox.x1 - bbox.x0, bbox.y1 - bbox.y0
+            rotation_point = bbox.x0 + width / 2., bbox.y0 + height / 2.
+        elif self.rotation_point == 'xy':
+            rotation_point = bbox.x0, bbox.y0
+        else:
+            rotation_point = self.rotation_point
+        return transforms.BboxTransformTo(bbox) \
+                + transforms.Affine2D() \
+                .translate(-rotation_point[0], -rotation_point[1]) \
+                .scale(1, self._aspect_ratio_correction) \
+                .rotate_deg(self.angle) \
+                .scale(1, 1 / self._aspect_ratio_correction) \
+                .translate(*rotation_point)
+
+    @property
+    def rotation_point(self):
+        """The rotation point of the patch."""
+        return self._rotation_point
+
+    @rotation_point.setter
+    def rotation_point(self, value):
+        if value in ['center', 'xy'] or (
+                isinstance(value, tuple) and len(value) == 2 and
+                isinstance(value[0], Number) and isinstance(value[1], Number)
+                ):
+            self._rotation_point = value
+        else:
+            raise ValueError("`rotation_point` must be one of "
+                             "{'xy', 'center', (number, number)}.")
 
     def get_x(self):
         """Return the left coordinate of the rectangle."""
@@ -1511,6 +1551,12 @@ def __init__(self, xy, width, height, angle=0, **kwargs):
         self._width, self._height = width, height
         self._angle = angle
         self._path = Path.unit_circle()
+        # Required for EllipseSelector with axes aspect ratio != 1
+        # The patch is defined in data coordinates and when changing the
+        # selector with square modifier and not in data coordinates, we need
+        # to correct for the aspect ratio difference between the data and
+        # display coordinate systems.
+        self._aspect_ratio_correction = 1.0
         # Note: This cannot be calculated until this is added to an Axes
         self._patch_transform = transforms.IdentityTransform()
 
@@ -1528,8 +1574,9 @@ def _recompute_transform(self):
         width = self.convert_xunits(self._width)
         height = self.convert_yunits(self._height)
         self._patch_transform = transforms.Affine2D() \
-            .scale(width * 0.5, height * 0.5) \
+            .scale(width * 0.5, height * 0.5 * self._aspect_ratio_correction) \
             .rotate_deg(self.angle) \
+            .scale(1, 1 / self._aspect_ratio_correction) \
             .translate(*center)
 
     def get_path(self):
diff --git a/lib/matplotlib/tests/test_widgets.py b/lib/matplotlib/tests/test_widgets.py
index 9955aad0512d..8ab3d71040dc 100644
--- a/lib/matplotlib/tests/test_widgets.py
+++ b/lib/matplotlib/tests/test_widgets.py
@@ -5,6 +5,7 @@
 from matplotlib.testing.decorators import check_figures_equal, image_comparison
 from matplotlib.testing.widgets import do_event, get_ax, mock_event
 
+import numpy as np
 from numpy.testing import assert_allclose
 
 import pytest
@@ -171,8 +172,28 @@ def onselect(epress, erelease):
     assert tool.extents == (xdata_new, extents[1], ydata_new, extents[3])
 
 
-@pytest.mark.parametrize('use_default_state', [True, False])
-def test_rectangle_resize_center(use_default_state):
+def test_rectangle_add_state():
+    ax = get_ax()
+
+    def onselect(epress, erelease):
+        pass
+
+    tool = widgets.RectangleSelector(ax, onselect, interactive=True)
+    # Create rectangle
+    _resize_rectangle(tool, 70, 65, 125, 130)
+
+    with pytest.raises(ValueError):
+        tool.add_state('unsupported_state')
+
+    with pytest.raises(ValueError):
+        tool.add_state('clear')
+    tool.add_state('move')
+    tool.add_state('square')
+    tool.add_state('center')
+
+
+@pytest.mark.parametrize('add_state', [True, False])
+def test_rectangle_resize_center(add_state):
     ax = get_ax()
 
     def onselect(epress, erelease):
@@ -183,8 +204,8 @@ def onselect(epress, erelease):
     _resize_rectangle(tool, 70, 65, 125, 130)
     assert tool.extents == (70.0, 125.0, 65.0, 130.0)
 
-    if use_default_state:
-        tool._default_state.add('center')
+    if add_state:
+        tool.add_state('center')
         use_key = None
     else:
         use_key = 'control'
@@ -244,8 +265,8 @@ def onselect(epress, erelease):
                             ydata_new, extents[3] - ydiff)
 
 
-@pytest.mark.parametrize('use_default_state', [True, False])
-def test_rectangle_resize_square(use_default_state):
+@pytest.mark.parametrize('add_state', [True, False])
+def test_rectangle_resize_square(add_state):
     ax = get_ax()
 
     def onselect(epress, erelease):
@@ -256,8 +277,8 @@ def onselect(epress, erelease):
     _resize_rectangle(tool, 70, 65, 120, 115)
     assert tool.extents == (70.0, 120.0, 65.0, 115.0)
 
-    if use_default_state:
-        tool._default_state.add('square')
+    if add_state:
+        tool.add_state('square')
         use_key = None
     else:
         use_key = 'shift'
@@ -326,9 +347,9 @@ def onselect(epress, erelease):
     tool = widgets.RectangleSelector(ax, onselect, interactive=True)
     # Create rectangle
     _resize_rectangle(tool, 70, 65, 120, 115)
-    tool._default_state.add('square')
-    tool._default_state.add('center')
-    assert tool.extents == (70.0, 120.0, 65.0, 115.0)
+    tool.add_state('square')
+    tool.add_state('center')
+    assert_allclose(tool.extents, (70.0, 120.0, 65.0, 115.0))
 
     # resize NE handle
     extents = tool.extents
@@ -336,8 +357,8 @@ def onselect(epress, erelease):
     xdiff, ydiff = 10, 5
     xdata_new, ydata_new = xdata + xdiff, ydata + ydiff
     _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
-    assert tool.extents == (extents[0] - xdiff, xdata_new,
-                            extents[2] - xdiff, extents[3] + xdiff)
+    assert_allclose(tool.extents, (extents[0] - xdiff, xdata_new,
+                                   extents[2] - xdiff, extents[3] + xdiff))
 
     # resize E handle
     extents = tool.extents
@@ -345,8 +366,8 @@ def onselect(epress, erelease):
     xdiff = 10
     xdata_new, ydata_new = xdata + xdiff, ydata
     _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
-    assert tool.extents == (extents[0] - xdiff, xdata_new,
-                            extents[2] - xdiff, extents[3] + xdiff)
+    assert_allclose(tool.extents, (extents[0] - xdiff, xdata_new,
+                                   extents[2] - xdiff, extents[3] + xdiff))
 
     # resize E handle negative diff
     extents = tool.extents
@@ -354,8 +375,8 @@ def onselect(epress, erelease):
     xdiff = -20
     xdata_new, ydata_new = xdata + xdiff, ydata
     _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
-    assert tool.extents == (extents[0] - xdiff, xdata_new,
-                            extents[2] - xdiff, extents[3] + xdiff)
+    assert_allclose(tool.extents, (extents[0] - xdiff, xdata_new,
+                                   extents[2] - xdiff, extents[3] + xdiff))
 
     # resize W handle
     extents = tool.extents
@@ -363,8 +384,8 @@ def onselect(epress, erelease):
     xdiff = 5
     xdata_new, ydata_new = xdata + xdiff, ydata
     _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
-    assert tool.extents == (xdata_new, extents[1] - xdiff,
-                            extents[2] + xdiff, extents[3] - xdiff)
+    assert_allclose(tool.extents, (xdata_new, extents[1] - xdiff,
+                                   extents[2] + xdiff, extents[3] - xdiff))
 
     # resize W handle negative diff
     extents = tool.extents
@@ -372,8 +393,8 @@ def onselect(epress, erelease):
     xdiff = -25
     xdata_new, ydata_new = xdata + xdiff, ydata
     _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
-    assert tool.extents == (xdata_new, extents[1] - xdiff,
-                            extents[2] + xdiff, extents[3] - xdiff)
+    assert_allclose(tool.extents, (xdata_new, extents[1] - xdiff,
+                                   extents[2] + xdiff, extents[3] - xdiff))
 
     # resize SW handle
     extents = tool.extents
@@ -381,8 +402,112 @@ def onselect(epress, erelease):
     xdiff, ydiff = 20, 25
     xdata_new, ydata_new = xdata + xdiff, ydata + ydiff
     _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
-    assert tool.extents == (extents[0] + ydiff, extents[1] - ydiff,
-                            ydata_new, extents[3] - ydiff)
+    assert_allclose(tool.extents, (extents[0] + ydiff, extents[1] - ydiff,
+                                   ydata_new, extents[3] - ydiff))
+
+
+@pytest.mark.parametrize('selector_class',
+                         [widgets.RectangleSelector, widgets.EllipseSelector])
+def test_rectangle_rotate(selector_class):
+    ax = get_ax()
+
+    def onselect(epress, erelease):
+        pass
+
+    tool = selector_class(ax, onselect=onselect, interactive=True)
+    # Draw rectangle
+    do_event(tool, 'press', xdata=100, ydata=100)
+    do_event(tool, 'onmove', xdata=130, ydata=140)
+    do_event(tool, 'release', xdata=130, ydata=140)
+    assert tool.extents == (100, 130, 100, 140)
+    assert len(tool._state) == 0
+
+    # Rotate anticlockwise using top-right corner
+    do_event(tool, 'on_key_press', key='r')
+    assert tool._state == set(['rotate'])
+    assert len(tool._state) == 1
+    do_event(tool, 'press', xdata=130, ydata=140)
+    do_event(tool, 'onmove', xdata=120, ydata=145)
+    do_event(tool, 'release', xdata=120, ydata=145)
+    do_event(tool, 'on_key_press', key='r')
+    assert len(tool._state) == 0
+    # Extents shouldn't change (as shape of rectangle hasn't changed)
+    assert tool.extents == (100, 130, 100, 140)
+    assert_allclose(tool.rotation, 25.56, atol=0.01)
+    tool.rotation = 45
+    assert tool.rotation == 45
+    # Corners should move
+    assert_allclose(tool.corners,
+                    np.array([[118.53, 139.75, 111.46, 90.25],
+                              [95.25, 116.46, 144.75, 123.54]]), atol=0.01)
+
+    # Scale using top-right corner
+    do_event(tool, 'press', xdata=110, ydata=145)
+    do_event(tool, 'onmove', xdata=110, ydata=160)
+    do_event(tool, 'release', xdata=110, ydata=160)
+    assert_allclose(tool.extents, (100, 139.75, 100, 151.82), atol=0.01)
+
+    if selector_class == widgets.RectangleSelector:
+        with pytest.raises(ValueError):
+            tool._selection_artist.rotation_point = 'unvalid_value'
+
+
+def test_rectange_add_remove_set():
+    ax = get_ax()
+
+    def onselect(epress, erelease):
+        pass
+
+    tool = widgets.RectangleSelector(ax, onselect=onselect, interactive=True)
+    # Draw rectangle
+    do_event(tool, 'press', xdata=100, ydata=100)
+    do_event(tool, 'onmove', xdata=130, ydata=140)
+    do_event(tool, 'release', xdata=130, ydata=140)
+    assert tool.extents == (100, 130, 100, 140)
+    assert len(tool._state) == 0
+    for state in ['rotate', 'square', 'center']:
+        tool.add_state(state)
+        assert len(tool._state) == 1
+        tool.remove_state(state)
+        assert len(tool._state) == 0
+
+
+@pytest.mark.parametrize('use_data_coordinates', [False, True])
+def test_rectangle_resize_square_center_aspect(use_data_coordinates):
+    ax = get_ax()
+    ax.set_aspect(0.8)
+
+    def onselect(epress, erelease):
+        pass
+
+    tool = widgets.RectangleSelector(ax, onselect, interactive=True,
+                                     use_data_coordinates=use_data_coordinates)
+    # Create rectangle
+    _resize_rectangle(tool, 70, 65, 120, 115)
+    assert tool.extents == (70.0, 120.0, 65.0, 115.0)
+    tool.add_state('square')
+    tool.add_state('center')
+
+    if use_data_coordinates:
+        # resize E handle
+        extents = tool.extents
+        xdata, ydata, width = extents[1], extents[3], extents[1] - extents[0]
+        xdiff, ycenter = 10,  extents[2] + (extents[3] - extents[2]) / 2
+        xdata_new, ydata_new = xdata + xdiff, ydata
+        ychange = width / 2 + xdiff
+        _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
+        assert_allclose(tool.extents, [extents[0] - xdiff, xdata_new,
+                                       ycenter - ychange, ycenter + ychange])
+    else:
+        # resize E handle
+        extents = tool.extents
+        xdata, ydata = extents[1], extents[3]
+        xdiff = 10
+        xdata_new, ydata_new = xdata + xdiff, ydata
+        ychange = xdiff * 1 / tool._aspect_ratio_correction
+        _resize_rectangle(tool, xdata, ydata, xdata_new, ydata_new)
+        assert_allclose(tool.extents, [extents[0] - xdiff, xdata_new,
+                                       46.25, 133.75])
 
 
 def test_ellipse():
@@ -423,7 +548,7 @@ def onselect(epress, erelease):
     do_event(tool, 'on_key_release', xdata=10, ydata=10, button=1,
              key='shift')
     extents = [int(e) for e in tool.extents]
-    assert extents == [10, 35, 10, 34]
+    assert extents == [10, 35, 10, 35]
 
     # create a square from center
     do_event(tool, 'on_key_press', xdata=100, ydata=100, button=1,
@@ -434,7 +559,7 @@ def onselect(epress, erelease):
     do_event(tool, 'on_key_release', xdata=100, ydata=100, button=1,
              key='ctrl+shift')
     extents = [int(e) for e in tool.extents]
-    assert extents == [70, 129, 70, 130]
+    assert extents == [70, 130, 70, 130]
 
     assert tool.geometry.shape == (2, 73)
     assert_allclose(tool.geometry[:, 0], [70., 100])
@@ -453,11 +578,10 @@ def onselect(epress, erelease):
                                                    'markeredgecolor': 'b'})
     tool.extents = (100, 150, 100, 150)
 
-    assert tool.corners == (
-        (100, 150, 150, 100), (100, 100, 150, 150))
+    assert_allclose(tool.corners, ((100, 150, 150, 100), (100, 100, 150, 150)))
     assert tool.extents == (100, 150, 100, 150)
-    assert tool.edge_centers == (
-        (100, 125.0, 150, 125.0), (125.0, 100, 125.0, 150))
+    assert_allclose(tool.edge_centers,
+                    ((100, 125.0, 150, 125.0), (125.0, 100, 125.0, 150)))
     assert tool.extents == (100, 150, 100, 150)
 
     # grab a corner and move it
@@ -803,6 +927,24 @@ def onselect(*args):
         assert tool.extents == (10, 50)
 
 
+def test_span_selector_add_state():
+    ax = get_ax()
+
+    def onselect(*args):
+        pass
+
+    tool = widgets.SpanSelector(ax, onselect, 'horizontal', interactive=True)
+
+    with pytest.raises(ValueError):
+        tool.add_state('unsupported_state')
+    with pytest.raises(ValueError):
+        tool.add_state('center')
+    with pytest.raises(ValueError):
+        tool.add_state('square')
+
+    tool.add_state('move')
+
+
 def test_tool_line_handle():
     ax = get_ax()
 
diff --git a/lib/matplotlib/widgets.py b/lib/matplotlib/widgets.py
index 34e6d9a4a948..826d990db776 100644
--- a/lib/matplotlib/widgets.py
+++ b/lib/matplotlib/widgets.py
@@ -20,7 +20,7 @@
 from . import _api, backend_tools, cbook, colors, ticker
 from .lines import Line2D
 from .patches import Circle, Rectangle, Ellipse
-from .transforms import TransformedPatchPath
+from .transforms import TransformedPatchPath, Affine2D
 
 
 class LockDraw:
@@ -1801,7 +1801,7 @@ def _update(self):
 class _SelectorWidget(AxesWidget):
 
     def __init__(self, ax, onselect, useblit=False, button=None,
-                 state_modifier_keys=None):
+                 state_modifier_keys=None, use_data_coordinates=False):
         super().__init__(ax)
 
         self.visible = True
@@ -1809,9 +1809,11 @@ def __init__(self, ax, onselect, useblit=False, button=None,
         self.useblit = useblit and self.canvas.supports_blit
         self.connect_default_events()
 
-        self.state_modifier_keys = dict(move=' ', clear='escape',
-                                        square='shift', center='control')
-        self.state_modifier_keys.update(state_modifier_keys or {})
+        self._state_modifier_keys = dict(move=' ', clear='escape',
+                                         square='shift', center='control',
+                                         rotate='r')
+        self._state_modifier_keys.update(state_modifier_keys or {})
+        self._use_data_coordinates = use_data_coordinates
 
         self.background = None
 
@@ -1829,11 +1831,11 @@ def __init__(self, ax, onselect, useblit=False, button=None,
         self._eventrelease = None
         self._prev_event = None
         self._state = set()
-        self._default_state = set()
 
     eventpress = _api.deprecate_privatize_attribute("3.5")
     eventrelease = _api.deprecate_privatize_attribute("3.5")
     state = _api.deprecate_privatize_attribute("3.5")
+    state_modifier_keys = _api.deprecate_privatize_attribute("3.6")
 
     def set_active(self, active):
         super().set_active(active)
@@ -1944,7 +1946,7 @@ def press(self, event):
             key = event.key or ''
             key = key.replace('ctrl', 'control')
             # move state is locked in on a button press
-            if key == self.state_modifier_keys['move']:
+            if key == self._state_modifier_keys['move']:
                 self._state.add('move')
             self._press(event)
             return True
@@ -1992,12 +1994,20 @@ def on_key_press(self, event):
         if self.active:
             key = event.key or ''
             key = key.replace('ctrl', 'control')
-            if key == self.state_modifier_keys['clear']:
+            if key == self._state_modifier_keys['clear']:
                 self.clear()
                 return
-            for (state, modifier) in self.state_modifier_keys.items():
-                if modifier in key:
-                    self._state.add(state)
+            for (state, modifier) in self._state_modifier_keys.items():
+                if modifier in key.split('+'):
+                    # 'rotate' is changing _state on press and is not removed
+                    # from _state when releasing
+                    if state == 'rotate':
+                        if state in self._state:
+                            self._state.discard(state)
+                        else:
+                            self._state.add(state)
+                    else:
+                        self._state.add(state)
             self._on_key_press(event)
 
     def _on_key_press(self, event):
@@ -2007,8 +2017,10 @@ def on_key_release(self, event):
         """Key release event handler and validator."""
         if self.active:
             key = event.key or ''
-            for (state, modifier) in self.state_modifier_keys.items():
-                if modifier in key:
+            for (state, modifier) in self._state_modifier_keys.items():
+                # 'rotate' is changing _state on press and is not removed
+                # from _state when releasing
+                if modifier in key.split('+') and state != 'rotate':
                     self._state.discard(state)
             self._on_key_release(event)
 
@@ -2062,6 +2074,53 @@ def set_handle_props(self, **handle_props):
             self.update()
         self._handle_props.update(handle_props)
 
+    def _validate_state(self, state):
+        supported_state = [
+            key for key, value in self._state_modifier_keys.items()
+            if key != 'clear' and value != 'not-applicable'
+            ]
+        _api.check_in_list(supported_state, state=state)
+
+    def add_state(self, state):
+        """
+        Add a state to define the widget's behavior. See the
+        `state_modifier_keys` parameters for details.
+
+        Parameters
+        ----------
+        state : str
+            Must be a supported state of the selector. See the
+            `state_modifier_keys` parameters for details.
+
+        Raises
+        ------
+        ValueError
+            When the state is not supported by the selector.
+
+        """
+        self._validate_state(state)
+        self._state.add(state)
+
+    def remove_state(self, state):
+        """
+        Remove a state to define the widget's behavior. See the
+        `state_modifier_keys` parameters for details.
+
+        Parameters
+        ----------
+        value : str
+            Must be a supported state of the selector. See the
+            `state_modifier_keys` parameters for details.
+
+        Raises
+        ------
+        ValueError
+            When the state is not supported by the selector.
+
+        """
+        self._validate_state(state)
+        self._state.remove(state)
+
 
 class SpanSelector(_SelectorWidget):
     """
@@ -2129,6 +2188,12 @@ def on_select(min: float, max: float) -> Any
         Distance in pixels within which the interactive tool handles can be
         activated.
 
+    state_modifier_keys : dict, optional
+        Keyboard modifiers which affect the widget's behavior.  Values
+        amend the defaults, which are:
+
+        - "clear": Clear the current shape, default: "escape".
+
     drag_from_anywhere : bool, default: False
         If `True`, the widget can be moved by clicking anywhere within
         its bounds.
@@ -2157,9 +2222,16 @@ def on_select(min: float, max: float) -> Any
     def __init__(self, ax, onselect, direction, minspan=0, useblit=False,
                  props=None, onmove_callback=None, interactive=False,
                  button=None, handle_props=None, grab_range=10,
-                 drag_from_anywhere=False, ignore_event_outside=False):
-
-        super().__init__(ax, onselect, useblit=useblit, button=button)
+                 state_modifier_keys=None, drag_from_anywhere=False,
+                 ignore_event_outside=False):
+
+        if state_modifier_keys is None:
+            state_modifier_keys = dict(clear='escape',
+                                       square='not-applicable',
+                                       center='not-applicable',
+                                       rotate='not-applicable')
+        super().__init__(ax, onselect, useblit=useblit, button=button,
+                         state_modifier_keys=state_modifier_keys)
 
         if props is None:
             props = dict(facecolor='red', alpha=0.5)
@@ -2725,15 +2797,18 @@ def onselect(eclick: MouseEvent, erelease: MouseEvent)
 
     state_modifier_keys : dict, optional
         Keyboard modifiers which affect the widget's behavior.  Values
-        amend the defaults.
+        amend the defaults, which are:
 
         - "move": Move the existing shape, default: no modifier.
         - "clear": Clear the current shape, default: "escape".
         - "square": Make the shape square, default: "shift".
-        - "center": Make the initial point the center of the shape,
-          default: "ctrl".
+        - "center": change the shape around its center, default: "ctrl".
+        - "rotate": Rotate the shape around its center between -45° and 45°,
+          default: "r".
 
-        "square" and "center" can be combined.
+        "square" and "center" can be combined. The square shape can be defined
+        in data or figure coordinates as determined by the
+        ``use_data_coordinates`` argument specified when creating the selector.
 
     drag_from_anywhere : bool, default: False
         If `True`, the widget can be moved by clicking anywhere within
@@ -2743,6 +2818,10 @@ def onselect(eclick: MouseEvent, erelease: MouseEvent)
         If `True`, the event triggered outside the span selector will be
         ignored.
 
+    use_data_coordinates : bool, default: False
+        If `True`, the "square" shape of the selector is defined in
+        data coordinates instead of figure coordinates.
+
     """
 
 
@@ -2774,11 +2853,11 @@ class RectangleSelector(_SelectorWidget):
                                           props=props)
     >>> fig.show()
 
+    >>> selector.add_state('square')
+
     See also: :doc:`/gallery/widgets/rectangle_selector`
     """
 
-    _shape_klass = Rectangle
-
     @_api.rename_parameter("3.5", "maxdist", "grab_range")
     @_api.rename_parameter("3.5", "marker_props", "handle_props")
     @_api.rename_parameter("3.5", "rectprops", "props")
@@ -2789,14 +2868,18 @@ def __init__(self, ax, onselect, drawtype='box',
                  lineprops=None, props=None, spancoords='data',
                  button=None, grab_range=10, handle_props=None,
                  interactive=False, state_modifier_keys=None,
-                 drag_from_anywhere=False, ignore_event_outside=False):
+                 drag_from_anywhere=False, ignore_event_outside=False,
+                 use_data_coordinates=False):
         super().__init__(ax, onselect, useblit=useblit, button=button,
-                         state_modifier_keys=state_modifier_keys)
+                         state_modifier_keys=state_modifier_keys,
+                         use_data_coordinates=use_data_coordinates)
 
         self.visible = True
         self._interactive = interactive
         self.drag_from_anywhere = drag_from_anywhere
         self.ignore_event_outside = ignore_event_outside
+        self._rotation = 0.0
+        self._aspect_ratio_correction = 1.0
 
         if drawtype == 'none':  # draw a line but make it invisible
             _api.warn_deprecated(
@@ -2814,8 +2897,7 @@ def __init__(self, ax, onselect, drawtype='box',
             props['animated'] = self.useblit
             self.visible = props.pop('visible', self.visible)
             self._props = props
-            to_draw = self._shape_klass((0, 0), 0, 1, visible=False,
-                                        **self._props)
+            to_draw = self._init_shape(**self._props)
             self.ax.add_patch(to_draw)
         if drawtype == 'line':
             _api.warn_deprecated(
@@ -2831,6 +2913,7 @@ def __init__(self, ax, onselect, drawtype='box',
             self.ax.add_line(to_draw)
 
         self._selection_artist = to_draw
+        self._set_aspect_ratio_correction()
 
         self.minspanx = minspanx
         self.minspany = minspany
@@ -2887,13 +2970,16 @@ def _handles_artists(self):
         return (*self._center_handle.artists, *self._corner_handles.artists,
                 *self._edge_handles.artists)
 
+    def _init_shape(self, **props):
+        return Rectangle((0, 0), 0, 1, visible=False,
+                         rotation_point='center', **props)
+
     def _press(self, event):
         """Button press event handler."""
         # make the drawn box/line visible get the click-coordinates,
         # button, ...
         if self._interactive and self._selection_artist.get_visible():
             self._set_active_handle(event)
-            self._extents_on_press = self.extents
         else:
             self._active_handle = None
 
@@ -2910,6 +2996,10 @@ def _press(self, event):
         else:
             self.set_visible(True)
 
+        self._extents_on_press = self.extents
+        self._rotation_on_press = self._rotation
+        self._set_aspect_ratio_correction()
+
         return False
 
     def _release(self, event):
@@ -2966,73 +3056,106 @@ def _release(self, event):
     def _onmove(self, event):
         """Motion notify event handler."""
 
-        state = self._state | self._default_state
+        state = self._state
+        rotate = ('rotate' in state and
+                  self._active_handle in self._corner_order)
+        eventpress = self._eventpress
+        # The calculations are done for rotation at zero: we apply inverse
+        # transformation to events except when we rotate and move
+        move = self._active_handle == 'C'
+        if not move and not rotate:
+            inv_tr = self._get_rotation_transform().inverted()
+            event.xdata, event.ydata = inv_tr.transform(
+                [event.xdata, event.ydata])
+            eventpress.xdata, eventpress.ydata = inv_tr.transform(
+                [eventpress.xdata, eventpress.ydata]
+                )
+
+        dx = event.xdata - eventpress.xdata
+        dy = event.ydata - eventpress.ydata
+        # refmax is used when moving the corner handle with the square state
+        # and is the maximum between refx and refy
+        refmax = None
+        if self._use_data_coordinates:
+            refx, refy = dx, dy
+        else:
+            # Add 1e-6 to avoid divided by zero error
+            refx = event.xdata / (eventpress.xdata or 1E-6)
+            refy = event.ydata / (eventpress.ydata or 1E-6)
+
+        x0, x1, y0, y1 = self._extents_on_press
+        # rotate an existing shape
+        if rotate:
+            # calculate angle abc
+            a = np.array([eventpress.xdata, eventpress.ydata])
+            b = np.array(self.center)
+            c = np.array([event.xdata, event.ydata])
+            angle = (np.arctan2(c[1]-b[1], c[0]-b[0]) -
+                     np.arctan2(a[1]-b[1], a[0]-b[0]))
+            self.rotation = np.rad2deg(self._rotation_on_press + angle)
+
         # resize an existing shape
-        if self._active_handle and self._active_handle != 'C':
-            x0, x1, y0, y1 = self._extents_on_press
+        elif self._active_handle and self._active_handle != 'C':
             size_on_press = [x1 - x0, y1 - y0]
             center = [x0 + size_on_press[0] / 2, y0 + size_on_press[1] / 2]
-            dx = event.xdata - self._eventpress.xdata
-            dy = event.ydata - self._eventpress.ydata
-
-            # change sign of relative changes to simplify calculation
-            # Switch variables so that only x1 and/or y1 are updated on move
-            x_factor = y_factor = 1
-            if 'W' in self._active_handle:
-                x_factor *= -1
-                dx *= x_factor
-                x0 = x1
-            if 'S' in self._active_handle:
-                y_factor *= -1
-                dy *= y_factor
-                y0 = y1
 
             # Keeping the center fixed
             if 'center' in state:
+                # hh, hw are half-height and half-width
                 if 'square' in state:
-                    # Force the same change in dx and dy
-                    if self._active_handle in ['E', 'W']:
-                        # using E, W handle we need to update dy accordingly
-                        dy = dx
-                    elif self._active_handle in ['S', 'N']:
-                        # using S, N handle, we need to update dx accordingly
-                        dx = dy
+                    # when using a corner, find which reference to use
+                    if self._active_handle in self._corner_order:
+                        refmax = max(refx, refy, key=abs)
+                    if self._active_handle in ['E', 'W'] or refmax == refx:
+                        hw = event.xdata - center[0]
+                        hh = hw / self._aspect_ratio_correction
                     else:
-                        dx = dy = max(dx, dy, key=abs)
-
-                # new half-width and half-height
-                hw = size_on_press[0] / 2 + dx
-                hh = size_on_press[1] / 2 + dy
-
-                if 'square' not in state:
+                        hh = event.ydata - center[1]
+                        hw = hh * self._aspect_ratio_correction
+                else:
+                    hw = size_on_press[0] / 2
+                    hh = size_on_press[1] / 2
                     # cancel changes in perpendicular direction
-                    if self._active_handle in ['E', 'W']:
-                        hh = size_on_press[1] / 2
-                    if self._active_handle in ['N', 'S']:
-                        hw = size_on_press[0] / 2
+                    if self._active_handle in ['E', 'W'] + self._corner_order:
+                        hw = abs(event.xdata - center[0])
+                    if self._active_handle in ['N', 'S'] + self._corner_order:
+                        hh = abs(event.ydata - center[1])
 
                 x0, x1, y0, y1 = (center[0] - hw, center[0] + hw,
                                   center[1] - hh, center[1] + hh)
 
             else:
-                # Keeping the opposite corner/edge fixed
+                # change sign of relative changes to simplify calculation
+                # Switch variables so that x1 and/or y1 are updated on move
+                x_factor = y_factor = 1
+                if 'W' in self._active_handle:
+                    x0 = x1
+                    x_factor *= -1
+                if 'S' in self._active_handle:
+                    y0 = y1
+                    y_factor *= -1
+                if self._active_handle in ['E', 'W'] + self._corner_order:
+                    x1 = event.xdata
+                if self._active_handle in ['N', 'S'] + self._corner_order:
+                    y1 = event.ydata
                 if 'square' in state:
-                    dx = dy = max(dx, dy, key=abs)
-                    x1 = x0 + x_factor * (dx + size_on_press[0])
-                    y1 = y0 + y_factor * (dy + size_on_press[1])
-                else:
-                    if self._active_handle in ['E', 'W'] + self._corner_order:
-                        x1 = event.xdata
-                    if self._active_handle in ['N', 'S'] + self._corner_order:
-                        y1 = event.ydata
+                    # when using a corner, find which reference to use
+                    if self._active_handle in self._corner_order:
+                        refmax = max(refx, refy, key=abs)
+                    if self._active_handle in ['E', 'W'] or refmax == refx:
+                        sign = np.sign(event.ydata - y0)
+                        y1 = y0 + sign * abs(x1 - x0) / \
+                            self._aspect_ratio_correction
+                    else:
+                        sign = np.sign(event.xdata - x0)
+                        x1 = x0 + sign * abs(y1 - y0) * \
+                            self._aspect_ratio_correction
 
         # move existing shape
-        elif (self._active_handle == 'C' or
-              (self.drag_from_anywhere and self._contains(event)) and
-              self._extents_on_press is not None):
+        elif self._active_handle == 'C':
             x0, x1, y0, y1 = self._extents_on_press
-            dx = event.xdata - self._eventpress.xdata
-            dy = event.ydata - self._eventpress.ydata
+            dx = event.xdata - eventpress.xdata
+            dy = event.ydata - eventpress.ydata
             x0 += dx
             x1 += dx
             y0 += dy
@@ -3040,26 +3163,22 @@ def _onmove(self, event):
 
         # new shape
         else:
+            self._rotation = 0
             # Don't create a new rectangle if there is already one when
             # ignore_event_outside=True
             if self.ignore_event_outside and self._selection_completed:
                 return
-            center = [self._eventpress.xdata, self._eventpress.ydata]
-            center_pix = [self._eventpress.x, self._eventpress.y]
+            center = [eventpress.xdata, eventpress.ydata]
             dx = (event.xdata - center[0]) / 2.
             dy = (event.ydata - center[1]) / 2.
 
             # square shape
             if 'square' in state:
-                dx_pix = abs(event.x - center_pix[0])
-                dy_pix = abs(event.y - center_pix[1])
-                if not dx_pix:
-                    return
-                maxd = max(abs(dx_pix), abs(dy_pix))
-                if abs(dx_pix) < maxd:
-                    dx *= maxd / (abs(dx_pix) + 1e-6)
-                if abs(dy_pix) < maxd:
-                    dy *= maxd / (abs(dy_pix) + 1e-6)
+                refmax = max(refx, refy, key=abs)
+                if refmax == refx:
+                    dy = np.sign(dy) * abs(dx) / self._aspect_ratio_correction
+                else:
+                    dx = np.sign(dx) * abs(dy) * self._aspect_ratio_correction
 
             # from center
             if 'center' in state:
@@ -3079,24 +3198,44 @@ def _onmove(self, event):
     @property
     def _rect_bbox(self):
         if self._drawtype == 'box':
-            x0 = self._selection_artist.get_x()
-            y0 = self._selection_artist.get_y()
-            width = self._selection_artist.get_width()
-            height = self._selection_artist.get_height()
-            return x0, y0, width, height
+            return self._selection_artist.get_bbox().bounds
         else:
             x, y = self._selection_artist.get_data()
             x0, x1 = min(x), max(x)
             y0, y1 = min(y), max(y)
             return x0, y0, x1 - x0, y1 - y0
 
+    def _set_aspect_ratio_correction(self):
+        aspect_ratio = self.ax._get_aspect_ratio()
+        if not hasattr(self._selection_artist, '_aspect_ratio_correction'):
+            # Aspect ratio correction is not supported with deprecated
+            # drawtype='line'. Remove this block in matplotlib 3.7
+            self._aspect_ratio_correction = 1
+            return
+
+        self._selection_artist._aspect_ratio_correction = aspect_ratio
+        if self._use_data_coordinates:
+            self._aspect_ratio_correction = 1
+        else:
+            self._aspect_ratio_correction = aspect_ratio
+
+    def _get_rotation_transform(self):
+        aspect_ratio = self.ax._get_aspect_ratio()
+        return Affine2D().translate(-self.center[0], -self.center[1]) \
+                .scale(1, aspect_ratio) \
+                .rotate(self._rotation) \
+                .scale(1, 1 / aspect_ratio) \
+                .translate(*self.center)
+
     @property
     def corners(self):
         """Corners of rectangle from lower left, moving clockwise."""
         x0, y0, width, height = self._rect_bbox
         xc = x0, x0 + width, x0 + width, x0
         yc = y0, y0, y0 + height, y0 + height
-        return xc, yc
+        transform = self._get_rotation_transform()
+        coords = transform.transform(np.array([xc, yc]).T).T
+        return coords[0], coords[1]
 
     @property
     def edge_centers(self):
@@ -3106,7 +3245,9 @@ def edge_centers(self):
         h = height / 2.
         xe = x0, x0 + w, x0 + width, x0 + w
         ye = y0 + h, y0, y0 + h, y0 + height
-        return xe, ye
+        transform = self._get_rotation_transform()
+        coords = transform.transform(np.array([xe, ye]).T).T
+        return coords[0], coords[1]
 
     @property
     def center(self):
@@ -3116,7 +3257,10 @@ def center(self):
 
     @property
     def extents(self):
-        """Return (xmin, xmax, ymin, ymax)."""
+        """
+        Return (xmin, xmax, ymin, ymax) as defined by the bounding box before
+        rotation.
+        """
         x0, y0, width, height = self._rect_bbox
         xmin, xmax = sorted([x0, x0 + width])
         ymin, ymax = sorted([y0, y0 + height])
@@ -3134,6 +3278,23 @@ def extents(self, extents):
         self.set_visible(self.visible)
         self.update()
 
+    @property
+    def rotation(self):
+        """
+        Rotation in degree in interval [-45°, 45°]. The rotation is limited in
+        range to keep the implementation simple.
+        """
+        return np.rad2deg(self._rotation)
+
+    @rotation.setter
+    def rotation(self, value):
+        # Restrict to a limited range of rotation [-45°, 45°] to avoid changing
+        # order of handles
+        if -45 <= value and value <= 45:
+            self._rotation = np.deg2rad(value)
+            # call extents setter to draw shape and update handles positions
+            self.extents = self.extents
+
     draw_shape = _api.deprecate_privatize_attribute('3.5')
 
     def _draw_shape(self, extents):
@@ -3153,6 +3314,7 @@ def _draw_shape(self, extents):
             self._selection_artist.set_y(ymin)
             self._selection_artist.set_width(xmax - xmin)
             self._selection_artist.set_height(ymax - ymin)
+            self._selection_artist.set_angle(self.rotation)
 
         elif self._drawtype == 'line':
             self._selection_artist.set_data([xmin, xmax], [ymin, ymax])
@@ -3170,8 +3332,7 @@ def _set_active_handle(self, event):
         elif m_dist < self.grab_range * 2:
             # Prioritise center handle over other handles
             self._active_handle = 'C'
-        elif (c_dist > self.grab_range and
-                  e_dist > self.grab_range):
+        elif c_dist > self.grab_range and e_dist > self.grab_range:
             # Not close to any handles
             if self.drag_from_anywhere and self._contains(event):
                 # Check if we've clicked inside the region
@@ -3226,9 +3387,11 @@ class EllipseSelector(RectangleSelector):
     :doc:`/gallery/widgets/rectangle_selector`
     """
 
-    _shape_klass = Ellipse
     draw_shape = _api.deprecate_privatize_attribute('3.5')
 
+    def _init_shape(self, **props):
+        return Ellipse((0, 0), 0, 1, visible=False, **props)
+
     def _draw_shape(self, extents):
         x0, x1, y0, y1 = extents
         xmin, xmax = sorted([x0, x1])
@@ -3241,6 +3404,7 @@ def _draw_shape(self, extents):
             self._selection_artist.center = center
             self._selection_artist.width = 2 * a
             self._selection_artist.height = 2 * b
+            self._selection_artist.angle = self.rotation
         else:
             rad = np.deg2rad(np.arange(31) * 12)
             x = a * np.cos(rad) + center[0]
@@ -3420,7 +3584,8 @@ def __init__(self, ax, onselect, useblit=False,
         state_modifier_keys = dict(clear='escape', move_vertex='control',
                                    move_all='shift', move='not-applicable',
                                    square='not-applicable',
-                                   center='not-applicable')
+                                   center='not-applicable',
+                                   rotate='not-applicable')
         super().__init__(ax, onselect, useblit=useblit,
                          state_modifier_keys=state_modifier_keys)
 
@@ -3592,13 +3757,13 @@ def _on_key_release(self, event):
         # 'move_all' mode (by checking the released key)
         if (not self._selection_completed
                 and
-                (event.key == self.state_modifier_keys.get('move_vertex')
-                 or event.key == self.state_modifier_keys.get('move_all'))):
+                (event.key == self._state_modifier_keys.get('move_vertex')
+                 or event.key == self._state_modifier_keys.get('move_all'))):
             self._xs.append(event.xdata)
             self._ys.append(event.ydata)
             self._draw_polygon()
         # Reset the polygon if the released key is the 'clear' key.
-        elif event.key == self.state_modifier_keys.get('clear'):
+        elif event.key == self._state_modifier_keys.get('clear'):
             event = self._clean_event(event)
             self._xs, self._ys = [event.xdata], [event.ydata]
             self._selection_completed = False