diff --git a/doc/users/next_whats_new/selector_improvement.rst b/doc/users/next_whats_new/selector_improvement.rst
index f19282335705..6691c2b23a38 100644
--- a/doc/users/next_whats_new/selector_improvement.rst
+++ b/doc/users/next_whats_new/selector_improvement.rst
@@ -2,8 +2,7 @@ Selectors improvement: rotation, aspect ratio correction and add/remove state
 -----------------------------------------------------------------------------
 
 The `~matplotlib.widgets.RectangleSelector` and
-`~matplotlib.widgets.EllipseSelector` can now be rotated interactively between
--45° and 45°. The range limits are currently dictated by the implementation.
+`~matplotlib.widgets.EllipseSelector` can now be rotated.
 The rotation is enabled or disabled by striking the *r* key
 ('r' is the default key mapped to 'rotate' in *state_modifier_keys*) or by calling
 ``selector.add_state('rotate')``.
diff --git a/lib/matplotlib/patches.py b/lib/matplotlib/patches.py
index 894717dcdd3e..5c4226134d3f 100644
--- a/lib/matplotlib/patches.py
+++ b/lib/matplotlib/patches.py
@@ -738,13 +738,6 @@ def __init__(self, xy, width, height, angle=0.0, *,
         self._height = height
         self.angle = float(angle)
         self.rotation_point = rotation_point
-        # Required for RectangleSelector with axes aspect ratio != 1
-        # The patch is defined in data coordinates and when changing the
-        # selector with square modifier and not in data coordinates, we need
-        # to correct for the aspect ratio difference between the data and
-        # display coordinate systems. Its value is typically provide by
-        # Axes._get_aspect_ratio()
-        self._aspect_ratio_correction = 1.0
         self._convert_units()  # Validate the inputs.
 
     def get_path(self):
@@ -772,13 +765,11 @@ def get_patch_transform(self):
             rotation_point = bbox.x0, bbox.y0
         else:
             rotation_point = self.rotation_point
-        return transforms.BboxTransformTo(bbox) \
-                + transforms.Affine2D() \
-                .translate(-rotation_point[0], -rotation_point[1]) \
-                .scale(1, self._aspect_ratio_correction) \
-                .rotate_deg(self.angle) \
-                .scale(1, 1 / self._aspect_ratio_correction) \
-                .translate(*rotation_point)
+        return (transforms.BboxTransformTo(bbox) +
+                transforms.Affine2D()
+                .translate(-rotation_point[0], -rotation_point[1])
+                .rotate_deg(self.angle)
+                .translate(*rotation_point))
 
     @property
     def rotation_point(self):
@@ -816,6 +807,14 @@ def get_corners(self):
         return self.get_patch_transform().transform(
             [(0, 0), (1, 0), (1, 1), (0, 1)])
 
+    def _get_edge_midpoints(self):
+        """
+        Return the edge midpoints of the rectangle, moving anti-clockwise from
+        the center of the left-hand edge.
+        """
+        return self.get_patch_transform().transform(
+            [(0, 0.5), (0.5, 0), (1, 0.5), (0.5, 1)])
+
     def get_center(self):
         """Return the centre of the rectangle."""
         return self.get_patch_transform().transform((0.5, 0.5))
@@ -1565,12 +1564,6 @@ def __init__(self, xy, width, height, angle=0, **kwargs):
         self._width, self._height = width, height
         self._angle = angle
         self._path = Path.unit_circle()
-        # Required for EllipseSelector with axes aspect ratio != 1
-        # The patch is defined in data coordinates and when changing the
-        # selector with square modifier and not in data coordinates, we need
-        # to correct for the aspect ratio difference between the data and
-        # display coordinate systems.
-        self._aspect_ratio_correction = 1.0
         # Note: This cannot be calculated until this is added to an Axes
         self._patch_transform = transforms.IdentityTransform()
 
@@ -1588,9 +1581,8 @@ def _recompute_transform(self):
         width = self.convert_xunits(self._width)
         height = self.convert_yunits(self._height)
         self._patch_transform = transforms.Affine2D() \
-            .scale(width * 0.5, height * 0.5 * self._aspect_ratio_correction) \
+            .scale(width * 0.5, height * 0.5) \
             .rotate_deg(self.angle) \
-            .scale(1, 1 / self._aspect_ratio_correction) \
             .translate(*center)
 
     def get_path(self):
@@ -1681,6 +1673,14 @@ def get_corners(self):
         return self.get_patch_transform().transform(
             [(-1, -1), (1, -1), (1, 1), (-1, 1)])
 
+    def _get_edge_midpoints(self):
+        """
+        Return the corners of the ellipse, moving anti-clockwise from
+        the center of the left-hand edge before rotation.
+        """
+        return self.get_patch_transform().transform(
+            [(-1, 0), (0, -1), (1, 0), (0, 1)])
+
 
 class Annulus(Patch):
     """
diff --git a/lib/matplotlib/tests/test_widgets.py b/lib/matplotlib/tests/test_widgets.py
index bb91684f305e..5a8808dad5fd 100644
--- a/lib/matplotlib/tests/test_widgets.py
+++ b/lib/matplotlib/tests/test_widgets.py
@@ -9,7 +9,6 @@
                                         mock_event, noop)
 
 import numpy as np
-from numpy.testing import assert_allclose
 
 import pytest
 
@@ -19,6 +18,13 @@ def ax():
     return get_ax()
 
 
+# Set default tolerances for checking coordinates. These are needed due to
+# small innacuracies in floating point conversions between data/display
+# coordinates
+assert_allclose = functools.partial(
+    np.testing.assert_allclose, atol=1e-12, rtol=1e-7)
+
+
 def check_rectangle(**kwargs):
     ax = get_ax()
 
@@ -26,20 +32,19 @@ def onselect(epress, erelease):
         ax._got_onselect = True
         assert epress.xdata == 100
         assert epress.ydata == 100
-        assert erelease.xdata == 199
-        assert erelease.ydata == 199
+        assert erelease.xdata == 200
+        assert erelease.ydata == 200
 
     tool = widgets.RectangleSelector(ax, onselect, **kwargs)
     do_event(tool, 'press', xdata=100, ydata=100, button=1)
-    do_event(tool, 'onmove', xdata=199, ydata=199, button=1)
-
+    do_event(tool, 'onmove', xdata=250, ydata=250, button=1)
     # purposely drag outside of axis for release
     do_event(tool, 'release', xdata=250, ydata=250, button=1)
 
     if kwargs.get('drawtype', None) not in ['line', 'none']:
         assert_allclose(tool.geometry,
-                        [[100., 100, 199, 199, 100],
-                         [100, 199, 199, 100, 100]],
+                        [[100., 100, 200, 200, 100],
+                         [100, 200, 200, 100, 100]],
                         err_msg=tool.geometry)
 
     assert ax._got_onselect
@@ -113,7 +118,7 @@ def test_rectangle_drag(ax, drag_from_anywhere, new_center):
                                      drag_from_anywhere=drag_from_anywhere)
     # Create rectangle
     click_and_drag(tool, start=(0, 10), end=(100, 120))
-    assert tool.center == (50, 65)
+    assert_allclose(tool.center, (50, 65))
     # Drag inside rectangle, but away from centre handle
     #
     # If drag_from_anywhere == True, this will move the rectangle by (10, 10),
@@ -122,11 +127,11 @@ def test_rectangle_drag(ax, drag_from_anywhere, new_center):
     # If drag_from_anywhere == False, this will create a new rectangle with
     # center (30, 20)
     click_and_drag(tool, start=(25, 15), end=(35, 25))
-    assert tool.center == new_center
+    assert_allclose(tool.center, new_center)
     # Check that in both cases, dragging outside the rectangle draws a new
     # rectangle
     click_and_drag(tool, start=(175, 185), end=(185, 195))
-    assert tool.center == (180, 190)
+    assert_allclose(tool.center, (180, 190))
 
 
 def test_rectangle_selector_set_props_handle_props(ax):
@@ -150,39 +155,49 @@ def test_rectangle_selector_set_props_handle_props(ax):
         assert artist.get_alpha() == 0.3
 
 
-def test_rectangle_resize(ax):
+# Should give same results if rectangle is created from any two
+# opposite corners
+@pytest.mark.parametrize('start, end', [[(0, 10), (100, 120)],
+                                        [(100, 120), (0, 10)],
+                                        [(0, 120), (100, 10)],
+                                        [(100, 10), (0, 120)]])
+def test_rectangle_resize(ax, start, end):
     tool = widgets.RectangleSelector(ax, onselect=noop, interactive=True)
     # Create rectangle
-    click_and_drag(tool, start=(0, 10), end=(100, 120))
-    assert tool.extents == (0.0, 100.0, 10.0, 120.0)
+    click_and_drag(tool, start=start, end=end)
+    assert_allclose(tool.extents, (0.0, 100.0, 10.0, 120.0))
 
     # resize NE handle
     extents = tool.extents
     xdata, ydata = extents[1], extents[3]
     xdata_new, ydata_new = xdata + 10, ydata + 5
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
-    assert tool.extents == (extents[0], xdata_new, extents[2], ydata_new)
+    assert_allclose(
+        tool.extents, (extents[0], xdata_new, extents[2], ydata_new))
 
     # resize E handle
     extents = tool.extents
     xdata, ydata = extents[1], extents[2] + (extents[3] - extents[2]) / 2
     xdata_new, ydata_new = xdata + 10, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
-    assert tool.extents == (extents[0], xdata_new, extents[2], extents[3])
+    np.testing.assert_allclose(
+        tool.extents, (extents[0], xdata_new, extents[2], extents[3]))
 
     # resize W handle
     extents = tool.extents
     xdata, ydata = extents[0], extents[2] + (extents[3] - extents[2]) / 2
     xdata_new, ydata_new = xdata + 15, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
-    assert tool.extents == (xdata_new, extents[1], extents[2], extents[3])
+    assert_allclose(
+        tool.extents, (xdata_new, extents[1], extents[2], extents[3]))
 
     # resize SW handle
     extents = tool.extents
     xdata, ydata = extents[0], extents[2]
     xdata_new, ydata_new = xdata + 20, ydata + 25
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
-    assert tool.extents == (xdata_new, extents[1], ydata_new, extents[3])
+    assert_allclose(
+        tool.extents, (xdata_new, extents[1], ydata_new, extents[3]))
 
 
 def test_rectangle_add_state(ax):
@@ -220,8 +235,8 @@ def test_rectangle_resize_center(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata + ydiff
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0] - xdiff, xdata_new,
-                            extents[2] - ydiff, ydata_new)
+    assert_allclose(tool.extents, (extents[0] - xdiff, xdata_new,
+                                   extents[2] - ydiff, ydata_new))
 
     # resize E handle
     extents = tool.extents
@@ -230,8 +245,8 @@ def test_rectangle_resize_center(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0] - xdiff, xdata_new,
-                            extents[2], extents[3])
+    assert_allclose(tool.extents, (extents[0] - xdiff, xdata_new,
+                                   extents[2], extents[3]))
 
     # resize E handle negative diff
     extents = tool.extents
@@ -240,8 +255,8 @@ def test_rectangle_resize_center(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0] - xdiff, xdata_new,
-                            extents[2], extents[3])
+    assert_allclose(tool.extents, (extents[0] - xdiff, xdata_new,
+                                   extents[2], extents[3]))
 
     # resize W handle
     extents = tool.extents
@@ -250,8 +265,8 @@ def test_rectangle_resize_center(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (xdata_new, extents[1] - xdiff,
-                            extents[2], extents[3])
+    assert_allclose(tool.extents, (xdata_new, extents[1] - xdiff,
+                                   extents[2], extents[3]))
 
     # resize W handle negative diff
     extents = tool.extents
@@ -260,8 +275,8 @@ def test_rectangle_resize_center(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (xdata_new, extents[1] - xdiff,
-                            extents[2], extents[3])
+    assert_allclose(tool.extents, (xdata_new, extents[1] - xdiff,
+                                   extents[2], extents[3]))
 
     # resize SW handle
     extents = tool.extents
@@ -270,8 +285,8 @@ def test_rectangle_resize_center(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata + ydiff
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (xdata_new, extents[1] - xdiff,
-                            ydata_new, extents[3] - ydiff)
+    assert_allclose(tool.extents, (xdata_new, extents[1] - xdiff,
+                                   ydata_new, extents[3] - ydiff))
 
 
 @pytest.mark.parametrize('add_state', [True, False])
@@ -279,7 +294,7 @@ def test_rectangle_resize_square(ax, add_state):
     tool = widgets.RectangleSelector(ax, onselect=noop, interactive=True)
     # Create rectangle
     click_and_drag(tool, start=(70, 65), end=(120, 115))
-    assert tool.extents == (70.0, 120.0, 65.0, 115.0)
+    assert_allclose(tool.extents, (70.0, 120.0, 65.0, 115.0))
 
     if add_state:
         tool.add_state('square')
@@ -294,8 +309,8 @@ def test_rectangle_resize_square(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata + ydiff
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0], xdata_new,
-                            extents[2], extents[3] + xdiff)
+    assert_allclose(tool.extents, (extents[0], xdata_new,
+                                   extents[2], extents[3] + xdiff))
 
     # resize E handle
     extents = tool.extents
@@ -304,8 +319,8 @@ def test_rectangle_resize_square(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0], xdata_new,
-                            extents[2], extents[3] + xdiff)
+    assert_allclose(tool.extents, (extents[0], xdata_new,
+                                   extents[2], extents[3] + xdiff))
 
     # resize E handle negative diff
     extents = tool.extents
@@ -314,8 +329,8 @@ def test_rectangle_resize_square(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0], xdata_new,
-                            extents[2], extents[3] + xdiff)
+    assert_allclose(tool.extents, (extents[0], xdata_new,
+                                   extents[2], extents[3] + xdiff))
 
     # resize W handle
     extents = tool.extents
@@ -324,8 +339,8 @@ def test_rectangle_resize_square(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (xdata_new, extents[1],
-                            extents[2], extents[3] - xdiff)
+    assert_allclose(tool.extents, (xdata_new, extents[1],
+                                   extents[2], extents[3] - xdiff))
 
     # resize W handle negative diff
     extents = tool.extents
@@ -334,8 +349,8 @@ def test_rectangle_resize_square(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (xdata_new, extents[1],
-                            extents[2], extents[3] - xdiff)
+    assert_allclose(tool.extents, (xdata_new, extents[1],
+                                   extents[2], extents[3] - xdiff))
 
     # resize SW handle
     extents = tool.extents
@@ -344,11 +359,12 @@ def test_rectangle_resize_square(ax, add_state):
     xdata_new, ydata_new = xdata + xdiff, ydata + ydiff
     click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new),
                    key=use_key)
-    assert tool.extents == (extents[0] + ydiff, extents[1],
-                            ydata_new, extents[3])
+    assert_allclose(tool.extents, (xdata_new, extents[1],
+                                   extents[2] + xdiff, extents[3]))
 
 
 def test_rectangle_resize_square_center(ax):
+    ax.set_aspect(1)
     tool = widgets.RectangleSelector(ax, onselect=noop, interactive=True)
     # Create rectangle
     click_and_drag(tool, start=(70, 65), end=(120, 115))
@@ -414,10 +430,11 @@ def test_rectangle_resize_square_center(ax):
 @pytest.mark.parametrize('selector_class',
                          [widgets.RectangleSelector, widgets.EllipseSelector])
 def test_rectangle_rotate(ax, selector_class):
+    ax.set_aspect(1)
     tool = selector_class(ax, onselect=noop, interactive=True)
     # Draw rectangle
     click_and_drag(tool, start=(100, 100), end=(130, 140))
-    assert tool.extents == (100, 130, 100, 140)
+    assert_allclose(tool.extents, (100, 130, 100, 140))
     assert len(tool._state) == 0
 
     # Rotate anticlockwise using top-right corner
@@ -427,19 +444,19 @@ def test_rectangle_rotate(ax, selector_class):
     click_and_drag(tool, start=(130, 140), end=(120, 145))
     do_event(tool, 'on_key_press', key='r')
     assert len(tool._state) == 0
-    # Extents shouldn't change (as shape of rectangle hasn't changed)
-    assert tool.extents == (100, 130, 100, 140)
+    # Extents change as the selector remains rigid in display coordinates
+    assert_allclose(tool.extents, (110.10, 119.90, 95.49, 144.51), atol=0.01)
     assert_allclose(tool.rotation, 25.56, atol=0.01)
     tool.rotation = 45
     assert tool.rotation == 45
     # Corners should move
     assert_allclose(tool.corners,
-                    np.array([[118.53, 139.75, 111.46, 90.25],
-                              [95.25, 116.46, 144.75, 123.54]]), atol=0.01)
+                    np.array([[110.10, 131.31, 103.03, 81.81],
+                              [95.49, 116.70, 144.98, 123.77]]), atol=0.01)
 
     # Scale using top-right corner
     click_and_drag(tool, start=(110, 145), end=(110, 160))
-    assert_allclose(tool.extents, (100, 139.75, 100, 151.82), atol=0.01)
+    assert_allclose(tool.extents, (110, 110, 145, 160), atol=0.01)
 
     if selector_class == widgets.RectangleSelector:
         with pytest.raises(ValueError):
@@ -450,7 +467,7 @@ def test_rectange_add_remove_set(ax):
     tool = widgets.RectangleSelector(ax, onselect=noop, interactive=True)
     # Draw rectangle
     click_and_drag(tool, start=(100, 100), end=(130, 140))
-    assert tool.extents == (100, 130, 100, 140)
+    assert_allclose(tool.extents, (100, 130, 100, 140))
     assert len(tool._state) == 0
     for state in ['rotate', 'square', 'center']:
         tool.add_state(state)
@@ -461,36 +478,38 @@ def test_rectange_add_remove_set(ax):
 
 @pytest.mark.parametrize('use_data_coordinates', [False, True])
 def test_rectangle_resize_square_center_aspect(ax, use_data_coordinates):
-    ax.set_aspect(0.8)
+    ax = get_ax()
+    ax.set_aspect(0.5)
+    # Need to call a draw to update ax.transData
+    plt.gcf().canvas.draw()
 
     tool = widgets.RectangleSelector(ax, onselect=noop, interactive=True,
                                      use_data_coordinates=use_data_coordinates)
-    # Create rectangle
-    click_and_drag(tool, start=(70, 65), end=(120, 115))
-    assert tool.extents == (70.0, 120.0, 65.0, 115.0)
     tool.add_state('square')
     tool.add_state('center')
+    # Create rectangle, width 50 in data coordinates
+    click_and_drag(tool, start=(70, 65), end=(120, 65))
 
     if use_data_coordinates:
-        # resize E handle
-        extents = tool.extents
-        xdata, ydata, width = extents[1], extents[3], extents[1] - extents[0]
-        xdiff, ycenter = 10,  extents[2] + (extents[3] - extents[2]) / 2
-        xdata_new, ydata_new = xdata + xdiff, ydata
-        ychange = width / 2 + xdiff
-        click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
-        assert_allclose(tool.extents, [extents[0] - xdiff, xdata_new,
-                                       ycenter - ychange, ycenter + ychange])
+        assert_allclose(tool.extents, (20, 120, 15, 115))
     else:
-        # resize E handle
-        extents = tool.extents
-        xdata, ydata = extents[1], extents[3]
-        xdiff = 10
-        xdata_new, ydata_new = xdata + xdiff, ydata
-        ychange = xdiff * 1 / tool._aspect_ratio_correction
-        click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
-        assert_allclose(tool.extents, [extents[0] - xdiff, xdata_new,
-                                       46.25, 133.75])
+        assert_allclose(tool.extents, (20, 120, -35, 165))
+
+    # resize E handle
+    extents = tool.extents
+    xdata, ydata = extents[1], extents[3]
+    xdiff = 10
+    xdata_new, ydata_new = xdata + xdiff, ydata
+    if use_data_coordinates:
+        # In data coordinates the difference should be equal in both directions
+        ydiff = xdiff
+    else:
+        # In display coordiantes, the change in data coordinates should be
+        # different in each direction
+        ydiff = xdiff / tool.ax._get_aspect_ratio()
+    click_and_drag(tool, start=(xdata, ydata), end=(xdata_new, ydata_new))
+    assert_allclose(tool.extents, [extents[0] - xdiff, xdata_new,
+                                   extents[2] - ydiff, extents[3] + ydiff])
 
 
 def test_ellipse(ax):
@@ -501,24 +520,22 @@ def test_ellipse(ax):
 
     # drag the rectangle
     click_and_drag(tool, start=(125, 125), end=(145, 145))
-    assert tool.extents == (120, 170, 120, 170)
+    assert_allclose(tool.extents, (120, 170, 120, 170))
 
     # create from center
     click_and_drag(tool, start=(100, 100), end=(125, 125), key='control')
-    assert tool.extents == (75, 125, 75, 125)
+    assert_allclose(tool.extents, (75, 125, 75, 125))
 
     # create a square
     click_and_drag(tool, start=(10, 10), end=(35, 30), key='shift')
-    extents = [int(e) for e in tool.extents]
-    assert extents == [10, 35, 10, 35]
+    assert_allclose(tool.extents, (10, 35, 10, 35))
 
     # create a square from center
     click_and_drag(tool, start=(100, 100), end=(125, 130), key='ctrl+shift')
-    extents = [int(e) for e in tool.extents]
-    assert extents == [70, 130, 70, 130]
+    assert_allclose(tool.extents, (70, 130, 70, 130))
 
     assert tool.geometry.shape == (2, 73)
-    assert_allclose(tool.geometry[:, 0], [70., 100])
+    assert_allclose(tool.geometry[:, 0], (70, 100))
 
 
 def test_rectangle_handles(ax):
@@ -530,22 +547,22 @@ def test_rectangle_handles(ax):
     tool.extents = (100, 150, 100, 150)
 
     assert_allclose(tool.corners, ((100, 150, 150, 100), (100, 100, 150, 150)))
-    assert tool.extents == (100, 150, 100, 150)
+    assert_allclose(tool.extents, (100, 150, 100, 150))
     assert_allclose(tool.edge_centers,
                     ((100, 125.0, 150, 125.0), (125.0, 100, 125.0, 150)))
-    assert tool.extents == (100, 150, 100, 150)
+    assert_allclose(tool.extents, (100, 150, 100, 150))
 
     # grab a corner and move it
     click_and_drag(tool, start=(100, 100), end=(120, 120))
-    assert tool.extents == (120, 150, 120, 150)
+    assert_allclose(tool.extents, (120, 150, 120, 150))
 
     # grab the center and move it
     click_and_drag(tool, start=(132, 132), end=(120, 120))
-    assert tool.extents == (108, 138, 108, 138)
+    assert_allclose(tool.extents, (108, 138, 108, 138))
 
     # create a new rectangle
     click_and_drag(tool, start=(10, 10), end=(100, 100))
-    assert tool.extents == (10, 100, 10, 100)
+    assert_allclose(tool.extents, (10, 100, 10, 100))
 
     # Check that marker_props worked.
     assert mcolors.same_color(
@@ -565,7 +582,7 @@ def onselect(vmin, vmax):
     click_and_drag(tool, start=(100, 110), end=(150, 120))
 
     assert tool.ax._got_onselect
-    assert tool.extents == (100.0, 150.0, 110.0, 120.0)
+    assert_allclose(tool.extents, (100.0, 150.0, 110.0, 120.0))
 
     # Reset tool.ax._got_onselect
     tool.ax._got_onselect = False
@@ -583,7 +600,7 @@ def onselect(vmin, vmax):
                                      ignore_event_outside=ignore_event_outside)
     click_and_drag(tool, start=(100, 110), end=(150, 120))
     assert tool.ax._got_onselect
-    assert tool.extents == (100.0, 150.0, 110.0, 120.0)
+    assert_allclose(tool.extents, (100.0, 150.0, 110.0, 120.0))
 
     # Reset
     ax._got_onselect = False
@@ -592,11 +609,11 @@ def onselect(vmin, vmax):
     if ignore_event_outside:
         # event have been ignored and span haven't changed.
         assert not ax._got_onselect
-        assert tool.extents == (100.0, 150.0, 110.0, 120.0)
+        assert_allclose(tool.extents, (100.0, 150.0, 110.0, 120.0))
     else:
         # A new shape is created
         assert ax._got_onselect
-        assert tool.extents == (150.0, 160.0, 150.0, 160.0)
+        assert_allclose(tool.extents, (150.0, 160.0, 150.0, 160.0))
 
 
 def check_span(*args, **kwargs):
@@ -1401,6 +1418,14 @@ def test_polygon_selector_box(ax):
                       polygon_place_vertex(*verts[3]) +
                       polygon_place_vertex(*verts[0]))
 
+    # Set smaller axes limits to reduce errors in converting from data to
+    # display coords. The canvas size is 640 x 640, so we need a tolerance of
+    # (data width / canvas width) = 50 / 640 ~ 0.08 when comparing points in
+    # data space
+    ax.set_xlim(0, 50)
+    ax.set_ylim(0, 50)
+    atol = 0.08
+
     # Create selector
     tool = widgets.PolygonSelector(ax, onselect=noop, draw_bounding_box=True)
     for (etype, event_args) in event_sequence:
@@ -1416,25 +1441,25 @@ def test_polygon_selector_box(ax):
     canvas.motion_notify_event(*t.transform((20, 20)))
     canvas.button_release_event(*t.transform((20, 20)), 1)
     np.testing.assert_allclose(
-        tool.verts, [(10, 0), (0, 10), (10, 20), (20, 10)])
+        tool.verts, [(10, 0), (0, 10), (10, 20), (20, 10)], atol=atol)
 
     # Move using the center of the bounding box
     canvas.button_press_event(*t.transform((10, 10)), 1)
     canvas.motion_notify_event(*t.transform((30, 30)))
     canvas.button_release_event(*t.transform((30, 30)), 1)
     np.testing.assert_allclose(
-        tool.verts, [(30, 20), (20, 30), (30, 40), (40, 30)])
+        tool.verts, [(30, 20), (20, 30), (30, 40), (40, 30)], atol=atol)
 
     # Remove a point from the polygon and check that the box extents update
     np.testing.assert_allclose(
-        tool._box.extents, (20.0, 40.0, 20.0, 40.0))
+        tool._box.extents, (20.0, 40.0, 20.0, 40.0), atol=atol)
 
     canvas.button_press_event(*t.transform((30, 20)), 3)
     canvas.button_release_event(*t.transform((30, 20)), 3)
     np.testing.assert_allclose(
-        tool.verts, [(20, 30), (30, 40), (40, 30)])
+        tool.verts, [(20, 30), (30, 40), (40, 30)], atol=atol)
     np.testing.assert_allclose(
-        tool._box.extents, (20.0, 40.0, 30.0, 40.0))
+        tool._box.extents, (20.0, 40.0, 30.0, 40.0), atol=atol)
 
 
 @pytest.mark.parametrize(
diff --git a/lib/matplotlib/widgets.py b/lib/matplotlib/widgets.py
index f9eb30f9cf8e..ef1affbc3651 100644
--- a/lib/matplotlib/widgets.py
+++ b/lib/matplotlib/widgets.py
@@ -11,6 +11,7 @@
 
 from contextlib import ExitStack
 import copy
+from dataclasses import dataclass
 from numbers import Integral, Number
 
 import numpy as np
@@ -2844,6 +2845,23 @@ def onselect(eclick: MouseEvent, erelease: MouseEvent)
     """
 
 
+@dataclass
+class _RectState:
+    x0: float
+    y0: float
+    width: float
+    height: float
+    rotation: float
+
+    @property
+    def xy(self):
+        return (self.x0, self.y0)
+
+    @xy.setter
+    def xy(self, xy):
+        self.x0, self.y0 = xy
+
+
 @docstring.Substitution(_RECTANGLESELECTOR_PARAMETERS_DOCSTRING.replace(
     '__ARTIST_NAME__', 'rectangle'))
 class RectangleSelector(_SelectorWidget):
@@ -2897,13 +2915,20 @@ def __init__(self, ax, onselect, drawtype='box',
         self._interactive = interactive
         self.drag_from_anywhere = drag_from_anywhere
         self.ignore_event_outside = ignore_event_outside
-        self._rotation = 0.0
-        self._aspect_ratio_correction = 1.0
+
+        # State that determines the position of the selector
+        # All of this state is defined in display coordinates
+        self._x0 = 0
+        self._y0 = 0
+        self._width = 0
+        self._height = 0
+        self._rotation = 0
 
         # State to allow the option of an interactive selector that can't be
         # interactively drawn. This is used in PolygonSelector as an
         # interactive bounding box to allow the polygon to be easily resized
         self._allow_creation = True
+        self._drawing_new = False
 
         if drawtype == 'none':  # draw a line but make it invisible
             _api.warn_deprecated(
@@ -2923,6 +2948,14 @@ def __init__(self, ax, onselect, drawtype='box',
             self._props = props
             to_draw = self._init_shape(**self._props)
             self.ax.add_patch(to_draw)
+            # ax.add_patch sets the transform to ax.transData. Override to None
+            # so the selector is defined in display coordinates, which makes
+            # it much easier to handle rotation and scaling
+            to_draw.set_transform(None)
+            # Becasue the transform in display coords, need to manually
+            # add a resize callback for when the axes are reszied
+            self.ax.figure.canvas.mpl_connect('resize_event', self._on_resize)
+
         if drawtype == 'line':
             _api.warn_deprecated(
                 "3.5", message="Support for drawtype='line' is deprecated "
@@ -2937,7 +2970,6 @@ def __init__(self, ax, onselect, drawtype='box',
             self.ax.add_line(to_draw)
 
         self._selection_artist = to_draw
-        self._set_aspect_ratio_correction()
 
         self.minspanx = minspanx
         self.minspany = minspany
@@ -2989,6 +3021,25 @@ def __init__(self, ax, onselect, drawtype='box',
         property(lambda self: self.grab_range,
                  lambda self, value: setattr(self, "grab_range", value)))
 
+    def _on_resize(self, event):
+        # Callback for an Axes resize
+        self._update_handles()
+
+    @property
+    def _position_state(self):
+        """Return a named tuple containing all position state attributes."""
+        return _RectState(
+            self._x0, self._y0, self._width, self._height, self._rotation)
+
+    @_position_state.setter
+    def _position_state(self, state):
+        self._x0 = state.x0
+        self._y0 = state.y0
+        self._width = state.width
+        self._height = state.height
+        self._rotation = state.rotation
+        self._update_selection_artist()
+
     @property
     def _handles_artists(self):
         return (*self._center_handle.artists, *self._corner_handles.artists,
@@ -2996,7 +3047,7 @@ def _handles_artists(self):
 
     def _init_shape(self, **props):
         return Rectangle((0, 0), 0, 1, visible=False,
-                         rotation_point='center', **props)
+                         rotation_point='xy', **props)
 
     def _press(self, event):
         """Button press event handler."""
@@ -3014,18 +3065,19 @@ def _press(self, event):
 
         if (self._active_handle is None and not self.ignore_event_outside and
                 self._allow_creation):
-            x = event.xdata
-            y = event.ydata
-            self.visible = False
-            self.extents = x, x, y, y
-            self.visible = True
-        else:
-            self.set_visible(True)
+            # Start drawing a new rectangle
+            self._x0 = event.x
+            self._y0 = event.y
+            self._width = 0
+            self._height = 0
+            self._rotation = 0
+            self._drawing_new = True
 
-        self._extents_on_press = self.extents
-        self._rotation_on_press = self._rotation
-        self._set_aspect_ratio_correction()
+        self.set_visible(True)
 
+        self._pos_state_on_press = self._position_state
+        if self._drawtype == 'box':
+            self._center_on_press = self._selection_artist.get_center()
         return False
 
     def _release(self, event):
@@ -3037,31 +3089,29 @@ def _release(self, event):
                 self.ignore_event_outside):
             return
 
-        # update the eventpress and eventrelease with the resulting extents
-        x0, x1, y0, y1 = self.extents
-        self._eventpress.xdata = x0
-        self._eventpress.ydata = y0
-        xy0 = self.ax.transData.transform([x0, y0])
-        self._eventpress.x, self._eventpress.y = xy0
-
-        self._eventrelease.xdata = x1
-        self._eventrelease.ydata = y1
-        xy1 = self.ax.transData.transform([x1, y1])
-        self._eventrelease.x, self._eventrelease.y = xy1
+        self._eventrelease.xdata = event.xdata
+        self._eventrelease.ydata = event.ydata
+        self._eventrelease.x = event.x
+        self._eventrelease.y = event.y
 
         # calculate dimensions of box or line
         if self.spancoords == 'data':
-            spanx = abs(self._eventpress.xdata - self._eventrelease.xdata)
-            spany = abs(self._eventpress.ydata - self._eventrelease.ydata)
+            # Can't use self.extents, as these are the tool handle locations
+            # that will be in old locations if a selector pre-exists
+            inv_tr = self.ax.transData.inverted()
+            x1, y1 = (self._x0 + self._width,
+                      self._y0 + self._height)
+            spanx, spany = (inv_tr.transform((x1, y1)) -
+                            inv_tr.transform((self._x0, self._y0)))
         elif self.spancoords == 'pixels':
-            spanx = abs(self._eventpress.x - self._eventrelease.x)
-            spany = abs(self._eventpress.y - self._eventrelease.y)
+            spanx = self._width
+            spany = self._height
         else:
             _api.check_in_list(['data', 'pixels'],
                                spancoords=self.spancoords)
         # check if drawn distance (if it exists) is not too small in
         # either x or y-direction
-        minspanxy = (spanx <= self.minspanx or spany <= self.minspany)
+        minspanxy = abs(spanx) <= self.minspanx or abs(spany) <= self.minspany
         if (self._drawtype != 'none' and minspanxy):
             for artist in self.artists:
                 artist.set_visible(False)
@@ -3072,10 +3122,23 @@ def _release(self, event):
         else:
             self.onselect(self._eventpress, self._eventrelease)
             self._selection_completed = True
+            if self._drawing_new and self._drawtype == 'box':
+                # When finished drawing, make sure width, height are positive,
+                # and put reference corner in lower left. This ensures that the
+                # orientation of the corners and edges is always anti-clockwise
+                c = self._selection_artist.get_corners()
+                cx, cy = c[:, 0], c[:, 1]
+                new_pos_state = self._position_state
+                new_pos_state.x0 = np.min(cx)
+                new_pos_state.y0 = np.min(cy)
+                new_pos_state.width = np.max(cx) - np.min(cx)
+                new_pos_state.height = np.max(cy) - np.min(cy)
+                self._position_state = new_pos_state
 
         self.update()
         self._active_handle = None
         self._extents_on_press = None
+        self._drawing_new = False
 
         return False
 
@@ -3090,126 +3153,161 @@ def _onmove(self, event):
         - Continue the creation of a new shape
         """
         eventpress = self._eventpress
-        # The calculations are done for rotation at zero: we apply inverse
-        # transformation to events except when we rotate and move
+        event.x, event.y = self._clip_to_axes(event.x, event.y)
+
+        # Decide which action to carry out
         state = self._state
         rotate = ('rotate' in state and
                   self._active_handle in self._corner_order)
         move = self._active_handle == 'C'
         resize = self._active_handle and not move
 
-        if resize:
-            inv_tr = self._get_rotation_transform().inverted()
-            event.xdata, event.ydata = inv_tr.transform(
-                [event.xdata, event.ydata])
-            eventpress.xdata, eventpress.ydata = inv_tr.transform(
-                [eventpress.xdata, eventpress.ydata]
-                )
-
-        dx = event.xdata - eventpress.xdata
-        dy = event.ydata - eventpress.ydata
-        # refmax is used when moving the corner handle with the square state
-        # and is the maximum between refx and refy
-        refmax = None
-        if self._use_data_coordinates:
-            refx, refy = dx, dy
-        else:
-            # Get dx/dy in display coordinates
-            refx = event.x - eventpress.x
-            refy = event.y - eventpress.y
+        # Create a variable for the new position after this move
+        new_pos_state = copy.copy(self._pos_state_on_press)
 
-        x0, x1, y0, y1 = self._extents_on_press
         # rotate an existing shape
         if rotate:
             # calculate angle abc
-            a = np.array([eventpress.xdata, eventpress.ydata])
-            b = np.array(self.center)
-            c = np.array([event.xdata, event.ydata])
+            a = [eventpress.x, eventpress.y]
+            b = self._center_on_press
+            c = [event.x, event.y]
             angle = (np.arctan2(c[1]-b[1], c[0]-b[0]) -
                      np.arctan2(a[1]-b[1], a[0]-b[0]))
-            self.rotation = np.rad2deg(self._rotation_on_press + angle)
+            new_pos_state.rotation = self._pos_state_on_press.rotation + angle
+            # Transform the rectangle corner so we are rotating about the
+            # center of the rectangle
+            new_pos_state.xy = Affine2D().rotate_around(*b, angle).transform(
+                self._pos_state_on_press.xy)
 
         elif resize:
-            size_on_press = [x1 - x0, y1 - y0]
-            center = [x0 + size_on_press[0] / 2, y0 + size_on_press[1] / 2]
+            # Do resizing in a de-rotated frame
+            t = Affine2D().rotate(-self._rotation)
+            press_x, press_y = t.transform((eventpress.x, eventpress.y))
+            event.x, event.y = t.transform((event.x, event.y))
+            x0, y0 = t.transform((self._pos_state_on_press.x0,
+                                  self._pos_state_on_press.y0))
+            dx = event.x - press_x
+            dy = event.y - press_y
 
             # Keeping the center fixed
             if 'center' in state:
+                size_on_press = [self._pos_state_on_press.width,
+                                 self._pos_state_on_press.height]
+                center = [x0 + size_on_press[0] / 2,
+                          y0 + size_on_press[1] / 2]
                 # hh, hw are half-height and half-width
                 if 'square' in state:
-                    # when using a corner, find which reference to use
+                    refmax = None
                     if self._active_handle in self._corner_order:
-                        refmax = max(refx, refy, key=abs)
-                    if self._active_handle in ['E', 'W'] or refmax == refx:
-                        hw = event.xdata - center[0]
-                        hh = hw / self._aspect_ratio_correction
+                        # When using a corner, use the maximum change in x/y
+                        refmax = max(dx, dy, key=abs)
+                    if self._active_handle in ['E', 'W'] or refmax == dx:
+                        hw = hh = abs(event.x - center[0])
+                        if self._use_data_coordinates:
+                            hh *= self.ax._get_aspect_ratio()
                     else:
-                        hh = event.ydata - center[1]
-                        hw = hh * self._aspect_ratio_correction
+                        hw = hh = abs(event.y - center[1])
+                        if self._use_data_coordinates:
+                            hw /= self.ax._get_aspect_ratio()
+
                 else:
                     hw = size_on_press[0] / 2
                     hh = size_on_press[1] / 2
                     # cancel changes in perpendicular direction
                     if self._active_handle in ['E', 'W'] + self._corner_order:
-                        hw = abs(event.xdata - center[0])
+                        hw = abs(event.x - center[0])
                     if self._active_handle in ['N', 'S'] + self._corner_order:
-                        hh = abs(event.ydata - center[1])
+                        hh = abs(event.y - center[1])
 
-                x0, x1, y0, y1 = (center[0] - hw, center[0] + hw,
-                                  center[1] - hh, center[1] + hh)
+                x0 = center[0] - hw
+                y0 = center[1] - hh
+                width = 2 * hw
+                height = 2 * hh
 
             else:
-                # change sign of relative changes to simplify calculation
-                # Switch variables so that x1 and/or y1 are updated on move
-                if 'W' in self._active_handle:
-                    x0 = x1
-                if 'S' in self._active_handle:
-                    y0 = y1
-                if self._active_handle in ['E', 'W'] + self._corner_order:
-                    x1 = event.xdata
-                if self._active_handle in ['N', 'S'] + self._corner_order:
-                    y1 = event.ydata
+                # center not in state
+                width = self._pos_state_on_press.width
+                height = self._pos_state_on_press.height
+
+                if 'N' in self._active_handle:
+                    height += dy
+                elif 'S' in self._active_handle:
+                    height -= dy
+                    y0 += dy
+
+                if 'E' in self._active_handle:
+                    width += dx
+                elif 'W' in self._active_handle:
+                    width -= dx
+                    x0 += dx
+
                 if 'square' in state:
-                    # when using a corner, find which reference to use
-                    if self._active_handle in self._corner_order:
-                        refmax = max(refx, refy, key=abs)
-                    if self._active_handle in ['E', 'W'] or refmax == refx:
-                        sign = np.sign(event.ydata - y0)
-                        y1 = y0 + sign * abs(x1 - x0) / \
-                            self._aspect_ratio_correction
-                    else:
-                        sign = np.sign(event.xdata - x0)
-                        x1 = x0 + sign * abs(y1 - y0) * \
-                            self._aspect_ratio_correction
+                    if self._active_handle in ['E', 'W']:
+                        height = width
+                    elif self._active_handle in ['N', 'S']:
+                        width = height
+                    elif self._active_handle == 'NE':
+                        width = height = max(event.x - x0, event.y - y0)
+                    elif self._active_handle == 'SW':
+                        # Keep x0 + width, y0 + height a fixed point
+                        new_wh = max(x0 + width - event.x,
+                                     y0 + height - event.y)
+                        x0 += width - new_wh
+                        y0 += height - new_wh
+                        width = height = new_wh
+                    elif self._active_handle == 'SE':
+                        # Keep x0, y0 + height a fixed point
+                        new_wh = max(event.x - x0, y0 + height - event.y)
+                        y0 += height - new_wh
+                        width = height = new_wh
+                    elif self._active_handle == 'NW':
+                        # Keep x0 + width, y0 a fixed point
+                        new_wh = max(x0 + width - event.x, event.y - y0)
+                        x0 += width - new_wh
+                        width = height = new_wh
+
+            # Transform back into de-rotated display coordiantes
+            new_pos_state.x0, new_pos_state.y0 = t.inverted().transform(
+                (x0, y0))
+            # Width and height are invariant under the rotation, so no need
+            # to transform
+            new_pos_state.width = width
+            new_pos_state.height = height
 
         elif move:
-            x0, x1, y0, y1 = self._extents_on_press
-            dx = event.xdata - eventpress.xdata
-            dy = event.ydata - eventpress.ydata
-            x0 += dx
-            x1 += dx
-            y0 += dy
-            y1 += dy
+            dx = event.x - eventpress.x
+            dy = event.y - eventpress.y
+            new_pos_state.x0 += dx
+            new_pos_state.y0 += dy
 
         else:
             # Create a new shape
-            self._rotation = 0
+
             # Don't create a new rectangle if there is already one when
             # ignore_event_outside=True
             if ((self.ignore_event_outside and self._selection_completed) or
                     not self._allow_creation):
                 return
-            center = [eventpress.xdata, eventpress.ydata]
-            dx = (event.xdata - center[0]) / 2.
-            dy = (event.ydata - center[1]) / 2.
+            center = [eventpress.x, eventpress.y]
+            dx = (event.x - center[0]) / 2
+            dy = (event.y - center[1]) / 2
 
             # square shape
             if 'square' in state:
+                refx = event.x - eventpress.x
+                refy = event.y - eventpress.y
                 refmax = max(refx, refy, key=abs)
+
                 if refmax == refx:
-                    dy = np.sign(dy) * abs(dx) / self._aspect_ratio_correction
+                    sign = np.sign(dy) or 1
+                    dy = sign * abs(dx)
+                    if self._use_data_coordinates:
+                        dy *= self.ax._get_aspect_ratio()
                 else:
-                    dx = np.sign(dx) * abs(dy) * self._aspect_ratio_correction
+                    sign = np.sign(dx) or 1
+                    dx = sign * abs(dy)
+                    if self._use_data_coordinates:
+                        dx /= self.ax._get_aspect_ratio()
 
             # from center
             if 'center' in state:
@@ -3221,41 +3319,17 @@ def _onmove(self, event):
                 center[0] += dx
                 center[1] += dy
 
-            x0, x1, y0, y1 = (center[0] - dx, center[0] + dx,
-                              center[1] - dy, center[1] + dy)
+            new_pos_state.x0 = center[0] - dx
+            new_pos_state.y0 = center[1] - dy
+            new_pos_state.width = 2 * dx
+            new_pos_state.height = 2 * dy
+            new_pos_state.rotation = 0
 
-        self.extents = x0, x1, y0, y1
-
-    @property
-    def _rect_bbox(self):
-        if self._drawtype == 'box':
-            return self._selection_artist.get_bbox().bounds
-        else:
-            x, y = self._selection_artist.get_data()
-            x0, x1 = min(x), max(x)
-            y0, y1 = min(y), max(y)
-            return x0, y0, x1 - x0, y1 - y0
-
-    def _set_aspect_ratio_correction(self):
-        aspect_ratio = self.ax._get_aspect_ratio()
-        if not hasattr(self._selection_artist, '_aspect_ratio_correction'):
-            # Aspect ratio correction is not supported with deprecated
-            # drawtype='line'. Remove this block in matplotlib 3.7
-            self._aspect_ratio_correction = 1
-            return
-
-        self._selection_artist._aspect_ratio_correction = aspect_ratio
-        if self._use_data_coordinates:
-            self._aspect_ratio_correction = 1
-        else:
-            self._aspect_ratio_correction = aspect_ratio
+        self._position_state = new_pos_state
 
     def _get_rotation_transform(self):
-        aspect_ratio = self.ax._get_aspect_ratio()
         return Affine2D().translate(-self.center[0], -self.center[1]) \
-                .scale(1, aspect_ratio) \
                 .rotate(self._rotation) \
-                .scale(1, 1 / aspect_ratio) \
                 .translate(*self.center)
 
     @property
@@ -3264,12 +3338,15 @@ def corners(self):
         Corners of rectangle in data coordinates from lower left,
         moving clockwise.
         """
-        x0, y0, width, height = self._rect_bbox
-        xc = x0, x0 + width, x0 + width, x0
-        yc = y0, y0, y0 + height, y0 + height
-        transform = self._get_rotation_transform()
-        coords = transform.transform(np.array([xc, yc]).T).T
-        return coords[0], coords[1]
+        if self._drawtype == 'box':
+            c = self._selection_artist.get_corners()
+            # Convert from display to data coordinates
+            c = self.ax.transData.inverted().transform(c)
+            return c[:, 0], c[:, 1]
+        elif self._drawtype == 'line':
+            x, y = self._selection_artist.get_data()
+            return (np.array([x[0], x[0], x[1], x[1]]),
+                    np.array([y[0], y[1], y[1], y[0]]))
 
     @property
     def edge_centers(self):
@@ -3277,20 +3354,16 @@ def edge_centers(self):
         Midpoint of rectangle edges in data coordiantes from left,
         moving anti-clockwise.
         """
-        x0, y0, width, height = self._rect_bbox
-        w = width / 2.
-        h = height / 2.
-        xe = x0, x0 + w, x0 + width, x0 + w
-        ye = y0 + h, y0, y0 + h, y0 + height
-        transform = self._get_rotation_transform()
-        coords = transform.transform(np.array([xe, ye]).T).T
-        return coords[0], coords[1]
+        c = self._selection_artist._get_edge_midpoints()
+        c = self.ax.transData.inverted().transform(c)
+        return c[:, 0], c[:, 1]
 
     @property
     def center(self):
         """Center of rectangle in data coordinates."""
-        x0, y0, width, height = self._rect_bbox
-        return x0 + width / 2., y0 + height / 2.
+        c = self._selection_artist.get_center()
+        # Convert from display to data coordinates
+        return self.ax.transData.inverted().transform(c)
 
     @property
     def extents(self):
@@ -3298,63 +3371,87 @@ def extents(self):
         Return (xmin, xmax, ymin, ymax) in data coordinates as defined by the
         bounding box before rotation.
         """
-        x0, y0, width, height = self._rect_bbox
-        xmin, xmax = sorted([x0, x0 + width])
-        ymin, ymax = sorted([y0, y0 + height])
-        return xmin, xmax, ymin, ymax
+        cx, cy = self.corners
+        return cx[0], cx[2], cy[0], cy[2]
 
     @extents.setter
     def extents(self, extents):
+        # Convert from data to figure coordinates
+        corner_min = self.ax.transData.transform((extents[0], extents[2]))
+        corner_max = self.ax.transData.transform((extents[1], extents[3]))
         # Update displayed shape
-        self._draw_shape(extents)
-        if self._interactive:
-            # Update displayed handles
-            self._corner_handles.set_data(*self.corners)
-            self._edge_handles.set_data(*self.edge_centers)
-            self._center_handle.set_data(*self.center)
+        self._draw_shape((corner_min[0], corner_max[0],
+                          corner_min[1], corner_max[1]))
         self.set_visible(self.visible)
         self.update()
 
     @property
     def rotation(self):
         """
-        Rotation in degree in interval [-45°, 45°]. The rotation is limited in
-        range to keep the implementation simple.
+        Rotation in degrees.
         """
         return np.rad2deg(self._rotation)
 
     @rotation.setter
     def rotation(self, value):
-        # Restrict to a limited range of rotation [-45°, 45°] to avoid changing
-        # order of handles
-        if -45 <= value and value <= 45:
-            self._rotation = np.deg2rad(value)
-            # call extents setter to draw shape and update handles positions
-            self.extents = self.extents
+        self._rotation = np.deg2rad(value)
+        self._update_selection_artist()
+
+    def _clip_to_axes(self, x, y):
+        """
+        Clip x and y values in diplay coordinates to the limits of the current
+        Axes.
+        """
+        xlim = sorted(self.ax.get_xlim())
+        ylim = sorted(self.ax.get_ylim())
+
+        min_lim = (xlim[0], ylim[0])
+        max_lim = (xlim[1], ylim[1])
+        # Axes limits in display coordinates
+        min_lim = self.ax.transData.transform(min_lim)
+        max_lim = self.ax.transData.transform(max_lim)
+
+        x = np.clip(x, min_lim[0], max_lim[0])
+        y = np.clip(y, min_lim[1], max_lim[1])
+        return x, y
 
+    # TODO: _draw_shape can be removed in 3.7
     draw_shape = _api.deprecate_privatize_attribute('3.5')
 
     def _draw_shape(self, extents):
         x0, x1, y0, y1 = extents
-        xmin, xmax = sorted([x0, x1])
-        ymin, ymax = sorted([y0, y1])
-        xlim = sorted(self.ax.get_xlim())
-        ylim = sorted(self.ax.get_ylim())
-
-        xmin = max(xlim[0], xmin)
-        ymin = max(ylim[0], ymin)
-        xmax = min(xmax, xlim[1])
-        ymax = min(ymax, ylim[1])
+        self._x0 = x0
+        self._y0 = y0
+        self._width = x1 - x0
+        self._height = y1 - y0
+        self._update_selection_artist()
 
+    def _update_selection_artist(self):
+        """
+        Update the selection artists from the current position state.
+        """
         if self._drawtype == 'box':
-            self._selection_artist.set_x(xmin)
-            self._selection_artist.set_y(ymin)
-            self._selection_artist.set_width(xmax - xmin)
-            self._selection_artist.set_height(ymax - ymin)
+            self._selection_artist.set_x(self._x0)
+            self._selection_artist.set_y(self._y0)
+            self._selection_artist.set_width(self._width)
+            self._selection_artist.set_height(self._height)
             self._selection_artist.set_angle(self.rotation)
 
         elif self._drawtype == 'line':
-            self._selection_artist.set_data([xmin, xmax], [ymin, ymax])
+            xy0 = self._x0, self._y0
+            xy1 = self._x0 + self._width, self._y0 + self._height
+            xy1 = Affine2D().rotate_around(*xy0, self._rotation).transform(xy1)
+            self._selection_artist.set_data([xy0[0], xy1[0]], [xy0[1], xy1[1]])
+
+        if self._interactive:
+            self._update_handles()
+
+        self.update()
+
+    def _update_handles(self):
+        self._corner_handles.set_data(*self.corners)
+        self._edge_handles.set_data(*self.edge_centers)
+        self._center_handle.set_data(*self.center)
 
     def _set_active_handle(self, event):
         """Set active handle based on the location of the mouse event."""
@@ -3428,37 +3525,32 @@ class EllipseSelector(RectangleSelector):
     def _init_shape(self, **props):
         return Ellipse((0, 0), 0, 1, visible=False, **props)
 
-    def _draw_shape(self, extents):
-        x0, x1, y0, y1 = extents
-        xmin, xmax = sorted([x0, x1])
-        ymin, ymax = sorted([y0, y1])
-        center = [x0 + (x1 - x0) / 2., y0 + (y1 - y0) / 2.]
-        a = (xmax - xmin) / 2.
-        b = (ymax - ymin) / 2.
-
+    def _update_selection_artist(self):
+        """
+        Update the selection artists from the current position state.
+        """
+        center = (self._x0 + self._width / 2,
+                  self._y0 + self._height / 2)
+        center = Affine2D().rotate_around(
+            self._x0, self._y0, self._rotation).transform(center)
         if self._drawtype == 'box':
             self._selection_artist.center = center
-            self._selection_artist.width = 2 * a
-            self._selection_artist.height = 2 * b
+            self._selection_artist.width = self._width
+            self._selection_artist.height = self._height
             self._selection_artist.angle = self.rotation
         else:
             rad = np.deg2rad(np.arange(31) * 12)
-            x = a * np.cos(rad) + center[0]
-            y = b * np.sin(rad) + center[1]
+            x = self._width / 2 * np.cos(rad) + center[0]
+            y = self._height / 2 * np.sin(rad) + center[1]
             self._selection_artist.set_data(x, y)
 
-    @property
-    def _rect_bbox(self):
-        if self._drawtype == 'box':
-            x, y = self._selection_artist.center
-            width = self._selection_artist.width
-            height = self._selection_artist.height
-            return x - width / 2., y - height / 2., width, height
-        else:
-            x, y = self._selection_artist.get_data()
-            x0, x1 = min(x), max(x)
-            y0, y1 = min(y), max(y)
-            return x0, y0, x1 - x0, y1 - y0
+        if self._interactive:
+            # Update displayed handles
+            self._corner_handles.set_data(*self.corners)
+            self._edge_handles.set_data(*self.edge_centers)
+            self._center_handle.set_data(*self.center)
+
+        self.update()
 
 
 class LassoSelector(_SelectorWidget):
@@ -3716,13 +3808,13 @@ def _scale_polygon(self, event):
             return
 
         # Create transform from old box to new box
-        x1, y1, w1, h1 = self._box._rect_bbox
+        xmin, xmax, ymin, ymax = self._box.extents
         old_bbox = self._get_bbox()
         t = (transforms.Affine2D()
              .translate(-old_bbox.x0, -old_bbox.y0)
              .scale(1 / old_bbox.width, 1 / old_bbox.height)
-             .scale(w1, h1)
-             .translate(x1, y1))
+             .scale(xmax - xmin, ymax - ymin)
+             .translate(xmin, ymin))
 
         # Update polygon verts.  Must be a list of tuples for consistency.
         new_verts = [(x, y) for x, y in t.transform(np.array(self.verts))]