From 3293164353d2b5168f83039a1158d54648a584ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Tue, 29 Aug 2023 12:45:49 +0200 Subject: [PATCH 1/2] API: Update lib.stride_tricks namespace --- numpy/__init__.py | 7 +- numpy/__init__.pyi | 2 +- numpy/core/_methods.py | 2 +- numpy/lib/__init__.py | 3 +- numpy/lib/__init__.pyi | 6 - numpy/lib/_function_base_impl.py | 4 +- numpy/lib/_stride_tricks_impl.py | 546 +++++++++++++++++++++++++ numpy/lib/_stride_tricks_impl.pyi | 80 ++++ numpy/lib/_twodim_base_impl.py | 2 +- numpy/lib/stride_tricks.py | 549 +------------------------- numpy/lib/stride_tricks.pyi | 83 +--- numpy/lib/tests/test_stride_tricks.py | 2 +- 12 files changed, 646 insertions(+), 640 deletions(-) create mode 100644 numpy/lib/_stride_tricks_impl.py create mode 100644 numpy/lib/_stride_tricks_impl.pyi diff --git a/numpy/__init__.py b/numpy/__init__.py index dc008c9067f2..9ee3c7bc7e97 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -184,8 +184,7 @@ from . import lib from .lib import ( DataSource, apply_along_axis, apply_over_axes, - array_split, broadcast_arrays, broadcast_shapes, - broadcast_to, c_, column_stack, diag_indices, + array_split, c_, column_stack, diag_indices, diag_indices_from, dsplit, dstack, emath, expand_dims, fill_diagonal, fromregex, get_array_wrap, genfromtxt, @@ -230,6 +229,9 @@ from .lib._utils_impl import ( byte_bounds, show_runtime, get_include, info ) + from .lib._stride_tricks_impl import ( + broadcast_arrays, broadcast_shapes, broadcast_to + ) from . import matrixlib as _mat from .matrixlib import ( asmatrix, bmat, matrix @@ -299,6 +301,7 @@ set(lib._ufunclike_impl.__all__) | set(lib._arraypad_impl.__all__) | set(lib._utils_impl.__all__) | + set(lib._stride_tricks_impl.__all__) | {"show_config", "__version__"} ) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 9ac02c2a63e6..ef483ced0354 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -536,7 +536,7 @@ from numpy.lib.shape_base import ( put_along_axis as put_along_axis, ) -from numpy.lib.stride_tricks import ( +from numpy.lib._stride_tricks_impl import ( broadcast_to as broadcast_to, broadcast_arrays as broadcast_arrays, broadcast_shapes as broadcast_shapes, diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 2050fa81a2f0..9675f9822aaa 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -82,7 +82,7 @@ def _count_reduce_items(arr, axis, keepdims=False, where=True): # axis and full sum is more excessive than needed. # guarded to protect circular imports - from numpy.lib.stride_tricks import broadcast_to + from numpy.lib._stride_tricks_impl import broadcast_to # count True values in (potentially broadcasted) boolean mask items = umr_sum(broadcast_to(where, arr.shape), axis, nt.intp, None, keepdims) diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index 6f90a6c059ce..cad38af841d5 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -24,6 +24,7 @@ from . import _nanfunctions_impl from . import _function_base_impl from . import shape_base +from . import _stride_tricks_impl from . import stride_tricks from . import _twodim_base_impl from . import _ufunclike_impl @@ -38,7 +39,6 @@ from .index_tricks import * from .shape_base import * -from .stride_tricks import * from .polynomial import * from .npyio import * from .arrayterator import Arrayterator @@ -50,7 +50,6 @@ __all__ = ['emath'] __all__ += index_tricks.__all__ __all__ += shape_base.__all__ -__all__ += stride_tricks.__all__ __all__ += polynomial.__all__ __all__ += npyio.__all__ diff --git a/numpy/lib/__init__.pyi b/numpy/lib/__init__.pyi index 9bd016050e87..6470622cbd44 100644 --- a/numpy/lib/__init__.pyi +++ b/numpy/lib/__init__.pyi @@ -89,12 +89,6 @@ from numpy.lib.shape_base import ( put_along_axis as put_along_axis, ) -from numpy.lib.stride_tricks import ( - broadcast_to as broadcast_to, - broadcast_arrays as broadcast_arrays, - broadcast_shapes as broadcast_shapes, -) - from numpy.core.multiarray import ( add_docstring as add_docstring, tracemalloc_domain as tracemalloc_domain, diff --git a/numpy/lib/_function_base_impl.py b/numpy/lib/_function_base_impl.py index 16a62896373f..f27259da49c5 100644 --- a/numpy/lib/_function_base_impl.py +++ b/numpy/lib/_function_base_impl.py @@ -2093,7 +2093,9 @@ def _parse_input_dimensions(args, input_core_dims): ndim = arg.ndim - len(core_dims) dummy_array = np.lib.stride_tricks.as_strided(0, arg.shape[:ndim]) broadcast_args.append(dummy_array) - broadcast_shape = np.lib.stride_tricks._broadcast_shape(*broadcast_args) + broadcast_shape = np.lib._stride_tricks_impl._broadcast_shape( + *broadcast_args + ) return broadcast_shape, dim_sizes diff --git a/numpy/lib/_stride_tricks_impl.py b/numpy/lib/_stride_tricks_impl.py new file mode 100644 index 000000000000..9798cd7056c9 --- /dev/null +++ b/numpy/lib/_stride_tricks_impl.py @@ -0,0 +1,546 @@ +""" +Utilities that manipulate strides to achieve desirable effects. + +An explanation of strides can be found in the :ref:`arrays.ndarray`. + +""" +import numpy as np +from numpy.core.numeric import normalize_axis_tuple +from numpy.core.overrides import array_function_dispatch, set_module + +__all__ = ['broadcast_to', 'broadcast_arrays', 'broadcast_shapes'] + + +class DummyArray: + """Dummy object that just exists to hang __array_interface__ dictionaries + and possibly keep alive a reference to a base array. + """ + + def __init__(self, interface, base=None): + self.__array_interface__ = interface + self.base = base + + +def _maybe_view_as_subclass(original_array, new_array): + if type(original_array) is not type(new_array): + # if input was an ndarray subclass and subclasses were OK, + # then view the result as that subclass. + new_array = new_array.view(type=type(original_array)) + # Since we have done something akin to a view from original_array, we + # should let the subclass finalize (if it has it implemented, i.e., is + # not None). + if new_array.__array_finalize__: + new_array.__array_finalize__(original_array) + return new_array + + +def as_strided(x, shape=None, strides=None, subok=False, writeable=True): + """ + Create a view into the array with the given shape and strides. + + .. warning:: This function has to be used with extreme care, see notes. + + Parameters + ---------- + x : ndarray + Array to create a new. + shape : sequence of int, optional + The shape of the new array. Defaults to ``x.shape``. + strides : sequence of int, optional + The strides of the new array. Defaults to ``x.strides``. + subok : bool, optional + .. versionadded:: 1.10 + + If True, subclasses are preserved. + writeable : bool, optional + .. versionadded:: 1.12 + + If set to False, the returned array will always be readonly. + Otherwise it will be writable if the original array was. It + is advisable to set this to False if possible (see Notes). + + Returns + ------- + view : ndarray + + See also + -------- + broadcast_to : broadcast an array to a given shape. + reshape : reshape an array. + lib.stride_tricks.sliding_window_view : + userfriendly and safe function for a creation of sliding window views. + + Notes + ----- + ``as_strided`` creates a view into the array given the exact strides + and shape. This means it manipulates the internal data structure of + ndarray and, if done incorrectly, the array elements can point to + invalid memory and can corrupt results or crash your program. + It is advisable to always use the original ``x.strides`` when + calculating new strides to avoid reliance on a contiguous memory + layout. + + Furthermore, arrays created with this function often contain self + overlapping memory, so that two elements are identical. + Vectorized write operations on such arrays will typically be + unpredictable. They may even give different results for small, large, + or transposed arrays. + + Since writing to these arrays has to be tested and done with great + care, you may want to use ``writeable=False`` to avoid accidental write + operations. + + For these reasons it is advisable to avoid ``as_strided`` when + possible. + """ + # first convert input to array, possibly keeping subclass + x = np.array(x, copy=False, subok=subok) + interface = dict(x.__array_interface__) + if shape is not None: + interface['shape'] = tuple(shape) + if strides is not None: + interface['strides'] = tuple(strides) + + array = np.asarray(DummyArray(interface, base=x)) + # The route via `__interface__` does not preserve structured + # dtypes. Since dtype should remain unchanged, we set it explicitly. + array.dtype = x.dtype + + view = _maybe_view_as_subclass(x, array) + + if view.flags.writeable and not writeable: + view.flags.writeable = False + + return view + + +def _sliding_window_view_dispatcher(x, window_shape, axis=None, *, + subok=None, writeable=None): + return (x,) + + +@array_function_dispatch(_sliding_window_view_dispatcher) +def sliding_window_view(x, window_shape, axis=None, *, + subok=False, writeable=False): + """ + Create a sliding window view into the array with the given window shape. + + Also known as rolling or moving window, the window slides across all + dimensions of the array and extracts subsets of the array at all window + positions. + + .. versionadded:: 1.20.0 + + Parameters + ---------- + x : array_like + Array to create the sliding window view from. + window_shape : int or tuple of int + Size of window over each axis that takes part in the sliding window. + If `axis` is not present, must have same length as the number of input + array dimensions. Single integers `i` are treated as if they were the + tuple `(i,)`. + axis : int or tuple of int, optional + Axis or axes along which the sliding window is applied. + By default, the sliding window is applied to all axes and + `window_shape[i]` will refer to axis `i` of `x`. + If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to + the axis `axis[i]` of `x`. + Single integers `i` are treated as if they were the tuple `(i,)`. + subok : bool, optional + If True, sub-classes will be passed-through, otherwise the returned + array will be forced to be a base-class array (default). + writeable : bool, optional + When true, allow writing to the returned view. The default is false, + as this should be used with caution: the returned view contains the + same memory location multiple times, so writing to one location will + cause others to change. + + Returns + ------- + view : ndarray + Sliding window view of the array. The sliding window dimensions are + inserted at the end, and the original dimensions are trimmed as + required by the size of the sliding window. + That is, ``view.shape = x_shape_trimmed + window_shape``, where + ``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less + than the corresponding window size. + + See Also + -------- + lib.stride_tricks.as_strided: A lower-level and less safe routine for + creating arbitrary views from custom shape and strides. + broadcast_to: broadcast an array to a given shape. + + Notes + ----- + For many applications using a sliding window view can be convenient, but + potentially very slow. Often specialized solutions exist, for example: + + - `scipy.signal.fftconvolve` + + - filtering functions in `scipy.ndimage` + + - moving window functions provided by + `bottleneck `_. + + As a rough estimate, a sliding window approach with an input size of `N` + and a window size of `W` will scale as `O(N*W)` where frequently a special + algorithm can achieve `O(N)`. That means that the sliding window variant + for a window size of 100 can be a 100 times slower than a more specialized + version. + + Nevertheless, for small window sizes, when no custom algorithm exists, or + as a prototyping and developing tool, this function can be a good solution. + + Examples + -------- + >>> x = np.arange(6) + >>> x.shape + (6,) + >>> v = sliding_window_view(x, 3) + >>> v.shape + (4, 3) + >>> v + array([[0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5]]) + + This also works in more dimensions, e.g. + + >>> i, j = np.ogrid[:3, :4] + >>> x = 10*i + j + >>> x.shape + (3, 4) + >>> x + array([[ 0, 1, 2, 3], + [10, 11, 12, 13], + [20, 21, 22, 23]]) + >>> shape = (2,2) + >>> v = sliding_window_view(x, shape) + >>> v.shape + (2, 3, 2, 2) + >>> v + array([[[[ 0, 1], + [10, 11]], + [[ 1, 2], + [11, 12]], + [[ 2, 3], + [12, 13]]], + [[[10, 11], + [20, 21]], + [[11, 12], + [21, 22]], + [[12, 13], + [22, 23]]]]) + + The axis can be specified explicitly: + + >>> v = sliding_window_view(x, 3, 0) + >>> v.shape + (1, 4, 3) + >>> v + array([[[ 0, 10, 20], + [ 1, 11, 21], + [ 2, 12, 22], + [ 3, 13, 23]]]) + + The same axis can be used several times. In that case, every use reduces + the corresponding original dimension: + + >>> v = sliding_window_view(x, (2, 3), (1, 1)) + >>> v.shape + (3, 1, 2, 3) + >>> v + array([[[[ 0, 1, 2], + [ 1, 2, 3]]], + [[[10, 11, 12], + [11, 12, 13]]], + [[[20, 21, 22], + [21, 22, 23]]]]) + + Combining with stepped slicing (`::step`), this can be used to take sliding + views which skip elements: + + >>> x = np.arange(7) + >>> sliding_window_view(x, 5)[:, ::2] + array([[0, 2, 4], + [1, 3, 5], + [2, 4, 6]]) + + or views which move by multiple elements + + >>> x = np.arange(7) + >>> sliding_window_view(x, 3)[::2, :] + array([[0, 1, 2], + [2, 3, 4], + [4, 5, 6]]) + + A common application of `sliding_window_view` is the calculation of running + statistics. The simplest example is the + `moving average `_: + + >>> x = np.arange(6) + >>> x.shape + (6,) + >>> v = sliding_window_view(x, 3) + >>> v.shape + (4, 3) + >>> v + array([[0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5]]) + >>> moving_average = v.mean(axis=-1) + >>> moving_average + array([1., 2., 3., 4.]) + + Note that a sliding window approach is often **not** optimal (see Notes). + """ + window_shape = (tuple(window_shape) + if np.iterable(window_shape) + else (window_shape,)) + # first convert input to array, possibly keeping subclass + x = np.array(x, copy=False, subok=subok) + + window_shape_array = np.array(window_shape) + if np.any(window_shape_array < 0): + raise ValueError('`window_shape` cannot contain negative values') + + if axis is None: + axis = tuple(range(x.ndim)) + if len(window_shape) != len(axis): + raise ValueError(f'Since axis is `None`, must provide ' + f'window_shape for all dimensions of `x`; ' + f'got {len(window_shape)} window_shape elements ' + f'and `x.ndim` is {x.ndim}.') + else: + axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True) + if len(window_shape) != len(axis): + raise ValueError(f'Must provide matching length window_shape and ' + f'axis; got {len(window_shape)} window_shape ' + f'elements and {len(axis)} axes elements.') + + out_strides = x.strides + tuple(x.strides[ax] for ax in axis) + + # note: same axis can be windowed repeatedly + x_shape_trimmed = list(x.shape) + for ax, dim in zip(axis, window_shape): + if x_shape_trimmed[ax] < dim: + raise ValueError( + 'window shape cannot be larger than input array shape') + x_shape_trimmed[ax] -= dim - 1 + out_shape = tuple(x_shape_trimmed) + window_shape + return as_strided(x, strides=out_strides, shape=out_shape, + subok=subok, writeable=writeable) + + +def _broadcast_to(array, shape, subok, readonly): + shape = tuple(shape) if np.iterable(shape) else (shape,) + array = np.array(array, copy=False, subok=subok) + if not shape and array.shape: + raise ValueError('cannot broadcast a non-scalar to a scalar array') + if any(size < 0 for size in shape): + raise ValueError('all elements of broadcast shape must be non-' + 'negative') + extras = [] + it = np.nditer( + (array,), flags=['multi_index', 'refs_ok', 'zerosize_ok'] + extras, + op_flags=['readonly'], itershape=shape, order='C') + with it: + # never really has writebackifcopy semantics + broadcast = it.itviews[0] + result = _maybe_view_as_subclass(array, broadcast) + # In a future version this will go away + if not readonly and array.flags._writeable_no_warn: + result.flags.writeable = True + result.flags._warn_on_write = True + return result + + +def _broadcast_to_dispatcher(array, shape, subok=None): + return (array,) + + +@array_function_dispatch(_broadcast_to_dispatcher, module='numpy') +def broadcast_to(array, shape, subok=False): + """Broadcast an array to a new shape. + + Parameters + ---------- + array : array_like + The array to broadcast. + shape : tuple or int + The shape of the desired array. A single integer ``i`` is interpreted + as ``(i,)``. + subok : bool, optional + If True, then sub-classes will be passed-through, otherwise + the returned array will be forced to be a base-class array (default). + + Returns + ------- + broadcast : array + A readonly view on the original array with the given shape. It is + typically not contiguous. Furthermore, more than one element of a + broadcasted array may refer to a single memory location. + + Raises + ------ + ValueError + If the array is not compatible with the new shape according to NumPy's + broadcasting rules. + + See Also + -------- + broadcast + broadcast_arrays + broadcast_shapes + + Notes + ----- + .. versionadded:: 1.10.0 + + Examples + -------- + >>> x = np.array([1, 2, 3]) + >>> np.broadcast_to(x, (3, 3)) + array([[1, 2, 3], + [1, 2, 3], + [1, 2, 3]]) + """ + return _broadcast_to(array, shape, subok=subok, readonly=True) + + +def _broadcast_shape(*args): + """Returns the shape of the arrays that would result from broadcasting the + supplied arrays against each other. + """ + # use the old-iterator because np.nditer does not handle size 0 arrays + # consistently + b = np.broadcast(*args[:32]) + # unfortunately, it cannot handle 32 or more arguments directly + for pos in range(32, len(args), 31): + # ironically, np.broadcast does not properly handle np.broadcast + # objects (it treats them as scalars) + # use broadcasting to avoid allocating the full array + b = broadcast_to(0, b.shape) + b = np.broadcast(b, *args[pos:(pos + 31)]) + return b.shape + + +@set_module('numpy') +def broadcast_shapes(*args): + """ + Broadcast the input shapes into a single shape. + + :ref:`Learn more about broadcasting here `. + + .. versionadded:: 1.20.0 + + Parameters + ---------- + *args : tuples of ints, or ints + The shapes to be broadcast against each other. + + Returns + ------- + tuple + Broadcasted shape. + + Raises + ------ + ValueError + If the shapes are not compatible and cannot be broadcast according + to NumPy's broadcasting rules. + + See Also + -------- + broadcast + broadcast_arrays + broadcast_to + + Examples + -------- + >>> np.broadcast_shapes((1, 2), (3, 1), (3, 2)) + (3, 2) + + >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) + (5, 6, 7) + """ + arrays = [np.empty(x, dtype=[]) for x in args] + return _broadcast_shape(*arrays) + + +def _broadcast_arrays_dispatcher(*args, subok=None): + return args + + +@array_function_dispatch(_broadcast_arrays_dispatcher, module='numpy') +def broadcast_arrays(*args, subok=False): + """ + Broadcast any number of arrays against each other. + + Parameters + ---------- + *args : array_likes + The arrays to broadcast. + + subok : bool, optional + If True, then sub-classes will be passed-through, otherwise + the returned arrays will be forced to be a base-class array (default). + + Returns + ------- + broadcasted : list of arrays + These arrays are views on the original arrays. They are typically + not contiguous. Furthermore, more than one element of a + broadcasted array may refer to a single memory location. If you need + to write to the arrays, make copies first. While you can set the + ``writable`` flag True, writing to a single output value may end up + changing more than one location in the output array. + + .. deprecated:: 1.17 + The output is currently marked so that if written to, a deprecation + warning will be emitted. A future version will set the + ``writable`` flag False so writing to it will raise an error. + + See Also + -------- + broadcast + broadcast_to + broadcast_shapes + + Examples + -------- + >>> x = np.array([[1,2,3]]) + >>> y = np.array([[4],[5]]) + >>> np.broadcast_arrays(x, y) + [array([[1, 2, 3], + [1, 2, 3]]), array([[4, 4, 4], + [5, 5, 5]])] + + Here is a useful idiom for getting contiguous copies instead of + non-contiguous views. + + >>> [np.array(a) for a in np.broadcast_arrays(x, y)] + [array([[1, 2, 3], + [1, 2, 3]]), array([[4, 4, 4], + [5, 5, 5]])] + + """ + # nditer is not used here to avoid the limit of 32 arrays. + # Otherwise, something like the following one-liner would suffice: + # return np.nditer(args, flags=['multi_index', 'zerosize_ok'], + # order='C').itviews + + args = [np.array(_m, copy=False, subok=subok) for _m in args] + + shape = _broadcast_shape(*args) + + if all(array.shape == shape for array in args): + # Common case where nothing needs to be broadcasted. + return args + + return [_broadcast_to(array, shape, subok=subok, readonly=False) + for array in args] diff --git a/numpy/lib/_stride_tricks_impl.pyi b/numpy/lib/_stride_tricks_impl.pyi new file mode 100644 index 000000000000..4c9a98e85f78 --- /dev/null +++ b/numpy/lib/_stride_tricks_impl.pyi @@ -0,0 +1,80 @@ +from collections.abc import Iterable +from typing import Any, TypeVar, overload, SupportsIndex + +from numpy import generic +from numpy._typing import ( + NDArray, + ArrayLike, + _ShapeLike, + _Shape, + _ArrayLike +) + +_SCT = TypeVar("_SCT", bound=generic) + +__all__: list[str] + +class DummyArray: + __array_interface__: dict[str, Any] + base: None | NDArray[Any] + def __init__( + self, + interface: dict[str, Any], + base: None | NDArray[Any] = ..., + ) -> None: ... + +@overload +def as_strided( + x: _ArrayLike[_SCT], + shape: None | Iterable[int] = ..., + strides: None | Iterable[int] = ..., + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def as_strided( + x: ArrayLike, + shape: None | Iterable[int] = ..., + strides: None | Iterable[int] = ..., + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[Any]: ... + +@overload +def sliding_window_view( + x: _ArrayLike[_SCT], + window_shape: int | Iterable[int], + axis: None | SupportsIndex = ..., + *, + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def sliding_window_view( + x: ArrayLike, + window_shape: int | Iterable[int], + axis: None | SupportsIndex = ..., + *, + subok: bool = ..., + writeable: bool = ..., +) -> NDArray[Any]: ... + +@overload +def broadcast_to( + array: _ArrayLike[_SCT], + shape: int | Iterable[int], + subok: bool = ..., +) -> NDArray[_SCT]: ... +@overload +def broadcast_to( + array: ArrayLike, + shape: int | Iterable[int], + subok: bool = ..., +) -> NDArray[Any]: ... + +def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... + +def broadcast_arrays( + *args: ArrayLike, + subok: bool = ..., +) -> list[NDArray[Any]]: ... diff --git a/numpy/lib/_twodim_base_impl.py b/numpy/lib/_twodim_base_impl.py index 048de1453284..1f3cd1f57447 100644 --- a/numpy/lib/_twodim_base_impl.py +++ b/numpy/lib/_twodim_base_impl.py @@ -12,7 +12,7 @@ from numpy.core.overrides import set_array_function_like_doc, set_module from numpy.core import overrides from numpy.core import iinfo -from numpy.lib.stride_tricks import broadcast_to +from numpy.lib._stride_tricks_impl import broadcast_to __all__ = [ diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index d8d083b0422d..65565c06689c 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -1,546 +1,3 @@ -""" -Utilities that manipulate strides to achieve desirable effects. - -An explanation of strides can be found in the :ref:`arrays.ndarray`. - -""" -import numpy as np -from numpy.core.numeric import normalize_axis_tuple -from numpy.core.overrides import array_function_dispatch, set_module - -__all__ = ['broadcast_to', 'broadcast_arrays', 'broadcast_shapes'] - - -class DummyArray: - """Dummy object that just exists to hang __array_interface__ dictionaries - and possibly keep alive a reference to a base array. - """ - - def __init__(self, interface, base=None): - self.__array_interface__ = interface - self.base = base - - -def _maybe_view_as_subclass(original_array, new_array): - if type(original_array) is not type(new_array): - # if input was an ndarray subclass and subclasses were OK, - # then view the result as that subclass. - new_array = new_array.view(type=type(original_array)) - # Since we have done something akin to a view from original_array, we - # should let the subclass finalize (if it has it implemented, i.e., is - # not None). - if new_array.__array_finalize__: - new_array.__array_finalize__(original_array) - return new_array - - -def as_strided(x, shape=None, strides=None, subok=False, writeable=True): - """ - Create a view into the array with the given shape and strides. - - .. warning:: This function has to be used with extreme care, see notes. - - Parameters - ---------- - x : ndarray - Array to create a new. - shape : sequence of int, optional - The shape of the new array. Defaults to ``x.shape``. - strides : sequence of int, optional - The strides of the new array. Defaults to ``x.strides``. - subok : bool, optional - .. versionadded:: 1.10 - - If True, subclasses are preserved. - writeable : bool, optional - .. versionadded:: 1.12 - - If set to False, the returned array will always be readonly. - Otherwise it will be writable if the original array was. It - is advisable to set this to False if possible (see Notes). - - Returns - ------- - view : ndarray - - See also - -------- - broadcast_to : broadcast an array to a given shape. - reshape : reshape an array. - lib.stride_tricks.sliding_window_view : - userfriendly and safe function for the creation of sliding window views. - - Notes - ----- - ``as_strided`` creates a view into the array given the exact strides - and shape. This means it manipulates the internal data structure of - ndarray and, if done incorrectly, the array elements can point to - invalid memory and can corrupt results or crash your program. - It is advisable to always use the original ``x.strides`` when - calculating new strides to avoid reliance on a contiguous memory - layout. - - Furthermore, arrays created with this function often contain self - overlapping memory, so that two elements are identical. - Vectorized write operations on such arrays will typically be - unpredictable. They may even give different results for small, large, - or transposed arrays. - - Since writing to these arrays has to be tested and done with great - care, you may want to use ``writeable=False`` to avoid accidental write - operations. - - For these reasons it is advisable to avoid ``as_strided`` when - possible. - """ - # first convert input to array, possibly keeping subclass - x = np.array(x, copy=False, subok=subok) - interface = dict(x.__array_interface__) - if shape is not None: - interface['shape'] = tuple(shape) - if strides is not None: - interface['strides'] = tuple(strides) - - array = np.asarray(DummyArray(interface, base=x)) - # The route via `__interface__` does not preserve structured - # dtypes. Since dtype should remain unchanged, we set it explicitly. - array.dtype = x.dtype - - view = _maybe_view_as_subclass(x, array) - - if view.flags.writeable and not writeable: - view.flags.writeable = False - - return view - - -def _sliding_window_view_dispatcher(x, window_shape, axis=None, *, - subok=None, writeable=None): - return (x,) - - -@array_function_dispatch(_sliding_window_view_dispatcher) -def sliding_window_view(x, window_shape, axis=None, *, - subok=False, writeable=False): - """ - Create a sliding window view into the array with the given window shape. - - Also known as rolling or moving window, the window slides across all - dimensions of the array and extracts subsets of the array at all window - positions. - - .. versionadded:: 1.20.0 - - Parameters - ---------- - x : array_like - Array to create the sliding window view from. - window_shape : int or tuple of int - Size of window over each axis that takes part in the sliding window. - If `axis` is not present, must have same length as the number of input - array dimensions. Single integers `i` are treated as if they were the - tuple `(i,)`. - axis : int or tuple of int, optional - Axis or axes along which the sliding window is applied. - By default, the sliding window is applied to all axes and - `window_shape[i]` will refer to axis `i` of `x`. - If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to - the axis `axis[i]` of `x`. - Single integers `i` are treated as if they were the tuple `(i,)`. - subok : bool, optional - If True, sub-classes will be passed-through, otherwise the returned - array will be forced to be a base-class array (default). - writeable : bool, optional - When true, allow writing to the returned view. The default is false, - as this should be used with caution: the returned view contains the - same memory location multiple times, so writing to one location will - cause others to change. - - Returns - ------- - view : ndarray - Sliding window view of the array. The sliding window dimensions are - inserted at the end, and the original dimensions are trimmed as - required by the size of the sliding window. - That is, ``view.shape = x_shape_trimmed + window_shape``, where - ``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less - than the corresponding window size. - - See Also - -------- - lib.stride_tricks.as_strided: A lower-level and less safe routine for - creating arbitrary views from custom shape and strides. - broadcast_to: broadcast an array to a given shape. - - Notes - ----- - For many applications using a sliding window view can be convenient, but - potentially very slow. Often specialized solutions exist, for example: - - - `scipy.signal.fftconvolve` - - - filtering functions in `scipy.ndimage` - - - moving window functions provided by - `bottleneck `_. - - As a rough estimate, a sliding window approach with an input size of `N` - and a window size of `W` will scale as `O(N*W)` where frequently a special - algorithm can achieve `O(N)`. That means that the sliding window variant - for a window size of 100 can be a 100 times slower than a more specialized - version. - - Nevertheless, for small window sizes, when no custom algorithm exists, or - as a prototyping and developing tool, this function can be a good solution. - - Examples - -------- - >>> x = np.arange(6) - >>> x.shape - (6,) - >>> v = sliding_window_view(x, 3) - >>> v.shape - (4, 3) - >>> v - array([[0, 1, 2], - [1, 2, 3], - [2, 3, 4], - [3, 4, 5]]) - - This also works in more dimensions, e.g. - - >>> i, j = np.ogrid[:3, :4] - >>> x = 10*i + j - >>> x.shape - (3, 4) - >>> x - array([[ 0, 1, 2, 3], - [10, 11, 12, 13], - [20, 21, 22, 23]]) - >>> shape = (2,2) - >>> v = sliding_window_view(x, shape) - >>> v.shape - (2, 3, 2, 2) - >>> v - array([[[[ 0, 1], - [10, 11]], - [[ 1, 2], - [11, 12]], - [[ 2, 3], - [12, 13]]], - [[[10, 11], - [20, 21]], - [[11, 12], - [21, 22]], - [[12, 13], - [22, 23]]]]) - - The axis can be specified explicitly: - - >>> v = sliding_window_view(x, 3, 0) - >>> v.shape - (1, 4, 3) - >>> v - array([[[ 0, 10, 20], - [ 1, 11, 21], - [ 2, 12, 22], - [ 3, 13, 23]]]) - - The same axis can be used several times. In that case, every use reduces - the corresponding original dimension: - - >>> v = sliding_window_view(x, (2, 3), (1, 1)) - >>> v.shape - (3, 1, 2, 3) - >>> v - array([[[[ 0, 1, 2], - [ 1, 2, 3]]], - [[[10, 11, 12], - [11, 12, 13]]], - [[[20, 21, 22], - [21, 22, 23]]]]) - - Combining with stepped slicing (`::step`), this can be used to take sliding - views which skip elements: - - >>> x = np.arange(7) - >>> sliding_window_view(x, 5)[:, ::2] - array([[0, 2, 4], - [1, 3, 5], - [2, 4, 6]]) - - or views which move by multiple elements - - >>> x = np.arange(7) - >>> sliding_window_view(x, 3)[::2, :] - array([[0, 1, 2], - [2, 3, 4], - [4, 5, 6]]) - - A common application of `sliding_window_view` is the calculation of running - statistics. The simplest example is the - `moving average `_: - - >>> x = np.arange(6) - >>> x.shape - (6,) - >>> v = sliding_window_view(x, 3) - >>> v.shape - (4, 3) - >>> v - array([[0, 1, 2], - [1, 2, 3], - [2, 3, 4], - [3, 4, 5]]) - >>> moving_average = v.mean(axis=-1) - >>> moving_average - array([1., 2., 3., 4.]) - - Note that a sliding window approach is often **not** optimal (see Notes). - """ - window_shape = (tuple(window_shape) - if np.iterable(window_shape) - else (window_shape,)) - # first convert input to array, possibly keeping subclass - x = np.array(x, copy=False, subok=subok) - - window_shape_array = np.array(window_shape) - if np.any(window_shape_array < 0): - raise ValueError('`window_shape` cannot contain negative values') - - if axis is None: - axis = tuple(range(x.ndim)) - if len(window_shape) != len(axis): - raise ValueError(f'Since axis is `None`, must provide ' - f'window_shape for all dimensions of `x`; ' - f'got {len(window_shape)} window_shape elements ' - f'and `x.ndim` is {x.ndim}.') - else: - axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True) - if len(window_shape) != len(axis): - raise ValueError(f'Must provide matching length window_shape and ' - f'axis; got {len(window_shape)} window_shape ' - f'elements and {len(axis)} axes elements.') - - out_strides = x.strides + tuple(x.strides[ax] for ax in axis) - - # note: same axis can be windowed repeatedly - x_shape_trimmed = list(x.shape) - for ax, dim in zip(axis, window_shape): - if x_shape_trimmed[ax] < dim: - raise ValueError( - 'window shape cannot be larger than input array shape') - x_shape_trimmed[ax] -= dim - 1 - out_shape = tuple(x_shape_trimmed) + window_shape - return as_strided(x, strides=out_strides, shape=out_shape, - subok=subok, writeable=writeable) - - -def _broadcast_to(array, shape, subok, readonly): - shape = tuple(shape) if np.iterable(shape) else (shape,) - array = np.array(array, copy=False, subok=subok) - if not shape and array.shape: - raise ValueError('cannot broadcast a non-scalar to a scalar array') - if any(size < 0 for size in shape): - raise ValueError('all elements of broadcast shape must be non-' - 'negative') - extras = [] - it = np.nditer( - (array,), flags=['multi_index', 'refs_ok', 'zerosize_ok'] + extras, - op_flags=['readonly'], itershape=shape, order='C') - with it: - # never really has writebackifcopy semantics - broadcast = it.itviews[0] - result = _maybe_view_as_subclass(array, broadcast) - # In a future version this will go away - if not readonly and array.flags._writeable_no_warn: - result.flags.writeable = True - result.flags._warn_on_write = True - return result - - -def _broadcast_to_dispatcher(array, shape, subok=None): - return (array,) - - -@array_function_dispatch(_broadcast_to_dispatcher, module='numpy') -def broadcast_to(array, shape, subok=False): - """Broadcast an array to a new shape. - - Parameters - ---------- - array : array_like - The array to broadcast. - shape : tuple or int - The shape of the desired array. A single integer ``i`` is interpreted - as ``(i,)``. - subok : bool, optional - If True, then sub-classes will be passed-through, otherwise - the returned array will be forced to be a base-class array (default). - - Returns - ------- - broadcast : array - A readonly view on the original array with the given shape. It is - typically not contiguous. Furthermore, more than one element of a - broadcasted array may refer to a single memory location. - - Raises - ------ - ValueError - If the array is not compatible with the new shape according to NumPy's - broadcasting rules. - - See Also - -------- - broadcast - broadcast_arrays - broadcast_shapes - - Notes - ----- - .. versionadded:: 1.10.0 - - Examples - -------- - >>> x = np.array([1, 2, 3]) - >>> np.broadcast_to(x, (3, 3)) - array([[1, 2, 3], - [1, 2, 3], - [1, 2, 3]]) - """ - return _broadcast_to(array, shape, subok=subok, readonly=True) - - -def _broadcast_shape(*args): - """Returns the shape of the arrays that would result from broadcasting the - supplied arrays against each other. - """ - # use the old-iterator because np.nditer does not handle size 0 arrays - # consistently - b = np.broadcast(*args[:32]) - # unfortunately, it cannot handle 32 or more arguments directly - for pos in range(32, len(args), 31): - # ironically, np.broadcast does not properly handle np.broadcast - # objects (it treats them as scalars) - # use broadcasting to avoid allocating the full array - b = broadcast_to(0, b.shape) - b = np.broadcast(b, *args[pos:(pos + 31)]) - return b.shape - - -@set_module('numpy') -def broadcast_shapes(*args): - """ - Broadcast the input shapes into a single shape. - - :ref:`Learn more about broadcasting here `. - - .. versionadded:: 1.20.0 - - Parameters - ---------- - *args : tuples of ints, or ints - The shapes to be broadcast against each other. - - Returns - ------- - tuple - Broadcasted shape. - - Raises - ------ - ValueError - If the shapes are not compatible and cannot be broadcast according - to NumPy's broadcasting rules. - - See Also - -------- - broadcast - broadcast_arrays - broadcast_to - - Examples - -------- - >>> np.broadcast_shapes((1, 2), (3, 1), (3, 2)) - (3, 2) - - >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) - (5, 6, 7) - """ - arrays = [np.empty(x, dtype=[]) for x in args] - return _broadcast_shape(*arrays) - - -def _broadcast_arrays_dispatcher(*args, subok=None): - return args - - -@array_function_dispatch(_broadcast_arrays_dispatcher, module='numpy') -def broadcast_arrays(*args, subok=False): - """ - Broadcast any number of arrays against each other. - - Parameters - ---------- - *args : array_likes - The arrays to broadcast. - - subok : bool, optional - If True, then sub-classes will be passed-through, otherwise - the returned arrays will be forced to be a base-class array (default). - - Returns - ------- - broadcasted : list of arrays - These arrays are views on the original arrays. They are typically - not contiguous. Furthermore, more than one element of a - broadcasted array may refer to a single memory location. If you need - to write to the arrays, make copies first. While you can set the - ``writable`` flag True, writing to a single output value may end up - changing more than one location in the output array. - - .. deprecated:: 1.17 - The output is currently marked so that if written to, a deprecation - warning will be emitted. A future version will set the - ``writable`` flag False so writing to it will raise an error. - - See Also - -------- - broadcast - broadcast_to - broadcast_shapes - - Examples - -------- - >>> x = np.array([[1,2,3]]) - >>> y = np.array([[4],[5]]) - >>> np.broadcast_arrays(x, y) - [array([[1, 2, 3], - [1, 2, 3]]), array([[4, 4, 4], - [5, 5, 5]])] - - Here is a useful idiom for getting contiguous copies instead of - non-contiguous views. - - >>> [np.array(a) for a in np.broadcast_arrays(x, y)] - [array([[1, 2, 3], - [1, 2, 3]]), array([[4, 4, 4], - [5, 5, 5]])] - - """ - # nditer is not used here to avoid the limit of 32 arrays. - # Otherwise, something like the following one-liner would suffice: - # return np.nditer(args, flags=['multi_index', 'zerosize_ok'], - # order='C').itviews - - args = [np.array(_m, copy=False, subok=subok) for _m in args] - - shape = _broadcast_shape(*args) - - if all(array.shape == shape for array in args): - # Common case where nothing needs to be broadcasted. - return args - - return [_broadcast_to(array, shape, subok=subok, readonly=False) - for array in args] +from ._stride_tricks_impl import ( + DummyArray, as_strided, sliding_window_view +) diff --git a/numpy/lib/stride_tricks.pyi b/numpy/lib/stride_tricks.pyi index 4c9a98e85f78..6b96a8fd643a 100644 --- a/numpy/lib/stride_tricks.pyi +++ b/numpy/lib/stride_tricks.pyi @@ -1,80 +1,5 @@ -from collections.abc import Iterable -from typing import Any, TypeVar, overload, SupportsIndex - -from numpy import generic -from numpy._typing import ( - NDArray, - ArrayLike, - _ShapeLike, - _Shape, - _ArrayLike +from numpy.lib._stride_tricks_impl import ( + DummyArray as DummyArray, + as_strided as as_strided, + sliding_window_view as sliding_window_view, ) - -_SCT = TypeVar("_SCT", bound=generic) - -__all__: list[str] - -class DummyArray: - __array_interface__: dict[str, Any] - base: None | NDArray[Any] - def __init__( - self, - interface: dict[str, Any], - base: None | NDArray[Any] = ..., - ) -> None: ... - -@overload -def as_strided( - x: _ArrayLike[_SCT], - shape: None | Iterable[int] = ..., - strides: None | Iterable[int] = ..., - subok: bool = ..., - writeable: bool = ..., -) -> NDArray[_SCT]: ... -@overload -def as_strided( - x: ArrayLike, - shape: None | Iterable[int] = ..., - strides: None | Iterable[int] = ..., - subok: bool = ..., - writeable: bool = ..., -) -> NDArray[Any]: ... - -@overload -def sliding_window_view( - x: _ArrayLike[_SCT], - window_shape: int | Iterable[int], - axis: None | SupportsIndex = ..., - *, - subok: bool = ..., - writeable: bool = ..., -) -> NDArray[_SCT]: ... -@overload -def sliding_window_view( - x: ArrayLike, - window_shape: int | Iterable[int], - axis: None | SupportsIndex = ..., - *, - subok: bool = ..., - writeable: bool = ..., -) -> NDArray[Any]: ... - -@overload -def broadcast_to( - array: _ArrayLike[_SCT], - shape: int | Iterable[int], - subok: bool = ..., -) -> NDArray[_SCT]: ... -@overload -def broadcast_to( - array: ArrayLike, - shape: int | Iterable[int], - subok: bool = ..., -) -> NDArray[Any]: ... - -def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... - -def broadcast_arrays( - *args: ArrayLike, - subok: bool = ..., -) -> list[NDArray[Any]]: ... diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index efec5d24dad4..d62661cb6530 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -4,7 +4,7 @@ assert_equal, assert_array_equal, assert_raises, assert_, assert_raises_regex, assert_warns, ) -from numpy.lib.stride_tricks import ( +from numpy.lib._stride_tricks_impl import ( as_strided, broadcast_arrays, _broadcast_shape, broadcast_to, broadcast_shapes, sliding_window_view, ) From 6e12461def5a8df7e36879ca1a1070fcf21efd00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Wed, 30 Aug 2023 11:06:48 +0200 Subject: [PATCH 2/2] Apply review comments --- numpy/core/tests/test_mem_overlap.py | 2 +- numpy/lib/stride_tricks.py | 2 +- numpy/lib/stride_tricks.pyi | 1 - numpy/typing/tests/data/reveal/stride_tricks.pyi | 2 -- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/numpy/core/tests/test_mem_overlap.py b/numpy/core/tests/test_mem_overlap.py index f720be7d3b4c..fc9840b1fa0b 100644 --- a/numpy/core/tests/test_mem_overlap.py +++ b/numpy/core/tests/test_mem_overlap.py @@ -568,7 +568,7 @@ def __array__(self): def view_element_first_byte(x): """Construct an array viewing the first byte of each element of `x`""" - from numpy.lib.stride_tricks import DummyArray + from numpy.lib._stride_tricks_impl import DummyArray interface = dict(x.__array_interface__) interface['typestr'] = '|b1' interface['descr'] = [('', '|b1')] diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index 65565c06689c..14e7646dcd58 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -1,3 +1,3 @@ from ._stride_tricks_impl import ( - DummyArray, as_strided, sliding_window_view + as_strided, sliding_window_view ) diff --git a/numpy/lib/stride_tricks.pyi b/numpy/lib/stride_tricks.pyi index 6b96a8fd643a..eb46f28ae5f4 100644 --- a/numpy/lib/stride_tricks.pyi +++ b/numpy/lib/stride_tricks.pyi @@ -1,5 +1,4 @@ from numpy.lib._stride_tricks_impl import ( - DummyArray as DummyArray, as_strided as as_strided, sliding_window_view as sliding_window_view, ) diff --git a/numpy/typing/tests/data/reveal/stride_tricks.pyi b/numpy/typing/tests/data/reveal/stride_tricks.pyi index 17769dc4bb39..b13235206862 100644 --- a/numpy/typing/tests/data/reveal/stride_tricks.pyi +++ b/numpy/typing/tests/data/reveal/stride_tricks.pyi @@ -6,8 +6,6 @@ AR_f8: npt.NDArray[np.float64] AR_LIKE_f: list[float] interface_dict: dict[str, Any] -reveal_type(np.lib.stride_tricks.DummyArray(interface_dict)) # E: lib.stride_tricks.DummyArray - reveal_type(np.lib.stride_tricks.as_strided(AR_f8)) # E: ndarray[Any, dtype[{float64}]] reveal_type(np.lib.stride_tricks.as_strided(AR_LIKE_f)) # E: ndarray[Any, dtype[Any]] reveal_type(np.lib.stride_tricks.as_strided(AR_f8, strides=(1, 5))) # E: ndarray[Any, dtype[{float64}]]