diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 480e229c2a74..0c2c1ff03016 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -1,34 +1,52 @@ +.. _neps.ufunc-overrides: + ================================= A Mechanism for Overriding Ufuncs ================================= +.. currentmodule:: numpy + :Author: Blake Griffith -:Contact: blake.g@utexas.edu +:Contact: blake.g@utexas.edu :Date: 2013-07-10 :Author: Pauli Virtanen :Author: Nathaniel Smith +:Author: Marten van Kerkwijk +:Date: 2017-03-31 Executive summary ================= NumPy's universal functions (ufuncs) currently have some limited -functionality for operating on user defined subclasses of ndarray using -``__array_prepare__`` and ``__array_wrap__`` [1]_, and there is little -to no support for arbitrary objects. e.g. SciPy's sparse matrices [2]_ -[3]_. +functionality for operating on user defined subclasses of +:class:`ndarray` using ``__array_prepare__`` and ``__array_wrap__`` +[1]_, and there is little to no support for arbitrary +objects. e.g. SciPy's sparse matrices [2]_ [3]_. Here we propose adding a mechanism to override ufuncs based on the ufunc -checking each of it's arguments for a ``__numpy_ufunc__`` method. -On discovery of ``__numpy_ufunc__`` the ufunc will hand off the -operation to the method. +checking each of it's arguments for a ``__array_ufunc__`` method. +On discovery of ``__array_ufunc__`` the ufunc will hand off the +operation to the method. This covers some of the same ground as Travis Oliphant's proposal to retro-fit NumPy with multi-methods [4]_, which would solve the same problem. The mechanism here follows more closely the way Python enables -classes to override ``__mul__`` and other binary operations. +classes to override ``__mul__`` and other binary operations. It also +specifically addresses how binary operators and ufuncs should interact. +(Note that in earlier iterations, the override was called +``__numpy_ufunc__``. An implementation was made, but had not quite the +right behaviour, hence the change in name.) + +The ``__array_ufunc__`` as described below requires that any +corresponding Python binary operations (``__mul__`` et al.) should be +implemented in a specific way and be compatible with Numpy's ndarray +semantics. Objects that do not satisfy this cannot override any Numpy +ufuncs. We do not specify a future-compatible path by which this +requirement can be relaxed --- any changes here require corresponding +changes in 3rd party code. .. [1] http://docs.python.org/doc/numpy/user/basics.subclassing.html .. [2] https://github.com/scipy/scipy/issues/2123 @@ -41,13 +59,14 @@ Motivation The current machinery for dispatching Ufuncs is generally agreed to be insufficient. There have been lengthy discussions and other proposed -solutions [5]_. +solutions [5]_, [6]_. -Using ufuncs with subclasses of ndarray is limited to ``__array_prepare__`` and -``__array_wrap__`` to prepare the arguments, but these don't allow you to for -example change the shape or the data of the arguments. Trying to ufunc things -that don't subclass ndarray is even more difficult, as the input arguments tend -to be cast to object arrays, which ends up producing surprising results. +Using ufuncs with subclasses of :class:`ndarray` is limited to +``__array_prepare__`` and ``__array_wrap__`` to prepare the output arguments, +but these don't allow you to for example change the shape or the data of +the arguments. Trying to ufunc things that don't subclass +:class:`ndarray` is even more difficult, as the input arguments tend to +be cast to object arrays, which ends up producing surprising results. Take this example of ufuncs interoperability with sparse matrices.:: @@ -77,11 +96,11 @@ Take this example of ufuncs interoperability with sparse matrices.:: Out[4]: matrix([[16, 0, 8], [ 8, 1, 5], [ 4, 1, 4]], dtype=int64) - + In [5]: np.multiply(a, bsp) # Returns NotImplemented to user, bad! Out[5]: NotImplemted -Returning ``NotImplemented`` to user should not happen. Moreover:: +Returning :obj:`NotImplemented` to user should not happen. Moreover:: In [6]: np.multiply(asp, b) Out[6]: array([[ <3x3 sparse matrix of type '' @@ -106,227 +125,477 @@ Returning ``NotImplemented`` to user should not happen. Moreover:: Here, it appears that the sparse matrix was converted to an object array scalar, which was then multiplied with all elements of the ``b`` array. However, this behavior is more confusing than useful, and having a -``TypeError`` would be preferable. +:exc:`TypeError` would be preferable. -Adding the ``__numpy_ufunc__`` functionality fixes this and would -deprecate the other ufunc modifying functions. +This proposal will *not* resolve the issue with scipy.sparse matrices, +which have multiplication semantics incompatible with numpy arrays. +However, the aim is to enable writing other custom array types that have +strictly ndarray compatible semantics. .. [5] http://mail.python.org/pipermail/numpy-discussion/2011-June/056945.html +.. [6] https://github.com/numpy/numpy/issues/5844 + Proposed interface ================== -Objects that want to override Ufuncs can define a ``__numpy_ufunc__`` method. -The method signature is:: +The standard array class :class:`ndarray` gains an ``__array_ufunc__`` +method and objects can override Ufuncs by overriding this method (if +they are :class:`ndarray` subclasses) or defining their own. The method +signature is:: - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs) + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) Here: -- *ufunc* is the ufunc object that was called. -- *method* is a string indicating which Ufunc method was called - (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, - ``"accumulate"``, ``"outer"``, ``"inner"``). -- *i* is the index of *self* in *inputs*. +- *ufunc* is the ufunc object that was called. +- *method* is a string indicating how the Ufunc was called, either + ``"__call__"`` to indicate it was called directly, or one of its + :ref:`methods`: ``"reduce"``, ``"accumulate"``, + ``"reduceat"``, ``"outer"``, or ``"at"``. - *inputs* is a tuple of the input arguments to the ``ufunc`` -- *kwargs* are the keyword arguments passed to the function. The ``out`` - arguments are always contained in *kwargs*, how positional variables - are passed is discussed below. - -The ufunc's arguments are first normalized into a tuple of input data -(``inputs``), and dict of keyword arguments. If there are output -arguments they are handled as follows: - -- One positional output variable x is passed in the kwargs dict as ``out : - x``. -- Multiple positional output variables ``x0, x1, ...`` are passed as a tuple - in the kwargs dict as ``out : (x0, x1, ...)``. -- Keyword output variables like ``out = x`` and ``out = (x0, x1, ...)`` are - passed unchanged to the kwargs dict like ``out : x`` and ``out : (x0, x1, - ...)`` respectively. -- Combinations of positional and keyword output variables are not - supported. +- *kwargs* contains any optional or keyword arguments passed to the + function. This includes any ``out`` arguments, which are always + contained in a tuple. + +Hence, the arguments are normalized: only the required input arguments +(``inputs``) are passed on as positional arguments, all the others are +passed on as a dict of keyword arguments (``kwargs``). In particular, if +there are output arguments, positional are otherwise, that are not +:obj:`None`, they are passed on as a tuple in the ``out`` keyword +argument. The function dispatch proceeds as follows: -- If one of the input arguments implements ``__numpy_ufunc__`` it is - executed instead of the Ufunc. +- If one of the input or output arguments implements + ``__array_ufunc__``, it is executed instead of the ufunc. -- If more than one of the input arguments implements ``__numpy_ufunc__``, +- If more than one of the arguments implements ``__array_ufunc__``, they are tried in the following order: subclasses before superclasses, - otherwise left to right. The first ``__numpy_ufunc__`` method returning - something else than ``NotImplemented`` determines the return value of - the Ufunc. + inputs before outputs, otherwise left to right. -- If all ``__numpy_ufunc__`` methods of the input arguments return - ``NotImplemented``, a ``TypeError`` is raised. +- The first ``__array_ufunc__`` method returning something else than + :obj:`NotImplemented` determines the return value of the Ufunc. + +- If all ``__array_ufunc__`` methods of the input arguments return + :obj:`NotImplemented`, a :exc:`TypeError` is raised. -- If a ``__numpy_ufunc__`` method raises an error, the error is propagated - immediately. +- If a ``__array_ufunc__`` method raises an error, the error is + propagated immediately. -If none of the input arguments has a ``__numpy_ufunc__`` method, the -execution falls back on the default ufunc behaviour. +- If none of the input arguments had an ``__array_ufunc__`` method, the + execution falls back on the default ufunc behaviour. +In the above, there is one proviso: if a class has an +``__array_ufunc__`` attribute but it is identical to +``ndarray.__array_ufunc__``, the attribute is ignored. This happens for +instances of `ndarray` and for `ndarray` subclasses that did not +override their inherited ``__array_ufunc__`` implementation. -In combination with Python's binary operations ----------------------------------------------- -The ``__numpy_ufunc__`` mechanism is fully independent of Python's -standard operator override mechanism, and the two do not interact -directly. +Type casting hierarchy +---------------------- -They however have indirect interactions, because NumPy's ``ndarray`` -type implements its binary operations via Ufuncs. Effectively, we have:: +The Python operator override mechanism gives much freedom in how to +write the override methods, and it requires some discipline in order to +achieve predictable results. Here, we discuss an approach for +understanding some of the implications, which can provide input in the +design. - class ndarray(object): - ... - def __mul__(self, other): - return np.multiply(self, other) +It is useful to maintain a clear idea of what types can be "upcast" to +others, possibly indirectly (e.g. indirect A->B->C is implemented but +direct A->C not). If the implementations of ``__array_ufunc__`` follow a +coherent type casting hierarchy, it can be used to understand results of +operations. -Suppose now we have a second class:: +Type casting can be expressed as a `graph `__ +defined as follows: - class MyObject(object): - def __numpy_ufunc__(self, *a, **kw): - return "ufunc" - def __mul__(self, other): - return 1234 - def __rmul__(self, other): - return 4321 + For each ``__array_ufunc__`` method, draw directed edges from each + possible input type to each possible output type. -In this case, standard Python override rules combined with the above -discussion imply:: + That is, in each case where ``y = x.__array_ufunc__(a, b, c, ...)`` + does something else than returning ``NotImplemented`` or raising an error, + draw edges ``type(a) -> type(y)``, ``type(b) -> type(y)``, ... - a = MyObject() - b = np.array([0]) +If the resulting graph is *acyclic*, it defines a coherent type casting +hierarchy (unambiguous partial ordering between types). In this case, +operations involving multiple types generally predictably produce result +of the "highest" type, or raise a :exc:`TypeError`. See examples at the +end of this section. - a * b # == 1234 OK - b * a # == "ufunc" surprising +If the graph has cycles, the ``__array_ufunc__`` type casting is not +well-defined, and things such as ``type(multiply(a, b)) != +type(multiply(b, a))`` or ``type(add(a, add(b, c))) != type(add(add(a, +b), c))`` are not excluded (and then probably always possible). -This is not what would be naively expected, and is therefore somewhat -undesirable behavior. +If the type casting hierarchy is well defined, for each class A, all +other classes that define ``__array_ufunc__`` belong to exactly one of +three groups: -The reason why this occurs is: because ``MyObject`` is not an ndarray -subclass, Python resolves the expression ``b * a`` by calling first -``b.__mul__``. Since NumPy implements this via an Ufunc, the call is -forwarded to ``__numpy_ufunc__`` and not to ``__rmul__``. Note that if -``MyObject`` is a subclass of ``ndarray``, Python calls ``a.__rmul__`` -first. The issue is therefore that ``__numpy_ufunc__`` implements -"virtual subclassing" of ndarray behavior, without actual subclassing. +- *Above A*: the types that A can be (indirectly) upcast to in ufuncs. -This issue can be resolved by a modification of the binary operation -methods in NumPy:: +- *Below A*: the types that can be (indirectly) upcast to A in ufuncs. - class ndarray(object): - ... - def __mul__(self, other): - if (not isinstance(other, self.__class__) - and hasattr(other, '__numpy_ufunc__') - and hasattr(other, '__rmul__')): - return NotImplemented - return np.multiply(self, other) +- *Incompatible*: neither above nor below A; types for which no + (indirect) upcasting is possible. - def __imul__(self, other): - if (other.__class__ is not self.__class__ - and hasattr(other, '__numpy_ufunc__') - and hasattr(other, '__rmul__')): - return NotImplemented - return np.multiply(self, other, out=self) +Note that the legacy behaviour of numpy ufuncs is to try to convert +unknown objects to :class:`ndarray` via :func:`np.asarray`. This is +equivalent to placing :class:`ndarray` above these objects in the graph. +Since we above defined :class:`ndarray` to return `NotImplemented` for +classes with custom ``__array_ufunc__``, this puts :class:`ndarray` +below such classes in the type hierarchy, allowing the operations to be +overridden. - b * a # == 4321 OK +In view of the above, binary ufuncs describing transitive operations +should aim to define a well-defined casting hierarchy. This is likely +also a sensible approach to all ufuncs --- exceptions to this should +consider carefully if any surprising behavior results. -The rationale here is the following: since the user class explicitly -defines both ``__numpy_ufunc__`` and ``__rmul__``, the implementor has -very likely made sure that the ``__rmul__`` method can process ndarrays. -If not, the special case is simple to deal with (just call -``np.multiply``). +.. admonition:: Example -The exclusion of subclasses of self can be made because Python itself -calls the right-hand method first in this case. Moreover, it is -desirable that ndarray subclasses are able to inherit the right-hand -binary operation methods from ndarray. + Type casting hierarchy. -The same priority shuffling needs to be done also for the in-place -operations, so that ``MyObject.__rmul__`` is prioritized over -``ndarray.__imul__``. + .. graphviz:: + digraph array_ufuncs { + rankdir=BT; + A -> C [label="C"]; + B -> C [label="C"]; + D -> B [label="B"]; + ndarray -> C [label="A"]; + ndarray -> B [label="B"]; + } -Demo -==== + The ``__array_ufunc__`` of type A can handle ndarrays returning C, + B can handle ndarray and D returning B, and C can handle A and B returning C, + but not ndarrays or D. The + result is a directed acyclic graph, and defines a type casting + hierarchy, with relations ``C > A``, ``C > ndarray``, ``C > B > ndarray``, + ``C > B > D``. The type A is incompatible with B, D, ndarray, + and D is incompatible with A and ndarray. Ufunc + expressions involving these classes should produce results of the + highest type involved or raise a :exc:`TypeError`. -A pull request[6]_ has been made including the changes proposed in this NEP. -Here is a demo highlighting the functionality.:: +.. admonition:: Example - In [1]: import numpy as np; + One-cycle in the ``__array_ufunc__`` graph. - In [2]: a = np.array([1]) + .. graphviz:: + + digraph array_ufuncs { + rankdir=BT; + A -> B [label="B"]; + B -> A [label="A"]; + } - In [3]: class B(): - ...: def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): - ...: return "B" - ...: - In [4]: b = B() + In this case, the ``__array_ufunc__`` relations have a cycle of length 1, + and a type casting hierarchy does not exist. Binary operations are not + commutative: ``type(a + b) is A`` but ``type(b + a) is B``. + +.. admonition:: Example + + Longer cycle in the ``__array_ufunc__`` graph. + + .. graphviz:: - In [5]: np.dot(a, b) - Out[5]: 'B' + digraph array_ufuncs { + rankdir=BT; + A -> B [label="B"]; + B -> C [label="C"]; + C -> A [label="A"]; + } - In [6]: np.multiply(a, b) - Out[6]: 'B' -A simple ``__numpy_ufunc__`` has been added to SciPy's sparse matrices -Currently this only handles ``np.dot`` and ``np.multiply`` because it was the -two most common cases where users would attempt to use sparse matrices with ufuncs. -The method is defined below:: + In this case, the ``__array_ufunc__`` relations have a longer cycle, and a + type casting hierarchy does not exist. Binary operations are still + commutative, but type transitivity is lost: ``type(a + (b + c)) is A`` but + ``type((a + b) + c) is C``. - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): - """Method for compatibility with NumPy's ufuncs and dot - functions. - """ - without_self = list(inputs) - del without_self[pos] - without_self = tuple(without_self) +Subclass hierarchies +-------------------- + +Generally, it is desirable to mirror the class hierarchy in the ufunc +type casting hierarchy. The recommendation is that an +``__array_ufunc__`` implementation of a class should generally return +`NotImplemented` unless the inputs are instances of the same class or +superclasses. This guarantees that in the type casting hierarchy, +superclasses are below, subclasses above, and other classes are +incompatible. Exceptions to this need to check they respect the +implicit type casting hierarchy. - if func == np.multiply: - return self.multiply(*without_self) +.. note:: - elif func == np.dot: - if pos == 0: - return self.__mul__(inputs[1]) - if pos == 1: - return self.__rmul__(inputs[0]) - else: + Note that type casting hierarchy and class hierarchy are here defined + to go the "opposite" directions. It would in principle also be + consistent to have ``__array_ufunc__`` handle also instances of + subclasses. In this case, the "subclasses first" dispatch rule would + ensure a relatively similar outcome. However, the behavior is then less + explicitly specified. + +Subclasses can be easily constructed if methods consistently use +:func:`super` to pass through the class hierarchy [7]_. To support +this, :class:`ndarray` has its own ``__array_ufunc__`` method, +equivalent to:: + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + # Cannot handle items that have __array_ufunc__ (other than our own). + outputs = kwargs.get('out', ()) + for item in inputs + outputs: + if (hasattr(item, '__array_ufunc__') and + type(item).__array_ufunc__ is not ndarray.__array_ufunc__): + return NotImplemented + + # If we didn't have to support legacy behaviour (__array_prepare__, + # __array_wrap__, etc.), we might here convert python floats, + # lists, etc, to arrays with + # items = [np.asarray(item) for item in inputs] + # and then start the right iterator for the given method. + # However, we do have to support legacy, so call back into the ufunc. + # Its arguments are now guaranteed not to have __array_ufunc__ + # overrides, and it will do the coercion to array for us. + return getattr(ufunc, method)(*items, **kwargs) + +Note that, as a special case, the ufunc dispatch mechanism does not call +this `ndarray.__array_ufunc__` method, even for `ndarray` subclasses +if they have not overridden the default `ndarray` implementation. As a +consequence, calling `ndarray.__array_ufunc__` will not result to a +nested ufunc dispatch cycle. + +The use of :func:`super` should be particularly useful for subclasses of +:class:`ndarray` that only add an attribute like a unit. In their +`__array_ufunc__` implementation, such classes can do possible +adjustment of the arguments relevant to their own class, and pass on to +the superclass implementation using :func:`super` until the ufunc is +actually done, and then do possible adjustments of the outputs. + +In general, custom implementations of `__array_ufunc__` should avoid +nested dispatch cycles, where one not just calls the ufunc via +``getattr(ufunc, method)(*items, **kwargs)``, but catches possible +exceptions, etc. As always, there may be exceptions. For instance, for a +class like :class:`MaskedArray`, which only cares that whatever +it contains is an :class:`ndarray` subclass, a reimplementation with +``__array_ufunc__`` may well be more easily done by directly applying +the ufunc to its data, and then adjusting the mask. Indeed, one can +think of this as part of the class determining whether it can handle the +other argument (i.e., where in the type hierarchy it sits). In this +case, one should return :obj:`NotImplemented` if the trial fails. So, +the implementation would be something like:: + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + # for simplicity, outputs are ignored here. + unmasked_items = tuple((item.data if isinstance(item, MaskedArray) + else item) for item in inputs) + try: + unmasked_result = getattr(ufunc, method)(*unmasked_items, **kwargs) + except TypeError: + return NotImplemented + # for simplicity, ignore that unmasked_result could be a tuple + # or a scalar. + if not isinstance(unmasked_result, np.ndarray): return NotImplemented + # now combine masks and view as MaskedArray instance + ... -So we now get the expected behavior when using ufuncs with sparse matrices.:: +As a specific example, consider a quantity and a masked array class +which both override ``__array_ufunc__``, with specific instances ``q`` +and ``ma``, where the latter contains a regular array. Executing +``np.multiply(q, ma)``, the ufunc will first dispatch to +``q.__array_ufunc__``, which returns :obj:`NotImplemented` (since the +quantity class turns itself into an array and calls :func:`super`, which +passes on to ``ndarray.__array_ufunc__``, which sees the override on +``ma``). Next, ``ma.__array_ufunc__`` gets a chance. It does not know +quantity, and if it were to just return :obj:`NotImplemented` as well, +an :exc:`TypeError` would result. But in our sample implementation, it +uses ``getattr(ufunc, method)`` to, effectively, evaluate +``np.multiply(q, ma.data)``. This again will pass to +``q.__array_ufunc__``, but this time, since ``ma.data`` is a regular +array, it will return a result that is also a quantity. Since this is a +subclass of :class:`ndarray`, ``ma.__array_ufunc__`` can turn this into +a masked array and thus return a result (obviously, if it was not a +array subclass, it could still return :obj:`NotImplemented`). + +Note that in the context of the type hierarchy discussed above this is a +somewhat tricky example, since :class:`MaskedArray` has a strange +position: it is above all subclasses of :class:`ndarray`, in that it can +cast them to its own type, but it does not itself know how to interact +with them in ufuncs. + + +Turning Ufuncs off +------------------ + +For some classes, Ufuncs make no sense, and, like for some other special +methods such as ``__hash__`` and ``__iter__`` [8]_, one can indicate +Ufuncs are not available by setting ``__array_ufunc__`` to :obj:`None`. +Inside a Ufunc, this is equivalent to unconditionally returning +:obj:`NotImplemented`, and thus will lead to a :exc:`TypeError` (unless +another operand implements ``__array_ufunc__`` and specifically knows +how to deal with the class). + +In the type casting hierarchy, this makes it explicit that the type is +incompatible relative to :class:`ndarray`. + +.. [7] https://rhettinger.wordpress.com/2011/05/26/super-considered-super/ + +.. [8] https://docs.python.org/3/reference/datamodel.html#specialnames - In [1]: import numpy as np; import scipy.sparse as sp +In combination with Python's binary operations +---------------------------------------------- - In [2]: a = np.random.randint(3, size=(3,3)) +The Python operator override mechanism in :class:`ndarray` is coupled to +the ``__array_ufunc__`` mechanism. :class:`ndarray` returns +:obj:`NotImplemented` from ``ndarray.__mul__(self, other)`` and other +binary operation methods if ``other.__array_ufunc__ is None``. If the +``__array_ufunc__`` attribute is absent, :obj:`NotImplemented` is +returned if ``other.__array_priority__ > self.__array_priority__``. In +other cases, :class:`ndarray` calls the corresponding ufunc. The +resulting behavior can modified by overriding the corresponding ufunc +via implementing ``__array_ufunc__``. + +A class wishing to modify the interaction with :class:`ndarray` in +binary operations has two options: + +1. Implement ``__array_ufunc__`` and follow Numpy semantics for Python + binary operations (see below). + +2. Set ``__array_ufunc__ = None``, and implement Python binary + operations freely. In this case, ufuncs will raise :exc:`TypeError` + in combination with ndarray inputs. + +For most numerical classes, the easiest way to override binary +operations is thus to define ``__array_ufunc__`` and override the +corresponding Ufunc. The class can then, like :class:`ndarray` itself, +define the binary operators in terms of Ufuncs. Here, one has to take +some care to ensure that one allows for other classes to indicate they +are not compatible, i.e., implementations should be something like:: + + class ArrayLike(object): + ... + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + ... + return result - In [3]: b = np.random.randint(3, size=(3,3)) + # Option 1: call ufunc directly + def __mul__(self, other): + if getattr(other, '__array_ufunc__', False) is None: + return NotImplemented + return np.multiply(self, other) - In [4]: asp = sp.csr_matrix(a); bsp = sp.csr_matrix(b) + def __rmul__(self, other): + if getattr(other, '__array_ufunc__', False) is None: + return NotImplemented + return np.multiply(other, self) + + def __imul__(self, other): + return np.multiply(self, other, out=(self,)) - In [5]: np.dot(a,b) - Out[5]: - array([[2, 4, 8], - [2, 4, 8], - [2, 2, 3]]) + # Option 2: call into one's own __array_ufunc__ + def __mul__(self, other): + return self.__array_ufunc__(np.multiply, '__call__', self, other) + + def __rmul__(self, other): + return self.__array_ufunc__(np.multiply, '__call__', other, self) + + def __imul__(self, other): + result = self.__array_ufunc__(np.multiply, '__call__', self, other, + out=(self,)) + if result is NotImplemented: + raise TypeError(...) + +To see why some care is necessary, consider another class ``other`` that +does not know how to deal with arrays and ufuncs, and thus has set +``__array_ufunc__`` to :obj:`None`, but does know how to do +multiplication:: + + class MyObject(object): + __array_ufunc__ = None + def __init__(self, value): + self.value = value + def __repr__(self): + return "MyObject({!r})".format(self.value) + def __mul__(self, other): + return MyObject(1234) + def __rmul__(self, other): + return MyObject(4321) + +For either option above, we get the expected result:: + + mine = MyObject(0) + arr = ArrayLike([0]) + + mine * arr # -> MyObject(1234) + mine *= arr # -> MyObject(1234) + arr * mine # -> MyObject(4321) + arr *= mine # -> TypeError + +Here, in the first and second example, ``mine.__mul__(arr)`` gets called +and the result arrives immediately. In the third example, first +``arr.__mul__(mine)`` is called. In option (1), the check on +``mine.__array_ufunc__ is None`` will succeed and thus +:obj:`NotImplemented` is returned, which causes ``mine.__rmul__(arg)`` +to be executed. In option (2), it is presumably inside +``arr.__array_ufunc__`` that it becomes clear that the other argument +cannot be dealt with, and again :obj:`NotImplemented` is returned, +causing control to pass to ``mine.__rmul__``. + +For the fourth example, with the in-place operators, we have here +followed :class:`ndarray` and ensure we never return +:obj:`NotImplemented`, but rather raise a :exc:`TypeError`. In +option (1) this happens indirectly: we pass to ``np.multiply``, which +calls ``arr.__array_ufunc__``. This, however, will not know what to do +with ``mine`` and will thus return :obj:`NotImplemented`. Then, the +ufunc turns to ``mine.__array_ufunc__``. But this is :obj:`None`, +equivalent to returning :obj:`NotImplemented`, so a :exc:`TypeError` is +raised. In option (2), we pass directly to ``arr.__array_ufunc__``, +which will return :obj:`NotImplemented`, which we catch. + +.. note :: the reason for not allowing in-place operations to return + :obj:`NotImplemented` is that these cannot generically be replaced by + a simple reverse operation: most array operations assume the contents + of the instance are changed in-place, and do not expect a new + instance. Also, what would ``ndarr[:] *= mine`` imply? Assuming it + means ``ndarr[:] = ndarr[:] * mine``, as python does by default if + the ``ndarr.__imul__`` were to return :obj:`NotImplemented`, is + likely to be wrong. + +Now consider what would happen if we had not added checks. For option +(1), the relevant case is if we had not checked whether +``__array_func__`` was set to :obj:`None`. In the third example, +``arr.__mul__(mine)`` is called, and without the check, this would go to +``np.multiply(arr, mine)``. This tries ``arr.__array_ufunc__``, which +returns :obj:`NotImplemented` and sees that ``mine.__array_ufunc__ is +None``, so a :exc:`TypeError` is raised. + +For option (2), the relevant example is the fourth, with ``arr *= +mine``: if we had let the :obj:`NotImplemented` pass, python would have +replaced this with ``arr = mine.__rmul__(arr)``, which is not wanted. + +Finally, we note that we had extensive discussion about whether it might +make more sense to ask classes like ``MyObject`` to implement a full +``__array_ufunc__`` [6]_. In the end, allowing classes to opt out was +preferred, and the above reasoning led us to agree on a similar +implementation for :class:`ndarray` itself. To help implement array-like +classes, the mixin class :class:`~numpy.lib.mixins.NDArrayOperatorsMixin` +provides overrides for all binary operators with corresponding ufuncs. + + +Future extensions to other functions +------------------------------------ + +Some numpy functions could be implemented as (generalized) Ufunc, in +which case it would be possible for them to be overridden by the +``__array_ufunc__`` method. A prime candidate is :func:`~numpy.matmul`, +which currently is not a Ufunc, but could be relatively easily be +rewritten as a (set of) generalized Ufuncs. The same may happen with +functions such as :func:`~numpy.median`, :func:`~numpy.min`, and +:func:`~numpy.argsort`. - In [6]: np.dot(asp,b) - Out[6]: - array([[2, 4, 8], - [2, 4, 8], - [2, 2, 3]], dtype=int64) - In [7]: np.dot(asp, bsp).A - Out[7]: - array([[2, 4, 8], - [2, 4, 8], - [2, 2, 3]], dtype=int64) - .. Local Variables: .. mode: rst .. coding: utf-8 diff --git a/doc/release/1.13.0-notes.rst b/doc/release/1.13.0-notes.rst index 2ee0b80ae4db..b436a96e6a76 100644 --- a/doc/release/1.13.0-notes.rst +++ b/doc/release/1.13.0-notes.rst @@ -7,10 +7,12 @@ This release supports Python 2.7 and 3.4 - 3.6. Highlights ========== - * Reuse of temporaries, operations like ``a + b + c`` will create fewer - temporaries on some platforms. + * Operations like ``a + b + c`` will reuse temporaries on some platforms, + resulting in less memory use and faster execution. * Inplace operations check if inputs overlap outputs and create temporaries to avoid problems. + * Improved ability for classes to override default ufunc behavior. See + ``__array_ufunc__`` below. Dropped Support @@ -96,6 +98,16 @@ used instead. New Features ============ +``__array_ufunc__`` added +------------------------- +This is the renamed and redesigned ``__numpy_ufunc__``. Any class, ndarray +subclass or not, can define this method or set it to ``None`` in order to +override the behavior of NumPy's ufuncs. This works quite similarly to Python's +``__mul__`` and other binary operation routines. See the documentation for a +more detailed description of the implementation and behavior of this new +option. The API is provisional, we do not yet guarantee backward compatibility +as modifications may be made pending feedback. + ``PyArray_MapIterArrayCopyIfOverlap`` added to NumPy C-API ---------------------------------------------------------- Similar to ``PyArray_MapIterArray`` but with an additional ``copy_if_overlap`` diff --git a/doc/source/conf.py b/doc/source/conf.py index 8c18e423a1ed..2bafc50ebfe5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -22,6 +22,7 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc', 'sphinx.ext.intersphinx', 'sphinx.ext.coverage', 'sphinx.ext.doctest', 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', 'matplotlib.sphinxext.plot_directive'] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 298e81717578..25105001ce68 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -39,77 +39,123 @@ Special attributes and methods NumPy provides several hooks that classes can customize: -.. method:: class.__numpy_ufunc__(ufunc, method, i, inputs, **kwargs) +.. py:method:: class.__array_ufunc__(ufunc, method, *inputs, **kwargs) - .. versionadded:: 1.11 + .. versionadded:: 1.13 - Any class (ndarray subclass or not) can define this method to - override behavior of NumPy's ufuncs. This works quite similarly to - Python's ``__mul__`` and other binary operation routines. + .. note:: The API is `provisional + `_, + i.e., we do not yet guarantee backward compatibility. + + Any class, ndarray subclass or not, can define this method or set it to + :obj:`None` in order to override the behavior of NumPy's ufuncs. This works + quite similarly to Python's ``__mul__`` and other binary operation routines. - *ufunc* is the ufunc object that was called. - *method* is a string indicating which Ufunc method was called (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, ``"accumulate"``, ``"outer"``, ``"inner"``). - - *i* is the index of *self* in *inputs*. - - *inputs* is a tuple of the input arguments to the ``ufunc`` + - *inputs* is a tuple of the input arguments to the ``ufunc``. - *kwargs* is a dictionary containing the optional input arguments - of the ufunc. The ``out`` argument is always contained in - *kwargs*, if given. See the discussion in :ref:`ufuncs` for - details. + of the ufunc. If given, any ``out`` arguments, both positional + and keyword, are passed as a :obj:`tuple` in *kwargs*. See the + discussion in :ref:`ufuncs` for details. The method should return either the result of the operation, or - :obj:`NotImplemented` if the operation requested is not - implemented. - - If one of the arguments has a :func:`__numpy_ufunc__` method, it is - executed *instead* of the ufunc. If more than one of the input - arguments implements :func:`__numpy_ufunc__`, they are tried in the - order: subclasses before superclasses, otherwise left to right. The - first routine returning something else than :obj:`NotImplemented` - determines the result. If all of the :func:`__numpy_ufunc__` - operations return :obj:`NotImplemented`, a :exc:`TypeError` is - raised. - - If an :class:`ndarray` subclass defines the :func:`__numpy_ufunc__` - method, this disables the :func:`__array_wrap__`, - :func:`__array_prepare__`, :data:`__array_priority__` mechanism - described below. - - .. note:: In addition to ufuncs, :func:`__numpy_ufunc__` also - overrides the behavior of :func:`numpy.dot` even though it is - not an Ufunc. - - .. note:: If you also define right-hand binary operator override - methods (such as ``__rmul__``) or comparison operations (such as - ``__gt__``) in your class, they take precedence over the - :func:`__numpy_ufunc__` mechanism when resolving results of - binary operations (such as ``ndarray_obj * your_obj``). - - The technical special case is: ``ndarray.__mul__`` returns - ``NotImplemented`` if the other object is *not* a subclass of - :class:`ndarray`, and defines both ``__numpy_ufunc__`` and - ``__rmul__``. Similar exception applies for the other operations - than multiplication. - - In such a case, when computing a binary operation such as - ``ndarray_obj * your_obj``, your ``__numpy_ufunc__`` method - *will not* be called. Instead, the execution passes on to your - right-hand ``__rmul__`` operation, as per standard Python - operator override rules. - - Similar special case applies to *in-place operations*: If you - define ``__rmul__``, then ``ndarray_obj *= your_obj`` *will not* - call your ``__numpy_ufunc__`` implementation. Instead, the - default Python behavior ``ndarray_obj = ndarray_obj * your_obj`` - occurs. - - Note that the above discussion applies only to Python's builtin - binary operation mechanism. ``np.multiply(ndarray_obj, - your_obj)`` always calls only your ``__numpy_ufunc__``, as - expected. - -.. method:: class.__array_finalize__(obj) + :obj:`NotImplemented` if the operation requested is not implemented. + + If one of the input or output arguments has a :func:`__array_ufunc__` + method, it is executed *instead* of the ufunc. If more than one of the + arguments implements :func:`__array_ufunc__`, they are tried in the + order: subclasses before superclasses, inputs before outputs, otherwise + left to right. The first routine returning something other than + :obj:`NotImplemented` determines the result. If all of the + :func:`__array_ufunc__` operations return :obj:`NotImplemented`, a + :exc:`TypeError` is raised. + + .. note:: We intend to re-implement numpy functions as (generalized) + Ufunc, in which case it will become possible for them to be + overridden by the ``__array_ufunc__`` method. A prime candidate is + :func:`~numpy.matmul`, which currently is not a Ufunc, but could be + relatively easily be rewritten as a (set of) generalized Ufuncs. The + same may happen with functions such as :func:`~numpy.median`, + :func:`~numpy.min`, and :func:`~numpy.argsort`. + + + Like with some other special methods in python, such as ``__hash__`` and + ``__iter__``, it is possible to indicate that your class does *not* + support ufuncs by setting ``__array_ufunc__ = None``. With this, + inside ufuncs, your class will be treated as if it returned + :obj:`NotImplemented` (which will lead to an :exc:`TypeError` + unless another class also provides a :func:`__array_ufunc__` method + which knows what to do with your class). + + The presence of :func:`__array_ufunc__` also influences how + :class:`ndarray` handles binary operations like ``arr + obj`` and ``arr + < obj`` when ``arr`` is an :class:`ndarray` and ``obj`` is an instance + of a custom class. There are two possibilities. If + ``obj.__array_ufunc__`` is present and not :obj:`None`, then + ``ndarray.__add__`` and friends will delegate to the ufunc machinery, + meaning that ``arr + obj`` becomes ``np.add(arr, obj)``, and then + :func:`~numpy.add` invokes ``obj.__array_ufunc__``. This is useful if you + want to define an object that acts like an array. + + Alternatively, if ``obj.__array_ufunc__`` is set to :obj:`None`, then as a + special case, special methods like ``ndarray.__add__`` will notice this + and *unconditionally* return :obj:`NotImplemented`, so that Python will + dispatch to ``obj.__radd__`` instead. This is useful if you want to define + a special object that interacts with arrays via binary operations, but + is not itself an array. For example, a units handling system might have + an object ``m`` representing the "meters" unit, and want to support the + syntax ``arr * m`` to represent that the array has units of "meters", but + not want to otherwise interact with arrays via ufuncs or otherwise. This + can be done by setting ``__array_ufunc__ = None`` and defining ``__mul__`` + and ``__rmul__`` methods. (Note that this means that writing an + ``__array_ufunc__`` that always returns :obj:`NotImplemented` is not + quite the same as setting ``__array_ufunc__ = None``: in the former + case, ``arr + obj`` will raise :exc:`TypeError`, while in the latter + case it is possible to define a ``__radd__`` method to prevent this.) + + The above does not hold for in-place operators, for which :class:`ndarray` + never returns :obj:`NotImplemented`. Hence, ``arr += obj`` would always + lead to a :exc:`TypeError`. This is because for arrays in-place operations + cannot generically be replaced by a simple reverse operation. (For + instance, by default, ``arr += obj`` would be translated to ``arr = + arr + obj``, i.e., ``arr`` would be replaced, contrary to what is expected + for in-place array operations.) + + .. note:: If you define ``__array_ufunc__``: + + - If you are not a subclass of :class:`ndarray`, we recommend your + class define special methods like ``__add__`` and ``__lt__`` that + delegate to ufuncs just like ndarray does. An easy way to do this + is to subclass from :class:`~numpy.lib.mixins.NDArrayOperatorsMixin`. + - If you subclass :class:`ndarray`, we recommend that you put all your + override logic in ``__array_ufunc__`` and not also override special + methods. This ensures the class hierarchy is determined in only one + place rather than separately by the ufunc machinery and by the binary + operation rules (which gives preference to special methods of + subclasses; the alternative way to enforce a one-place only hierarchy, + of setting :func:`__array_ufunc__` to :obj:`None`, would seem very + unexpected and thus confusing, as then the subclass would not work at + all with ufuncs). + - :class:`ndarray` defines its own :func:`__array_ufunc__`, which, + evaluates the ufunc if no arguments have overrides, and returns + :obj:`NotImplemented` otherwise. This may be useful for subclasses + for which :func:`__array_ufunc__` converts any instances of its own + class to :class:`ndarray`: it can then pass these on to its + superclass using ``super().__array_ufunc__(*inputs, **kwargs)``, + and finally return the results after possible back-conversion. The + advantage of this practice is that it ensures that it is possible + to have a hierarchy of subclasses that extend the behaviour. See + :ref:`Subclassing ndarray ` for details. + + .. note:: If a class defines the :func:`__array_ufunc__` method, + this disables the :func:`__array_wrap__`, + :func:`__array_prepare__`, :data:`__array_priority__` mechanism + described below for ufuncs (which may eventually be deprecated). + +.. py:method:: class.__array_finalize__(obj) This method is called whenever the system internally allocates a new array from *obj*, where *obj* is a subclass (subtype) of the @@ -118,7 +164,7 @@ NumPy provides several hooks that classes can customize: to update meta-information from the "parent." Subclasses inherit a default implementation of this method that does nothing. -.. method:: class.__array_prepare__(array, context=None) +.. py:method:: class.__array_prepare__(array, context=None) At the beginning of every :ref:`ufunc `, this method is called on the input object with the highest array @@ -130,7 +176,10 @@ NumPy provides several hooks that classes can customize: the subclass and update metadata before returning the array to the ufunc for computation. -.. method:: class.__array_wrap__(array, context=None) + .. note:: For ufuncs, it is hoped to eventually deprecate this method in + favour of :func:`__array_ufunc__`. + +.. py:method:: class.__array_wrap__(array, context=None) At the end of every :ref:`ufunc `, this method is called on the input object with the highest array priority, or @@ -142,14 +191,20 @@ NumPy provides several hooks that classes can customize: into an instance of the subclass and update metadata before returning the array to the user. -.. data:: class.__array_priority__ + .. note:: For ufuncs, it is hoped to eventually deprecate this method in + favour of :func:`__array_ufunc__`. + +.. py:attribute:: class.__array_priority__ The value of this attribute is used to determine what type of object to return in situations where there is more than one possibility for the Python type of the returned object. Subclasses inherit a default value of 0.0 for this attribute. -.. method:: class.__array__([dtype]) + .. note:: For ufuncs, it is hoped to eventually deprecate this method in + favour of :func:`__array_ufunc__`. + +.. py:method:: class.__array__([dtype]) If a class (ndarray subclass or not) having the :func:`__array__` method is used as the output object of an :ref:`ufunc diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index 5a5f2b81855d..45b9ac3d94da 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -30,6 +30,13 @@ Memory ranges shares_memory may_share_memory +Array mixins +------------ +.. autosummary:: + :toctree: generated/ + + lib.mixins.NDArrayOperatorsMixin + NumPy version comparison ------------------------ .. autosummary:: diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index 4dd1b3e1823a..bcd3d5f0aaa2 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -416,6 +416,8 @@ possess. None of the attributes can be set. ufunc.types ufunc.identity +.. _ufuncs.methods: + Methods ------- diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index f25159d8e95a..01741cd1afb6 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -645,3 +645,15 @@ def __init__(self, axis, ndim=None, msg_prefix=None): msg = "{}: {}".format(msg_prefix, msg) super(AxisError, self).__init__(msg) + + +def array_ufunc_errmsg_formatter(ufunc, method, *inputs, **kwargs): + """ Format the error message for when __array_ufunc__ gives up. """ + args_string = ', '.join(['{!r}'.format(arg) for arg in inputs] + + ['{}={!r}'.format(k, v) + for k, v in kwargs.items()]) + args = inputs + kwargs.get('out', ()) + types_string = ', '.join(repr(type(arg).__name__) for arg in args) + return ('operand type(s) do not implement __array_ufunc__' + '({!r}, {!r}, {}): {}' + .format(ufunc, method, args_string, types_string)) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 20d4c77922b4..e057c56141d3 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -748,6 +748,8 @@ def get_mathlib_info(*args): join('src', 'private', 'templ_common.h.src'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), + join('src', 'private', 'ufunc_override.h'), + join('src', 'private', 'binop_override.h'), join('src', 'private', 'npy_extint128.h'), join('include', 'numpy', 'arrayobject.h'), join('include', 'numpy', '_neighborhood_iterator_imp.h'), @@ -818,6 +820,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'vdot.c'), join('src', 'private', 'templ_common.h.src'), join('src', 'private', 'mem_overlap.c'), + join('src', 'private', 'ufunc_override.c'), ] blas_info = get_info('blas_opt', 0) @@ -871,7 +874,9 @@ def generate_umath_c(ext, build_dir): join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'scalarmath.c.src'), join('src', 'umath', 'ufunc_type_resolution.c'), - join('src', 'private', 'mem_overlap.c')] + join('src', 'umath', 'override.c'), + join('src', 'private', 'mem_overlap.c'), + join('src', 'private', 'ufunc_override.c')] umath_deps = [ generate_umath_py, @@ -880,10 +885,12 @@ def generate_umath_c(ext, build_dir): join('src', 'multiarray', 'common.h'), join('src', 'private', 'templ_common.h.src'), join('src', 'umath', 'simd.inc.src'), + join('src', 'umath', 'override.h'), join(codegen_dir, 'generate_ufunc_api.py'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), - join('src', 'private', 'ufunc_override.h')] + npymath_sources + join('src', 'private', 'ufunc_override.h'), + join('src', 'private', 'binop_override.h')] + npymath_sources config.add_extension('umath', sources=umath_src + diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 8946ff2556ee..df389020128c 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -54,6 +54,8 @@ maintainer email: oliphant.travis@ieee.org #include "mem_overlap.h" #include "numpyos.h" +#include "binop_override.h" + /*NUMPY_API Compute the size of an array (in number of items) */ @@ -1335,23 +1337,12 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) switch (cmp_op) { case Py_LT: - if (needs_right_binop_forward(obj_self, other, "__gt__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - /* See discussion in number.c */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyArray_GenericBinaryFunction(self, other, - n_ops.less); + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); + result = PyArray_GenericBinaryFunction(self, other, n_ops.less); break; case Py_LE: - if (needs_right_binop_forward(obj_self, other, "__ge__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyArray_GenericBinaryFunction(self, other, - n_ops.less_equal); + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); + result = PyArray_GenericBinaryFunction(self, other, n_ops.less_equal); break; case Py_EQ: /* @@ -1401,11 +1392,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - if (needs_right_binop_forward(obj_self, other, "__eq__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, (PyObject *)other, n_ops.equal); @@ -1478,11 +1465,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - if (needs_right_binop_forward(obj_self, other, "__ne__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, (PyObject *)other, n_ops.not_equal); if (result == NULL) { @@ -1502,20 +1485,12 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) } break; case Py_GT: - if (needs_right_binop_forward(obj_self, other, "__lt__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, other, n_ops.greater); break; case Py_GE: - if (needs_right_binop_forward(obj_self, other, "__le__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, other, n_ops.greater_equal); break; diff --git a/numpy/core/src/multiarray/cblasfuncs.c b/numpy/core/src/multiarray/cblasfuncs.c index 4b11be947ff4..3b0b2f4f6d75 100644 --- a/numpy/core/src/multiarray/cblasfuncs.c +++ b/numpy/core/src/multiarray/cblasfuncs.c @@ -237,7 +237,7 @@ _bad_strides(PyArrayObject *ap) * This is for use by PyArray_MatrixProduct2. It is assumed on entry that * the arrays ap1 and ap2 have a common data type given by typenum that is * float, double, cfloat, or cdouble and have dimension <= 2. The - * __numpy_ufunc__ nonsense is also assumed to have been taken care of. + * __array_ufunc__ nonsense is also assumed to have been taken care of. */ NPY_NO_EXPORT PyObject * cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index dc9b2edec51d..1df3b8b48c41 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -14,6 +14,8 @@ #include "common.h" #include "buffer.h" +#include "get_attr_string.h" + /* * The casting to use for implicit assignment operations resulting from * in-place operations (like +=) and out= arguments. (Notice that this @@ -29,61 +31,6 @@ * warning (that people's code will be broken in a future release.) */ -/* - * PyArray_GetAttrString_SuppressException: - * - * Stripped down version of PyObject_GetAttrString, - * avoids lookups for None, tuple, and List objects, - * and doesn't create a PyErr since this code ignores it. - * - * This can be much faster then PyObject_GetAttrString where - * exceptions are not used by caller. - * - * 'obj' is the object to search for attribute. - * - * 'name' is the attribute to search for. - * - * Returns attribute value on success, 0 on failure. - */ -PyObject * -PyArray_GetAttrString_SuppressException(PyObject *obj, char *name) -{ - PyTypeObject *tp = Py_TYPE(obj); - PyObject *res = (PyObject *)NULL; - - /* We do not need to check for special attributes on trivial types */ - if (_is_basic_python_type(obj)) { - return NULL; - } - - /* Attribute referenced by (char *)name */ - if (tp->tp_getattr != NULL) { - res = (*tp->tp_getattr)(obj, name); - if (res == NULL) { - PyErr_Clear(); - } - } - /* Attribute referenced by (PyObject *)name */ - else if (tp->tp_getattro != NULL) { -#if defined(NPY_PY3K) - PyObject *w = PyUnicode_InternFromString(name); -#else - PyObject *w = PyString_InternFromString(name); -#endif - if (w == NULL) { - return (PyObject *)NULL; - } - res = (*tp->tp_getattro)(obj, w); - Py_DECREF(w); - if (res == NULL) { - PyErr_Clear(); - } - } - return res; -} - - - NPY_NO_EXPORT NPY_CASTING NPY_DEFAULT_ASSIGN_CASTING = NPY_SAME_KIND_CASTING; diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 8da317856d06..ae9b960c86f0 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -40,9 +40,6 @@ NPY_NO_EXPORT int PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyArray_Descr **out_dtype, int string_status); -NPY_NO_EXPORT PyObject * -PyArray_GetAttrString_SuppressException(PyObject *v, char *name); - /* * Returns NULL without setting an exception if no scalar is matched, a * new dtype reference otherwise. @@ -255,34 +252,6 @@ npy_memchr(char * haystack, char needle, return p; } -static NPY_INLINE int -_is_basic_python_type(PyObject * obj) -{ - if (obj == Py_None || - PyBool_Check(obj) || - /* Basic number types */ -#if !defined(NPY_PY3K) - PyInt_CheckExact(obj) || - PyString_CheckExact(obj) || -#endif - PyLong_CheckExact(obj) || - PyFloat_CheckExact(obj) || - PyComplex_CheckExact(obj) || - /* Basic sequence types */ - PyList_CheckExact(obj) || - PyTuple_CheckExact(obj) || - PyDict_CheckExact(obj) || - PyAnySet_CheckExact(obj) || - PyUnicode_CheckExact(obj) || - PyBytes_CheckExact(obj) || - PySlice_Check(obj)) { - - return 1; - } - - return 0; -} - /* * Convert NumPy stride to BLAS stride. Returns 0 if conversion cannot be done * (BLAS won't handle negative or zero strides the way we want). diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0506d3dee3e7..f7a5f3deb4d3 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -29,6 +29,8 @@ #include "alloc.h" #include +#include "get_attr_string.h" + /* * Reading from a file or a string. * diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index dc02b29f4959..946dc542f090 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1007,6 +1007,48 @@ array_getarray(PyArrayObject *self, PyObject *args) } +static PyObject * +array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *ufunc, *method_name, *normal_args, *ufunc_method; + PyObject *result = NULL; + + if (PyTuple_Size(args) < 2) { + PyErr_SetString(PyExc_TypeError, + "__array_ufunc__ requires at least 2 arguments"); + return NULL; + } + normal_args = PyTuple_GetSlice(args, 2, PyTuple_GET_SIZE(args)); + if (normal_args == NULL) { + return NULL; + } + /* ndarray cannot handle overrides itself */ + if (PyUFunc_WithOverride(normal_args, kwds, NULL)) { + result = Py_NotImplemented; + Py_INCREF(Py_NotImplemented); + goto cleanup; + } + + ufunc = PyTuple_GET_ITEM(args, 0); + method_name = PyTuple_GET_ITEM(args, 1); + /* + * TODO(?): call into UFunc code at a later point, since here arguments are + * already normalized and we do not have to look for __array_ufunc__ again. + */ + ufunc_method = PyObject_GetAttr(ufunc, method_name); + if (ufunc_method == NULL) { + goto cleanup; + } + result = PyObject_Call(ufunc_method, normal_args, kwds); + Py_DECREF(ufunc_method); + +cleanup: + Py_DECREF(normal_args); + /* no need to DECREF borrowed references ufunc and method_name */ + return result; +} + + static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) { @@ -2030,11 +2072,7 @@ array_cumprod(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) { - static PyUFuncObject *cached_npy_dot = NULL; - int errval; - PyObject *override = NULL; - PyObject *a = (PyObject *)self, *b, *o = Py_None; - PyObject *newargs; + PyObject *a = (PyObject *)self, *b, *o = NULL; PyArrayObject *ret; char* kwlist[] = {"b", "out", NULL }; @@ -2043,36 +2081,15 @@ array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (cached_npy_dot == NULL) { - PyObject *module = PyImport_ImportModule("numpy.core.multiarray"); - cached_npy_dot = (PyUFuncObject*)PyDict_GetItemString( - PyModule_GetDict(module), "dot"); - - Py_INCREF(cached_npy_dot); - Py_DECREF(module); - } - - if ((newargs = PyTuple_Pack(3, a, b, o)) == NULL) { - return NULL; - } - errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", - newargs, NULL, &override, 2); - Py_DECREF(newargs); - - if (errval) { - return NULL; - } - else if (override) { - return override; - } - - if (o == Py_None) { - o = NULL; - } - if (o != NULL && !PyArray_Check(o)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (o != NULL) { + if (o == Py_None) { + o = NULL; + } + else if (!PyArray_Check(o)) { + PyErr_SetString(PyExc_TypeError, + "'out' must be an array"); + return NULL; + } } ret = (PyArrayObject *)PyArray_MatrixProduct2(a, b, (PyArrayObject *)o); return PyArray_Return(ret); @@ -2471,6 +2488,9 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { {"__array_wrap__", (PyCFunction)array_wraparray, METH_VARARGS, NULL}, + {"__array_ufunc__", + (PyCFunction)array_ufunc, + METH_VARARGS | METH_KEYWORDS, NULL}, /* for the sys module */ {"__sizeof__", diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 2fdabb18705e..73ba8c5c4ee9 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -61,6 +61,8 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "compiled_base.h" #include "mem_overlap.h" +#include "get_attr_string.h" + /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; @@ -2177,41 +2179,22 @@ array_innerproduct(PyObject *NPY_UNUSED(dummy), PyObject *args) static PyObject * array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) { - static PyUFuncObject *cached_npy_dot = NULL; - int errval; - PyObject *override = NULL; PyObject *v, *a, *o = NULL; PyArrayObject *ret; char* kwlist[] = {"a", "b", "out", NULL }; - if (cached_npy_dot == NULL) { - PyObject *module = PyImport_ImportModule("numpy.core.multiarray"); - cached_npy_dot = (PyUFuncObject*)PyDict_GetItemString( - PyModule_GetDict(module), "dot"); - - Py_INCREF(cached_npy_dot); - Py_DECREF(module); - } - - errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", args, kwds, - &override, 2); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", kwlist, &a, &v, &o)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", + kwlist, &a, &v, &o)) { return NULL; } - if (o == Py_None) { - o = NULL; - } - if (o != NULL && !PyArray_Check(o)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (o != NULL) { + if (o == Py_None) { + o = NULL; + } + else if (!PyArray_Check(o)) { + PyErr_SetString(PyExc_TypeError, "'out' must be an array"); + return NULL; + } } ret = (PyArrayObject *)PyArray_MatrixProduct2(a, v, (PyArrayObject *)o); return PyArray_Return(ret); @@ -2344,14 +2327,10 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *args) * out: Either NULL, or an array into which the output should be placed. * * Returns NULL on error. - * Returns NotImplemented on priority override. */ static PyObject * array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) { - static PyObject *matmul = NULL; - int errval; - PyObject *override = NULL; PyObject *in1, *in2, *out = NULL; char* kwlist[] = {"a", "b", "out", NULL }; PyArrayObject *ap1, *ap2, *ret = NULL; @@ -2362,39 +2341,25 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) char *subscripts; PyArrayObject *ops[2]; - npy_cache_import("numpy.core.multiarray", "matmul", &matmul); - if (matmul == NULL) { - return NULL; - } - - errval = PyUFunc_CheckOverride((PyUFuncObject*)matmul, "__call__", - args, kwds, &override, 2); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matmul", kwlist, &in1, &in2, &out)) { return NULL; } - if (out == Py_None) { - out = NULL; - } - if (out != NULL && !PyArray_Check(out)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (out != NULL) { + if (out == Py_None) { + out = NULL; + } + else if (!PyArray_Check(out)) { + PyErr_SetString(PyExc_TypeError, "'out' must be an array"); + return NULL; + } } dtype = PyArray_DescrFromObject(in1, NULL); dtype = PyArray_DescrFromObject(in2, dtype); if (dtype == NULL) { - PyErr_SetString(PyExc_ValueError, - "Cannot find a common data type."); + PyErr_SetString(PyExc_ValueError, "Cannot find a common data type."); return NULL; } typenum = dtype->type_num; @@ -2402,7 +2367,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) if (typenum == NPY_OBJECT) { /* matmul is not currently implemented for object arrays */ PyErr_SetString(PyExc_TypeError, - "Object arrays are not currently supported"); + "Object arrays are not currently supported"); Py_DECREF(dtype); return NULL; } @@ -2424,7 +2389,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) if (PyArray_NDIM(ap1) == 0 || PyArray_NDIM(ap2) == 0) { /* Scalars are rejected */ PyErr_SetString(PyExc_ValueError, - "Scalar operands are not allowed, use '*' instead"); + "Scalar operands are not allowed, use '*' instead"); return NULL; } @@ -4530,7 +4495,7 @@ intern_strings(void) npy_ma_str_array_wrap = PyUString_InternFromString("__array_wrap__"); npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); npy_ma_str_buffer = PyUString_InternFromString("__buffer__"); - npy_ma_str_ufunc = PyUString_InternFromString("__numpy_ufunc__"); + npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); npy_ma_str_order = PyUString_InternFromString("order"); npy_ma_str_copy = PyUString_InternFromString("copy"); npy_ma_str_dtype = PyUString_InternFromString("dtype"); diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index f0b5637d2b13..ad1d43178c60 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -14,6 +14,8 @@ #include "number.h" #include "temp_elide.h" +#include "binop_override.h" + /************************************************************************* **************** Implement Number Protocol **************************** *************************************************************************/ @@ -87,88 +89,6 @@ PyArray_SetNumericOps(PyObject *dict) (PyDict_SetItemString(dict, #op, n_ops.op)==-1)) \ goto fail; -static int -has_ufunc_attr(PyObject * obj) { - /* attribute check is expensive for scalar operations, avoid if possible */ - if (PyArray_CheckExact(obj) || PyArray_CheckAnyScalarExact(obj) || - _is_basic_python_type(obj)) { - return 0; - } - else { - return PyObject_HasAttrString(obj, "__numpy_ufunc__"); - } -} - -/* - * Check whether the operation needs to be forwarded to the right-hand binary - * operation. - * - * This is the case when all of the following conditions apply: - * - * (i) the other object defines __numpy_ufunc__ - * (ii) the other object defines the right-hand operation __r*__ - * (iii) Python hasn't already called the right-hand operation - * [occurs if the other object is a strict subclass provided - * the operation is not in-place] - * - * An additional check is made in GIVE_UP_IF_HAS_RIGHT_BINOP macro below: - * - * (iv) other.__class__.__r*__ is not self.__class__.__r*__ - * - * This is needed, because CPython does not call __rmul__ if - * the tp_number slots of the two objects are the same. - * - * This always prioritizes the __r*__ routines over __numpy_ufunc__, independent - * of whether the other object is an ndarray subclass or not. - */ - -NPY_NO_EXPORT int -needs_right_binop_forward(PyObject *self, PyObject *other, - const char *right_name, int inplace_op) -{ - if (other == NULL || - self == NULL || - Py_TYPE(self) == Py_TYPE(other) || - PyArray_CheckExact(other) || - PyArray_CheckAnyScalar(other)) { - /* - * Quick cases - */ - return 0; - } - if ((!inplace_op && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) || - !PyArray_Check(self)) { - /* - * Bail out if Python would already have called the right-hand - * operation. - */ - return 0; - } - if (has_ufunc_attr(other) && - PyObject_HasAttrString(other, right_name)) { - return 1; - } - else { - return 0; - } -} - -/* In pure-Python, SAME_SLOTS can be replaced by - getattr(m1, op_name) is getattr(m2, op_name) */ -#define SAME_SLOTS(m1, m2, slot_name) \ - (Py_TYPE(m1)->tp_as_number != NULL && Py_TYPE(m2)->tp_as_number != NULL && \ - Py_TYPE(m1)->tp_as_number->slot_name == Py_TYPE(m2)->tp_as_number->slot_name) - -#define GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, left_name, right_name, inplace, slot_name) \ - do { \ - if (needs_right_binop_forward((PyObject *)m1, m2, right_name, inplace) && \ - (inplace || !SAME_SLOTS(m1, m2, slot_name))) { \ - Py_INCREF(Py_NotImplemented); \ - return Py_NotImplemented; \ - } \ - } while (0) - - /*NUMPY_API Get dictionary showing number functions that all arrays will use */ @@ -289,36 +209,18 @@ PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, NPY_NO_EXPORT PyObject * PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op) { + /* + * I suspect that the next few lines are buggy and cause NotImplemented to + * be returned at weird times... but if we raise an error here, then + * *everything* breaks. (Like, 'arange(10) + 1' and just + * 'repr(arange(10))' both blow up with an error here.) Not sure what's + * going on with that, but I'll leave it alone for now. - njs, 2015-06-21 + */ if (op == NULL) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (!PyArray_Check(m2) && !has_ufunc_attr(m2)) { - /* - * Catch priority inversion and punt, but only if it's guaranteed - * that we were called through m1 and the other guy is not an array - * at all. Note that some arrays need to pass through here even - * with priorities inverted, for example: float(17) * np.matrix(...) - * - * See also: - * - https://github.com/numpy/numpy/issues/3502 - * - https://github.com/numpy/numpy/issues/3503 - * - * NB: there's another copy of this code in - * numpy.ma.core.MaskedArray._delegate_binop - * which should possibly be updated when this is. - */ - double m1_prio = PyArray_GetPriority((PyObject *)m1, - NPY_SCALAR_PRIORITY); - double m2_prio = PyArray_GetPriority((PyObject *)m2, - NPY_SCALAR_PRIORITY); - if (m1_prio < m2_prio) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - } - return PyObject_CallFunctionObjArgs(op, m1, m2, NULL); } @@ -381,8 +283,9 @@ array_inplace_right_shift(PyArrayObject *m1, PyObject *m2); static PyObject * array_add(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__add__", "__radd__", 0, nb_add); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_add, array_add); if (try_binary_elide(m1, m2, &array_inplace_add, &res, 1)) { return res; } @@ -392,8 +295,9 @@ array_add(PyArrayObject *m1, PyObject *m2) static PyObject * array_subtract(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__sub__", "__rsub__", 0, nb_subtract); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_subtract, array_subtract); if (try_binary_elide(m1, m2, &array_inplace_subtract, &res, 0)) { return res; } @@ -403,8 +307,9 @@ array_subtract(PyArrayObject *m1, PyObject *m2) static PyObject * array_multiply(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__mul__", "__rmul__", 0, nb_multiply); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, array_multiply); if (try_binary_elide(m1, m2, &array_inplace_multiply, &res, 1)) { return res; } @@ -415,8 +320,9 @@ array_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__div__", "__rdiv__", 0, nb_divide); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_divide, array_divide); if (try_binary_elide(m1, m2, &array_inplace_divide, &res, 0)) { return res; } @@ -427,7 +333,7 @@ array_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_remainder(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__mod__", "__rmod__", 0, nb_remainder); + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_remainder, array_remainder); return PyArray_GenericBinaryFunction(m1, m2, n_ops.remainder); } @@ -443,8 +349,7 @@ array_matrix_multiply(PyArrayObject *m1, PyObject *m2) if (matmul == NULL) { return NULL; } - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__matmul__", "__rmatmul__", - 0, nb_matrix_multiply); + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_matrix_multiply, array_matrix_multiply); return PyArray_GenericBinaryFunction(m1, m2, matmul); } @@ -458,11 +363,12 @@ array_inplace_matrix_multiply(PyArrayObject *m1, PyObject *m2) } #endif -/* Determine if object is a scalar and if so, convert the object - * to a double and place it in the out_exponent argument - * and return the "scalar kind" as a result. If the object is - * not a scalar (or if there are other error conditions) - * return NPY_NOSCALAR, and out_exponent is undefined. +/* + * Determine if object is a scalar and if so, convert the object + * to a double and place it in the out_exponent argument + * and return the "scalar kind" as a result. If the object is + * not a scalar (or if there are other error conditions) + * return NPY_NOSCALAR, and out_exponent is undefined. */ static NPY_SCALARKIND is_scalar_with_conversion(PyObject *o2, double* out_exponent) @@ -573,7 +479,8 @@ fast_scalar_power(PyArrayObject *a1, PyObject *o2, int inplace) if (inplace || can_elide_temp_unary(a1)) { return PyArray_GenericInplaceUnaryFunction(a1, fastop); - } else { + } + else { return PyArray_GenericUnaryFunction(a1, fastop); } } @@ -615,12 +522,14 @@ static PyObject * array_power(PyArrayObject *a1, PyObject *o2, PyObject *modulo) { PyObject *value; + if (modulo != Py_None) { /* modular exponentiation is not implemented (gh-8804) */ Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - GIVE_UP_IF_HAS_RIGHT_BINOP(a1, o2, "__pow__", "__rpow__", 0, nb_power); + + BINOP_GIVE_UP_IF_NEEDED(a1, o2, nb_power, array_power); value = fast_scalar_power(a1, o2, 0); if (!value) { value = PyArray_GenericBinaryFunction(a1, o2, n_ops.power); @@ -659,8 +568,9 @@ array_invert(PyArrayObject *m1) static PyObject * array_left_shift(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__lshift__", "__rlshift__", 0, nb_lshift); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_lshift, array_left_shift); if (try_binary_elide(m1, m2, &array_inplace_left_shift, &res, 0)) { return res; } @@ -670,8 +580,9 @@ array_left_shift(PyArrayObject *m1, PyObject *m2) static PyObject * array_right_shift(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__rshift__", "__rrshift__", 0, nb_rshift); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_rshift, array_right_shift); if (try_binary_elide(m1, m2, &array_inplace_right_shift, &res, 0)) { return res; } @@ -681,8 +592,9 @@ array_right_shift(PyArrayObject *m1, PyObject *m2) static PyObject * array_bitwise_and(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__and__", "__rand__", 0, nb_and); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_and, array_bitwise_and); if (try_binary_elide(m1, m2, &array_inplace_bitwise_and, &res, 1)) { return res; } @@ -692,8 +604,9 @@ array_bitwise_and(PyArrayObject *m1, PyObject *m2) static PyObject * array_bitwise_or(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__or__", "__ror__", 0, nb_or); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_or, array_bitwise_or); if (try_binary_elide(m1, m2, &array_inplace_bitwise_or, &res, 1)) { return res; } @@ -703,8 +616,9 @@ array_bitwise_or(PyArrayObject *m1, PyObject *m2) static PyObject * array_bitwise_xor(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__xor__", "__rxor__", 0, nb_xor); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_xor, array_bitwise_xor); if (try_binary_elide(m1, m2, &array_inplace_bitwise_xor, &res, 1)) { return res; } @@ -714,21 +628,18 @@ array_bitwise_xor(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_add(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__iadd__", "__radd__", 1, nb_inplace_add); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.add); } static PyObject * array_inplace_subtract(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__isub__", "__rsub__", 1, nb_inplace_subtract); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.subtract); } static PyObject * array_inplace_multiply(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__imul__", "__rmul__", 1, nb_inplace_multiply); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.multiply); } @@ -736,7 +647,6 @@ array_inplace_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__idiv__", "__rdiv__", 1, nb_inplace_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.divide); } #endif @@ -744,7 +654,6 @@ array_inplace_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_remainder(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__imod__", "__rmod__", 1, nb_inplace_remainder); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.remainder); } @@ -753,7 +662,6 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo { /* modulo is ignored! */ PyObject *value; - GIVE_UP_IF_HAS_RIGHT_BINOP(a1, o2, "__ipow__", "__rpow__", 1, nb_inplace_power); value = fast_scalar_power(a1, o2, 1); if (!value) { value = PyArray_GenericInplaceBinaryFunction(a1, o2, n_ops.power); @@ -764,43 +672,39 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo static PyObject * array_inplace_left_shift(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ilshift__", "__rlshift__", 1, nb_inplace_lshift); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.left_shift); } static PyObject * array_inplace_right_shift(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__irshift__", "__rrshift__", 1, nb_inplace_rshift); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.right_shift); } static PyObject * array_inplace_bitwise_and(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__iand__", "__rand__", 1, nb_inplace_and); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_and); } static PyObject * array_inplace_bitwise_or(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ior__", "__ror__", 1, nb_inplace_or); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_or); } static PyObject * array_inplace_bitwise_xor(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ixor__", "__rxor__", 1, nb_inplace_xor); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_xor); } static PyObject * array_floor_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__floordiv__", "__rfloordiv__", 0, nb_floor_divide); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_floor_divide, array_floor_divide); if (try_binary_elide(m1, m2, &array_inplace_floor_divide, &res, 0)) { return res; } @@ -810,8 +714,9 @@ array_floor_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_true_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__truediv__", "__rtruediv__", 0, nb_true_divide); + PyObject *res; + + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_true_divide, array_true_divide); if (PyArray_CheckExact(m1) && (PyArray_ISFLOAT(m1) || PyArray_ISCOMPLEX(m1)) && try_binary_elide(m1, m2, &array_inplace_true_divide, &res, 0)) { @@ -823,7 +728,6 @@ array_true_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_floor_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ifloordiv__", "__rfloordiv__", 1, nb_inplace_floor_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.floor_divide); } @@ -831,7 +735,6 @@ array_inplace_floor_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_true_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__itruediv__", "__rtruediv__", 1, nb_inplace_true_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.true_divide); } @@ -864,7 +767,8 @@ static PyObject * array_divmod(PyArrayObject *op1, PyObject *op2) { PyObject *divp, *modp, *result; - GIVE_UP_IF_HAS_RIGHT_BINOP(op1, op2, "__divmod__", "__rdivmod__", 0, nb_divmod); + + BINOP_GIVE_UP_IF_NEEDED(op1, op2, nb_divmod, array_divmod); divp = array_floor_divide(op1, op2); if (divp == NULL) { diff --git a/numpy/core/src/multiarray/number.h b/numpy/core/src/multiarray/number.h index 0c8355e3170d..86f681c10e65 100644 --- a/numpy/core/src/multiarray/number.h +++ b/numpy/core/src/multiarray/number.h @@ -65,8 +65,4 @@ NPY_NO_EXPORT PyObject * PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, int rtype, PyArrayObject *out); -NPY_NO_EXPORT int -needs_right_binop_forward(PyObject *self, PyObject *other, - const char *right_name, int is_inplace); - #endif diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 7edf3b71d88c..f6bd5f5a7a52 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -27,6 +27,8 @@ #include +#include "binop_override.h" + NPY_NO_EXPORT PyBoolScalarObject _PyArrayScalar_BoolValues[] = { {PyObject_HEAD_INIT(&PyBoolArrType_Type) 0}, {PyObject_HEAD_INIT(&PyBoolArrType_Type) 1}, @@ -151,63 +153,14 @@ gentype_free(PyObject *v) static PyObject * gentype_power(PyObject *m1, PyObject *m2, PyObject *modulo) { - PyObject *arr, *ret, *arg2; - char *msg="unsupported operand type(s) for ** or pow()"; - if (modulo != Py_None) { /* modular exponentiation is not implemented (gh-8804) */ Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (!PyArray_IsScalar(m1, Generic)) { - if (PyArray_Check(m1)) { - ret = Py_TYPE(m1)->tp_as_number->nb_power(m1,m2, Py_None); - } - else { - if (!PyArray_IsScalar(m2, Generic)) { - PyErr_SetString(PyExc_TypeError, msg); - return NULL; - } - arr = PyArray_FromScalar(m2, NULL); - if (arr == NULL) { - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(m1, arr, Py_None); - Py_DECREF(arr); - } - return ret; - } - if (!PyArray_IsScalar(m2, Generic)) { - if (PyArray_Check(m2)) { - ret = Py_TYPE(m2)->tp_as_number->nb_power(m1,m2, Py_None); - } - else { - if (!PyArray_IsScalar(m1, Generic)) { - PyErr_SetString(PyExc_TypeError, msg); - return NULL; - } - arr = PyArray_FromScalar(m1, NULL); - if (arr == NULL) { - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(arr, m2, Py_None); - Py_DECREF(arr); - } - return ret; - } - arr = arg2 = NULL; - arr = PyArray_FromScalar(m1, NULL); - arg2 = PyArray_FromScalar(m2, NULL); - if (arr == NULL || arg2 == NULL) { - Py_XDECREF(arr); - Py_XDECREF(arg2); - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(arr, arg2, Py_None); - Py_DECREF(arr); - Py_DECREF(arg2); - return ret; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_power, gentype_power); + return PyArray_Type.tp_as_number->nb_power(m1, m2, Py_None); } static PyObject * @@ -249,6 +202,7 @@ gentype_generic_method(PyObject *self, PyObject *args, PyObject *kwds, static PyObject * gentype_@name@(PyObject *m1, PyObject *m2) { + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_@name@, gentype_@name@); return PyArray_Type.tp_as_number->nb_@name@(m1, m2); } @@ -262,6 +216,7 @@ gentype_@name@(PyObject *m1, PyObject *m2) static PyObject * gentype_@name@(PyObject *m1, PyObject *m2) { + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_@name@, gentype_@name@); return PyArray_Type.tp_as_number->nb_@name@(m1, m2); } /**end repeat**/ @@ -306,8 +261,8 @@ gentype_multiply(PyObject *m1, PyObject *m2) } return PySequence_Repeat(m2, repeat); } - /* All normal cases are handled by PyArray's multiply */ + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, gentype_multiply); return PyArray_Type.tp_as_number->nb_multiply(m1, m2); } diff --git a/numpy/core/src/private/binop_override.h b/numpy/core/src/private/binop_override.h new file mode 100644 index 000000000000..8b4458777d5a --- /dev/null +++ b/numpy/core/src/private/binop_override.h @@ -0,0 +1,196 @@ +#ifndef __BINOP_OVERRIDE_H +#define __BINOP_OVERRIDE_H + +#include +#include +#include "numpy/arrayobject.h" + +#include "get_attr_string.h" + +/* + * Logic for deciding when binops should return NotImplemented versus when + * they should go ahead and call a ufunc (or similar). + * + * The interaction between binop methods (ndarray.__add__ and friends) and + * ufuncs (which dispatch to __array_ufunc__) is both complicated in its own + * right, and also has complicated historical constraints. + * + * In the very old days, the rules were: + * - If the other argument has a higher __array_priority__, then return + * NotImplemented + * - Otherwise, call the corresponding ufunc. + * - And the ufunc might return NotImplemented based on some complex + * criteria that I won't reproduce here. + * + * Ufuncs no longer return NotImplemented (except in a few marginal situations + * which are being phased out -- see https://github.com/numpy/numpy/pull/5864) + * + * So as of 1.9, the effective rules were: + * - If the other argument has a higher __array_priority__, and is *not* a + * subclass of ndarray, then return NotImplemented. (If it is a subclass, + * the regular Python rules have already given it a chance to run; so if we + * are running, then it means the other argument has already returned + * NotImplemented and is basically asking us to take care of things.) + * - Otherwise call the corresponding ufunc. + * + * We would like to get rid of __array_priority__, and __array_ufunc__ + * provides a large part of a replacement for it. Once __array_ufunc__ is + * widely available, the simplest dispatch rules that might possibly work + * would be: + * - Always call the corresponding ufunc. + * + * But: + * - Doing this immediately would break backwards compatibility -- there's a + * lot of code using __array_priority__ out there. + * - It's not at all clear whether __array_ufunc__ actually is sufficient for + * all use cases. (See https://github.com/numpy/numpy/issues/5844 for lots + * of discussion of this, and in particular + * https://github.com/numpy/numpy/issues/5844#issuecomment-112014014 + * for a summary of some conclusions.) Also, python 3.6 defines a standard + * where setting a special-method name to None is a signal that that method + * cannot be used. + * + * So for 1.13, we are going to try the following rules. a.__add__(b) will + * be implemented as follows: + * - If b does not define __array_ufunc__, apply the legacy rule: + * - If not isinstance(b, a.__class__), and b.__array_priority__ is higher + * than a.__array_priority__, return NotImplemented + * - If b does define __array_ufunc__ but it is None, return NotImplemented + * - Otherwise, call the corresponding ufunc. + * + * For reversed operations like b.__radd__(a), and for in-place operations + * like a.__iadd__(b), we: + * - Call the corresponding ufunc + * + * Rationale for __radd__: This is because by the time the reversed operation + * is called, there are only two possibilities: The first possibility is that + * the current class is a strict subclass of the other class. In practice, the + * only way this will happen is if b is a strict subclass of a, and a is + * ndarray or a subclass of ndarray, and neither a nor b has actually + * overridden this method. In this case, Python will never call a.__add__ + * (because it's identical to b.__radd__), so we have no-one to defer to; + * there's no reason to return NotImplemented. The second possibility is that + * b.__add__ has already been called and returned NotImplemented. Again, in + * this case there is no point in returning NotImplemented. + * + * Rationale for __iadd__: In-place operations do not take all the trouble + * above, because if __iadd__ returns NotImplemented then Python will silently + * convert the operation into an out-of-place operation, i.e. 'a += b' will + * silently become 'a = a + b'. We don't want to allow this for arrays, + * because it will create unexpected memory allocations, break views, + * etc. + * + * In the future we might change these rules further. For example, we plan to + * eventually deprecate __array_priority__ in cases where __array_ufunc__ is + * not present. + */ + +static int +binop_override_forward_binop_should_defer(PyObject *self, PyObject *other) +{ + /* + * This function assumes that self.__binop__(other) is underway and + * implements the rules described above. Python's C API is funny, and + * makes it tricky to tell whether a given slot is called for __binop__ + * ("forward") or __rbinop__ ("reversed"). You are responsible for + * determining this before calling this function; it only provides the + * logic for forward binop implementations. + */ + + /* + * NB: there's another copy of this code in + * numpy.ma.core.MaskedArray._delegate_binop + * which should possibly be updated when this is. + */ + + PyObject *attr; + double self_prio, other_prio; + int defer; + /* + * attribute check is expensive for scalar operations, avoid if possible + */ + if (other == NULL || + self == NULL || + Py_TYPE(self) == Py_TYPE(other) || + PyArray_CheckExact(other) || + PyArray_CheckAnyScalarExact(other) || + _is_basic_python_type(other)) { + return 0; + } + /* + * Classes with __array_ufunc__ are living in the future, and only need to + * check whether __array_ufunc__ equals None. + */ + attr = PyArray_GetAttrString_SuppressException(other, "__array_ufunc__"); + if (attr) { + defer = (attr == Py_None); + Py_DECREF(attr); + return defer; + } + /* + * Otherwise, we need to check for the legacy __array_priority__. But if + * other.__class__ is a subtype of self.__class__, then it's already had + * a chance to run, so no need to defer to it. + */ + if(PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { + return 0; + } + self_prio = PyArray_GetPriority((PyObject *)self, NPY_SCALAR_PRIORITY); + other_prio = PyArray_GetPriority((PyObject *)other, NPY_SCALAR_PRIORITY); + return self_prio < other_prio; +} + +/* + * A CPython slot like ->tp_as_number->nb_add gets called for *both* forward + * and reversed operations. E.g. + * a + b + * may call + * a->tp_as_number->nb_add(a, b) + * and + * b + a + * may call + * a->tp_as_number->nb_add(b, a) + * and the only way to tell which is which is for a slot implementation 'f' to + * check + * arg1->tp_as_number->nb_add == f + * arg2->tp_as_number->nb_add == f + * If both are true, then CPython will as a special case only call the + * operation once (i.e., it performs both the forward and reversed binops + * simultaneously). This function is mostly intended for figuring out + * whether we are a forward binop that might want to return NotImplemented, + * and in the both-at-once case we never want to return NotImplemented, so in + * that case BINOP_IS_FORWARD returns false. + * + * This is modeled on the checks in CPython's typeobject.c SLOT1BINFULL + * macro. + */ +#define BINOP_IS_FORWARD(m1, m2, SLOT_NAME, test_func) \ + (Py_TYPE(m2)->tp_as_number != NULL && \ + (void*)(Py_TYPE(m2)->tp_as_number->SLOT_NAME) != (void*)(test_func)) + +#define BINOP_GIVE_UP_IF_NEEDED(m1, m2, slot_expr, test_func) \ + do { \ + if (BINOP_IS_FORWARD(m1, m2, slot_expr, test_func) && \ + binop_override_forward_binop_should_defer((PyObject*)m1, (PyObject*)m2)) { \ + Py_INCREF(Py_NotImplemented); \ + return Py_NotImplemented; \ + } \ + } while (0) + +/* + * For rich comparison operations, it's impossible to distinguish + * between a forward comparison and a reversed/reflected + * comparison. So we assume they are all forward. This only works because the + * logic in binop_override_forward_binop_should_defer is essentially + * asymmetric -- you can never have two duck-array types that each decide to + * defer to the other. + */ +#define RICHCMP_GIVE_UP_IF_NEEDED(m1, m2) \ + do { \ + if (binop_override_forward_binop_should_defer((PyObject*)m1, (PyObject*)m2)) { \ + Py_INCREF(Py_NotImplemented); \ + return Py_NotImplemented; \ + } \ + } while (0) + +#endif diff --git a/numpy/core/src/private/get_attr_string.h b/numpy/core/src/private/get_attr_string.h new file mode 100644 index 000000000000..b32be28f7a42 --- /dev/null +++ b/numpy/core/src/private/get_attr_string.h @@ -0,0 +1,85 @@ +#ifndef __GET_ATTR_STRING_H +#define __GET_ATTR_STRING_H + +static NPY_INLINE int +_is_basic_python_type(PyObject * obj) +{ + if (obj == Py_None || + PyBool_Check(obj) || + /* Basic number types */ +#if !defined(NPY_PY3K) + PyInt_CheckExact(obj) || + PyString_CheckExact(obj) || +#endif + PyLong_CheckExact(obj) || + PyFloat_CheckExact(obj) || + PyComplex_CheckExact(obj) || + /* Basic sequence types */ + PyList_CheckExact(obj) || + PyTuple_CheckExact(obj) || + PyDict_CheckExact(obj) || + PyAnySet_CheckExact(obj) || + PyUnicode_CheckExact(obj) || + PyBytes_CheckExact(obj) || + PySlice_Check(obj)) { + + return 1; + } + + return 0; +} + +/* + * PyArray_GetAttrString_SuppressException: + * + * Stripped down version of PyObject_GetAttrString, + * avoids lookups for None, tuple, and List objects, + * and doesn't create a PyErr since this code ignores it. + * + * This can be much faster then PyObject_GetAttrString where + * exceptions are not used by caller. + * + * 'obj' is the object to search for attribute. + * + * 'name' is the attribute to search for. + * + * Returns attribute value on success, 0 on failure. + */ +static PyObject * +PyArray_GetAttrString_SuppressException(PyObject *obj, char *name) +{ + PyTypeObject *tp = Py_TYPE(obj); + PyObject *res = (PyObject *)NULL; + + /* We do not need to check for special attributes on trivial types */ + if (_is_basic_python_type(obj)) { + return NULL; + } + + /* Attribute referenced by (char *)name */ + if (tp->tp_getattr != NULL) { + res = (*tp->tp_getattr)(obj, name); + if (res == NULL) { + PyErr_Clear(); + } + } + /* Attribute referenced by (PyObject *)name */ + else if (tp->tp_getattro != NULL) { +#if defined(NPY_PY3K) + PyObject *w = PyUnicode_InternFromString(name); +#else + PyObject *w = PyString_InternFromString(name); +#endif + if (w == NULL) { + return (PyObject *)NULL; + } + res = (*tp->tp_getattro)(obj, w); + Py_DECREF(w); + if (res == NULL) { + PyErr_Clear(); + } + } + return res; +} + +#endif diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c new file mode 100644 index 000000000000..b5cd46b898f5 --- /dev/null +++ b/numpy/core/src/private/ufunc_override.c @@ -0,0 +1,130 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NO_IMPORT_ARRAY + +#include "npy_pycompat.h" +#include "get_attr_string.h" +#include "npy_import.h" + +#include "ufunc_override.h" + +/* + * Check whether an object has __array_ufunc__ defined on its class and it + * is not the default, i.e., the object is not an ndarray, and its + * __array_ufunc__ is not the same as that of ndarray. + * + * Note that since this module is used with both multiarray and umath, we do + * not have access to PyArray_Type and therewith neither to PyArray_CheckExact + * nor to the default __array_ufunc__ method, so instead we import locally. + * TODO: Can this really not be done more smartly? + */ +static int +has_non_default_array_ufunc(PyObject *obj) +{ + static PyObject *ndarray = NULL; + static PyObject *ndarray_array_ufunc = NULL; + PyObject *cls_array_ufunc; + int non_default; + + /* on first entry, import and cache ndarray and its __array_ufunc__ */ + if (ndarray == NULL) { + npy_cache_import("numpy.core.multiarray", "ndarray", &ndarray); + ndarray_array_ufunc = PyObject_GetAttrString(ndarray, + "__array_ufunc__"); + } + + /* Fast return for ndarray */ + if ((PyObject *)Py_TYPE(obj) == ndarray) { + return 0; + } + /* does the class define __array_ufunc__? */ + cls_array_ufunc = PyArray_GetAttrString_SuppressException( + (PyObject *)Py_TYPE(obj), "__array_ufunc__"); + if (cls_array_ufunc == NULL) { + return 0; + } + /* is it different from ndarray.__array_ufunc__? */ + non_default = (cls_array_ufunc != ndarray_array_ufunc); + Py_DECREF(cls_array_ufunc); + return non_default; +} + +/* + * Check whether a set of input and output args have a non-default + * `__array_ufunc__` method. Return the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL) + * using borrowed references. + * + * returns -1 on failure. + */ +NPY_NO_EXPORT int +PyUFunc_WithOverride(PyObject *args, PyObject *kwds, + PyObject **with_override) +{ + int i; + + int nargs; + int nout_kwd = 0; + int out_kwd_is_tuple = 0; + int noa = 0; /* Number of overriding args.*/ + + PyObject *obj; + PyObject *out_kwd_obj = NULL; + /* + * Check inputs + */ + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_TypeError, + "Internal Numpy error: call to PyUFunc_HasOverride " + "with non-tuple"); + goto fail; + } + nargs = PyTuple_GET_SIZE(args); + if (nargs > NPY_MAXARGS) { + PyErr_SetString(PyExc_TypeError, + "Internal Numpy error: too many arguments in call " + "to PyUFunc_HasOverride"); + goto fail; + } + /* be sure to include possible 'out' keyword argument. */ + if (kwds && PyDict_CheckExact(kwds)) { + out_kwd_obj = PyDict_GetItemString(kwds, "out"); + if (out_kwd_obj != NULL) { + out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); + if (out_kwd_is_tuple) { + nout_kwd = PyTuple_GET_SIZE(out_kwd_obj); + } + else { + nout_kwd = 1; + } + } + } + + for (i = 0; i < nargs + nout_kwd; ++i) { + if (i < nargs) { + obj = PyTuple_GET_ITEM(args, i); + } + else { + if (out_kwd_is_tuple) { + obj = PyTuple_GET_ITEM(out_kwd_obj, i - nargs); + } + else { + obj = out_kwd_obj; + } + } + /* + * Now see if the object provides an __array_ufunc__. However, we should + * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as + * any ndarray subclass instances that did not override __array_ufunc__. + */ + if (has_non_default_array_ufunc(obj)) { + if (with_override != NULL) { + with_override[noa] = obj; + } + ++noa; + } + } + return noa; + +fail: + return -1; +} diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 59a90c770542..15a932174cfc 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -1,420 +1,15 @@ #ifndef __UFUNC_OVERRIDE_H #define __UFUNC_OVERRIDE_H -#include -#include "numpy/arrayobject.h" -#include "common.h" -#include -#include "numpy/ufuncobject.h" -static void -normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds, - int nin) -{ - /* ufunc.__call__(*args, **kwds) */ - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj = PyDict_GetItemString(*normal_kwds, "sig"); - - /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - if (obj != NULL) { - Py_INCREF(obj); - PyDict_SetItemString(*normal_kwds, "signature", obj); - PyDict_DelItemString(*normal_kwds, "sig"); - } - - *normal_args = PyTuple_GetSlice(args, 0, nin); - - /* If we have more args than nin, they must be the output variables.*/ - if (nargs > nin) { - if ((nargs - nin) == 1) { - obj = PyTuple_GET_ITEM(args, nargs - 1); - PyDict_SetItemString(*normal_kwds, "out", obj); - } - else { - obj = PyTuple_GetSlice(args, nin, nargs); - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); - } - } -} - -static void -normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); - } - else if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else if (i == 3) { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - else { - /* keepdims */ - PyDict_SetItemString(*normal_kwds, "keepdims", obj); - } - } - return; -} - -static void -normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.accumulate(a[, axis, dtype, out]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); - } - else if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - } - return; -} - -static void -normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ - int i; - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - /* a and indicies */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - } - else if (i == 1) { - /* Handled above, when i == 0. */ - continue; - } - else if (i == 2) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 3) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - } - return; -} - -static void -normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.outer(A, B) - * This has no kwds so we don't need to do any kwd stuff. - */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - return; -} - -static void -normalize_at_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.at(a, indices[, b]) */ - int nargs = PyTuple_GET_SIZE(args); - - *normal_args = PyTuple_GetSlice(args, 0, nargs); - return; -} +#include "npy_config.h" /* - * Check a set of args for the `__numpy_ufunc__` method. If more than one of - * the input arguments implements `__numpy_ufunc__`, they are tried in the - * order: subclasses before superclasses, otherwise left to right. The first - * routine returning something other than `NotImplemented` determines the - * result. If all of the `__numpy_ufunc__` operations returns `NotImplemented`, - * a `TypeError` is raised. - * - * Returns 0 on success and 1 on exception. On success, *result contains the - * result of the operation, if any. If *result is NULL, there is no override. + * Check whether a set of input and output args have a non-default + * `__array_ufunc__` method. Returns the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL). + * returns -1 on failure. */ -static int -PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result, - int nin) -{ - int i; - int override_pos; /* Position of override in args.*/ - int j; - - int nargs; - int nout_kwd = 0; - int out_kwd_is_tuple = 0; - int noa = 0; /* Number of overriding args.*/ - - PyObject *obj; - PyObject *out_kwd_obj = NULL; - PyObject *other_obj; - - PyObject *method_name = NULL; - PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ - PyObject *normal_kwds = NULL; - - PyObject *with_override[NPY_MAXARGS]; - - /* Pos of each override in args */ - int with_override_pos[NPY_MAXARGS]; - - /* 2016-01-29: Disable for now in master -- can re-enable once details are - * sorted out. All commented bits are tagged NUMPY_UFUNC_DISABLED. -njs - */ - result = NULL; - return 0; - - /* - * Check inputs - */ - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_ValueError, - "Internal Numpy error: call to PyUFunc_CheckOverride " - "with non-tuple"); - goto fail; - } - nargs = PyTuple_GET_SIZE(args); - if (nargs > NPY_MAXARGS) { - PyErr_SetString(PyExc_ValueError, - "Internal Numpy error: too many arguments in call " - "to PyUFunc_CheckOverride"); - goto fail; - } - - /* be sure to include possible 'out' keyword argument. */ - if ((kwds)&& (PyDict_CheckExact(kwds))) { - out_kwd_obj = PyDict_GetItemString(kwds, "out"); - if (out_kwd_obj != NULL) { - out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); - if (out_kwd_is_tuple) { - nout_kwd = PyTuple_GET_SIZE(out_kwd_obj); - } - else { - nout_kwd = 1; - } - } - } - - for (i = 0; i < nargs + nout_kwd; ++i) { - if (i < nargs) { - obj = PyTuple_GET_ITEM(args, i); - } - else { - if (out_kwd_is_tuple) { - obj = PyTuple_GET_ITEM(out_kwd_obj, i-nargs); - } - else { - obj = out_kwd_obj; - } - } - /* - * TODO: could use PyArray_GetAttrString_SuppressException if it - * weren't private to multiarray.so - */ - if (PyArray_CheckExact(obj) || PyArray_IsScalar(obj, Generic) || - _is_basic_python_type(obj)) { - continue; - } - if (PyObject_HasAttrString(obj, "__numpy_ufunc__")) { - with_override[noa] = obj; - with_override_pos[noa] = i; - ++noa; - } - } - - /* No overrides, bail out.*/ - if (noa == 0) { - *result = NULL; - return 0; - } - - method_name = PyUString_FromString(method); - if (method_name == NULL) { - goto fail; - } - - /* - * Normalize ufunc arguments. - */ - - /* Build new kwds */ - if (kwds && PyDict_CheckExact(kwds)) { - normal_kwds = PyDict_Copy(kwds); - } - else { - normal_kwds = PyDict_New(); - } - if (normal_kwds == NULL) { - goto fail; - } - - /* decide what to do based on the method. */ - /* ufunc.__call__ */ - if (strcmp(method, "__call__") == 0) { - normalize___call___args(ufunc, args, &normal_args, &normal_kwds, nin); - } - - /* ufunc.reduce */ - else if (strcmp(method, "reduce") == 0) { - normalize_reduce_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.accumulate */ - else if (strcmp(method, "accumulate") == 0) { - normalize_accumulate_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.reduceat */ - else if (strcmp(method, "reduceat") == 0) { - normalize_reduceat_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.outer */ - else if (strcmp(method, "outer") == 0) { - normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.at */ - else if (strcmp(method, "at") == 0) { - normalize_at_args(ufunc, args, &normal_args, &normal_kwds); - } - - if (normal_args == NULL) { - goto fail; - } - - /* - * Call __numpy_ufunc__ functions in correct order - */ - while (1) { - PyObject *numpy_ufunc; - PyObject *override_args; - PyObject *override_obj; - - override_obj = NULL; - *result = NULL; - - /* Choose an overriding argument */ - for (i = 0; i < noa; i++) { - obj = with_override[i]; - if (obj == NULL) { - continue; - } - - /* Get the first instance of an overriding arg.*/ - override_pos = with_override_pos[i]; - override_obj = obj; - - /* Check for sub-types to the right of obj. */ - for (j = i + 1; j < noa; j++) { - other_obj = with_override[j]; - if (PyObject_Type(other_obj) != PyObject_Type(obj) && - PyObject_IsInstance(other_obj, - PyObject_Type(override_obj))) { - override_obj = NULL; - break; - } - } - - /* override_obj had no subtypes to the right. */ - if (override_obj) { - with_override[i] = NULL; /* We won't call this one again */ - break; - } - } - - /* Check if there is a method left to call */ - if (!override_obj) { - /* No acceptable override found. */ - PyErr_SetString(PyExc_TypeError, - "__numpy_ufunc__ not implemented for this type."); - goto fail; - } - - /* Call the override */ - numpy_ufunc = PyObject_GetAttrString(override_obj, - "__numpy_ufunc__"); - if (numpy_ufunc == NULL) { - goto fail; - } - - override_args = Py_BuildValue("OOiO", ufunc, method_name, - override_pos, normal_args); - if (override_args == NULL) { - Py_DECREF(numpy_ufunc); - goto fail; - } - - *result = PyObject_Call(numpy_ufunc, override_args, normal_kwds); - - Py_DECREF(numpy_ufunc); - Py_DECREF(override_args); - - if (*result == NULL) { - /* Exception occurred */ - goto fail; - } - else if (*result == Py_NotImplemented) { - /* Try the next one */ - Py_DECREF(*result); - continue; - } - else { - /* Good result. */ - break; - } - } - - /* Override found, return it. */ - Py_XDECREF(method_name); - Py_XDECREF(normal_args); - Py_XDECREF(normal_kwds); - return 0; - -fail: - Py_XDECREF(method_name); - Py_XDECREF(normal_args); - Py_XDECREF(normal_kwds); - return 1; -} +NPY_NO_EXPORT int +PyUFunc_WithOverride(PyObject *args, PyObject *kwds, + PyObject **with_override); #endif diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c new file mode 100644 index 000000000000..1faf2568b4f1 --- /dev/null +++ b/numpy/core/src/umath/override.c @@ -0,0 +1,590 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NO_IMPORT_ARRAY + +#include "npy_pycompat.h" +#include "numpy/ufuncobject.h" +#include "npy_import.h" + +#include "ufunc_override.h" +#include "override.h" + +/* + * The following functions normalize ufunc arguments. The work done is similar + * to what is done inside ufunc_object by get_ufunc_arguments for __call__ and + * generalized ufuncs, and by PyUFunc_GenericReduction for the other methods. + * It would be good to unify (see gh-8892). + */ + +/* + * ufunc() and ufunc.outer() accept 'sig' or 'signature'; + * normalize to 'signature' + */ +static int +normalize_signature_keyword(PyObject *normal_kwds) +{ + PyObject* obj = PyDict_GetItemString(normal_kwds, "sig"); + if (obj != NULL) { + if (PyDict_GetItemString(normal_kwds, "signature")) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'sig' and 'signature'"); + return -1; + } + Py_INCREF(obj); + PyDict_SetItemString(normal_kwds, "signature", obj); + PyDict_DelItemString(normal_kwds, "sig"); + } + return 0; +} + +static int +normalize___call___args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.__call__(*args, **kwds) + */ + int i; + int not_all_none; + int nin = ufunc->nin; + int nout = ufunc->nout; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "ufunc() missing %d of %d required positional argument(s)", + nin - nargs, nin); + return -1; + } + if (nargs > nin+nout) { + PyErr_Format(PyExc_TypeError, + "ufunc() takes from %d to %d arguments but %d were given", + nin, nin+nout, nargs); + return -1; + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } + + /* If we have more args than nin, they must be the output variables.*/ + if (nargs > nin) { + if(PyDict_GetItemString(*normal_kwds, "out")) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('out') and position (%d)", + nin); + return -1; + } + for (i = nin; i < nargs; i++) { + not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); + if (not_all_none) { + break; + } + } + if (not_all_none) { + if (nargs - nin == nout) { + obj = PyTuple_GetSlice(args, nin, nargs); + } + else { + PyObject *item; + + obj = PyTuple_New(nout); + if (obj == NULL) { + return -1; + } + for (i = 0; i < nout; i++) { + if (i + nin < nargs) { + item = PyTuple_GET_ITEM(args, nin+i); + } + else { + item = Py_None; + } + Py_INCREF(item); + PyTuple_SET_ITEM(obj, i, item); + } + } + PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); + } + } + /* finally, ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + return normalize_signature_keyword(*normal_kwds); +} + +static int +normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.reduce(a[, axis, dtype, out, keepdims]) + */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + + if (nargs < 1 || nargs > 5) { + PyErr_Format(PyExc_TypeError, + "ufunc.reduce() takes from 1 to 5 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, 1); + if (*normal_args == NULL) { + return -1; + } + + for (i = 1; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.accumulate(a[, axis, dtype, out]) + */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + + if (nargs < 1 || nargs > 4) { + PyErr_Format(PyExc_TypeError, + "ufunc.accumulate() takes from 1 to 4 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, 1); + if (*normal_args == NULL) { + return -1; + } + + for (i = 1; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.reduceat(a, indicies[, axis, dtype, out]) + * the number of arguments has been checked in PyUFunc_GenericReduction. + */ + int i; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; + + if (nargs < 2 || nargs > 5) { + PyErr_Format(PyExc_TypeError, + "ufunc.reduceat() takes from 2 to 4 positional " + "arguments but %d were given", nargs); + return -1; + } + /* a and indicies */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + if (*normal_args == NULL) { + return -1; + } + + for (i = 2; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 4) { + obj = PyTuple_GetSlice(args, 4, 5); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 4) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.outer(*args, **kwds) + * all positional arguments should be inputs. + * for the keywords, we only need to check 'sig' vs 'signature'. + */ + int nin = ufunc->nin; + int nargs = PyTuple_GET_SIZE(args); + + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "ufunc.outer() missing %d of %d required positional " + "argument(s)", nin - nargs, nin); + return -1; + } + if (nargs > nin) { + PyErr_Format(PyExc_TypeError, + "ufunc.outer() takes %d arguments but %d were given", + nin, nargs); + return -1; + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } + + /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + return normalize_signature_keyword(*normal_kwds); +} + +static int +normalize_at_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.at(a, indices[, b]) */ + int nargs = PyTuple_GET_SIZE(args); + + if (nargs < 2 || nargs > 3) { + PyErr_Format(PyExc_TypeError, + "ufunc.at() takes from 2 to 3 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, nargs); + return (*normal_args == NULL); +} + +/* + * Check a set of args for the `__array_ufunc__` method. If more than one of + * the input arguments implements `__array_ufunc__`, they are tried in the + * order: subclasses before superclasses, otherwise left to right. The first + * (non-None) routine returning something other than `NotImplemented` + * determines the result. If all of the `__array_ufunc__` operations return + * `NotImplemented` (or are None), a `TypeError` is raised. + * + * Returns 0 on success and 1 on exception. On success, *result contains the + * result of the operation, if any. If *result is NULL, there is no override. + */ +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result) +{ + int i; + int j; + int status; + + int noa; + PyObject *with_override[NPY_MAXARGS]; + + PyObject *obj; + PyObject *other_obj; + PyObject *out; + + PyObject *method_name = NULL; + PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ + PyObject *normal_kwds = NULL; + + PyObject *override_args = NULL; + Py_ssize_t len; + + /* + * Check inputs for overrides + */ + noa = PyUFunc_WithOverride(args, kwds, with_override); + /* No overrides, bail out.*/ + if (noa == 0) { + *result = NULL; + return 0; + } + + /* + * Normalize ufunc arguments. + */ + + /* Build new kwds */ + if (kwds && PyDict_CheckExact(kwds)) { + + /* ensure out is always a tuple */ + normal_kwds = PyDict_Copy(kwds); + out = PyDict_GetItemString(normal_kwds, "out"); + if (out != NULL) { + int nout = ufunc->nout; + + if (PyTuple_Check(out)) { + int all_none = 1; + + if (PyTuple_GET_SIZE(out) != nout) { + PyErr_Format(PyExc_TypeError, + "The 'out' tuple must have exactly " + "%d entries: one per ufunc output", nout); + goto fail; + } + for (i = 0; i < PyTuple_GET_SIZE(out); i++) { + all_none = (PyTuple_GET_ITEM(out, i) == Py_None); + if (!all_none) { + break; + } + } + if (all_none) { + PyDict_DelItemString(normal_kwds, "out"); + } + } + else { + /* not a tuple */ + if (nout > 1 && DEPRECATE("passing a single argument to the " + "'out' keyword argument of a " + "ufunc with\n" + "more than one output will " + "result in an error in the " + "future") < 0) { + /* + * If the deprecation is removed, also remove the loop + * below setting tuple items to None (but keep this future + * error message.) + */ + PyErr_SetString(PyExc_TypeError, + "'out' must be a tuple of arguments"); + goto fail; + } + if (out != Py_None) { + /* not already a tuple and not None */ + PyObject *out_tuple = PyTuple_New(nout); + + if (out_tuple == NULL) { + goto fail; + } + for (i = 1; i < nout; i++) { + Py_INCREF(Py_None); + PyTuple_SET_ITEM(out_tuple, i, Py_None); + } + /* out was borrowed ref; make it permanent */ + Py_INCREF(out); + /* steals reference */ + PyTuple_SET_ITEM(out_tuple, 0, out); + PyDict_SetItemString(normal_kwds, "out", out_tuple); + Py_DECREF(out_tuple); + } + else { + /* out=None; remove it */ + PyDict_DelItemString(normal_kwds, "out"); + } + } + } + } + else { + normal_kwds = PyDict_New(); + } + if (normal_kwds == NULL) { + goto fail; + } + + /* decide what to do based on the method. */ + + /* ufunc.__call__ */ + if (strcmp(method, "__call__") == 0) { + status = normalize___call___args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.reduce */ + else if (strcmp(method, "reduce") == 0) { + status = normalize_reduce_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.accumulate */ + else if (strcmp(method, "accumulate") == 0) { + status = normalize_accumulate_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.reduceat */ + else if (strcmp(method, "reduceat") == 0) { + status = normalize_reduceat_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.outer */ + else if (strcmp(method, "outer") == 0) { + status = normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); + } + /* ufunc.at */ + else if (strcmp(method, "at") == 0) { + status = normalize_at_args(ufunc, args, &normal_args, &normal_kwds); + } + /* unknown method */ + else { + PyErr_Format(PyExc_TypeError, + "Internal Numpy error: unknown ufunc method '%s' in call " + "to PyUFunc_CheckOverride", method); + status = -1; + } + if (status != 0) { + Py_XDECREF(normal_args); + goto fail; + } + + len = PyTuple_GET_SIZE(normal_args); + override_args = PyTuple_New(len + 2); + if (override_args == NULL) { + goto fail; + } + + Py_INCREF(ufunc); + /* PyTuple_SET_ITEM steals reference */ + PyTuple_SET_ITEM(override_args, 0, (PyObject *)ufunc); + method_name = PyUString_FromString(method); + if (method_name == NULL) { + goto fail; + } + Py_INCREF(method_name); + PyTuple_SET_ITEM(override_args, 1, method_name); + for (i = 0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(normal_args, i); + + Py_INCREF(item); + PyTuple_SET_ITEM(override_args, i + 2, item); + } + Py_DECREF(normal_args); + + /* Call __array_ufunc__ functions in correct order */ + while (1) { + PyObject *array_ufunc; + PyObject *override_obj; + + override_obj = NULL; + *result = NULL; + + /* Choose an overriding argument */ + for (i = 0; i < noa; i++) { + obj = with_override[i]; + if (obj == NULL) { + continue; + } + + /* Get the first instance of an overriding arg.*/ + override_obj = obj; + + /* Check for sub-types to the right of obj. */ + for (j = i + 1; j < noa; j++) { + other_obj = with_override[j]; + if (other_obj != NULL && + PyObject_Type(other_obj) != PyObject_Type(obj) && + PyObject_IsInstance(other_obj, + PyObject_Type(override_obj))) { + override_obj = NULL; + break; + } + } + + /* override_obj had no subtypes to the right. */ + if (override_obj) { + /* We won't call this one again */ + with_override[i] = NULL; + break; + } + } + + /* Check if there is a method left to call */ + if (!override_obj) { + /* No acceptable override found. */ + static PyObject *errmsg_formatter = NULL; + PyObject *errmsg; + + npy_cache_import("numpy.core._internal", + "array_ufunc_errmsg_formatter", + &errmsg_formatter); + if (errmsg_formatter != NULL) { + errmsg = PyObject_Call(errmsg_formatter, override_args, + normal_kwds); + if (errmsg != NULL) { + PyErr_SetObject(PyExc_TypeError, errmsg); + Py_DECREF(errmsg); + } + } + goto fail; + } + + /* Access the override */ + array_ufunc = PyObject_GetAttrString(override_obj, + "__array_ufunc__"); + if (array_ufunc == NULL) { + goto fail; + } + + /* If None, try next one (i.e., as if it returned NotImplemented) */ + if (array_ufunc == Py_None) { + Py_DECREF(array_ufunc); + continue; + } + + *result = PyObject_Call(array_ufunc, override_args, normal_kwds); + Py_DECREF(array_ufunc); + + if (*result == NULL) { + /* Exception occurred */ + goto fail; + } + else if (*result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(*result); + continue; + } + else { + /* Good result. */ + break; + } + } + + /* Override found, return it. */ + Py_XDECREF(method_name); + Py_XDECREF(normal_kwds); + Py_DECREF(override_args); + return 0; + +fail: + Py_XDECREF(method_name); + Py_XDECREF(normal_kwds); + Py_XDECREF(override_args); + return 1; +} diff --git a/numpy/core/src/umath/override.h b/numpy/core/src/umath/override.h new file mode 100644 index 000000000000..68f3c6ef0814 --- /dev/null +++ b/numpy/core/src/umath/override.h @@ -0,0 +1,11 @@ +#ifndef _NPY_UMATH_OVERRIDE_H +#define _NPY_UMATH_OVERRIDE_H + +#include "npy_config.h" +#include "numpy/ufuncobject.h" + +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result); +#endif diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index 723ee998ae2d..eb75b73753d3 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -23,6 +23,8 @@ #include "numpy/halffloat.h" #include "templ_common.h" +#include "binop_override.h" + /* Basic operations: * * BINARY: @@ -827,6 +829,8 @@ static PyObject * int first; #endif + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_@oper@, @name@_@oper@); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -964,6 +968,8 @@ static PyObject * int first; @type@ out = {@zero@, @zero@}; + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1041,6 +1047,8 @@ static PyObject * PyObject *ret; @type@ arg1, arg2, out; + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1102,6 +1110,9 @@ static PyObject * int first; @type@ out = @zero@; + + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1506,6 +1517,8 @@ static PyObject* npy_@name@ arg1, arg2; int out=0; + RICHCMP_GIVE_UP_IF_NEEDED(self, other); + switch(_@name@_convert2_to_ctypes(self, &arg1, other, &arg2)) { case 0: break; diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 22a73e6ba919..137a93781a93 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -44,7 +44,7 @@ #include "mem_overlap.h" #include "ufunc_object.h" -#include "ufunc_override.h" +#include "override.h" /********** PRINTF DEBUG TRACING **************/ #define NPY_UF_DBG_TRACING 0 @@ -4370,8 +4370,7 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) mps[i] = NULL; } - errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override, - ufunc->nin); + errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override); if (errval) { return NULL; } @@ -5068,6 +5067,14 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *new_args, *tmp; PyObject *shape1, *shape2, *newshape; + errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override); + if (errval) { + return NULL; + } + else if (override) { + return override; + } + if (ufunc->core_enabled) { PyErr_Format(PyExc_TypeError, "method outer is not allowed in ufunc with non-trivial"\ @@ -5087,15 +5094,6 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) return NULL; } - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override, 0); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - tmp = PySequence_GetItem(args, 0); if (tmp == NULL) { return NULL; @@ -5165,8 +5163,7 @@ ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override); if (errval) { return NULL; } @@ -5182,8 +5179,7 @@ ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override); if (errval) { return NULL; } @@ -5199,8 +5195,7 @@ ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override); if (errval) { return NULL; } @@ -5264,8 +5259,7 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) char * err_msg = NULL; NPY_BEGIN_THREADS_DEF; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override, 0); + errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override); if (errval) { return NULL; } diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 2419c31f8f6f..1a6cee030c1b 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -266,7 +266,7 @@ intern_strings(void) npy_um_str_array_prepare = PyUString_InternFromString("__array_prepare__"); npy_um_str_array_wrap = PyUString_InternFromString("__array_wrap__"); npy_um_str_array_finalize = PyUString_InternFromString("__array_finalize__"); - npy_um_str_ufunc = PyUString_InternFromString("__numpy_ufunc__"); + npy_um_str_ufunc = PyUString_InternFromString("__array_ufunc__"); npy_um_str_pyvals_name = PyUString_InternFromString(UFUNC_PYVALS_NAME); return npy_um_str_out && npy_um_str_subok && npy_um_str_array_prepare && diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 82b8fec14c5e..6d9a8fdc3c47 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2406,27 +2406,6 @@ def test_dot(self): a.dot(b=b, out=c) assert_equal(c, np.dot(a, b)) - def test_dot_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return "A" - - class B(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return NotImplemented - - a = A() - b = B() - c = np.array([[1]]) - - assert_equal(np.dot(a, b), "A") - assert_equal(c.dot(a), "A") - assert_raises(TypeError, np.dot, b, c) - assert_raises(TypeError, c.dot, b) - def test_dot_type_mismatch(self): c = 1. A = np.array((1,1), dtype='i,i') @@ -2886,241 +2865,192 @@ def test_elide_scalar(self): a = np.bool_() assert_(type(~(a & a)) is np.bool_) - def test_ufunc_override_rop_precedence(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - # Check that __rmul__ and other right-hand operations have - # precedence over __numpy_ufunc__ - + # ndarray.__rop__ always calls ufunc + # ndarray.__iop__ always calls ufunc + # ndarray.__op__, __rop__: + # - defer if other has __array_ufunc__ and it is None + # or other is not a subclass and has higher array priority + # - else, call ufunc + def test_ufunc_binop_interaction(self): + # Python method name (without underscores) + # -> (numpy ufunc, has_in_place_version, preferred_dtype) ops = { - '__add__': ('__radd__', np.add, True), - '__sub__': ('__rsub__', np.subtract, True), - '__mul__': ('__rmul__', np.multiply, True), - '__truediv__': ('__rtruediv__', np.true_divide, True), - '__floordiv__': ('__rfloordiv__', np.floor_divide, True), - '__mod__': ('__rmod__', np.remainder, True), - '__divmod__': ('__rdivmod__', None, False), - '__pow__': ('__rpow__', np.power, True), - '__lshift__': ('__rlshift__', np.left_shift, True), - '__rshift__': ('__rrshift__', np.right_shift, True), - '__and__': ('__rand__', np.bitwise_and, True), - '__xor__': ('__rxor__', np.bitwise_xor, True), - '__or__': ('__ror__', np.bitwise_or, True), - '__ge__': ('__le__', np.less_equal, False), - '__gt__': ('__lt__', np.less, False), - '__le__': ('__ge__', np.greater_equal, False), - '__lt__': ('__gt__', np.greater, False), - '__eq__': ('__eq__', np.equal, False), - '__ne__': ('__ne__', np.not_equal, False), + 'add': (np.add, True, float), + 'sub': (np.subtract, True, float), + 'mul': (np.multiply, True, float), + 'truediv': (np.true_divide, True, float), + 'floordiv': (np.floor_divide, True, float), + 'mod': (np.remainder, True, float), + 'divmod': (None, False, float), + 'pow': (np.power, True, int), + 'lshift': (np.left_shift, True, int), + 'rshift': (np.right_shift, True, int), + 'and': (np.bitwise_and, True, int), + 'xor': (np.bitwise_xor, True, int), + 'or': (np.bitwise_or, True, int), + # 'ge': (np.less_equal, False), + # 'gt': (np.less, False), + # 'le': (np.greater_equal, False), + # 'lt': (np.greater, False), + # 'eq': (np.equal, False), + # 'ne': (np.not_equal, False), } - class OtherNdarraySubclass(np.ndarray): + class Coerced(Exception): pass - class OtherNdarraySubclassWithOverride(np.ndarray): - def __numpy_ufunc__(self, *a, **kw): - raise AssertionError(("__numpy_ufunc__ %r %r shouldn't have " - "been called!") % (a, kw)) - - def check(op_name, ndsubclass): - rop_name, np_op, has_iop = ops[op_name] - - if has_iop: - iop_name = '__i' + op_name[2:] - iop = getattr(operator, iop_name) - - if op_name == "__divmod__": - op = divmod - else: - op = getattr(operator, op_name) - - # Dummy class - def __init__(self, *a, **kw): - pass - - def __numpy_ufunc__(self, *a, **kw): - raise AssertionError(("__numpy_ufunc__ %r %r shouldn't have " - "been called!") % (a, kw)) - - def __op__(self, *other): - return "op" - - def __rop__(self, *other): - return "rop" - - if ndsubclass: - bases = (np.ndarray,) + def array_impl(self): + raise Coerced + + def op_impl(self, other): + return "forward" + + def rop_impl(self, other): + return "reverse" + + def iop_impl(self, other): + return "in-place" + + def array_ufunc_impl(self, ufunc, method, *args, **kwargs): + return ("__array_ufunc__", ufunc, method, args, kwargs) + + # Create an object with the given base, in the given module, with a + # bunch of placeholder __op__ methods, and optionally a + # __array_ufunc__ and __array_priority__. + def make_obj(base, array_priority=False, array_ufunc=False, + alleged_module="__main__"): + class_namespace = {"__array__": array_impl} + if array_priority is not False: + class_namespace["__array_priority__"] = array_priority + for op in ops: + class_namespace["__{0}__".format(op)] = op_impl + class_namespace["__r{0}__".format(op)] = rop_impl + class_namespace["__i{0}__".format(op)] = iop_impl + if array_ufunc is not False: + class_namespace["__array_ufunc__"] = array_ufunc + eval_namespace = {"base": base, + "class_namespace": class_namespace, + "__name__": alleged_module, + } + MyType = eval("type('MyType', (base,), class_namespace)", + eval_namespace) + if issubclass(MyType, np.ndarray): + # Use this range to avoid special case weirdnesses around + # divide-by-0, pow(x, 2), overflow due to pow(big, big), etc. + return np.arange(3, 5).view(MyType) else: - bases = (object,) - - dct = {'__init__': __init__, - '__numpy_ufunc__': __numpy_ufunc__, - op_name: __op__} - if op_name != rop_name: - dct[rop_name] = __rop__ - - cls = type("Rop" + rop_name, bases, dct) - - # Check behavior against both bare ndarray objects and a - # ndarray subclasses with and without their own override - obj = cls((1,), buffer=np.ones(1,)) - - arr_objs = [np.array([1]), - np.array([2]).view(OtherNdarraySubclass), - np.array([3]).view(OtherNdarraySubclassWithOverride), - ] - - for arr in arr_objs: - err_msg = "%r %r" % (op_name, arr,) - - # Check that ndarray op gives up if it sees a non-subclass - if not isinstance(obj, arr.__class__): - assert_equal(getattr(arr, op_name)(obj), - NotImplemented, err_msg=err_msg) - - # Check that the Python binops have priority - assert_equal(op(obj, arr), "op", err_msg=err_msg) - if op_name == rop_name: - assert_equal(op(arr, obj), "op", err_msg=err_msg) - else: - assert_equal(op(arr, obj), "rop", err_msg=err_msg) - - # Check that Python binops have priority also for in-place ops - if has_iop: - assert_equal(getattr(arr, iop_name)(obj), - NotImplemented, err_msg=err_msg) - if op_name != "__pow__": - # inplace pow requires the other object to be - # integer-like? - assert_equal(iop(arr, obj), "rop", err_msg=err_msg) - - # Check that ufunc call __numpy_ufunc__ normally - if np_op is not None: - assert_raises(AssertionError, np_op, arr, obj, - err_msg=err_msg) - assert_raises(AssertionError, np_op, obj, arr, - err_msg=err_msg) - - # Check all binary operations - for op_name in sorted(ops.keys()): - yield check, op_name, True - yield check, op_name, False - - def test_ufunc_override_rop_simple(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - # Check parts of the binary op overriding behavior in an - # explicit test case that is easier to understand. - class SomeClass(object): - def __numpy_ufunc__(self, *a, **kw): - return "ufunc" - - def __mul__(self, other): - return 123 - - def __rmul__(self, other): - return 321 - - def __rsub__(self, other): - return "no subs for me" - - def __gt__(self, other): - return "yep" - - def __lt__(self, other): - return "nope" - - class SomeClass2(SomeClass, np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): - if ufunc is np.multiply or ufunc is np.bitwise_and: - return "ufunc" - else: - inputs = list(inputs) - if i < len(inputs): - inputs[i] = np.asarray(self) - func = getattr(ufunc, method) - if ('out' in kw) and (kw['out'] is not None): - kw['out'] = np.asarray(kw['out']) - r = func(*inputs, **kw) - x = self.__class__(r.shape, dtype=r.dtype) - x[...] = r - return x - - class SomeClass3(SomeClass2): - def __rsub__(self, other): - return "sub for me" - - arr = np.array([0]) - obj = SomeClass() - obj2 = SomeClass2((1,), dtype=np.int_) - obj2[0] = 9 - obj3 = SomeClass3((1,), dtype=np.int_) - obj3[0] = 4 - - # obj is first, so should get to define outcome. - assert_equal(obj * arr, 123) - # obj is second, but has __numpy_ufunc__ and defines __rmul__. - assert_equal(arr * obj, 321) - # obj is second, but has __numpy_ufunc__ and defines __rsub__. - assert_equal(arr - obj, "no subs for me") - # obj is second, but has __numpy_ufunc__ and defines __lt__. - assert_equal(arr > obj, "nope") - # obj is second, but has __numpy_ufunc__ and defines __gt__. - assert_equal(arr < obj, "yep") - # Called as a ufunc, obj.__numpy_ufunc__ is used. - assert_equal(np.multiply(arr, obj), "ufunc") - # obj is second, but has __numpy_ufunc__ and defines __rmul__. - arr *= obj - assert_equal(arr, 321) - - # obj2 is an ndarray subclass, so CPython takes care of the same rules. - assert_equal(obj2 * arr, 123) - assert_equal(arr * obj2, 321) - assert_equal(arr - obj2, "no subs for me") - assert_equal(arr > obj2, "nope") - assert_equal(arr < obj2, "yep") - # Called as a ufunc, obj2.__numpy_ufunc__ is called. - assert_equal(np.multiply(arr, obj2), "ufunc") - # Also when the method is not overridden. - assert_equal(arr & obj2, "ufunc") - arr *= obj2 - assert_equal(arr, 321) - - obj2 += 33 - assert_equal(obj2[0], 42) - assert_equal(obj2.sum(), 42) - assert_(isinstance(obj2, SomeClass2)) - - # Obj3 is subclass that defines __rsub__. CPython calls it. - assert_equal(arr - obj3, "sub for me") - assert_equal(obj2 - obj3, "sub for me") - # obj3 is a subclass that defines __rmul__. CPython calls it. - assert_equal(arr * obj3, 321) - # But not here, since obj3.__rmul__ is obj2.__rmul__. - assert_equal(obj2 * obj3, 123) - # And of course, here obj3.__mul__ should be called. - assert_equal(obj3 * obj2, 123) - # obj3 defines __numpy_ufunc__ but obj3.__radd__ is obj2.__radd__. - # (and both are just ndarray.__radd__); see #4815. - res = obj2 + obj3 - assert_equal(res, 46) - assert_(isinstance(res, SomeClass2)) - # Since obj3 is a subclass, it should have precedence, like CPython - # would give, even though obj2 has __numpy_ufunc__ and __radd__. - # See gh-4815 and gh-5747. - res = obj3 + obj2 - assert_equal(res, 46) - assert_(isinstance(res, SomeClass3)) + return MyType() + + def check(obj, binop_override_expected, ufunc_override_expected, + check_scalar=True): + for op, (ufunc, has_inplace, dtype) in ops.items(): + check_objs = [np.arange(3, 5, dtype=dtype)] + if check_scalar: + check_objs.append(check_objs[0][0]) + for arr in check_objs: + arr_method = getattr(arr, "__{0}__".format(op)) + + def norm(result): + if op == "divmod": + assert_(isinstance(result, tuple)) + return result[0] + else: + return result + + if binop_override_expected: + assert_equal(arr_method(obj), NotImplemented) + elif ufunc_override_expected: + assert_equal(norm(arr_method(obj))[0], + "__array_ufunc__") + else: + if (isinstance(obj, np.ndarray) and + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): + # __array__ gets ignored + res = norm(arr_method(obj)) + assert_(res.__class__ is obj.__class__) + else: + assert_raises((TypeError, Coerced), + arr_method, obj) + + arr_rmethod = getattr(arr, "__r{0}__".format(op)) + if ufunc_override_expected: + res = norm(arr_rmethod(obj)) + assert_equal(res[0], "__array_ufunc__") + if ufunc is not None: + assert_equal(res[1], ufunc) + else: + if (isinstance(obj, np.ndarray) and + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): + # __array__ gets ignored + res = norm(arr_rmethod(obj)) + assert_(res.__class__ is obj.__class__) + else: + # __array_ufunc__ = "asdf" creates a TypeError + assert_raises((TypeError, Coerced), + arr_rmethod, obj) + + # array scalars don't have in-place operators + if has_inplace and isinstance(arr, np.ndarray): + arr_imethod = getattr(arr, "__i{0}__".format(op)) + if ufunc_override_expected: + res = arr_imethod(obj) + assert_equal(res[0], "__array_ufunc__") + if ufunc is not None: + assert_equal(res[1], ufunc) + assert_(type(res[-1]["out"]) is tuple) + assert_(res[-1]["out"][0] is arr) + else: + if (isinstance(obj, np.ndarray) and + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): + # __array__ gets ignored + assert_(arr_imethod(obj) is arr) + else: + assert_raises((TypeError, Coerced), + arr_imethod, obj) + + op_fn = getattr(operator, op, None) + if op_fn is None: + op_fn = getattr(operator, op + "_", None) + if op_fn is None: + op_fn = getattr(builtins, op) + assert_equal(op_fn(obj, arr), "forward") + if not isinstance(obj, np.ndarray): + if binop_override_expected: + assert_equal(op_fn(arr, obj), "reverse") + elif ufunc_override_expected: + assert_equal(norm(op_fn(arr, obj))[0], + "__array_ufunc__") + if ufunc_override_expected and ufunc is not None: + assert_equal(norm(ufunc(obj, arr))[0], + "__array_ufunc__") + + # No array priority, no numpy ufunc -> nothing called + check(make_obj(object), False, False) + # Negative array priority, no numpy ufunc -> nothing called + # (has to be very negative, because scalar priority is -1000000.0) + check(make_obj(object, array_priority=-2**30), False, False) + # Positive array priority, no numpy ufunc -> binops only + check(make_obj(object, array_priority=1), True, False) + # ndarray ignores array priority for ndarray subclasses + check(make_obj(np.ndarray, array_priority=1), False, False, + check_scalar=False) + # Positive array priority and numpy ufunc -> numpy ufunc only + check(make_obj(object, array_priority=1, + array_ufunc=array_ufunc_impl), False, True) + check(make_obj(np.ndarray, array_priority=1, + array_ufunc=array_ufunc_impl), False, True) + # array_ufunc set to None -> defer binops only + check(make_obj(object, array_ufunc=None), True, False) + check(make_obj(np.ndarray, array_ufunc=None), True, False, + check_scalar=False) def test_ufunc_override_normalize_signature(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # gh-5674 class SomeClass(object): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): return kw a = SomeClass() @@ -3133,58 +3063,63 @@ def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): assert_('sig' not in kw and 'signature' in kw) assert_equal(kw['signature'], 'ii->i') - def test_numpy_ufunc_index(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - + def test_array_ufunc_index(self): # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) + # This also checks implicitly that 'out' is always a tuple. class CheckIndex(object): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): - return i + def __array_ufunc__(self, ufunc, method, *inputs, **kw): + for i, a in enumerate(inputs): + if a is self: + return i + # calls below mean we must be in an output. + for j, a in enumerate(kw['out']): + if a is self: + return (j,) a = CheckIndex() dummy = np.arange(2.) # 1 input, 1 output assert_equal(np.sin(a), 0) - assert_equal(np.sin(dummy, a), 1) - assert_equal(np.sin(dummy, out=a), 1) - assert_equal(np.sin(dummy, out=(a,)), 1) + assert_equal(np.sin(dummy, a), (0,)) + assert_equal(np.sin(dummy, out=a), (0,)) + assert_equal(np.sin(dummy, out=(a,)), (0,)) assert_equal(np.sin(a, a), 0) assert_equal(np.sin(a, out=a), 0) assert_equal(np.sin(a, out=(a,)), 0) # 1 input, 2 outputs - assert_equal(np.modf(dummy, a), 1) - assert_equal(np.modf(dummy, None, a), 2) - assert_equal(np.modf(dummy, dummy, a), 2) - assert_equal(np.modf(dummy, out=a), 1) - assert_equal(np.modf(dummy, out=(a,)), 1) - assert_equal(np.modf(dummy, out=(a, None)), 1) - assert_equal(np.modf(dummy, out=(a, dummy)), 1) - assert_equal(np.modf(dummy, out=(None, a)), 2) - assert_equal(np.modf(dummy, out=(dummy, a)), 2) + assert_equal(np.modf(dummy, a), (0,)) + assert_equal(np.modf(dummy, None, a), (1,)) + assert_equal(np.modf(dummy, dummy, a), (1,)) + assert_equal(np.modf(dummy, out=(a, None)), (0,)) + assert_equal(np.modf(dummy, out=(a, dummy)), (0,)) + assert_equal(np.modf(dummy, out=(None, a)), (1,)) + assert_equal(np.modf(dummy, out=(dummy, a)), (1,)) assert_equal(np.modf(a, out=(dummy, a)), 0) + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', '', DeprecationWarning) + assert_equal(np.modf(dummy, out=a), (0,)) + assert_(w[0].category is DeprecationWarning) + assert_raises(TypeError, np.modf, dummy, out=(a,)) + # 2 inputs, 1 output assert_equal(np.add(a, dummy), 0) assert_equal(np.add(dummy, a), 1) - assert_equal(np.add(dummy, dummy, a), 2) + assert_equal(np.add(dummy, dummy, a), (0,)) assert_equal(np.add(dummy, a, a), 1) - assert_equal(np.add(dummy, dummy, out=a), 2) - assert_equal(np.add(dummy, dummy, out=(a,)), 2) + assert_equal(np.add(dummy, dummy, out=a), (0,)) + assert_equal(np.add(dummy, dummy, out=(a,)), (0,)) assert_equal(np.add(a, dummy, out=a), 0) def test_out_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # regression test for github bug 4753 class OutClass(np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): if 'out' in kw: tmp_kw = kw.copy() tmp_kw.pop('out') func = getattr(ufunc, method) - kw['out'][...] = func(*inputs, **tmp_kw) + kw['out'][0][...] = func(*inputs, **tmp_kw) A = np.array([0]).view(OutClass) B = np.array([5]) @@ -5319,31 +5254,6 @@ def test_matrix_matrix_values(self): res = self.matmul(m12, m21) assert_equal(res, tgt12_21) - def test_numpy_ufunc_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - - class A(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return "A" - - class B(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return NotImplemented - - a = A([1, 2]) - b = B([1, 2]) - c = np.ones(2) - assert_equal(self.matmul(a, b), "A") - assert_equal(self.matmul(b, a), "A") - assert_raises(TypeError, self.matmul, b, c) - class TestMatmul(MatmulCommon, TestCase): matmul = np.matmul diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 26e15539eb0e..1d0518f88417 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1225,7 +1225,7 @@ def test_structured_equal(self): # https://github.com/numpy/numpy/issues/4855 class MyA(np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return getattr(ufunc, method)(*(input.view(np.ndarray) for input in inputs), **kwargs) a = np.arange(12.).reshape(4,3) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index d3b379a524fa..41108ab5f8d2 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -3,15 +3,18 @@ import sys import platform import warnings +import fnmatch import itertools from numpy.testing.utils import _gen_alignment_data import numpy.core.umath as ncu +from numpy.core import umath_tests as ncu_tests import numpy as np from numpy.testing import ( TestCase, run_module_suite, assert_, assert_equal, assert_raises, - assert_array_equal, assert_almost_equal, assert_array_almost_equal, - dec, assert_allclose, assert_no_warnings, suppress_warnings + assert_raises_regex, assert_array_equal, assert_almost_equal, + assert_array_almost_equal, dec, assert_allclose, assert_no_warnings, + suppress_warnings ) @@ -1568,51 +1571,30 @@ def __array__(self): assert_equal(ncu.maximum(a, B()), 0) assert_equal(ncu.maximum(a, C()), 0) - def test_ufunc_override_disabled(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - # This test should be removed when __numpy_ufunc__ is re-enabled. - - class MyArray(object): - def __numpy_ufunc__(self, *args, **kwargs): - self._numpy_ufunc_called = True - - my_array = MyArray() - real_array = np.ones(10) - assert_raises(TypeError, lambda: real_array + my_array) - assert_raises(TypeError, np.add, real_array, my_array) - assert not hasattr(my_array, "_numpy_ufunc_called") - - def test_ufunc_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): - return self, func, method, pos, inputs, kwargs + def __array_ufunc__(self, func, method, *inputs, **kwargs): + return self, func, method, inputs, kwargs a = A() b = np.matrix([1]) res0 = np.multiply(a, b) - res1 = np.dot(a, b) + res1 = np.multiply(b, b, out=a) # self assert_equal(res0[0], a) assert_equal(res1[0], a) assert_equal(res0[1], np.multiply) - assert_equal(res1[1], np.dot) + assert_equal(res1[1], np.multiply) assert_equal(res0[2], '__call__') assert_equal(res1[2], '__call__') - assert_equal(res0[3], 0) - assert_equal(res1[3], 0) - assert_equal(res0[4], (a, b)) - assert_equal(res1[4], (a, b)) - assert_equal(res0[5], {}) - assert_equal(res1[5], {}) + assert_equal(res0[3], (a, b)) + assert_equal(res1[3], (b, b)) + assert_equal(res0[4], {}) + assert_equal(res1[4], {'out': (a,)}) def test_ufunc_override_mro(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return # Some multi arg functions for testing. def tres_mul(a, b, c): @@ -1626,23 +1608,23 @@ def quatro_mul(a, b, c, d): four_mul_ufunc = np.frompyfunc(quatro_mul, 4, 1) class A(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "A" class ASub(A): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "ASub" class B(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "B" class C(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented - class CSub(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + class CSub(C): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented a = A() @@ -1704,12 +1686,10 @@ def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): assert_raises(TypeError, four_mul_ufunc, 1, c, c_sub, c) def test_ufunc_override_methods(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return self, ufunc, method, pos, inputs, kwargs + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + return self, ufunc, method, inputs, kwargs # __call__ a = A() @@ -1717,21 +1697,24 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], '__call__') - assert_equal(res[3], 1) - assert_equal(res[4], (1, a)) - assert_equal(res[5], {'foo': 'bar', 'answer': 42}) + assert_equal(res[3], (1, a)) + assert_equal(res[4], {'foo': 'bar', 'answer': 42}) + + # __call__, wrong args + assert_raises(TypeError, np.multiply, a) + assert_raises(TypeError, np.multiply, a, a, a, a) + assert_raises(TypeError, np.multiply, a, a, sig='a', signature='a') # reduce, positional args res = np.multiply.reduce(a, 'axis0', 'dtype0', 'out0', 'keep0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'keepdims': 'keep0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'keepdims': 'keep0', + 'axis': 'axis0'}) # reduce, kwargs res = np.multiply.reduce(a, axis='axis0', dtype='dtype0', out='out0', @@ -1739,23 +1722,32 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'keepdims': 'keep0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'keepdims': 'keep0', + 'axis': 'axis0'}) + + # reduce, output equal to None removed. + res = np.multiply.reduce(a, out=None) + assert_equal(res[4], {}) + res = np.multiply.reduce(a, out=(None,)) + assert_equal(res[4], {}) + + # reduce, wrong args + assert_raises(TypeError, np.multiply.reduce, a, out=()) + assert_raises(TypeError, np.multiply.reduce, a, out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.reduce, a, 'axis0', axis='axis0') # accumulate, pos args res = np.multiply.accumulate(a, 'axis0', 'dtype0', 'out0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'accumulate') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) # accumulate, kwargs res = np.multiply.accumulate(a, axis='axis0', dtype='dtype0', @@ -1763,22 +1755,33 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'accumulate') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) + + # accumulate, output equal to None removed. + res = np.multiply.accumulate(a, out=None) + assert_equal(res[4], {}) + res = np.multiply.accumulate(a, out=(None,)) + assert_equal(res[4], {}) + + # accumulate, wrong args + assert_raises(TypeError, np.multiply.accumulate, a, out=()) + assert_raises(TypeError, np.multiply.accumulate, a, + out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.accumulate, a, + 'axis0', axis='axis0') # reduceat, pos args res = np.multiply.reduceat(a, [4, 2], 'axis0', 'dtype0', 'out0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduceat') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2])) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a, [4, 2])) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) # reduceat, kwargs res = np.multiply.reduceat(a, [4, 2], axis='axis0', dtype='dtype0', @@ -1786,39 +1789,55 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduceat') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2])) - assert_equal(res[5], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + assert_equal(res[3], (a, [4, 2])) + assert_equal(res[4], {'dtype':'dtype0', + 'out': ('out0',), + 'axis': 'axis0'}) + + # reduceat, output equal to None removed. + res = np.multiply.reduceat(a, [4, 2], out=None) + assert_equal(res[4], {}) + res = np.multiply.reduceat(a, [4, 2], out=(None,)) + assert_equal(res[4], {}) + + # reduceat, wrong args + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], out=()) + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], + out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], + 'axis0', axis='axis0') # outer res = np.multiply.outer(a, 42) assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'outer') - assert_equal(res[3], 0) - assert_equal(res[4], (a, 42)) - assert_equal(res[5], {}) + assert_equal(res[3], (a, 42)) + assert_equal(res[4], {}) + + # outer, wrong args + assert_raises(TypeError, np.multiply.outer, a) + assert_raises(TypeError, np.multiply.outer, a, a, a, a) # at res = np.multiply.at(a, [4, 2], 'b0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'at') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2], 'b0')) + assert_equal(res[3], (a, [4, 2], 'b0')) + + # at, wrong args + assert_raises(TypeError, np.multiply.at, a) + assert_raises(TypeError, np.multiply.at, a, a, a, a) def test_ufunc_override_out(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return kwargs class B(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return kwargs a = A() @@ -1830,12 +1849,12 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): res4 = np.multiply(a, 4, 'out_arg') res5 = np.multiply(a, 5, out='out_arg') - assert_equal(res0['out'], 'out_arg') - assert_equal(res1['out'], 'out_arg') - assert_equal(res2['out'], 'out_arg') - assert_equal(res3['out'], 'out_arg') - assert_equal(res4['out'], 'out_arg') - assert_equal(res5['out'], 'out_arg') + assert_equal(res0['out'][0], 'out_arg') + assert_equal(res1['out'][0], 'out_arg') + assert_equal(res2['out'][0], 'out_arg') + assert_equal(res3['out'][0], 'out_arg') + assert_equal(res4['out'][0], 'out_arg') + assert_equal(res5['out'][0], 'out_arg') # ufuncs with multiple output modf and frexp. res6 = np.modf(a, 'out0', 'out1') @@ -1845,17 +1864,192 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res7['out'][0], 'out0') assert_equal(res7['out'][1], 'out1') + # While we're at it, check that default output is never passed on. + assert_(np.sin(a, None) == {}) + assert_(np.sin(a, out=None) == {}) + assert_(np.sin(a, out=(None,)) == {}) + assert_(np.modf(a, None) == {}) + assert_(np.modf(a, None, None) == {}) + assert_(np.modf(a, out=(None, None)) == {}) + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', '', DeprecationWarning) + assert_(np.modf(a, out=None) == {}) + assert_(w[0].category is DeprecationWarning) + + # don't give positional and output argument, or too many arguments. + # wrong number of arguments in the tuple is an error too. + assert_raises(TypeError, np.multiply, a, b, 'one', out='two') + assert_raises(TypeError, np.multiply, a, b, 'one', 'two') + assert_raises(TypeError, np.multiply, a, b, out=('one', 'two')) + assert_raises(TypeError, np.multiply, a, out=()) + assert_raises(TypeError, np.modf, a, 'one', out=('two', 'three')) + assert_raises(TypeError, np.modf, a, 'one', 'two', 'three') + assert_raises(TypeError, np.modf, a, out=('one', 'two', 'three')) + assert_raises(TypeError, np.modf, a, out=('one',)) + def test_ufunc_override_exception(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): - def __numpy_ufunc__(self, *a, **kwargs): + def __array_ufunc__(self, *a, **kwargs): raise ValueError("oops") a = A() - for func in [np.divide, np.dot]: - assert_raises(ValueError, func, a, a) + assert_raises(ValueError, np.negative, 1, out=a) + assert_raises(ValueError, np.negative, a) + assert_raises(ValueError, np.divide, 1., a) + + def test_ufunc_override_not_implemented(self): + + class A(object): + __array_ufunc__ = None + + msg = ("operand type(s) do not implement __array_ufunc__(" + ", '__call__', <*>): 'A'") + with assert_raises_regex(TypeError, fnmatch.translate(msg)): + np.negative(A()) + + msg = ("operand type(s) do not implement __array_ufunc__(" + ", '__call__', <*>, , out=(1,)): " + "'A', 'object', 'int'") + with assert_raises_regex(TypeError, fnmatch.translate(msg)): + np.add(A(), object(), out=1) + + def test_gufunc_override(self): + # gufunc are just ufunc instances, but follow a different path, + # so check __array_ufunc__ overrides them properly. + class A(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + return self, ufunc, method, inputs, kwargs + + inner1d = ncu_tests.inner1d + a = A() + res = inner1d(a, a) + assert_equal(res[0], a) + assert_equal(res[1], inner1d) + assert_equal(res[2], '__call__') + assert_equal(res[3], (a, a)) + assert_equal(res[4], {}) + + res = inner1d(1, 1, out=a) + assert_equal(res[0], a) + assert_equal(res[1], inner1d) + assert_equal(res[2], '__call__') + assert_equal(res[3], (1, 1)) + assert_equal(res[4], {'out': (a,)}) + + # wrong number of arguments in the tuple is an error too. + assert_raises(TypeError, inner1d, a, out='two') + assert_raises(TypeError, inner1d, a, a, 'one', out='two') + assert_raises(TypeError, inner1d, a, a, 'one', 'two') + assert_raises(TypeError, inner1d, a, a, out=('one', 'two')) + assert_raises(TypeError, inner1d, a, a, out=()) + + def test_ufunc_override_with_super(self): + + class A(np.ndarray): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + args = [] + in_no = [] + for i, input_ in enumerate(inputs): + if isinstance(input_, A): + in_no.append(i) + args.append(input_.view(np.ndarray)) + else: + args.append(input_) + + outputs = kwargs.pop('out', None) + out_no = [] + if outputs: + out_args = [] + for j, output in enumerate(outputs): + if isinstance(output, A): + out_no.append(j) + out_args.append(output.view(np.ndarray)) + else: + out_args.append(output) + kwargs['out'] = tuple(out_args) + else: + outputs = (None,) * ufunc.nout + + info = {} + if in_no: + info['inputs'] = in_no + if out_no: + info['outputs'] = out_no + + results = super(A, self).__array_ufunc__(ufunc, method, + *args, **kwargs) + if results is NotImplemented: + return NotImplemented + + if method == 'at': + return + + if ufunc.nout == 1: + results = (results,) + + results = tuple((np.asarray(result).view(A) + if output is None else output) + for result, output in zip(results, outputs)) + if results and isinstance(results[0], A): + results[0].info = info + + return results[0] if len(results) == 1 else results + + class B(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + if any(isinstance(input_, A) for input_ in inputs): + return "A!" + else: + return NotImplemented + + d = np.arange(5.) + # 1 input, 1 output + a = np.arange(5.).view(A) + b = np.sin(a) + check = np.sin(d) + assert_(np.all(check == b)) + assert_equal(b.info, {'inputs': [0]}) + b = np.sin(d, out=(a,)) + assert_(np.all(check == b)) + assert_equal(b.info, {'outputs': [0]}) + assert_(b is a) + a = np.arange(5.).view(A) + b = np.sin(a, out=a) + assert_(np.all(check == b)) + assert_equal(b.info, {'inputs': [0], 'outputs': [0]}) + + # 1 input, 2 outputs + a = np.arange(5.).view(A) + b1, b2 = np.modf(a) + assert_equal(b1.info, {'inputs': [0]}) + b1, b2 = np.modf(d, out=(None, a)) + assert_(b2 is a) + assert_equal(b1.info, {'outputs': [1]}) + a = np.arange(5.).view(A) + b = np.arange(5.).view(A) + c1, c2 = np.modf(a, out=(a, b)) + assert_(c1 is a) + assert_(c2 is b) + assert_equal(c1.info, {'inputs': [0], 'outputs': [0, 1]}) + + # 2 input, 1 output + a = np.arange(5.).view(A) + b = np.arange(5.).view(A) + c = np.add(a, b, out=a) + assert_(c is a) + assert_equal(c.info, {'inputs': [0, 1], 'outputs': [0]}) + # some tests with a non-ndarray subclass + a = np.arange(5.) + b = B() + assert_(a.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_(b.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_raises(TypeError, np.add, a, b) + a = a.view(A) + assert_(a.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_(b.__array_ufunc__(np.add, '__call__', a, b) == "A!") + assert_(np.add(a, b) == "A!") + class TestChoose(TestCase): def test_mixed(self): diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index b6c742a2b175..36d8ff97d285 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -1,5 +1,4 @@ -""" -============================= +"""============================= Subclassing ndarray in python ============================= @@ -220,8 +219,9 @@ class other than the class in which it is defined, the ``__init__`` * For the explicit constructor call, our subclass will need to create a new ndarray instance of its own class. In practice this means that we, the authors of the code, will need to make a call to - ``ndarray.__new__(MySubClass,...)``, or do view casting of an existing - array (see below) + ``ndarray.__new__(MySubClass,...)``, a class-hierarchy prepared call to + ``super(MySubClass, cls).__new__(cls, ...)``, or do view casting of an + existing array (see below) * For view casting and new-from-template, the equivalent of ``ndarray.__new__(MySubClass,...`` is called, at the C level. @@ -237,7 +237,7 @@ class other than the class in which it is defined, the ``__init__`` class C(np.ndarray): def __new__(cls, *args, **kwargs): print('In __new__ with class %s' % cls) - return np.ndarray.__new__(cls, *args, **kwargs) + return super(C, cls).__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): # in practice you probably will not need or want an __init__ @@ -275,7 +275,8 @@ def __array_finalize__(self, obj): def __array_finalize__(self, obj): -``ndarray.__new__`` passes ``__array_finalize__`` the new object, of our +One sees that the ``super`` call, which goes to +``ndarray.__new__``, passes ``__array_finalize__`` the new object, of our own class (``self``) as well as the object from which the view has been taken (``obj``). As you can see from the output above, the ``self`` is always a newly created instance of our subclass, and the type of ``obj`` @@ -303,13 +304,14 @@ def __array_finalize__(self, obj): class InfoArray(np.ndarray): def __new__(subtype, shape, dtype=float, buffer=None, offset=0, - strides=None, order=None, info=None): + strides=None, order=None, info=None): # Create the ndarray instance of our type, given the usual # ndarray input arguments. This will call the standard # ndarray constructor, but return an object of our type. # It also triggers a call to InfoArray.__array_finalize__ - obj = np.ndarray.__new__(subtype, shape, dtype, buffer, offset, strides, - order) + obj = super(InfoArray, subtype).__new__(subtype, shape, dtype, + buffer, offset, strides, + order) # set the new 'info' attribute to the value passed obj.info = info # Finally, we must return the newly created object: @@ -412,15 +414,162 @@ def __array_finalize__(self, obj): >>> v.info 'information' -.. _array-wrap: +.. _array-ufunc: + +``__array_ufunc__`` for ufuncs +------------------------------ + + .. versionadded:: 1.13 + +A subclass can override what happens when executing numpy ufuncs on it by +overriding the default ``ndarray.__array_ufunc__`` method. This method is +executed *instead* of the ufunc and should return either the result of the +operation, or :obj:`NotImplemented` if the operation requested is not +implemented. + +The signature of ``__array_ufunc__`` is:: + + def __array_ufunc__(ufunc, method, *inputs, **kwargs): -``__array_wrap__`` for ufuncs -------------------------------------------------------- + - *ufunc* is the ufunc object that was called. + - *method* is a string indicating how the Ufunc was called, either + ``"__call__"`` to indicate it was called directly, or one of its + :ref:`methods`: ``"reduce"``, ``"accumulate"``, + ``"reduceat"``, ``"outer"``, or ``"at"``. + - *inputs* is a tuple of the input arguments to the ``ufunc`` + - *kwargs* contains any optional or keyword arguments passed to the + function. This includes any ``out`` arguments, which are always + contained in a tuple. + +A typical implementation would convert any inputs or ouputs that are +instances of one's own class, pass everything on to a superclass using +``super()``, and finally return the results after possible +back-conversion. An example, taken from the test case +``test_ufunc_override_with_super`` in ``core/tests/test_umath.py``, is the +following. + +.. testcode:: + + input numpy as np + + class A(np.ndarray): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + args = [] + in_no = [] + for i, input_ in enumerate(inputs): + if isinstance(input_, A): + in_no.append(i) + args.append(input_.view(np.ndarray)) + else: + args.append(input_) + + outputs = kwargs.pop('out', None) + out_no = [] + if outputs: + out_args = [] + for j, output in enumerate(outputs): + if isinstance(output, A): + out_no.append(j) + out_args.append(output.view(np.ndarray)) + else: + out_args.append(output) + kwargs['out'] = tuple(out_args) + else: + outputs = (None,) * ufunc.nout + + info = {} + if in_no: + info['inputs'] = in_no + if out_no: + info['outputs'] = out_no + + results = super(A, self).__array_ufunc__(ufunc, method, + *args, **kwargs) + if results is NotImplemented: + return NotImplemented + + if method == 'at': + return + + if ufunc.nout == 1: + results = (results,) + + results = tuple((np.asarray(result).view(A) + if output is None else output) + for result, output in zip(results, outputs)) + if results and isinstance(results[0], A): + results[0].info = info + + return results[0] if len(results) == 1 else results + +So, this class does not actually do anything interesting: it just +converts any instances of its own to regular ndarray (otherwise, we'd +get infinite recursion!), and adds an ``info`` dictionary that tells +which inputs and outputs it converted. Hence, e.g., + +>>> a = np.arange(5.).view(A) +>>> b = np.sin(a) +>>> b.info +{'inputs': [0]} +>>> b = np.sin(np.arange(5.), out=(a,)) +>>> b.info +{'outputs': [0]} +>>> a = np.arange(5.).view(A) +>>> b = np.ones(1).view(A) +>>> c = a + b +>>> c.info +{'inputs': [0, 1]} +>>> a += b +>>> a.info +{'inputs': [0, 1], 'outputs': [0]} + +Note that another approach would be to to use ``getattr(ufunc, +methods)(*inputs, **kwargs)`` instead of the ``super`` call. For this example, +the result would be identical, but there is a difference if another operand +also defines ``__array_ufunc__``. E.g., lets assume that we evalulate +``np.add(a, b)``, where ``b`` is an instance of another class ``B`` that has +an override. If you use ``super`` as in the example, +``ndarray.__array_ufunc__`` will notice that ``b`` has an override, which +means it cannot evaluate the result itself. Thus, it will return +`NotImplemented` and so will our class ``A``. Then, control will be passed +over to ``b``, which either knows how to deal with us and produces a result, +or does not and returns `NotImplemented`, raising a ``TypeError``. + +If instead, we replace our ``super`` call with ``getattr(ufunc, method)``, we +effectively do ``np.add(a.view(np.ndarray), b)``. Again, ``B.__array_ufunc__`` +will be called, but now it sees an ``ndarray`` as the other argument. Likely, +it will know how to handle this, and return a new instance of the ``B`` class +to us. Our example class is not set up to handle this, but it might well be +the best approach if, e.g., one were to re-implement ``MaskedArray`` using + ``__array_ufunc__``. + +As a final note: if the ``super`` route is suited to a given class, an +advantage of using it is that it helps in constructing class hierarchies. +E.g., suppose that our other class ``B`` also used the ``super`` in its +``__array_ufunc__`` implementation, and we created a class ``C`` that depended +on both, i.e., ``class C(A, B)`` (with, for simplicity, not another +``__array_ufunc__`` override). Then any ufunc on an instance of ``C`` would +pass on to ``A.__array_ufunc__``, the ``super`` call in ``A`` would go to +``B.__array_ufunc__``, and the ``super`` call in ``B`` would go to +``ndarray.__array_ufunc__``, thus allowing ``A`` and ``B`` to collaborate. + +.. _array-wrap: -``__array_wrap__`` gets called at the end of numpy ufuncs and other numpy -functions, to allow a subclass to set the type of the return value -and update attributes and metadata. Let's show how this works with an example. -First we make the same subclass as above, but with a different name and +``__array_wrap__`` for ufuncs and other functions +------------------------------------------------- + +Prior to numpy 1.13, the behaviour of ufuncs could only be tuned using +``__array_wrap__`` and ``__array_prepare__``. These two allowed one to +change the output type of a ufunc, but, in constrast to +``__array_ufunc__``, did not allow one to make any changes to the inputs. +It is hoped to eventually deprecate these, but ``__array_wrap__`` is also +used by other numpy functions and methods, such as ``squeeze``, so at the +present time is still needed for full functionality. + +Conceptually, ``__array_wrap__`` "wraps up the action" in the sense of +allowing a subclass to set the type of the return value and update +attributes and metadata. Let's show how this works with an example. First +we return to the simpler example subclass, but with a different name and some print statements: .. testcode:: @@ -446,7 +595,7 @@ def __array_wrap__(self, out_arr, context=None): print(' self is %s' % repr(self)) print(' arr is %s' % repr(out_arr)) # then just call the parent - return np.ndarray.__array_wrap__(self, out_arr, context) + return super(MySubClass, self).__array_wrap__(self, out_arr, context) We run a ufunc on an instance of our new array: @@ -467,13 +616,12 @@ def __array_wrap__(self, out_arr, context=None): >>> ret.info 'spam' -Note that the ufunc (``np.add``) has called the ``__array_wrap__`` method of the -input with the highest ``__array_priority__`` value, in this case -``MySubClass.__array_wrap__``, with arguments ``self`` as ``obj``, and -``out_arr`` as the (ndarray) result of the addition. In turn, the -default ``__array_wrap__`` (``ndarray.__array_wrap__``) has cast the -result to class ``MySubClass``, and called ``__array_finalize__`` - -hence the copying of the ``info`` attribute. This has all happened at the C level. +Note that the ufunc (``np.add``) has called the ``__array_wrap__`` method +with arguments ``self`` as ``obj``, and ``out_arr`` as the (ndarray) result +of the addition. In turn, the default ``__array_wrap__`` +(``ndarray.__array_wrap__``) has cast the result to class ``MySubClass``, +and called ``__array_finalize__`` - hence the copying of the ``info`` +attribute. This has all happened at the C level. But, we could do anything we wanted: @@ -494,11 +642,12 @@ def __array_wrap__(self, arr, context=None): So, by defining a specific ``__array_wrap__`` method for our subclass, we can tweak the output from ufuncs. The ``__array_wrap__`` method requires ``self``, then an argument - which is the result of the ufunc - -and an optional parameter *context*. This parameter is returned by some -ufuncs as a 3-element tuple: (name of the ufunc, argument of the ufunc, -domain of the ufunc). ``__array_wrap__`` should return an instance of -its containing class. See the masked array subclass for an -implementation. +and an optional parameter *context*. This parameter is returned by +ufuncs as a 3-element tuple: (name of the ufunc, arguments of the ufunc, +domain of the ufunc), but is not set by other numpy functions. Though, +as seen above, it is possible to do otherwise, ``__array_wrap__`` should +return an instance of its containing class. See the masked array +subclass for an implementation. In addition to ``__array_wrap__``, which is called on the way out of the ufunc, there is also an ``__array_prepare__`` method which is called on diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index 1d65db55e18e..4cdb76b20ef9 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -8,6 +8,7 @@ from .type_check import * from .index_tricks import * from .function_base import * +from .mixins import * from .nanfunctions import * from .shape_base import * from .stride_tricks import * @@ -29,6 +30,7 @@ __all__ += type_check.__all__ __all__ += index_tricks.__all__ __all__ += function_base.__all__ +__all__ += mixins.__all__ __all__ += shape_base.__all__ __all__ += stride_tricks.__all__ __all__ += twodim_base.__all__ diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py new file mode 100644 index 000000000000..21e4b346f37d --- /dev/null +++ b/numpy/lib/mixins.py @@ -0,0 +1,171 @@ +"""Mixin classes for custom array types that don't inherit from ndarray.""" +from __future__ import division, absolute_import, print_function + +import sys + +from numpy.core import umath as um + +# Nothing should be exposed in the top-level NumPy module. +__all__ = [] + + +def _disables_array_ufunc(obj): + """True when __array_ufunc__ is set to None.""" + try: + return obj.__array_ufunc__ is None + except AttributeError: + return False + + +def _binary_method(ufunc): + """Implement a forward binary method with a ufunc, e.g., __add__.""" + def func(self, other): + if _disables_array_ufunc(other): + return NotImplemented + return ufunc(self, other) + return func + + +def _reflected_binary_method(ufunc): + """Implement a reflected binary method with a ufunc, e.g., __radd__.""" + def func(self, other): + if _disables_array_ufunc(other): + return NotImplemented + return ufunc(other, self) + return func + + +def _inplace_binary_method(ufunc): + """Implement an in-place binary method with a ufunc, e.g., __iadd__.""" + def func(self, other): + return ufunc(self, other, out=(self,)) + return func + + +def _numeric_methods(ufunc): + """Implement forward, reflected and inplace binary methods with a ufunc.""" + return (_binary_method(ufunc), + _reflected_binary_method(ufunc), + _inplace_binary_method(ufunc)) + + +def _unary_method(ufunc): + """Implement a unary special method with a ufunc.""" + def func(self): + return ufunc(self) + return func + + +class NDArrayOperatorsMixin(object): + """Mixin defining all operator special methods using __array_ufunc__. + + This class implements the special methods for almost all of Python's + builtin operators defined in the `operator` module, including comparisons + (``==``, ``>``, etc.) and arithmetic (``+``, ``*``, ``-``, etc.), by + deferring to the ``__array_ufunc__`` method, which subclasses must + implement. + + This class does not yet implement the special operators corresponding + to ``divmod``, unary ``+`` or ``matmul`` (``@``), because these operation + do not yet have corresponding NumPy ufuncs. + + It is useful for writing classes that do not inherit from `numpy.ndarray`, + but that should support arithmetic and numpy universal functions like + arrays as described in :ref:`A Mechanism for Overriding Ufuncs + `. + + As an trivial example, consider this implementation of an ``ArrayLike`` + class that simply wraps a NumPy array and ensures that the result of any + arithmetic operation is also an ``ArrayLike`` object:: + + class ArrayLike(np.lib.mixins.NDArrayOperatorsMixin): + def __init__(self, value): + self.value = np.asarray(value) + + # One might also consider adding the built-in list type to this + # list, to support operations like np.add(array_like, list) + _HANDLED_TYPES = (np.ndarray, numbers.Number) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.get('out', ()) + for x in inputs + out: + # Only support operations with instances of _HANDLED_TYPES. + # Use ArrayLike instead of type(self) for isinstance to + # allow subclasses that don't override __array_ufunc__ to + # handle ArrayLike objects. + if not isinstance(x, self._HANDLED_TYPES + (ArrayLike,)): + return NotImplemented + + # Defer to the implementation of the ufunc on unwrapped values. + inputs = tuple(x.value if isinstance(x, ArrayLike) else x + for x in inputs) + if out: + kwargs['out'] = tuple( + x.value if isinstance(x, ArrayLike) else x + for x in out) + result = getattr(ufunc, method)(*inputs, **kwargs) + + if type(result) is tuple: + # multiple return values + return tuple(type(self)(x) for x in result) + elif method == 'at': + # no return value + return None + else: + # one return value + return type(self)(result) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.value) + + In interactions between ``ArrayLike`` objects and numbers or numpy arrays, + the result is always another ``ArrayLike``: + + >>> x = ArrayLike([1, 2, 3]) + >>> x - 1 + ArrayLike(array([0, 1, 2])) + >>> 1 - x + ArrayLike(array([ 0, -1, -2])) + >>> np.arange(3) - x + ArrayLike(array([-1, -1, -1])) + >>> x - np.arange(3) + ArrayLike(array([1, 1, 1])) + + Note that unlike ``numpy.ndarray``, ``ArrayLike`` does not allow operations + with arbitrary, unrecognized types. This ensures that interactions with + ArrayLike preserve a well-defined casting hierarchy. + """ + # Like np.ndarray, this mixin class implements "Option 1" from the ufunc + # overrides NEP. + + # comparisons don't have reflected and in-place versions + __lt__ = _binary_method(um.less) + __le__ = _binary_method(um.less_equal) + __eq__ = _binary_method(um.equal) + __ne__ = _binary_method(um.not_equal) + __gt__ = _binary_method(um.greater) + __ge__ = _binary_method(um.greater_equal) + + # numeric methods + __add__, __radd__, __iadd__ = _numeric_methods(um.add) + __sub__, __rsub__, __isub__ = _numeric_methods(um.subtract) + __mul__, __rmul__, __imul__ = _numeric_methods(um.multiply) + if sys.version_info.major < 3: + # Python 3 uses only __truediv__ and __floordiv__ + __div__, __rdiv__, __idiv__ = _numeric_methods(um.divide) + __truediv__, __rtruediv__, __itruediv__ = _numeric_methods(um.true_divide) + __floordiv__, __rfloordiv__, __ifloordiv__ = _numeric_methods( + um.floor_divide) + __mod__, __rmod__, __imod__ = _numeric_methods(um.mod) + # TODO: handle the optional third argument for __pow__? + __pow__, __rpow__, __ipow__ = _numeric_methods(um.power) + __lshift__, __rlshift__, __ilshift__ = _numeric_methods(um.left_shift) + __rshift__, __rrshift__, __irshift__ = _numeric_methods(um.right_shift) + __and__, __rand__, __iand__ = _numeric_methods(um.bitwise_and) + __xor__, __rxor__, __ixor__ = _numeric_methods(um.bitwise_xor) + __or__, __ror__, __ior__ = _numeric_methods(um.bitwise_or) + + # unary methods + __neg__ = _unary_method(um.negative) + __abs__ = _unary_method(um.absolute) + __invert__ = _unary_method(um.invert) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py new file mode 100644 index 000000000000..57c4a4cd80e0 --- /dev/null +++ b/numpy/lib/tests/test_mixins.py @@ -0,0 +1,200 @@ +from __future__ import division, absolute_import, print_function + +import numbers +import operator +import sys + +import numpy as np +from numpy.testing import ( + TestCase, run_module_suite, assert_, assert_equal, assert_raises) + + +PY2 = sys.version_info.major < 3 + + +# NOTE: This class should be kept as an exact copy of the example from the +# docstring for NDArrayOperatorsMixin. + +class ArrayLike(np.lib.mixins.NDArrayOperatorsMixin): + def __init__(self, value): + self.value = np.asarray(value) + + # One might also consider adding the built-in list type to this + # list, to support operations like np.add(array_like, list) + _HANDLED_TYPES = (np.ndarray, numbers.Number) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.get('out', ()) + for x in inputs + out: + # Only support operations with instances of _HANDLED_TYPES. + # Use ArrayLike instead of type(self) for isinstance to + # allow subclasses that don't override __array_ufunc__ to + # handle ArrayLike objects. + if not isinstance(x, self._HANDLED_TYPES + (ArrayLike,)): + return NotImplemented + + # Defer to the implementation of the ufunc on unwrapped values. + inputs = tuple(x.value if isinstance(x, ArrayLike) else x + for x in inputs) + if out: + kwargs['out'] = tuple( + x.value if isinstance(x, ArrayLike) else x + for x in out) + result = getattr(ufunc, method)(*inputs, **kwargs) + + if type(result) is tuple: + # multiple return values + return tuple(type(self)(x) for x in result) + elif method == 'at': + # no return value + return None + else: + # one return value + return type(self)(result) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.value) + + +def _assert_equal_type_and_value(result, expected, err_msg=None): + assert_equal(type(result), type(expected), err_msg=err_msg) + assert_equal(result.value, expected.value, err_msg=err_msg) + assert_equal(getattr(result.value, 'dtype', None), + getattr(expected.value, 'dtype', None), err_msg=err_msg) + + +class TestNDArrayOperatorsMixin(TestCase): + + def test_array_like_add(self): + + def check(result): + _assert_equal_type_and_value(result, ArrayLike(0)) + + check(ArrayLike(0) + 0) + check(0 + ArrayLike(0)) + + check(ArrayLike(0) + np.array(0)) + check(np.array(0) + ArrayLike(0)) + + check(ArrayLike(np.array(0)) + 0) + check(0 + ArrayLike(np.array(0))) + + check(ArrayLike(np.array(0)) + np.array(0)) + check(np.array(0) + ArrayLike(np.array(0))) + + def test_inplace(self): + array_like = ArrayLike(np.array([0])) + array_like += 1 + _assert_equal_type_and_value(array_like, ArrayLike(np.array([1]))) + + array = np.array([0]) + array += ArrayLike(1) + _assert_equal_type_and_value(array, ArrayLike(np.array([1]))) + + def test_opt_out(self): + + class OptOut(object): + """Object that opts out of __array_ufunc__.""" + __array_ufunc__ = None + + def __add__(self, other): + return self + + def __radd__(self, other): + return self + + array_like = ArrayLike(1) + opt_out = OptOut() + + # supported operations + assert_(array_like + opt_out is opt_out) + assert_(opt_out + array_like is opt_out) + + # not supported + with assert_raises(TypeError): + # don't use the Python default, array_like = array_like + opt_out + array_like += opt_out + with assert_raises(TypeError): + array_like - opt_out + with assert_raises(TypeError): + opt_out - array_like + + def test_subclass(self): + + class SubArrayLike(ArrayLike): + """Should take precedence over ArrayLike.""" + + x = ArrayLike(0) + y = SubArrayLike(1) + _assert_equal_type_and_value(x + y, y) + _assert_equal_type_and_value(y + x, y) + + def test_object(self): + x = ArrayLike(0) + obj = object() + with assert_raises(TypeError): + x + obj + with assert_raises(TypeError): + obj + x + with assert_raises(TypeError): + x += obj + + def test_unary_methods(self): + array = np.array([-1, 0, 1, 2]) + array_like = ArrayLike(array) + for op in [operator.neg, + # pos is not yet implemented + abs, + operator.invert]: + _assert_equal_type_and_value(op(array_like), ArrayLike(op(array))) + + def test_binary_methods(self): + array = np.array([-1, 0, 1, 2]) + array_like = ArrayLike(array) + operators = [ + operator.lt, + operator.le, + operator.eq, + operator.ne, + operator.gt, + operator.ge, + operator.add, + operator.sub, + operator.mul, + operator.truediv, + operator.floordiv, + # TODO: test div on Python 2, only + operator.mod, + # divmod is not yet implemented + pow, + operator.lshift, + operator.rshift, + operator.and_, + operator.xor, + operator.or_, + ] + for op in operators: + expected = ArrayLike(op(array, 1)) + actual = op(array_like, 1) + err_msg = 'failed for operator {}'.format(op) + _assert_equal_type_and_value(expected, actual, err_msg=err_msg) + + def test_ufunc_at(self): + array = ArrayLike(np.array([1, 2, 3, 4])) + assert_(np.negative.at(array, np.array([0, 1])) is None) + _assert_equal_type_and_value(array, ArrayLike([-1, -2, 3, 4])) + + def test_ufunc_two_outputs(self): + def check(result): + assert_(type(result) is tuple) + assert_equal(len(result), 2) + mantissa, exponent = np.frexp(2 ** -3) + _assert_equal_type_and_value(result[0], ArrayLike(mantissa)) + _assert_equal_type_and_value(result[1], ArrayLike(exponent)) + + check(np.frexp(ArrayLike(2 ** -3))) + check(np.frexp(ArrayLike(np.array(2 ** -3)))) + + +if __name__ == "__main__": + run_module_suite() diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 554fd6dc517e..cb0bfdde296a 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -3933,13 +3933,17 @@ def __repr__(self): def _delegate_binop(self, other): # This emulates the logic in - # multiarray/number.c:PyArray_GenericBinaryFunction - if (not isinstance(other, np.ndarray) - and not hasattr(other, "__numpy_ufunc__")): + # private/binop_override.h:forward_binop_should_defer + if isinstance(other, type(self)): + return False + array_ufunc = getattr(other, "__array_ufunc__", False) + if array_ufunc is False: other_priority = getattr(other, "__array_priority__", -1000000) - if self.__array_priority__ < other_priority: - return True - return False + return self.__array_priority__ < other_priority + else: + # If array_ufunc is not None, it will be called inside the ufunc; + # None explicitly tells us to not call the ufunc, i.e., defer. + return array_ufunc is None def _comparison(self, other, compare): """Compare self with other using operator.eq or operator.ne.