From 8aeb08af1b657d753ae6281f0e3b890b4687632f Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Nov 2023 17:26:08 +0100 Subject: [PATCH 1/9] API: Remove PyArray_REFCNT and NPY_REFCOUNT These don't really make sense, even less as public API. --- doc/release/upcoming_changes/25156.c_api_removal.rst | 1 + doc/source/reference/c-api/array.rst | 4 ---- numpy/__init__.pxd | 1 - numpy/_core/include/numpy/ndarrayobject.h | 4 ---- numpy/_core/src/multiarray/shape.c | 2 +- 5 files changed, 2 insertions(+), 10 deletions(-) create mode 100644 doc/release/upcoming_changes/25156.c_api_removal.rst diff --git a/doc/release/upcoming_changes/25156.c_api_removal.rst b/doc/release/upcoming_changes/25156.c_api_removal.rst new file mode 100644 index 000000000000..a0d6962862aa --- /dev/null +++ b/doc/release/upcoming_changes/25156.c_api_removal.rst @@ -0,0 +1 @@ +* ``PyArray_REFCNT`` and ``NPY_REFCOUNT`` are removed. Use ``Py_REFCNT`` instead. diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index b3083f0a8c4b..d1a794c13879 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3453,10 +3453,6 @@ Miscellaneous Macros Returns the minimum of *a* and *b*. If (*a*) or (*b*) are expressions they are evaluated twice. -.. c:function:: npy_intp PyArray_REFCOUNT(PyObject* op) - - Returns the reference count of any Python object. - .. c:function:: void PyArray_DiscardWritebackIfCopy(PyArrayObject* obj) If ``obj->flags`` has :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, this function diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index b8cc791369da..153a5b8c6885 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -460,7 +460,6 @@ cdef extern from "numpy/arrayobject.h": object PyArray_ZEROS(int nd, npy_intp* dims, int type, int fortran) object PyArray_EMPTY(int nd, npy_intp* dims, int type, int fortran) void PyArray_FILLWBYTE(object, int val) - npy_intp PyArray_REFCOUNT(object) object PyArray_ContiguousFromAny(op, int, int min_depth, int max_depth) unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2) bint PyArray_EquivByteorders(int b1, int b2) nogil diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h index c8da3850b29d..39b56d9051b4 100644 --- a/numpy/_core/include/numpy/ndarrayobject.h +++ b/numpy/_core/include/numpy/ndarrayobject.h @@ -103,10 +103,6 @@ extern "C" { #define PyArray_FILLWBYTE(obj, val) memset(PyArray_DATA(obj), val, \ PyArray_NBYTES(obj)) -#ifndef PYPY_VERSION -#define PyArray_REFCOUNT(obj) (((PyObject *)(obj))->ob_refcnt) -#define NPY_REFCOUNT PyArray_REFCOUNT -#endif #define NPY_MAX_ELSIZE (2 * NPY_SIZEOF_LONGDOUBLE) #define PyArray_ContiguousFromAny(op, type, min_depth, max_depth) \ diff --git a/numpy/_core/src/multiarray/shape.c b/numpy/_core/src/multiarray/shape.c index 2c91edfcddd8..370deba4a90f 100644 --- a/numpy/_core/src/multiarray/shape.c +++ b/numpy/_core/src/multiarray/shape.c @@ -105,7 +105,7 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, "Use the np.resize function or refcheck=False"); return NULL; #else - refcnt = PyArray_REFCOUNT(self); + refcnt = Py_REFCNT(self); #endif /* PYPY_VERSION */ } else { From 1d7ed0a96b84bedb00807a97abf59cc3b91ae373 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 4 Oct 2023 19:00:17 +0200 Subject: [PATCH 2/9] API: Make `intp` `ssize_t` and introduce characters nN This also removes `NPY_INTPLTR` to avoid confusion as it would have to be `p` on old NumPy but should be `n` on new NumPy versions. I suspect all users would use `p` explicitly anyway. Since `p` is pretty clearly defined as pointer, retain that meaning. --- doc/source/reference/arrays.interface.rst | 4 +- doc/source/reference/c-api/config.rst | 4 +- doc/source/reference/c-api/dtype.rst | 15 ++++-- .../code_generators/generate_numpy_api.py | 11 +++++ numpy/_core/include/numpy/_numpyconfig.h.in | 2 + numpy/_core/include/numpy/ndarraytypes.h | 13 ++--- numpy/_core/include/numpy/npy_common.h | 47 +++++++------------ numpy/_core/include/numpy/numpyconfig.h | 3 -- numpy/_core/meson.build | 2 + numpy/_core/numerictypes.py | 8 ++-- numpy/_core/src/multiarray/arraytypes.c.src | 13 ++++- numpy/_core/tests/test_dtype.py | 11 +++-- 12 files changed, 79 insertions(+), 54 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index 163abd59b4dd..b78e8e75cb1f 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -220,8 +220,8 @@ as:: int itemsize; /* size of each element */ int flags; /* flags indicating how the data should be interpreted */ /* must set ARR_HAS_DESCR bit to validate descr */ - Py_intptr_t *shape; /* A length-nd array of shape information */ - Py_intptr_t *strides; /* A length-nd array of stride information */ + Py_ssize_t *shape; /* A length-nd array of shape information */ + Py_ssize_t *strides; /* A length-nd array of stride information */ void *data; /* A pointer to the first element of the array */ PyObject *descr; /* NULL or data-description (same as descr key of __array_interface__) -- must set ARR_HAS_DESCR diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst index 963a106f8a76..097eba9b7089 100644 --- a/doc/source/reference/c-api/config.rst +++ b/doc/source/reference/c-api/config.rst @@ -58,9 +58,11 @@ information is available to the pre-processor. .. c:macro:: NPY_SIZEOF_PY_INTPTR_T + Size of a pointer ``void *`` and ``intptr_t``/``Py_intptr_t``. + .. c:macro:: NPY_SIZEOF_INTP - Size of a pointer on this platform (sizeof(void \*)) + Size of a ``size_t`` on this platform (``sizeof(size_t)``) Platform information diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst index 3a57a4d0d5c7..9e66e172bd52 100644 --- a/doc/source/reference/c-api/dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -363,13 +363,20 @@ to the front of the integer name. .. c:type:: npy_intp - Py_intptr_t (an integer that is the size of a pointer on - the platform). + ``Py_ssize_t`` (a signed integer with the same size as the C ``size_t``). + This is the correct integer for lengths or indexing. In practice this is + normally the size of a pointer, but this is not guaranteed. + + ..note:: + Before NumPy 2.0, this was the same as ``Py_intptr_t``. + While a better match, this did not match actual usage in practice. + On the Python side, we still support ``np.dtype('p')`` to fetch a dtype + compatible with storing pointers, while ``n`` is the correct character + for the ``ssize_t``. .. c:type:: npy_uintp - unsigned Py_intptr_t (an integer that is the size of a pointer on - the platform). + The C ``size_t``/``Py_size_t``. (Complex) Floating point diff --git a/numpy/_core/code_generators/generate_numpy_api.py b/numpy/_core/code_generators/generate_numpy_api.py index 07cda9e3b69d..f8a822511161 100644 --- a/numpy/_core/code_generators/generate_numpy_api.py +++ b/numpy/_core/code_generators/generate_numpy_api.py @@ -81,6 +81,17 @@ return -1; } + /* + * On exceedingly few platforms these sizes may not match, in which case + * We do not support older NumPy versions at all. + */ + if (sizeof(Py_ssize_t) != sizeof(Py_intptr_t) && + PyArray_GetNDArrayCFeatureVersion() < NPY_2_0_API_VERSION) { + PyErr_Format(PyExc_RuntimeError, + "module compiled against NumPy 2.0 but running on NumPy 1.x. " + "Unfortunately, this is not supported on niche platforms where " + "`sizeof(size_t) != sizeof(inptr_t)`."); + } /* * Perform runtime check of C API version. As of now NumPy 2.0 is ABI * backwards compatible (in the exposed feature subset!) for all practical diff --git a/numpy/_core/include/numpy/_numpyconfig.h.in b/numpy/_core/include/numpy/_numpyconfig.h.in index c8b26f59d767..ada5f00b489b 100644 --- a/numpy/_core/include/numpy/_numpyconfig.h.in +++ b/numpy/_core/include/numpy/_numpyconfig.h.in @@ -10,6 +10,8 @@ #mesondefine NPY_SIZEOF_LONGDOUBLE #mesondefine NPY_SIZEOF_COMPLEX_LONGDOUBLE #mesondefine NPY_SIZEOF_PY_INTPTR_T +#mesondefine NPY_SIZEOF_INTP +#mesondefine NPY_SIZEOF_UINTP #mesondefine NPY_SIZEOF_OFF_T #mesondefine NPY_SIZEOF_PY_LONG_LONG #mesondefine NPY_SIZEOF_LONGLONG diff --git a/numpy/_core/include/numpy/ndarraytypes.h b/numpy/_core/include/numpy/ndarraytypes.h index 3ff8c239bb94..f88b9da5c7d0 100644 --- a/numpy/_core/include/numpy/ndarraytypes.h +++ b/numpy/_core/include/numpy/ndarraytypes.h @@ -120,13 +120,14 @@ enum NPY_TYPECHAR { NPY_CHARLTR = 'c', /* - * No Descriptor, just a define -- this let's - * Python users specify an array of integers - * large enough to hold a pointer on the - * platform + * Note, we removed `NPY_INTPLTR` due to changing its definition + * to 'n', rather than 'p'. On any typical platform this is the + * same integer. 'n' should be used for the `np.intp` with the same + * size as `size_t` while 'p' remains pointer sized. + * + * 'p', 'P', 'n', and 'N' are valid and defined explicitly + * in `arraytypes.c.src`. */ - NPY_INTPLTR = 'p', - NPY_UINTPLTR = 'P', /* * These are for dtype 'kinds', not dtype 'typecodes' diff --git a/numpy/_core/include/numpy/npy_common.h b/numpy/_core/include/numpy/npy_common.h index 986e99621af0..13a403491afe 100644 --- a/numpy/_core/include/numpy/npy_common.h +++ b/numpy/_core/include/numpy/npy_common.h @@ -195,11 +195,11 @@ enum { }; /* - * This is to typedef npy_intp to the appropriate pointer size for this - * platform. Py_intptr_t, Py_uintptr_t are defined in pyport.h. + * This is to typedef npy_intp to the appropriate size for Py_ssize_t. + * (Before NumPy 2.0 we used Py_intptr_t and Py_uintptr_t from `pyport.h`.) */ -typedef Py_intptr_t npy_intp; -typedef Py_uintptr_t npy_uintp; +typedef Py_ssize_t npy_intp; +typedef size_t npy_uintp; /* * Define sizes that were not defined in numpyconfig.h. @@ -208,8 +208,6 @@ typedef Py_uintptr_t npy_uintp; #define NPY_SIZEOF_BYTE 1 #define NPY_SIZEOF_DATETIME 8 #define NPY_SIZEOF_TIMEDELTA 8 -#define NPY_SIZEOF_INTP NPY_SIZEOF_PY_INTPTR_T -#define NPY_SIZEOF_UINTP NPY_SIZEOF_PY_INTPTR_T #define NPY_SIZEOF_HALF 2 #define NPY_SIZEOF_CFLOAT NPY_SIZEOF_COMPLEX_FLOAT #define NPY_SIZEOF_CDOUBLE NPY_SIZEOF_COMPLEX_DOUBLE @@ -228,17 +226,7 @@ typedef Py_uintptr_t npy_uintp; * functions use different formatting codes that are portably specified * according to the Python documentation. See issue gh-2388. */ -#if NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONG - #define NPY_INTP NPY_LONG - #define NPY_UINTP NPY_ULONG - #define PyIntpArrType_Type PyLongArrType_Type - #define PyUIntpArrType_Type PyULongArrType_Type - #define NPY_MAX_INTP NPY_MAX_LONG - #define NPY_MIN_INTP NPY_MIN_LONG - #define NPY_MAX_UINTP NPY_MAX_ULONG - #define NPY_INTP_FMT "ld" -#elif NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_INT - /* NumPy always prefers long over int if the size is identical! */ +#if NPY_SIZEOF_INTP == NPY_SIZEOF_INT #define NPY_INTP NPY_INT #define NPY_UINTP NPY_UINT #define PyIntpArrType_Type PyIntArrType_Type @@ -247,7 +235,16 @@ typedef Py_uintptr_t npy_uintp; #define NPY_MIN_INTP NPY_MIN_INT #define NPY_MAX_UINTP NPY_MAX_UINT #define NPY_INTP_FMT "d" -#elif defined(PY_LONG_LONG) && (NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONGLONG) +#elif NPY_SIZEOF_INTP == NPY_SIZEOF_LONG + #define NPY_INTP NPY_LONG + #define NPY_UINTP NPY_ULONG + #define PyIntpArrType_Type PyLongArrType_Type + #define PyUIntpArrType_Type PyULongArrType_Type + #define NPY_MAX_INTP NPY_MAX_LONG + #define NPY_MIN_INTP NPY_MIN_LONG + #define NPY_MAX_UINTP NPY_MAX_ULONG + #define NPY_INTP_FMT "ld" +#elif defined(PY_LONG_LONG) && (NPY_SIZEOF_INTP == NPY_SIZEOF_LONGLONG) #define NPY_INTP NPY_LONGLONG #define NPY_UINTP NPY_ULONGLONG #define PyIntpArrType_Type PyLongLongArrType_Type @@ -256,18 +253,8 @@ typedef Py_uintptr_t npy_uintp; #define NPY_MIN_INTP NPY_MIN_LONGLONG #define NPY_MAX_UINTP NPY_MAX_ULONGLONG #define NPY_INTP_FMT "lld" -#endif - -/* - * We can only use C99 formats for npy_int_p if it is the same as - * intp_t, hence the condition on HAVE_UNITPTR_T - */ -#if (NPY_USE_C99_FORMATS) == 1 \ - && (defined HAVE_UINTPTR_T) \ - && (defined HAVE_INTTYPES_H) - #include - #undef NPY_INTP_FMT - #define NPY_INTP_FMT PRIdPTR +#else + #error "Failed to correctly define NPY_INTP and NPY_UINTP" #endif diff --git a/numpy/_core/include/numpy/numpyconfig.h b/numpy/_core/include/numpy/numpyconfig.h index 61898f0ebdbe..19cffc13f77f 100644 --- a/numpy/_core/include/numpy/numpyconfig.h +++ b/numpy/_core/include/numpy/numpyconfig.h @@ -16,14 +16,11 @@ */ #ifdef __APPLE__ #undef NPY_SIZEOF_LONG - #undef NPY_SIZEOF_PY_INTPTR_T #ifdef __LP64__ #define NPY_SIZEOF_LONG 8 - #define NPY_SIZEOF_PY_INTPTR_T 8 #else #define NPY_SIZEOF_LONG 4 - #define NPY_SIZEOF_PY_INTPTR_T 4 #endif #undef NPY_SIZEOF_LONGDOUBLE diff --git a/numpy/_core/meson.build b/numpy/_core/meson.build index 8ed696531b4c..cc668c62f007 100644 --- a/numpy/_core/meson.build +++ b/numpy/_core/meson.build @@ -109,6 +109,8 @@ types_to_check = [ ['NPY_SIZEOF_FLOAT', 'float'], ['NPY_SIZEOF_DOUBLE', 'double'], ['NPY_SIZEOF_LONGDOUBLE', 'long double'], + ['NPY_SIZEOF_INTP', 'size_t'], + ['NPY_SIZEOF_UINTP', 'size_t'], ] foreach symbol_type: types_to_check cdata.set(symbol_type[0], cc.sizeof(symbol_type[1])) diff --git a/numpy/_core/numerictypes.py b/numpy/_core/numerictypes.py index 14312403838d..620c37ac56da 100644 --- a/numpy/_core/numerictypes.py +++ b/numpy/_core/numerictypes.py @@ -497,14 +497,14 @@ def _scalar_type_key(typ): del key typecodes = {'Character': 'c', - 'Integer': 'bhilqp', - 'UnsignedInteger': 'BHILQP', + 'Integer': 'bhilqnp', + 'UnsignedInteger': 'BHILQNP', 'Float': 'efdg', 'Complex': 'FDG', - 'AllInteger': 'bBhHiIlLqQpP', + 'AllInteger': 'bBhHiIlLqQnNpP', 'AllFloat': 'efdgFDG', 'Datetime': 'Mm', - 'All': '?bhilqpBHILQPefdgFDGSUVOMm'} + 'All': '?bhilqnpBHILQNPefdgFDGSUVOMm'} # backwards compatibility --- deprecated name # Formal deprecation: Numpy 1.20.0, 2020-10-19 (see numpy/__init__.py) diff --git a/numpy/_core/src/multiarray/arraytypes.c.src b/numpy/_core/src/multiarray/arraytypes.c.src index 51ea6d29168f..98e47cb159eb 100644 --- a/numpy/_core/src/multiarray/arraytypes.c.src +++ b/numpy/_core/src/multiarray/arraytypes.c.src @@ -4711,7 +4711,6 @@ set_typeinfo(PyObject *dict) * * #name = BOOL, * BYTE, UBYTE, SHORT, USHORT, INT, UINT, - * INTP, UINTP, * LONG, ULONG, LONGLONG, ULONGLONG, * HALF, FLOAT, DOUBLE, LONGDOUBLE, * CFLOAT, CDOUBLE, CLONGDOUBLE, @@ -4722,6 +4721,18 @@ set_typeinfo(PyObject *dict) _letter_to_num[NPY_@name@LTR] = NPY_@name@; /**end repeat**/ + _letter_to_num['n'] = NPY_INTP; + _letter_to_num['N'] = NPY_UINTP; + +#if NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_INTP + _letter_to_num['p'] = NPY_INTP; + _letter_to_num['P'] = NPY_UINTP; +#elif NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONGLONG + _letter_to_num['p'] = NPY_LONGLONG; + _letter_to_num['P'] = NPY_ULONGLONG; +#else + #error "Did not find correct pointer sized integer." +#endif /**begin repeat * #name = BOOL, diff --git a/numpy/_core/tests/test_dtype.py b/numpy/_core/tests/test_dtype.py index 9fb062b5e4c6..f3fcb30b0b51 100644 --- a/numpy/_core/tests/test_dtype.py +++ b/numpy/_core/tests/test_dtype.py @@ -1666,8 +1666,9 @@ class TestFromCTypes: @staticmethod def check(ctype, dtype): dtype = np.dtype(dtype) - assert_equal(np.dtype(ctype), dtype) - assert_equal(np.dtype(ctype()), dtype) + assert np.dtype(ctype) == dtype + assert np.dtype(ctype()) == dtype + assert ctypes.sizeof(ctype) == dtype.itemsize def test_array(self): c8 = ctypes.c_uint8 @@ -1702,8 +1703,12 @@ def test_pointer(self): p_uint8 = ctypes.POINTER(ctypes.c_uint8) assert_raises(TypeError, np.dtype, p_uint8) + def test_size_t(self): + assert np.dtype(np.uintp) is np.dtype("N") + self.check(ctypes.c_size_t, np.uintp) + def test_void_pointer(self): - self.check(ctypes.c_void_p, np.uintp) + self.check(ctypes.c_void_p, "P") def test_union(self): class Union(ctypes.Union): From 17ac2eeae4b27921688e0dbf008c70565de8c7b7 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 9 Oct 2023 16:51:59 +0200 Subject: [PATCH 3/9] DOC: Add release snippets for intp/uintp redefinition --- .../upcoming_changes/24888.c_api_removal.rst | 2 ++ doc/release/upcoming_changes/24888.change.rst | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 doc/release/upcoming_changes/24888.c_api_removal.rst create mode 100644 doc/release/upcoming_changes/24888.change.rst diff --git a/doc/release/upcoming_changes/24888.c_api_removal.rst b/doc/release/upcoming_changes/24888.c_api_removal.rst new file mode 100644 index 000000000000..1ca24ac8dd31 --- /dev/null +++ b/doc/release/upcoming_changes/24888.c_api_removal.rst @@ -0,0 +1,2 @@ +* ``NPY_INTPLTR`` has been removed to avoid confusion (see ``intp`` + redefinition). diff --git a/doc/release/upcoming_changes/24888.change.rst b/doc/release/upcoming_changes/24888.change.rst new file mode 100644 index 000000000000..b547cc51ddc3 --- /dev/null +++ b/doc/release/upcoming_changes/24888.change.rst @@ -0,0 +1,16 @@ +Redefinition of ``np.intp``/``np.uintp`` (almost never a change) +---------------------------------------------------------------- +Due to the actual use of these types almost always matching the use of +``size_t``/``Py_ssize_t`` this is now the definition in C. +Previously, it matched ``intptr_t`` and ``uintptr_t`` which would often +have been subtly incorrect in principle. +This has no effect on the vast majority of machines since the size +of these types only differ on extremely niche platforms. + +However, in general it means that: +* Pointers may not necessarily fit into an ``intp`` typed + array anymore. The ``p`` and ``P`` character codes can + still be used, however. +* The new character codes ``nN`` were introduced. +* It is now correct to use the Python C-API functions when parsing + to ``npy_intp`` typed arguments. From ab5c7f61e1912f43d2be6706df5b57e4884e2c46 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 17 Oct 2023 13:16:36 +0200 Subject: [PATCH 4/9] MAINT: Fixup release note to explain transition and support n in buffers --- doc/release/upcoming_changes/24888.change.rst | 11 ++++++----- numpy/_core/_internal.py | 2 +- numpy/_core/src/multiarray/buffer.c | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/release/upcoming_changes/24888.change.rst b/doc/release/upcoming_changes/24888.change.rst index b547cc51ddc3..e1b9a572a220 100644 --- a/doc/release/upcoming_changes/24888.change.rst +++ b/doc/release/upcoming_changes/24888.change.rst @@ -3,14 +3,15 @@ Redefinition of ``np.intp``/``np.uintp`` (almost never a change) Due to the actual use of these types almost always matching the use of ``size_t``/``Py_ssize_t`` this is now the definition in C. Previously, it matched ``intptr_t`` and ``uintptr_t`` which would often -have been subtly incorrect in principle. +have been subtly incorrect. This has no effect on the vast majority of machines since the size of these types only differ on extremely niche platforms. -However, in general it means that: -* Pointers may not necessarily fit into an ``intp`` typed - array anymore. The ``p`` and ``P`` character codes can - still be used, however. +However, it means that: +* Pointers may not necessarily fit into an ``intp`` typed array anymore. + The ``p`` and ``P`` character codes can still be used, however. +* Creating ``intptr_t`` or ``uintptr_t`` typed arrays in C remains possible + in a cross-platform way via ``PyArray_DescrFromType('p')``. * The new character codes ``nN`` were introduced. * It is now correct to use the Python C-API functions when parsing to ``npy_intp`` typed arguments. diff --git a/numpy/_core/_internal.py b/numpy/_core/_internal.py index d878ff50182d..1c2bfac6554f 100644 --- a/numpy/_core/_internal.py +++ b/numpy/_core/_internal.py @@ -219,7 +219,7 @@ def _getintp_ctype(): import numpy as np val = dummy_ctype(np.intp) else: - char = dtype('p').char + char = dtype('n').char if char == 'i': val = ctypes.c_int elif char == 'l': diff --git a/numpy/_core/src/multiarray/buffer.c b/numpy/_core/src/multiarray/buffer.c index b3f6397fa3a1..181af007dd7b 100644 --- a/numpy/_core/src/multiarray/buffer.c +++ b/numpy/_core/src/multiarray/buffer.c @@ -1081,6 +1081,8 @@ _pep3118_letter_to_type(char letter, int native, int is_complex) case 'L': return native ? NPY_ULONG : NPY_UINT32; case 'q': return native ? NPY_LONGLONG : NPY_INT64; case 'Q': return native ? NPY_ULONGLONG : NPY_UINT64; + case 'n': return native ? NPY_INTP : -1; + case 'N': return native ? NPY_UINTP : -1; case 'e': return NPY_HALF; case 'f': return is_complex ? NPY_CFLOAT : NPY_FLOAT; case 'd': return is_complex ? NPY_CDOUBLE : NPY_DOUBLE; From 27248ba7040e4c9a1d5891d4929ac18b87ce3a13 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 17 Oct 2023 13:48:13 +0200 Subject: [PATCH 5/9] MAINT: Use Py_intptr_t for NpyIter allocation sizes This is probably unnecessary but for good sport seems to make sense. OpenVMS can have intptr_t at size 8 while size_t and pointers are 4 sized (because you can change the pointer size and the intptr supports either size). In general it probably doesn't care here, in principle, size_t could also be smaller than a pointer though. --- numpy/_core/src/multiarray/nditer_impl.h | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/numpy/_core/src/multiarray/nditer_impl.h b/numpy/_core/src/multiarray/nditer_impl.h index a33e4f90fa4b..0332d78ec913 100644 --- a/numpy/_core/src/multiarray/nditer_impl.h +++ b/numpy/_core/src/multiarray/nditer_impl.h @@ -68,11 +68,11 @@ #endif /**********************************************/ -/* Rounds up a number of bytes to be divisible by sizeof intp */ -#if NPY_SIZEOF_INTP == 4 -#define NPY_INTP_ALIGNED(size) ((size + 0x3)&(-0x4)) +/* Rounds up a number of bytes to be divisible by sizeof intptr_t */ +#if NPY_SIZEOF_PY_INTPTR_T == 4 +#define NPY_PTR_ALIGNED(size) ((size + 0x3)&(-0x4)) #else -#define NPY_INTP_ALIGNED(size) ((size + 0x7)&(-0x8)) +#define NPY_PTR_ALIGNED(size) ((size + 0x7)&(-0x8)) #endif /* Internal iterator flags */ @@ -165,20 +165,20 @@ typedef npy_int16 npyiter_opitflags; /* Byte sizes of the iterator members */ #define NIT_PERM_SIZEOF(itflags, ndim, nop) \ - NPY_INTP_ALIGNED(NPY_MAXDIMS) + NPY_PTR_ALIGNED(NPY_MAXDIMS) #define NIT_DTYPES_SIZEOF(itflags, ndim, nop) \ - ((NPY_SIZEOF_INTP)*(nop)) + ((NPY_SIZEOF_PY_INTPTR_T)*(nop)) #define NIT_RESETDATAPTR_SIZEOF(itflags, ndim, nop) \ - ((NPY_SIZEOF_INTP)*(nop+1)) + ((NPY_SIZEOF_PY_INTPTR_T)*(nop+1)) #define NIT_BASEOFFSETS_SIZEOF(itflags, ndim, nop) \ - ((NPY_SIZEOF_INTP)*(nop+1)) + ((NPY_SIZEOF_PY_INTPTR_T)*(nop+1)) /* Could be sizeof intp */ #define NIT_OPERANDS_SIZEOF(itflags, ndim, nop) \ - ((NPY_SIZEOF_INTP)*(nop)) + ((NPY_SIZEOF_PY_INTPTR_T)*(nop)) #define NIT_OPITFLAGS_SIZEOF(itflags, ndim, nop) \ - (NPY_INTP_ALIGNED(sizeof(npyiter_opitflags) * nop)) + (NPY_PTR_ALIGNED(sizeof(npyiter_opitflags) * nop)) #define NIT_BUFFERDATA_SIZEOF(itflags, ndim, nop) \ ((itflags&NPY_ITFLAG_BUFFER) ? ( \ - (NPY_SIZEOF_INTP)*(6 + 5*nop) + sizeof(NpyIter_TransferInfo) * nop) : 0) + (NPY_SIZEOF_PY_INTPTR_T)*(6 + 5*nop) + sizeof(NpyIter_TransferInfo) * nop) : 0) /* Byte offsets of the iterator members starting from iter->iter_flexdata */ #define NIT_PERM_OFFSET() \ @@ -245,14 +245,14 @@ struct NpyIter_TransferInfo_tag { NPY_cast_info read; NPY_cast_info write; NPY_traverse_info clear; - /* Probably unnecessary, but make sure what follows is intp aligned: */ - npy_intp _unused_ensure_alignment[]; + /* Probably unnecessary, but make sure what follows is intptr aligned: */ + Py_intptr_t _unused_ensure_alignment[]; }; struct NpyIter_BufferData_tag { npy_intp buffersize, size, bufiterend, reduce_pos, reduce_outersize, reduce_outerdim; - npy_intp bd_flexdata; + Py_intptr_t bd_flexdata; }; #define NBF_BUFFERSIZE(bufferdata) ((bufferdata)->buffersize) @@ -277,7 +277,7 @@ struct NpyIter_BufferData_tag { /* Internal-only AXISDATA MEMBER ACCESS. */ struct NpyIter_AxisData_tag { npy_intp shape, index; - npy_intp ad_flexdata; + Py_intptr_t ad_flexdata; }; #define NAD_SHAPE(axisdata) ((axisdata)->shape) #define NAD_INDEX(axisdata) ((axisdata)->index) @@ -297,7 +297,7 @@ struct NpyIter_AxisData_tag { 1 + \ /* intp stride[nop+1] AND char* ptr[nop+1] */ \ 2*((nop)+1) \ - )*(size_t)NPY_SIZEOF_INTP) + )*(size_t)NPY_SIZEOF_PY_INTPTR_T) /* * Macro to advance an AXISDATA pointer by a specified count. From 0df27eb1ad40509975292ec74f5a70befe9815ed Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 17 Oct 2023 13:52:26 +0200 Subject: [PATCH 6/9] TYP: Update intp typing codes (unfortunately p and P are now gone) --- numpy/_typing/_char_codes.py | 4 ++-- numpy/typing/mypy_plugin.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/_typing/_char_codes.py b/numpy/_typing/_char_codes.py index a9546123ed1e..e5c4fa5d1bd2 100644 --- a/numpy/_typing/_char_codes.py +++ b/numpy/_typing/_char_codes.py @@ -22,7 +22,7 @@ _ByteCodes = Literal["byte", "b", "=b", "b"] _ShortCodes = Literal["short", "h", "=h", "h"] _IntCCodes = Literal["intc", "i", "=i", "i"] -_IntPCodes = Literal["intp", "int", "int_", "p", "=p", "p"] +_IntPCodes = Literal["intp", "int", "int_", "n", "=n", "n"] _LongCodes = Literal["long", "l", "=l", "l"] _IntCodes = _IntPCodes _LongLongCodes = Literal["longlong", "q", "=q", "q"] @@ -30,7 +30,7 @@ _UByteCodes = Literal["ubyte", "B", "=B", "B"] _UShortCodes = Literal["ushort", "H", "=H", "H"] _UIntCCodes = Literal["uintc", "I", "=I", "I"] -_UIntPCodes = Literal["uintp", "uint", "P", "=P", "P"] +_UIntPCodes = Literal["uintp", "uint", "N", "=N", "N"] _ULongCodes = Literal["ulong", "L", "=L", "L"] _UIntCodes = _UIntPCodes _ULongLongCodes = Literal["ulonglong", "Q", "=Q", "Q"] diff --git a/numpy/typing/mypy_plugin.py b/numpy/typing/mypy_plugin.py index 42c5ad17cba7..63f063ccc795 100644 --- a/numpy/typing/mypy_plugin.py +++ b/numpy/typing/mypy_plugin.py @@ -94,8 +94,8 @@ def _get_extended_precision_list() -> list[str]: def _get_c_intp_name() -> str: - # Adapted from `np._core._internal._getintp_ctype` - char = np.dtype('p').char + # Adapted from `np.core._internal._getintp_ctype` + char = np.dtype('n').char if char == 'i': return "c_int" elif char == 'l': From a736d41272d2327fdc8b34a3e6068b7773ae4d53 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 17 Oct 2023 14:49:18 +0200 Subject: [PATCH 7/9] DOC: Adjust casting table to include the new `nN` --- doc/source/user/basics.ufuncs.rst | 56 ++++++++++++++++--------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/doc/source/user/basics.ufuncs.rst b/doc/source/user/basics.ufuncs.rst index 58bcfdb35ce5..afe0043f2187 100644 --- a/doc/source/user/basics.ufuncs.rst +++ b/doc/source/user/basics.ufuncs.rst @@ -217,33 +217,35 @@ a different table. ... print() ... >>> print_table(np.typecodes['All']) -X ? b h i l q p B H I L Q P e f d g F D G S U V O M m -? Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y -b - Y Y Y Y Y Y - - - - - - Y Y Y Y Y Y Y Y Y Y Y - Y -h - - Y Y Y Y Y - - - - - - - Y Y Y Y Y Y Y Y Y Y - Y -i - - - Y Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y -l - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y -q - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y -p - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y -B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y -H - - - Y Y Y Y - Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y -I - - - - Y Y Y - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y -L - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - - -Q - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - - -P - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - - -e - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - - -f - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - - -d - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - - -g - - - - - - - - - - - - - - - - Y - - Y Y Y Y Y - - -F - - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y - - -D - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y - - -G - - - - - - - - - - - - - - - - - - - Y Y Y Y Y - - -S - - - - - - - - - - - - - - - - - - - - Y Y Y Y - - -U - - - - - - - - - - - - - - - - - - - - - Y Y Y - - -V - - - - - - - - - - - - - - - - - - - - - - Y Y - - -O - - - - - - - - - - - - - - - - - - - - - - - Y - - -M - - - - - - - - - - - - - - - - - - - - - - Y Y Y - -m - - - - - - - - - - - - - - - - - - - - - - Y Y - Y +X ? b h i l q n p B H I L Q N P e f d g F D G S U V O M m +? Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y +b - Y Y Y Y Y Y Y - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - Y +h - - Y Y Y Y Y Y - - - - - - - - Y Y Y Y Y Y Y Y Y Y - Y +i - - - Y Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y +l - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y +q - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y +n - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y +p - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y +B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y +H - - - Y Y Y Y Y - Y Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y +I - - - - Y Y Y Y - - Y Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y +L - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - - +Q - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - - +N - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - - +P - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - - +e - - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - - +f - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - - +d - - - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - - +g - - - - - - - - - - - - - - - - - - Y - - Y Y Y Y Y - - +F - - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y - - +D - - - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y - - +G - - - - - - - - - - - - - - - - - - - - - Y Y Y Y Y - - +S - - - - - - - - - - - - - - - - - - - - - - Y Y Y Y - - +U - - - - - - - - - - - - - - - - - - - - - - - Y Y Y - - +V - - - - - - - - - - - - - - - - - - - - - - - - Y Y - - +O - - - - - - - - - - - - - - - - - - - - - - - - - Y - - +M - - - - - - - - - - - - - - - - - - - - - - - - Y Y Y - +m - - - - - - - - - - - - - - - - - - - - - - - - Y Y - Y You should note that, while included in the table for completeness, the 'S', 'U', and 'V' types cannot be operated on by ufuncs. Also, From b6328e9793ae62d8d6d17288e412087484432555 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 27 Oct 2023 10:27:19 +0200 Subject: [PATCH 8/9] DOC: Update the type table to mention the codes p and P --- doc/source/user/basics.types.rst | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/doc/source/user/basics.types.rst b/doc/source/user/basics.types.rst index 0deb05fdb7ff..337227a629c6 100644 --- a/doc/source/user/basics.types.rst +++ b/doc/source/user/basics.types.rst @@ -67,13 +67,24 @@ confusion with builtin python type names, such as `numpy.bool_`. * - `numpy.intp` - N/A - - ``intptr_t`` (defined in ``stdint.h``) - - Platform-defined integer type capable of holding a pointer. + - ``ssize_t``/``Py_ssize_t`` + - Platform-defined integer of size ``size_t``; used e.g. for sizes. * - `numpy.uintp` - N/A - - ``uintptr_t`` (defined in ``stdint.h``) - - Platform-defined integer type capable of holding a pointer without sign. + - ``size_t`` + - Platform-defined integer type capable of storing the maximum + allocation size. + + * - N/A + - ``'p'`` + - ``intptr_t`` + - Guaranteed to hold pointers. Character code only (Python and C). + + * - N/A + - ``'P'`` + - ``uintptr_t`` + - Guaranteed to hold pointers. Character code only (Python and C). * - `numpy.int32` or `numpy.int64` - `numpy.long` From 82d0941b78687b9f4edaff2631e8a826fe7235b3 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 3 Nov 2023 12:49:21 +0100 Subject: [PATCH 9/9] BUG: Fix order: long should be preferred over int for `intp`/default (Might have gotten confused in the merge conflict?) --- numpy/_core/include/numpy/npy_common.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/numpy/_core/include/numpy/npy_common.h b/numpy/_core/include/numpy/npy_common.h index 13a403491afe..8e40a44000ff 100644 --- a/numpy/_core/include/numpy/npy_common.h +++ b/numpy/_core/include/numpy/npy_common.h @@ -226,16 +226,7 @@ typedef size_t npy_uintp; * functions use different formatting codes that are portably specified * according to the Python documentation. See issue gh-2388. */ -#if NPY_SIZEOF_INTP == NPY_SIZEOF_INT - #define NPY_INTP NPY_INT - #define NPY_UINTP NPY_UINT - #define PyIntpArrType_Type PyIntArrType_Type - #define PyUIntpArrType_Type PyUIntArrType_Type - #define NPY_MAX_INTP NPY_MAX_INT - #define NPY_MIN_INTP NPY_MIN_INT - #define NPY_MAX_UINTP NPY_MAX_UINT - #define NPY_INTP_FMT "d" -#elif NPY_SIZEOF_INTP == NPY_SIZEOF_LONG +#if NPY_SIZEOF_INTP == NPY_SIZEOF_LONG #define NPY_INTP NPY_LONG #define NPY_UINTP NPY_ULONG #define PyIntpArrType_Type PyLongArrType_Type @@ -244,6 +235,15 @@ typedef size_t npy_uintp; #define NPY_MIN_INTP NPY_MIN_LONG #define NPY_MAX_UINTP NPY_MAX_ULONG #define NPY_INTP_FMT "ld" +#elif NPY_SIZEOF_INTP == NPY_SIZEOF_INT + #define NPY_INTP NPY_INT + #define NPY_UINTP NPY_UINT + #define PyIntpArrType_Type PyIntArrType_Type + #define PyUIntpArrType_Type PyUIntArrType_Type + #define NPY_MAX_INTP NPY_MAX_INT + #define NPY_MIN_INTP NPY_MIN_INT + #define NPY_MAX_UINTP NPY_MAX_UINT + #define NPY_INTP_FMT "d" #elif defined(PY_LONG_LONG) && (NPY_SIZEOF_INTP == NPY_SIZEOF_LONGLONG) #define NPY_INTP NPY_LONGLONG #define NPY_UINTP NPY_ULONGLONG