numpy · mattip · Nov 22, 2023 · Nov 15, 2023 · Oct 4, 2023 · Oct 9, 2023
diff --git a/doc/release/upcoming_changes/24888.c_api_removal.rst b/doc/release/upcoming_changes/24888.c_api_removal.rst
@@ -0,0 +1,2 @@
+* ``NPY_INTPLTR`` has been removed to avoid confusion (see ``intp``
+  redefinition).
diff --git a/doc/release/upcoming_changes/24888.change.rst b/doc/release/upcoming_changes/24888.change.rst
@@ -0,0 +1,17 @@
+Redefinition of ``np.intp``/``np.uintp`` (almost never a change)
+----------------------------------------------------------------
+Due to the actual use of these types almost always matching the use of
+``size_t``/``Py_ssize_t`` this is now the definition in C.
+Previously, it matched ``intptr_t`` and ``uintptr_t`` which would often
+have been subtly incorrect.
+This has no effect on the vast majority of machines since the size
+of these types only differ on extremely niche platforms.
+
+However, it means that:
+* Pointers may not necessarily fit into an ``intp`` typed array anymore.
+  The ``p`` and ``P`` character codes can still be used, however.
+* Creating ``intptr_t`` or ``uintptr_t`` typed arrays in C remains possible
+  in a cross-platform way via ``PyArray_DescrFromType('p')``.
+* The new character codes ``nN`` were introduced.
+* It is now correct to use the Python C-API functions when parsing
+  to ``npy_intp`` typed arguments.
diff --git a/doc/release/upcoming_changes/25156.c_api_removal.rst b/doc/release/upcoming_changes/25156.c_api_removal.rst
@@ -0,0 +1 @@
+* ``PyArray_REFCNT`` and ``NPY_REFCOUNT`` are removed. Use ``Py_REFCNT`` instead.
diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst
@@ -220,8 +220,8 @@ as::
     int itemsize;         /* size of each element */
     int flags;            /* flags indicating how the data should be interpreted */
                           /*   must set ARR_HAS_DESCR bit to validate descr */
-    Py_intptr_t *shape;   /* A length-nd array of shape information */
-    Py_intptr_t *strides; /* A length-nd array of stride information */
+    Py_ssize_t *shape;    /* A length-nd array of shape information */
+    Py_ssize_t *strides;  /* A length-nd array of stride information */
     void *data;           /* A pointer to the first element of the array */
     PyObject *descr;      /* NULL or data-description (same as descr key
                                   of __array_interface__) -- must set ARR_HAS_DESCR

diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst
@@ -3453,10 +3453,6 @@ Miscellaneous Macros
     Returns the minimum of *a* and *b*. If (*a*) or (*b*) are
     expressions they are evaluated twice.
 
-.. c:function:: npy_intp PyArray_REFCOUNT(PyObject* op)
-
-    Returns the reference count of any Python object.
-
 .. c:function:: void PyArray_DiscardWritebackIfCopy(PyArrayObject* obj)
 
     If ``obj->flags`` has :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, this function

diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst
@@ -58,9 +58,11 @@ information is available to the pre-processor.
 
 .. c:macro:: NPY_SIZEOF_PY_INTPTR_T
 
+    Size of a pointer ``void *`` and ``intptr_t``/``Py_intptr_t``.
+
 .. c:macro:: NPY_SIZEOF_INTP
 
-    Size of a pointer on this platform (sizeof(void \*))
+    Size of a ``size_t`` on this platform (``sizeof(size_t)``)
 
 
 Platform information

diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst
@@ -363,13 +363,20 @@ to the front of the integer name.
 
 .. c:type:: npy_intp
 
-    Py_intptr_t (an integer that is the size of a pointer on
-    the platform).
+    ``Py_ssize_t`` (a signed integer with the same size as the C ``size_t``).
+    This is the correct integer for lengths or indexing.  In practice this is
+    normally the size of a pointer, but this is not guaranteed.
+
+    ..note::
+        Before NumPy 2.0, this was the same as ``Py_intptr_t``.
+        While a better match, this did not match actual usage in practice.
+        On the Python side, we still support ``np.dtype('p')`` to fetch a dtype
+        compatible with storing pointers, while ``n`` is the correct character
+        for the ``ssize_t``.
 
 .. c:type:: npy_uintp
 
-    unsigned Py_intptr_t (an integer that is the size of a pointer on
-    the platform).
+    The C ``size_t``/``Py_size_t``.
 
 
 (Complex) Floating point

diff --git a/doc/source/user/basics.types.rst b/doc/source/user/basics.types.rst
@@ -67,13 +67,24 @@ confusion with builtin python type names, such as `numpy.bool_`.
 
     * - `numpy.intp`
       - N/A
-      - ``intptr_t`` (defined in ``stdint.h``)
-      - Platform-defined integer type capable of holding a pointer.
+      - ``ssize_t``/``Py_ssize_t``
+      - Platform-defined integer of size ``size_t``; used e.g. for sizes.
 
     * - `numpy.uintp`
       - N/A
-      - ``uintptr_t`` (defined in ``stdint.h``)
-      - Platform-defined integer type capable of holding a pointer without sign.
+      - ``size_t``
+      - Platform-defined integer type capable of storing the maximum
+        allocation size.
+
+    * - N/A
+      - ``'p'``
+      - ``intptr_t``
+      - Guaranteed to hold pointers. Character code only (Python and C).
+
+    * - N/A
+      - ``'P'``
+      - ``uintptr_t``
+      - Guaranteed to hold pointers. Character code only (Python and C).
 
     * - `numpy.int32` or `numpy.int64`
       - `numpy.long`

diff --git a/doc/source/user/basics.ufuncs.rst b/doc/source/user/basics.ufuncs.rst
@@ -217,33 +217,35 @@ a different table.
 ...         print()
 ...
 >>> print_table(np.typecodes['All'])
-X ? b h i l q p B H I L Q P e f d g F D G S U V O M m
-? Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y
-b - Y Y Y Y Y Y - - - - - - Y Y Y Y Y Y Y Y Y Y Y - Y
-h - - Y Y Y Y Y - - - - - - - Y Y Y Y Y Y Y Y Y Y - Y
-i - - - Y Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y
-l - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y
-q - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y
-p - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y
-B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y
-H - - - Y Y Y Y - Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y
-I - - - - Y Y Y - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y
-L - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - -
-Q - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - -
-P - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - -
-e - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - -
-f - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - -
-d - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - -
-g - - - - - - - - - - - - - - - - Y - - Y Y Y Y Y - -
-F - - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y - -
-D - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y - -
-G - - - - - - - - - - - - - - - - - - - Y Y Y Y Y - -
-S - - - - - - - - - - - - - - - - - - - - Y Y Y Y - -
-U - - - - - - - - - - - - - - - - - - - - - Y Y Y - -
-V - - - - - - - - - - - - - - - - - - - - - - Y Y - -
-O - - - - - - - - - - - - - - - - - - - - - - - Y - -
-M - - - - - - - - - - - - - - - - - - - - - - Y Y Y -
-m - - - - - - - - - - - - - - - - - - - - - - Y Y - Y
+X ? b h i l q n p B H I L Q N P e f d g F D G S U V O M m
+? Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y
+b - Y Y Y Y Y Y Y - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - Y
+h - - Y Y Y Y Y Y - - - - - - - - Y Y Y Y Y Y Y Y Y Y - Y
+i - - - Y Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y
+l - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y
+q - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y
+n - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y
+p - - - - Y Y Y Y - - - - - - - - - Y Y - Y Y Y Y Y Y - Y
+B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y
+H - - - Y Y Y Y Y - Y Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y
+I - - - - Y Y Y Y - - Y Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y
+L - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - -
+Q - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - -
+N - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - -
+P - - - - - - - - - - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - -
+e - - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - -
+f - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - -
+d - - - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - -
+g - - - - - - - - - - - - - - - - - - Y - - Y Y Y Y Y - -
+F - - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y - -
+D - - - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y - -
+G - - - - - - - - - - - - - - - - - - - - - Y Y Y Y Y - -
+S - - - - - - - - - - - - - - - - - - - - - - Y Y Y Y - -
+U - - - - - - - - - - - - - - - - - - - - - - - Y Y Y - -
+V - - - - - - - - - - - - - - - - - - - - - - - - Y Y - -
+O - - - - - - - - - - - - - - - - - - - - - - - - - Y - -
+M - - - - - - - - - - - - - - - - - - - - - - - - Y Y Y -
+m - - - - - - - - - - - - - - - - - - - - - - - - Y Y - Y
 
 You should note that, while included in the table for completeness,
 the 'S', 'U', and 'V' types cannot be operated on by ufuncs. Also,

diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd
@@ -460,7 +460,6 @@ cdef extern from "numpy/arrayobject.h":
     object PyArray_ZEROS(int nd, npy_intp* dims, int type, int fortran)
     object PyArray_EMPTY(int nd, npy_intp* dims, int type, int fortran)
     void PyArray_FILLWBYTE(object, int val)
-    npy_intp PyArray_REFCOUNT(object)
     object PyArray_ContiguousFromAny(op, int, int min_depth, int max_depth)
     unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2)
     bint PyArray_EquivByteorders(int b1, int b2) nogil

diff --git a/numpy/_core/_internal.py b/numpy/_core/_internal.py
@@ -219,7 +219,7 @@ def _getintp_ctype():
         import numpy as np
         val = dummy_ctype(np.intp)
     else:
-        char = dtype('p').char
+        char = dtype('n').char
         if char == 'i':
             val = ctypes.c_int
         elif char == 'l':

diff --git a/numpy/_core/code_generators/generate_numpy_api.py b/numpy/_core/code_generators/generate_numpy_api.py
@@ -81,6 +81,17 @@
       return -1;
   }
 
+  /*
+   * On exceedingly few platforms these sizes may not match, in which case
+   * We do not support older NumPy versions at all.
+   */
+  if (sizeof(Py_ssize_t) != sizeof(Py_intptr_t) &&
+        PyArray_GetNDArrayCFeatureVersion() < NPY_2_0_API_VERSION) {
+    PyErr_Format(PyExc_RuntimeError,
+        "module compiled against NumPy 2.0 but running on NumPy 1.x. "
+        "Unfortunately, this is not supported on niche platforms where "
+        "`sizeof(size_t) != sizeof(inptr_t)`.");
+  }
   /*
    * Perform runtime check of C API version.  As of now NumPy 2.0 is ABI
    * backwards compatible (in the exposed feature subset!) for all practical

diff --git a/numpy/_core/include/numpy/_numpyconfig.h.in b/numpy/_core/include/numpy/_numpyconfig.h.in
@@ -10,6 +10,8 @@
 #mesondefine NPY_SIZEOF_LONGDOUBLE
 #mesondefine NPY_SIZEOF_COMPLEX_LONGDOUBLE
 #mesondefine NPY_SIZEOF_PY_INTPTR_T
+#mesondefine NPY_SIZEOF_INTP
+#mesondefine NPY_SIZEOF_UINTP
 #mesondefine NPY_SIZEOF_OFF_T
 #mesondefine NPY_SIZEOF_PY_LONG_LONG
 #mesondefine NPY_SIZEOF_LONGLONG

diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h
@@ -103,10 +103,6 @@ extern "C" {
 
 #define PyArray_FILLWBYTE(obj, val) memset(PyArray_DATA(obj), val, \
                                            PyArray_NBYTES(obj))
-#ifndef PYPY_VERSION
-#define PyArray_REFCOUNT(obj) (((PyObject *)(obj))->ob_refcnt)
-#define NPY_REFCOUNT PyArray_REFCOUNT
-#endif
 #define NPY_MAX_ELSIZE (2 * NPY_SIZEOF_LONGDOUBLE)
 
 #define PyArray_ContiguousFromAny(op, type, min_depth, max_depth) \

diff --git a/numpy/_core/include/numpy/ndarraytypes.h b/numpy/_core/include/numpy/ndarraytypes.h
@@ -120,13 +120,14 @@ enum NPY_TYPECHAR {
         NPY_CHARLTR = 'c',
 
         /*
-         * No Descriptor, just a define -- this let's
-         * Python users specify an array of integers
-         * large enough to hold a pointer on the
-         * platform
+         * Note, we removed `NPY_INTPLTR` due to changing its definition
+         * to 'n', rather than 'p'.  On any typical platform this is the
+         * same integer.  'n' should be used for the `np.intp` with the same
+         * size as `size_t` while 'p' remains pointer sized.
+         *
+         * 'p', 'P', 'n', and 'N' are valid and defined explicitly
+         * in `arraytypes.c.src`.
          */
-        NPY_INTPLTR = 'p',
-        NPY_UINTPLTR = 'P',
 
         /*
          * These are for dtype 'kinds', not dtype 'typecodes'

diff --git a/numpy/_core/include/numpy/npy_common.h b/numpy/_core/include/numpy/npy_common.h
@@ -195,11 +195,11 @@ enum {
 };
 
 /*
- * This is to typedef npy_intp to the appropriate pointer size for this
- * platform.  Py_intptr_t, Py_uintptr_t are defined in pyport.h.
+ * This is to typedef npy_intp to the appropriate size for Py_ssize_t.
+ * (Before NumPy 2.0 we used Py_intptr_t and Py_uintptr_t from `pyport.h`.)
  */
-typedef Py_intptr_t npy_intp;
-typedef Py_uintptr_t npy_uintp;
+typedef Py_ssize_t npy_intp;
+typedef size_t npy_uintp;
 
 /*
  * Define sizes that were not defined in numpyconfig.h.
@@ -208,8 +208,6 @@ typedef Py_uintptr_t npy_uintp;
 #define NPY_SIZEOF_BYTE 1
 #define NPY_SIZEOF_DATETIME 8
 #define NPY_SIZEOF_TIMEDELTA 8
-#define NPY_SIZEOF_INTP NPY_SIZEOF_PY_INTPTR_T
-#define NPY_SIZEOF_UINTP NPY_SIZEOF_PY_INTPTR_T
 #define NPY_SIZEOF_HALF 2
 #define NPY_SIZEOF_CFLOAT NPY_SIZEOF_COMPLEX_FLOAT
 #define NPY_SIZEOF_CDOUBLE NPY_SIZEOF_COMPLEX_DOUBLE
@@ -228,7 +226,7 @@ typedef Py_uintptr_t npy_uintp;
  *      functions use different formatting codes that are portably specified
  *      according to the Python documentation. See issue gh-2388.
  */
-#if NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONG
+#if NPY_SIZEOF_INTP == NPY_SIZEOF_LONG
         #define NPY_INTP NPY_LONG
         #define NPY_UINTP NPY_ULONG
         #define PyIntpArrType_Type PyLongArrType_Type
@@ -237,8 +235,7 @@ typedef Py_uintptr_t npy_uintp;
         #define NPY_MIN_INTP NPY_MIN_LONG
         #define NPY_MAX_UINTP NPY_MAX_ULONG
         #define NPY_INTP_FMT "ld"
-#elif NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_INT
-        /* NumPy always prefers long over int if the size is identical! */
+#elif NPY_SIZEOF_INTP == NPY_SIZEOF_INT
         #define NPY_INTP NPY_INT
         #define NPY_UINTP NPY_UINT
         #define PyIntpArrType_Type PyIntArrType_Type
@@ -247,7 +244,7 @@ typedef Py_uintptr_t npy_uintp;
         #define NPY_MIN_INTP NPY_MIN_INT
         #define NPY_MAX_UINTP NPY_MAX_UINT
         #define NPY_INTP_FMT "d"
-#elif defined(PY_LONG_LONG) && (NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONGLONG)
+#elif defined(PY_LONG_LONG) && (NPY_SIZEOF_INTP == NPY_SIZEOF_LONGLONG)
         #define NPY_INTP NPY_LONGLONG
         #define NPY_UINTP NPY_ULONGLONG
         #define PyIntpArrType_Type PyLongLongArrType_Type
@@ -256,18 +253,8 @@ typedef Py_uintptr_t npy_uintp;
         #define NPY_MIN_INTP NPY_MIN_LONGLONG
         #define NPY_MAX_UINTP NPY_MAX_ULONGLONG
         #define NPY_INTP_FMT "lld"
-#endif
-
-/*
- * We can only use C99 formats for npy_int_p if it is the same as
- * intp_t, hence the condition on HAVE_UNITPTR_T
- */
-#if (NPY_USE_C99_FORMATS) == 1 \
-        && (defined HAVE_UINTPTR_T) \
-        && (defined HAVE_INTTYPES_H)
-        #include <inttypes.h>
-        #undef NPY_INTP_FMT
-        #define NPY_INTP_FMT PRIdPTR
+#else
+    #error "Failed to correctly define NPY_INTP and NPY_UINTP"
 #endif
 
 

diff --git a/numpy/_core/include/numpy/numpyconfig.h b/numpy/_core/include/numpy/numpyconfig.h
@@ -16,14 +16,11 @@
  */
 #ifdef __APPLE__
     #undef NPY_SIZEOF_LONG
-    #undef NPY_SIZEOF_PY_INTPTR_T
 
     #ifdef __LP64__
         #define NPY_SIZEOF_LONG         8
-        #define NPY_SIZEOF_PY_INTPTR_T  8
     #else
         #define NPY_SIZEOF_LONG         4
-        #define NPY_SIZEOF_PY_INTPTR_T  4
     #endif
 
     #undef NPY_SIZEOF_LONGDOUBLE

diff --git a/numpy/_core/meson.build b/numpy/_core/meson.build
@@ -109,6 +109,8 @@ types_to_check = [
   ['NPY_SIZEOF_FLOAT', 'float'],
   ['NPY_SIZEOF_DOUBLE', 'double'],
   ['NPY_SIZEOF_LONGDOUBLE', 'long double'],
+  ['NPY_SIZEOF_INTP', 'size_t'],
+  ['NPY_SIZEOF_UINTP', 'size_t'],
 ]
 foreach symbol_type: types_to_check
   cdata.set(symbol_type[0], cc.sizeof(symbol_type[1]))

diff --git a/numpy/_core/numerictypes.py b/numpy/_core/numerictypes.py
@@ -497,14 +497,14 @@ def _scalar_type_key(typ):
 del key
 
 typecodes = {'Character': 'c',
-             'Integer': 'bhilqp',
-             'UnsignedInteger': 'BHILQP',
+             'Integer': 'bhilqnp',
+             'UnsignedInteger': 'BHILQNP',
              'Float': 'efdg',
              'Complex': 'FDG',
-             'AllInteger': 'bBhHiIlLqQpP',
+             'AllInteger': 'bBhHiIlLqQnNpP',
              'AllFloat': 'efdgFDG',
              'Datetime': 'Mm',
-             'All': '?bhilqpBHILQPefdgFDGSUVOMm'}
+             'All': '?bhilqnpBHILQNPefdgFDGSUVOMm'}
 
 # backwards compatibility --- deprecated name
 # Formal deprecation: Numpy 1.20.0, 2020-10-19 (see numpy/__init__.py)

diff --git a/numpy/_core/src/multiarray/arraytypes.c.src b/numpy/_core/src/multiarray/arraytypes.c.src
@@ -4711,7 +4711,6 @@ set_typeinfo(PyObject *dict)
      *
      * #name = BOOL,
      *         BYTE, UBYTE, SHORT, USHORT, INT, UINT,
-     *         INTP, UINTP,
      *         LONG, ULONG, LONGLONG, ULONGLONG,
      *         HALF, FLOAT, DOUBLE, LONGDOUBLE,
      *         CFLOAT, CDOUBLE, CLONGDOUBLE,
@@ -4722,6 +4721,18 @@ set_typeinfo(PyObject *dict)
     _letter_to_num[NPY_@name@LTR] = NPY_@name@;
 
     /**end repeat**/
+    _letter_to_num['n'] = NPY_INTP;
+    _letter_to_num['N'] = NPY_UINTP;
+
+#if NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_INTP
+    _letter_to_num['p'] = NPY_INTP;
+    _letter_to_num['P'] = NPY_UINTP;
+#elif NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONGLONG
+    _letter_to_num['p'] = NPY_LONGLONG;
+    _letter_to_num['P'] = NPY_ULONGLONG;
+#else
+    #error "Did not find correct pointer sized integer."
+#endif
 
     /**begin repeat
       * #name = BOOL,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		* ``NPY_INTPLTR`` has been removed to avoid confusion (see ``intp``
		redefinition).
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		* ``PyArray_REFCNT`` and ``NPY_REFCOUNT`` are removed. Use ``Py_REFCNT`` instead.