gh-123497: New limit for Python integers on 64-bit platforms (GH-123724)

serhiy-storchaka · web-flow · commit d08c7888229e · 2024-09-29T10:40:20.000+03:00
Instead of be limited just by the size of addressable memory (2**63
bytes), Python integers are now also limited by the number of bits, so
the number of bit now always fit in a 64-bit integer.

Both limits are much larger than what might be available in practice,
so it doesn't affect users.

_PyLong_NumBits() and _PyLong_Frexp() are now always successful.
diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h
@@ -71,10 +71,9 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v);
    absolute value of a long.  For example, this returns 1 for 1 and -1, 2
    for 2 and -2, and 2 for 3 and -3.  It returns 0 for 0.
    v must not be NULL, and must be a normalized long.
-   (uint64_t)-1 is returned and OverflowError set if the true result doesn't
-   fit in a size_t.
+   Always successful.
 */
-PyAPI_FUNC(uint64_t) _PyLong_NumBits(PyObject *v);
+PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v);
 
 /* _PyLong_FromByteArray:  View the n unsigned bytes as a binary integer in
    base 256, and return a Python int with the same numeric value.
diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
@@ -79,11 +79,10 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i)
 }
 
 // _PyLong_Frexp returns a double x and an exponent e such that the
-// true value is approximately equal to x * 2**e.  e is >= 0.  x is
+// true value is approximately equal to x * 2**e.  x is
 // 0.0 if and only if the input is 0 (in which case, e and x are both
-// zeroes); otherwise, 0.5 <= abs(x) < 1.0.  On overflow, which is
-// possible if the number of bits doesn't fit into a Py_ssize_t, sets
-// OverflowError and returns -1.0 for x, 0 for e.
+// zeroes); otherwise, 0.5 <= abs(x) < 1.0.
+// Always successful.
 //
 // Export for 'math' shared extension
 PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e);
@@ -105,10 +104,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *);
 PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base);
 
 // Export for 'math' shared extension
-PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, uint64_t);
+PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t);
 
 // Export for 'math' shared extension
-PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, uint64_t);
+PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t);
 
 PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right);
 PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right);
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
@@ -2146,7 +2146,7 @@ save_long(PicklerObject *self, PyObject *obj)
 
     if (self->proto >= 2) {
         /* Linear-time pickling. */
-        uint64_t nbits;
+        int64_t nbits;
         size_t nbytes;
         unsigned char *pdata;
         char header[5];
@@ -2161,8 +2161,8 @@ save_long(PicklerObject *self, PyObject *obj)
             return 0;
         }
         nbits = _PyLong_NumBits(obj);
-        if (nbits == (uint64_t)-1 && PyErr_Occurred())
-            goto error;
+        assert(nbits >= 0);
+        assert(!PyErr_Occurred());
         /* How many bytes do we need?  There are nbits >> 3 full
          * bytes of data, and nbits & 7 leftover bits.  If there
          * are any leftover bits, then we clearly need another
diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
@@ -295,7 +295,7 @@ random_seed(RandomObject *self, PyObject *arg)
     int result = -1;  /* guilty until proved innocent */
     PyObject *n = NULL;
     uint32_t *key = NULL;
-    uint64_t bits;
+    int64_t bits;
     size_t keyused;
     int res;
 
@@ -335,8 +335,8 @@ random_seed(RandomObject *self, PyObject *arg)
 
     /* Now split n into 32-bit chunks, from the right. */
     bits = _PyLong_NumBits(n);
-    if (bits == (uint64_t)-1 && PyErr_Occurred())
-        goto Done;
+    assert(bits >= 0);
+    assert(!PyErr_Occurred());
 
     /* Figure out how many 32-bit chunks this gives us. */
     keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1);
diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c
@@ -1657,7 +1657,7 @@ math_isqrt(PyObject *module, PyObject *n)
 /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/
 {
     int a_too_large, c_bit_length;
-    uint64_t c, d;
+    int64_t c, d;
     uint64_t m;
     uint32_t u;
     PyObject *a = NULL, *b;
@@ -1680,14 +1680,13 @@ math_isqrt(PyObject *module, PyObject *n)
 
     /* c = (n.bit_length() - 1) // 2 */
     c = _PyLong_NumBits(n);
-    if (c == (uint64_t)(-1)) {
-        goto error;
-    }
-    c = (c - 1U) / 2U;
+    assert(c > 0);
+    assert(!PyErr_Occurred());
+    c = (c - 1) / 2;
 
     /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a
        fast, almost branch-free algorithm. */
-    if (c <= 31U) {
+    if (c <= 31) {
         int shift = 31 - (int)c;
         m = (uint64_t)PyLong_AsUnsignedLongLong(n);
         Py_DECREF(n);
@@ -1704,13 +1703,13 @@ math_isqrt(PyObject *module, PyObject *n)
 
     /* From n >= 2**64 it follows that c.bit_length() >= 6. */
     c_bit_length = 6;
-    while ((c >> c_bit_length) > 0U) {
+    while ((c >> c_bit_length) > 0) {
         ++c_bit_length;
     }
 
     /* Initialise d and a. */
     d = c >> (c_bit_length - 5);
-    b = _PyLong_Rshift(n, 2U*c - 62U);
+    b = _PyLong_Rshift(n, 2*c - 62);
     if (b == NULL) {
         goto error;
     }
@@ -1727,12 +1726,12 @@ math_isqrt(PyObject *module, PyObject *n)
 
     for (int s = c_bit_length - 6; s >= 0; --s) {
         PyObject *q;
-        uint64_t e = d;
+        int64_t e = d;
 
         d = c >> s;
 
         /* q = (n >> 2*c - e - d + 1) // a */
-        q = _PyLong_Rshift(n, 2U*c - d - e + 1U);
+        q = _PyLong_Rshift(n, 2*c - d - e + 1);
         if (q == NULL) {
             goto error;
         }
@@ -1742,7 +1741,7 @@ math_isqrt(PyObject *module, PyObject *n)
         }
 
         /* a = (a << d - 1 - e) + q */
-        Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e));
+        Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e));
         if (a == NULL) {
             Py_DECREF(q);
             goto error;
@@ -2202,8 +2201,8 @@ loghelper(PyObject* arg, double (*func)(double))
                to compute the log anyway.  Clear the exception and continue. */
             PyErr_Clear();
             x = _PyLong_Frexp((PyLongObject *)arg, &e);
-            if (x == -1.0 && PyErr_Occurred())
-                return NULL;
+            assert(e >= 0);
+            assert(!PyErr_Occurred());
             /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */
             result = func(x) + func(2.0) * e;
         }
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
@@ -406,19 +406,16 @@ float_richcompare(PyObject *v, PyObject *w, int op)
         }
         /* The signs are the same. */
         /* Convert w to a double if it fits.  In particular, 0 fits. */
-        uint64_t nbits64 = _PyLong_NumBits(w);
-        if (nbits64 > (unsigned int)DBL_MAX_EXP) {
+        int64_t nbits64 = _PyLong_NumBits(w);
+        assert(nbits64 >= 0);
+        assert(!PyErr_Occurred());
+        if (nbits64 > DBL_MAX_EXP) {
             /* This Python integer is larger than any finite C double.
              * Replace with little doubles
              * that give the same outcome -- w is so large that
              * its magnitude must exceed the magnitude of any
              * finite float.
              */
-            if (nbits64 == (uint64_t)-1 && PyErr_Occurred()) {
-                /* This Python integer is so large that uint64_t isn't
-                 * big enough to hold the # of bits. */
-                PyErr_Clear();
-            }
             i = (double)vsign;
             assert(wsign != 0);
             j = wsign * 2.0;
diff --git a/Objects/longobject.c b/Objects/longobject.c
diff --git a/Python/ast_opt.c b/Python/ast_opt.c
