From 40426744f9dedab3e03997bb916abdeaa69667b0 Mon Sep 17 00:00:00 2001 From: Edward E Date: Thu, 30 Mar 2023 16:52:12 -0500 Subject: [PATCH 1/2] BUG: Use 2GiB chunking code for fwrite() on mingw32/64 Addresses #2256 --- numpy/core/include/numpy/npy_common.h | 2 +- numpy/core/src/multiarray/convert.c | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h index 3b31bcf2dfd0..22d29686caed 100644 --- a/numpy/core/include/numpy/npy_common.h +++ b/numpy/core/include/numpy/npy_common.h @@ -129,7 +129,7 @@ #define NPY_STEALS_REF_TO_ARG(n) #endif -/* 64 bit file position support, also on win-amd64. Ticket #1660 */ +/* 64 bit file position support, also on win-amd64. Issue gh-2256 */ #if defined(_MSC_VER) && defined(_WIN64) && (_MSC_VER > 1400) || \ defined(__MINGW32__) || defined(__MINGW64__) #include diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index 9e0c9fb604ba..565cce773924 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -157,11 +157,15 @@ PyArray_ToFile(PyArrayObject *self, FILE *fp, char *sep, char *format) size = PyArray_SIZE(self); NPY_BEGIN_ALLOW_THREADS; -#if defined (_MSC_VER) && defined(_WIN64) - /* Workaround Win64 fwrite() bug. Issue gh-2556 +#if defined(_MSC_VER) && defined(_WIN64) || \ + defined(__MINGW32__) || defined(__MINGW64__) + /* Workaround Win64 fwrite() bug. Issue gh-2256 * If you touch this code, please run this test which is so slow - * it was removed from the test suite + * it was removed from the test suite. Note that the original + * failure mode involves an infinite loop during tofile() * + * import tempfile, numpy as np + * from numpy.testing import (assert_) * fourgbplus = 2**32 + 2**16 * testbytes = np.arange(8, dtype=np.int8) * n = len(testbytes) From 1c1779490376077b8fef643f65222bb4834567b1 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 16 May 2023 10:54:57 +0200 Subject: [PATCH 2/2] MAINT: Use fwrite chunking generally on Windows 64 and size_t as type This is a bit too broad because msys UCRT runtime may actually not need it. Changes the type in the loop to `size_t`. This is not necessary but the current constants are buggy without it if the branch is accidentally used on 32bit. --- numpy/core/src/multiarray/convert.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index 565cce773924..8a6b517a46bd 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -157,9 +157,12 @@ PyArray_ToFile(PyArrayObject *self, FILE *fp, char *sep, char *format) size = PyArray_SIZE(self); NPY_BEGIN_ALLOW_THREADS; -#if defined(_MSC_VER) && defined(_WIN64) || \ - defined(__MINGW32__) || defined(__MINGW64__) - /* Workaround Win64 fwrite() bug. Issue gh-2256 +#if defined(NPY_OS_WIN64) + /* + * Workaround Win64 fwrite() bug. Issue gh-2256 + * The native 64 windows runtime has this issue, the above will + * also trigger UCRT (which doesn't), so it could be more precise. + * * If you touch this code, please run this test which is so slow * it was removed from the test suite. Note that the original * failure mode involves an infinite loop during tofile() @@ -181,8 +184,8 @@ PyArray_ToFile(PyArrayObject *self, FILE *fp, char *sep, char *format) * assert_((a[-n:] == testbytes).all()) */ { - npy_intp maxsize = 2147483648 / PyArray_DESCR(self)->elsize; - npy_intp chunksize; + size_t maxsize = 2147483648 / (size_t)PyArray_DESCR(self)->elsize; + size_t chunksize; n = 0; while (size > 0) { @@ -190,7 +193,7 @@ PyArray_ToFile(PyArrayObject *self, FILE *fp, char *sep, char *format) n2 = fwrite((const void *) ((char *)PyArray_DATA(self) + (n * PyArray_DESCR(self)->elsize)), (size_t) PyArray_DESCR(self)->elsize, - (size_t) chunksize, fp); + chunksize, fp); if (n2 < chunksize) { break; }