From 247dc339ad7d9a1befb151bd92c2b276dda0acd6 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Thu, 1 Jan 2015 23:13:03 -0500 Subject: [PATCH 1/4] BUG: Fixes issue #5412. Added support for PEP 3118 PIL-style buffers in multiarray constructor. --- numpy/core/src/multiarray/ctors.c | 89 +++++++++++++++++++++++++---- numpy/core/tests/test_multiarray.py | 14 +++++ 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0017de0ad6e7..6e6efd72ab7a 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1259,6 +1259,52 @@ PyArray_New(PyTypeObject *subtype, int nd, npy_intp *dims, int type_num, return new; } +int +_copy_from_pil_style(char* dest, int ndim, char *buf, npy_intp *shape, + npy_intp *strides, npy_intp *suboffsets, npy_intp itemsize) +{ + int ax, last_axis; + npy_intp indices[NPY_MAXDIMS]; + void **pointers[NPY_MAXDIMS]; + char *pointer = (char*)buf; + npy_intp blocksize; + /* Initialize indices and pointers. */ + for (ax = 0; ax < ndim; ++ax) { + indices[ax] = 0; + pointers[ax] = (void **)pointer; + if (suboffsets[ax] >= 0) { + last_axis = ax; + pointer = *((char**)pointer) + suboffsets[ax]; + } + } + /* compute blocksize */ + blocksize = itemsize; + for (ax = last_axis + 1; ax < ndim; ++ax) + blocksize *= shape[ax]; + /* main loop */ + for (;;) { + memcpy(dest, pointer, blocksize); + dest += blocksize; + ax = last_axis; + /* Update indices and pointers for the next block. */ + while (indices[ax] == shape[ax] - 1) { + if (ax == 0) + return 0; + indices[ax--] = 0; + } + ++indices[ax]; + /* update pointers */ + pointer = (char *)pointers[ax] + strides[ax]; + for (; ax <= last_axis; ++ax) { + pointers[ax] = (void **)pointer; + if (suboffsets[ax] >=0 ) { + pointer = *((char**)pointer) + suboffsets[ax]; + } + } + } + return -1; +} + NPY_NO_EXPORT int _array_from_buffer_3118(PyObject *obj, PyObject **out) @@ -1344,18 +1390,41 @@ _array_from_buffer_3118(PyObject *obj, PyObject **out) } } - flags = NPY_ARRAY_BEHAVED & (view->readonly ? ~NPY_ARRAY_WRITEABLE : ~0); - r = PyArray_NewFromDescr(&PyArray_Type, descr, - nd, shape, strides, view->buf, - flags, NULL); - if (r == NULL || - PyArray_SetBaseObject((PyArrayObject *)r, memoryview) < 0) { - Py_XDECREF(r); - Py_DECREF(memoryview); - return -1; + if (view->suboffsets == NULL) { + flags = NPY_ARRAY_BEHAVED & (view->readonly ? ~NPY_ARRAY_WRITEABLE : ~0); + r = PyArray_NewFromDescr(&PyArray_Type, descr, + nd, shape, strides, view->buf, + flags, NULL); + if (r == NULL || + PyArray_SetBaseObject((PyArrayObject *)r, memoryview) < 0) { + Py_XDECREF(r); + Py_DECREF(memoryview); + return -1; + } + PyArray_UpdateFlags((PyArrayObject *)r, NPY_ARRAY_UPDATE_ALL); } - PyArray_UpdateFlags((PyArrayObject *)r, NPY_ARRAY_UPDATE_ALL); + else { + /* Recompute strides */ + npy_intp blocksize = view->itemsize; + for (k = nd - 1; k >= 0; --k) { + if (view->suboffsets[k] >= 0) { + strides[k] = blocksize; + } + blocksize *= shape[k]; + } + r = PyArray_NewFromDescr(&PyArray_Type, descr, + nd, shape, strides, NULL, 0, NULL); + if (r == NULL) + goto fail; + if (PyArray_SetBaseObject((PyArrayObject *)r, memoryview) < 0) { + Py_DECREF(r); + goto fail; + } + _copy_from_pil_style((char *)PyArray_DATA((PyArrayObject *)r), nd, + (char *)view->buf, shape, view->strides, + view->suboffsets, view->itemsize); + } *out = r; return 0; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 262fbc0c05cd..470f7ce55b42 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -5921,6 +5921,20 @@ def test_relaxed_strides(self): shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) assert_(strides[-1] == 8) + def test_pil_style(self): + try: + import _testbuffer + except ImportError: + raise SkipTest("_testbuffer is not available") + + for shape in [(2, 3), (2, 3, 4)]: + data = list(range(np.prod(shape))) + buffer = _testbuffer.ndarray(data, shape, format='i', + flags=_testbuffer.ND_PIL) + a = np.array(buffer) + b = np.array(data, dtype='i').reshape(shape) + yield assert_equal, a, b + class TestArrayAttributeDeletion(object): From ec9e11a1641be8881d4daf82acfcde6dee15b542 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 27 Mar 2016 16:51:35 -0400 Subject: [PATCH 2/4] FIX: PyArray_NewFromDescr steals descr, no need to set base since data is copied. --- numpy/core/src/multiarray/ctors.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 6e6efd72ab7a..0e8d2017456f 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1415,11 +1415,7 @@ _array_from_buffer_3118(PyObject *obj, PyObject **out) r = PyArray_NewFromDescr(&PyArray_Type, descr, nd, shape, strides, NULL, 0, NULL); if (r == NULL) - goto fail; - if (PyArray_SetBaseObject((PyArrayObject *)r, memoryview) < 0) { - Py_DECREF(r); - goto fail; - } + goto fail2; _copy_from_pil_style((char *)PyArray_DATA((PyArrayObject *)r), nd, (char *)view->buf, shape, view->strides, view->suboffsets, view->itemsize); @@ -1430,6 +1426,7 @@ _array_from_buffer_3118(PyObject *obj, PyObject **out) fail: Py_XDECREF(descr); +fail2: Py_DECREF(memoryview); return -1; From 76689cf57edbf044af4aa5cbece9ed1a3c30de0e Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Mon, 28 Mar 2016 14:48:30 -0400 Subject: [PATCH 3/4] FIX: Initialize last_axis to make compiler happy. --- numpy/core/src/multiarray/ctors.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0e8d2017456f..8b00be10d64f 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1263,7 +1263,7 @@ int _copy_from_pil_style(char* dest, int ndim, char *buf, npy_intp *shape, npy_intp *strides, npy_intp *suboffsets, npy_intp itemsize) { - int ax, last_axis; + int ax, last_axis = 0; npy_intp indices[NPY_MAXDIMS]; void **pointers[NPY_MAXDIMS]; char *pointer = (char*)buf; @@ -1274,8 +1274,8 @@ _copy_from_pil_style(char* dest, int ndim, char *buf, npy_intp *shape, pointers[ax] = (void **)pointer; if (suboffsets[ax] >= 0) { last_axis = ax; - pointer = *((char**)pointer) + suboffsets[ax]; - } + pointer = *((char**)pointer) + suboffsets[ax]; + } } /* compute blocksize */ blocksize = itemsize; From 730aa71f13a6b0b9ec51f409648bc3d6246b81b8 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Mon, 28 Mar 2016 15:32:46 -0400 Subject: [PATCH 4/4] FIX: Added a comment explaining initialization for last_axis. --- numpy/core/src/multiarray/ctors.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 8b00be10d64f..cec9318157f7 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1263,11 +1263,16 @@ int _copy_from_pil_style(char* dest, int ndim, char *buf, npy_intp *shape, npy_intp *strides, npy_intp *suboffsets, npy_intp itemsize) { - int ax, last_axis = 0; npy_intp indices[NPY_MAXDIMS]; void **pointers[NPY_MAXDIMS]; char *pointer = (char*)buf; npy_intp blocksize; + int ax; + /* Since suboffsets cannot be all negative, last_axis will be inevitably + * reset in the initialization loop below. However, the compiler cannot + * know this, so we preset last_axis to an arbitrary value. */ + int last_axis = 0; + /* Initialize indices and pointers. */ for (ax = 0; ax < ndim; ++ax) { indices[ax] = 0;