From 6363bb7cbb1d0e292e8af87f92a28faf781d3d95 Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:35:52 -0500 Subject: [PATCH 1/8] Add ability to get view when selecting subset of fields. Add ability to get view when selecting subset of fields in a struct array, for numpy 1.8.0. Currently a copy is still returned - set WARN_ON_WRITE flag which will be removed in a future version of numpy. --- numpy/core/_internal.py | 7 +++++++ numpy/core/src/multiarray/mapping.c | 1 + 2 files changed, 8 insertions(+) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 309b53c44761..29d21a9bc861 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -298,6 +298,13 @@ def _index_fields(ary, fields): for name in fields: newarray[name] = ary[name] + names = [name for name in fields if name in dt.names] + formats = dt.fields[name][0] for name in fields if name in dt.names] + offsets = [dt.fields[name][1] for name in fields if name in dt.names] + + view_dtype = {'names':names, 'formats':formats, 'offsets':offsets} + view = ary.view(dtype=view_dtype) + return newarray # Given a string containing a PEP 3118 format specifier, diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index cdefb9982481..b2643f89f1fc 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -975,6 +975,7 @@ array_subscript(PyArrayObject *self, PyObject *op) } obj = PyObject_CallMethod(_numpy_internal, "_index_fields", "OO", self, op); + PyArray_ENABLEFLAGS((PyArrayObject*)obj, NPY_ARRAY_WARN_ON_WRITE); Py_DECREF(_numpy_internal); return obj; } From 0c09e1f184fde335f3533b0b9ef6ea46379309db Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:37:28 -0500 Subject: [PATCH 2/8] fix previous commit to return copy of view instead of view --- numpy/core/_internal.py | 17 ++++------------- numpy/core/src/multiarray/mapping.c | 5 ++++- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 29d21a9bc861..88a41fd0d53c 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -286,26 +286,17 @@ def _newnames(datatype, order): # construct a new array with just those fields copied over def _index_fields(ary, fields): from multiarray import empty, dtype + from numpy import copy dt = ary.dtype - new_dtype = [(name, dt[name]) for name in fields if name in dt.names] - if ary.flags.f_contiguous: - order = 'F' - else: - order = 'C' - - newarray = empty(ary.shape, dtype=new_dtype, order=order) - - for name in fields: - newarray[name] = ary[name] names = [name for name in fields if name in dt.names] - formats = dt.fields[name][0] for name in fields if name in dt.names] + formats = [dt.fields[name][0] for name in fields if name in dt.names] offsets = [dt.fields[name][1] for name in fields if name in dt.names] - view_dtype = {'names':names, 'formats':formats, 'offsets':offsets} + view_dtype = {'names':names, 'formats':formats, 'offsets':offsets, 'itemsize':dt.itemsize} view = ary.view(dtype=view_dtype) - return newarray + return copy(view) # Given a string containing a PEP 3118 format specifier, # construct a Numpy dtype diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index b2643f89f1fc..663a3ef7f509 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -975,8 +975,11 @@ array_subscript(PyArrayObject *self, PyObject *op) } obj = PyObject_CallMethod(_numpy_internal, "_index_fields", "OO", self, op); - PyArray_ENABLEFLAGS((PyArrayObject*)obj, NPY_ARRAY_WARN_ON_WRITE); Py_DECREF(_numpy_internal); + if (obj == NULL) { + return NULL; + } + PyArray_ENABLEFLAGS((PyArrayObject*)obj, NPY_ARRAY_WARN_ON_WRITE); return obj; } } From 91cb088047bd357a1b266fa8cb9d0deefea5820c Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:38:17 -0500 Subject: [PATCH 3/8] Change WARN_ON_WRITE Deprecation Warning Change WARN_ON_WRITE Deprecation Warning to include returning a copy when selecting muultiple fields of a structured array. --- numpy/core/src/multiarray/arrayobject.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index e8bc6b7b638c..4a219e70e194 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -694,9 +694,10 @@ array_might_be_written(PyArrayObject *obj) { const char *msg = "Numpy has detected that you (may be) writing to an array returned\n" - "by numpy.diagonal. This code will likely break in the next numpy\n" - "release -- see numpy.diagonal docs for details. The quick fix is\n" - "to make an explicit copy (e.g., do arr.diagonal().copy())."; + "by numpy.diagonal or by selecting multiple fields in a structured\n" + "array. This code will likely break in the next numpy release --\n" + "see numpy.diagonal or structured array docs for details. The quick\n" + "fix is to make an explicit copy (e.g., do arr.diagonal().copy())."; if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { if (DEPRECATE_FUTUREWARNING(msg) < 0) { return -1; From a32325d1d5fc5e23622db5027a6dd35d42193e95 Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:39:19 -0500 Subject: [PATCH 4/8] Call view object's copy method --- numpy/core/_internal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 88a41fd0d53c..92ab0c8b08ff 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -286,7 +286,6 @@ def _newnames(datatype, order): # construct a new array with just those fields copied over def _index_fields(ary, fields): from multiarray import empty, dtype - from numpy import copy dt = ary.dtype names = [name for name in fields if name in dt.names] @@ -296,7 +295,7 @@ def _index_fields(ary, fields): view_dtype = {'names':names, 'formats':formats, 'offsets':offsets, 'itemsize':dt.itemsize} view = ary.view(dtype=view_dtype) - return copy(view) + return view.copy() # Given a string containing a PEP 3118 format specifier, # construct a Numpy dtype From 781468b5938ce1a79804613b222063d81e99963d Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 11:59:56 -0500 Subject: [PATCH 5/8] Updated reference docs for DeprecationWarning --- doc/source/reference/arrays.indexing.rst | 10 ++++++++++ numpy/core/src/multiarray/arrayobject.c | 7 ++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 8da4ecca7dde..bc12c5d0e241 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -335,6 +335,16 @@ sub-array) but of data type ``x.dtype['field-name']`` and contains only the part of the data in the specified field. Also record array scalars can be "indexed" this way. +Indexing into a record array can also be done with a list of field names, +*e.g.* ``x[['field-name1','field-name2']]``. Currently this returns a new +array containing a copy of the values in the fields specified in the list. +As of NumPy 1.7, returning a copy is being deprecated in favor of returning +a view. A copy will continue to be returned for now, but a DeprecationWarning +will be issued when writing to the copy. If you depend on the current +behavior, then we suggest copying the returned array explicitly, i.e. use +x[['field-name1','field-name2']].copy(). This will work with both past and +future versions of NumPy. + If the accessed field is a sub-array, the dimensions of the sub-array are appended to the shape of the result. diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 4a219e70e194..f0e9e36a5fb1 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -694,10 +694,11 @@ array_might_be_written(PyArrayObject *obj) { const char *msg = "Numpy has detected that you (may be) writing to an array returned\n" - "by numpy.diagonal or by selecting multiple fields in a structured\n" + "by numpy.diagonal or by selecting multiple fields in a record\n" "array. This code will likely break in the next numpy release --\n" - "see numpy.diagonal or structured array docs for details. The quick\n" - "fix is to make an explicit copy (e.g., do arr.diagonal().copy())."; + "see numpy.diagonal or arrays.indexing reference docs for details.\n" + "The quick fix is to make an explicit copy (e.g., do\n" + "arr.diagonal().copy() or arr[['f0','f1']].copy())."; if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { if (DEPRECATE_FUTUREWARNING(msg) < 0) { return -1; From bf58296eae309e92d8632bad4f52d602cb8c0f22 Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 13:43:18 -0500 Subject: [PATCH 6/8] update 1.7 release notes --- doc/release/1.7.0-notes.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/release/1.7.0-notes.rst b/doc/release/1.7.0-notes.rst index e8b1de72de9d..824b8f14b6bf 100644 --- a/doc/release/1.7.0-notes.rst +++ b/doc/release/1.7.0-notes.rst @@ -26,6 +26,13 @@ functions. To facilitate this transition, numpy 1.7 produces a FutureWarning if it detects that you may be attempting to write to such an array. See the documentation for np.diagonal for details. +Similar to np.diagonal above, in a future version of numpy, indexing +a record array by a list of field names will return a view onto the +original array, instead of producing a copy as they do now. As with +np.diagonal, numpy 1.7 produces a DeprecationWarning if it detects +that you may be attemping to write to such an array. See the documentation +for array indexing for details. + The default casting rule for UFunc out= parameters has been changed from 'unsafe' to 'same_kind'. Most usages which violate the 'same_kind' rule are likely bugs, so this change may expose previously undetected From 2eb9610acab872538742ce7db5cbbae6cb23360e Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 15:42:01 -0500 Subject: [PATCH 7/8] Add tests for record array indexing - add tests for record array indexing with multiple field names - add tests for DeprecationWarning when writing to array returned by record array indexing with multiple field names --- numpy/core/tests/test_multiarray.py | 60 ++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 3427800a2d26..0fc238a49722 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1909,7 +1909,8 @@ def test_unicode_field_names(self): def test_field_names(self): # Test unicode and 8-bit / byte strings can be used a = np.zeros((1,), dtype=[('f1', 'i4'), - ('f2', [('sf1', 'i4')])]) + ('f2', 'i4'), + ('f3', [('sf1', 'i4')])]) is_py3 = sys.version_info[0] >= 3 if is_py3: funcs = (str,) @@ -1934,12 +1935,18 @@ def test_field_names(self): assert_raises(IndexError, b[0].__setitem__, fnn, 1) assert_raises(IndexError, b[0].__getitem__, fnn) # Subfield - fn2 = func('f2') + fn3 = func('f3') sfn1 = func('sf1') - b[fn2][sfn1] = 1 - assert_equal(b[fn2][sfn1], 1) - assert_raises(ValueError, b[fn2].__setitem__, fnn, 1) - assert_raises(ValueError, b[fn2].__getitem__, fnn) + b[fn3][sfn1] = 1 + assert_equal(b[fn3][sfn1], 1) + assert_raises(ValueError, b[fn3].__setitem__, fnn, 1) + assert_raises(ValueError, b[fn3].__getitem__, fnn) + # multiple Subfields + fn2 = func('f2') + b[fn2] = 3 + assert_equal(b[['f1','f2']][0].tolist(), (2, 3)) + assert_equal(b[['f2','f1']][0].tolist(), (3, 2)) + assert_equal(b[['f1','f3']][0].tolist(), (2, (1,))) # non-ascii unicode field indexing is well behaved if not is_py3: raise SkipTest('non ascii unicode field indexing skipped; ' @@ -1948,6 +1955,47 @@ def test_field_names(self): assert_raises(ValueError, a.__setitem__, u'\u03e0', 1) assert_raises(ValueError, a.__getitem__, u'\u03e0') + def test_field_names_deprecation(self): + import warnings + from numpy.testing.utils import WarningManager + def collect_warning_types(f, *args, **kwargs): + ctx = WarningManager(record=True) + warning_log = ctx.__enter__() + warnings.simplefilter("always") + try: + f(*args, **kwargs) + finally: + ctx.__exit__() + return [w.category for w in warning_log] + a = np.zeros((1,), dtype=[('f1', 'i4'), + ('f2', 'i4'), + ('f3', [('sf1', 'i4')])]) + a['f1'][0] = 1 + a['f2'][0] = 2 + a['f3'][0] = (3,) + b = np.zeros((1,), dtype=[('f1', 'i4'), + ('f2', 'i4'), + ('f3', [('sf1', 'i4')])]) + b['f1'][0] = 1 + b['f2'][0] = 2 + b['f3'][0] = (3,) + + # All the different functions raise a warning, but not an error, and + # 'a' is not modified: + assert_equal(collect_warning_types(a[['f1','f2']].__setitem__, 0, (10,20)), + [DeprecationWarning]) + assert_equal(a, b) + # Views also warn + subset = a[['f1','f2']] + subset_view = subset.view() + assert_equal(collect_warning_types(subset_view['f1'].__setitem__, 0, 10), + [DeprecationWarning]) + # But the write goes through: + assert_equal(subset['f1'][0], 10) + # Only one warning per multiple field indexing, though (even if there are + # multiple views involved): + assert_equal(collect_warning_types(subset['f1'].__setitem__, 0, 10), + []) class TestView(TestCase): def test_basic(self): From a03e8b4d286e91ef5823c059dcfb7a52ce420725 Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 16:03:41 -0500 Subject: [PATCH 8/8] change DeprecationWarning to FutureWarning --- doc/release/1.7.0-notes.rst | 2 +- doc/source/reference/arrays.indexing.rst | 2 +- numpy/core/tests/test_multiarray.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/release/1.7.0-notes.rst b/doc/release/1.7.0-notes.rst index 824b8f14b6bf..f8f54219cac3 100644 --- a/doc/release/1.7.0-notes.rst +++ b/doc/release/1.7.0-notes.rst @@ -29,7 +29,7 @@ such an array. See the documentation for np.diagonal for details. Similar to np.diagonal above, in a future version of numpy, indexing a record array by a list of field names will return a view onto the original array, instead of producing a copy as they do now. As with -np.diagonal, numpy 1.7 produces a DeprecationWarning if it detects +np.diagonal, numpy 1.7 produces a FutureWarning if it detects that you may be attemping to write to such an array. See the documentation for array indexing for details. diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index bc12c5d0e241..f8966f5c1be1 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -339,7 +339,7 @@ Indexing into a record array can also be done with a list of field names, *e.g.* ``x[['field-name1','field-name2']]``. Currently this returns a new array containing a copy of the values in the fields specified in the list. As of NumPy 1.7, returning a copy is being deprecated in favor of returning -a view. A copy will continue to be returned for now, but a DeprecationWarning +a view. A copy will continue to be returned for now, but a FutureWarning will be issued when writing to the copy. If you depend on the current behavior, then we suggest copying the returned array explicitly, i.e. use x[['field-name1','field-name2']].copy(). This will work with both past and diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 0fc238a49722..e3e24fae156f 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1983,13 +1983,13 @@ def collect_warning_types(f, *args, **kwargs): # All the different functions raise a warning, but not an error, and # 'a' is not modified: assert_equal(collect_warning_types(a[['f1','f2']].__setitem__, 0, (10,20)), - [DeprecationWarning]) + [FutureWarning]) assert_equal(a, b) # Views also warn subset = a[['f1','f2']] subset_view = subset.view() assert_equal(collect_warning_types(subset_view['f1'].__setitem__, 0, 10), - [DeprecationWarning]) + [FutureWarning]) # But the write goes through: assert_equal(subset['f1'][0], 10) # Only one warning per multiple field indexing, though (even if there are