From 2dd352cef771dc5efd053fbd31b75a151110e84f Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 11 Jul 2020 18:08:36 +0200 Subject: [PATCH 01/25] Added a benchmark for `trim_zeros()` --- benchmarks/benchmarks/bench_trim_zeros.py | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 benchmarks/benchmarks/bench_trim_zeros.py diff --git a/benchmarks/benchmarks/bench_trim_zeros.py b/benchmarks/benchmarks/bench_trim_zeros.py new file mode 100644 index 000000000000..b8715c156f22 --- /dev/null +++ b/benchmarks/benchmarks/bench_trim_zeros.py @@ -0,0 +1,25 @@ +from .common import Benchmark + +import numpy as np + + +class TrimZeros(Benchmark): + def setup(self): + self.ar_float = np.hstack([ + np.zeros(100_000), + np.random.uniform(size=100_000), + np.zeros(100_000), + ]) + + dtype = [('a', 'int64'), ('b', 'float64'), ('c', bool)] + self.ar_void = self.ar_float.astype(dtype) + self.ar_str = self.ar_float.astype(str) + + def time_trim_zeros_float(self): + np.trim_zeros(self.ar_float) + + def time_trim_zeros_void(self): + np.trim_zeros(self.ar_void) + + def time_trim_zeros_str(self): + np.trim_zeros(self.ar_str) From 59e7cde66404b71e1517299182b65fcc8719e8c8 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 11 Jul 2020 18:53:41 +0200 Subject: [PATCH 02/25] Updated the benchmark with `param_names` and `params` --- benchmarks/benchmarks/bench_trim_zeros.py | 39 +++++++++++++---------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/benchmarks/benchmarks/bench_trim_zeros.py b/benchmarks/benchmarks/bench_trim_zeros.py index b8715c156f22..d72ed817bb1a 100644 --- a/benchmarks/benchmarks/bench_trim_zeros.py +++ b/benchmarks/benchmarks/bench_trim_zeros.py @@ -2,24 +2,29 @@ import numpy as np +_FLOAT = np.dtype('float64') +_COMPLEX = np.dtype('complex128') +_INT = np.dtype('int64') +_BOOL = np.dtype('bool') +_VOID = np.dtype([('a', 'int64'), ('b', 'float64'), ('c', bool)]) +_STR = np.dtype('U32') +_BYTES = np.dtype('S32') -class TrimZeros(Benchmark): - def setup(self): - self.ar_float = np.hstack([ - np.zeros(100_000), - np.random.uniform(size=100_000), - np.zeros(100_000), - ]) - - dtype = [('a', 'int64'), ('b', 'float64'), ('c', bool)] - self.ar_void = self.ar_float.astype(dtype) - self.ar_str = self.ar_float.astype(str) - def time_trim_zeros_float(self): - np.trim_zeros(self.ar_float) +class TrimZeros(Benchmark): + param_names = ["dtype", "size"] + params = [ + [_INT, _FLOAT, _COMPLEX, _BOOL, _STR, _BYTES, _VOID], + [3000, 30_000, 300_000] + ] - def time_trim_zeros_void(self): - np.trim_zeros(self.ar_void) + def setup(self, dtype, size): + n = size // 3 + self.array = np.hstack([ + np.zeros(n), + np.random.uniform(size=n), + np.zeros(n), + ]).astype(dtype) - def time_trim_zeros_str(self): - np.trim_zeros(self.ar_str) + def time_trim_zeros(self, dtype, size): + np.trim_zeros(self.array) From b6fa460c33bf73ed96d067564fb491f5e8779b33 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 11 Jul 2020 18:59:47 +0200 Subject: [PATCH 03/25] Improve the performance of `np.trim_zeros()` --- numpy/lib/function_base.py | 56 +++++++++++++++++++-------- numpy/lib/tests/test_function_base.py | 46 ++++++++++++++++------ 2 files changed, 74 insertions(+), 28 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 6ea9cc4de3e1..4772feed60c9 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1598,7 +1598,7 @@ def trim_zeros(filt, trim='fb'): Parameters ---------- - filt : 1-D array or sequence + filt : array_like, 1 dimension Input array. trim : str, optional A string with 'f' representing trim from front and 'b' to trim from @@ -1619,27 +1619,51 @@ def trim_zeros(filt, trim='fb'): >>> np.trim_zeros(a, 'b') array([0, 0, 0, ..., 0, 2, 1]) + In practice all leading and/or trailing elements will be trimmed as + long as they evaluate to ``False``. + + >>> b = np.array(('', '', '' ,'a', 'b', 'c', '', 'b', 'a', '')) + >>> np.trim_zeros(b) + array(['a', 'b', 'c', '', 'b', 'a']) + The input data type is preserved, list/tuple in means list/tuple out. >>> np.trim_zeros([0, 1, 2, 0]) [1, 2] """ - first = 0 - trim = trim.upper() - if 'F' in trim: - for i in filt: - if i != 0.: - break - else: - first = first + 1 - last = len(filt) - if 'B' in trim: - for i in filt[::-1]: - if i != 0.: - break - else: - last = last - 1 + try: + arr = np.asanyarray(filt, dtype=bool) + + # str/bytes and structured arrays cannot be directly converted into bool arrays + except (TypeError, ValueError): + arr_any = np.asanyarray(filt) + _arr = arr_any.view(bool) + _arr.shape = arr_any.shape + (arr_any.dtype.itemsize,) + arr = _arr.any(axis=-1) + + if arr.ndim != 1: + raise ValueError('trim_zeros requires an array of exactly one dimension') + + trim_upper = trim.upper() + len_arr = len(arr) + first = last = None + + if 'F' in trim_upper: + first = arr.argmax() + # If `arr[first] is False` then so are all other elements + if not arr[first]: + return filt[len_arr:] + + if 'B' in trim_upper: + last = len_arr -arr[::-1].argmax() + # `last == len(arr)` if all elements in `arr` are zero; + # `arr[last]` will thus raise an IndexError + try: + arr[last] + except IndexError: + return filt[len_arr:] + return filt[first:last] diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index eb2fc3311aca..706b48c67670 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1166,25 +1166,47 @@ def test_subclass(self): class TestTrimZeros: - """ - Only testing for integer splits. + a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) + b = a.astype(float) + c = a.astype(complex) + d = a.astype([('a', int), ('b', float), ('c', bool)]) - """ + e = np.array([None, [], 1, False, 'b', 3.0, range(4), b''], dtype=object) + f = np.array(['', '', 'a', '', 'b', 'c', 'd', '']) + g = f.astype(bytes) + + def values(self): + attr_names = ('a', 'b', 'c', 'd', 'e', 'f', 'g') + return (getattr(self, name) for name in attr_names) def test_basic(self): - a = np.array([0, 0, 1, 2, 3, 4, 0]) - res = trim_zeros(a) - assert_array_equal(res, np.array([1, 2, 3, 4])) + slc = np.s_[2:-1] + for arr in self.values(): + res = trim_zeros(arr) + assert_array_equal(res, arr[slc]) def test_leading_skip(self): - a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) - res = trim_zeros(a) - assert_array_equal(res, np.array([1, 0, 2, 3, 4])) + slc = np.s_[:-1] + for arr in self.values(): + res = trim_zeros(arr, trim='b') + assert_array_equal(res, arr[slc]) def test_trailing_skip(self): - a = np.array([0, 0, 1, 0, 2, 3, 0, 4, 0]) - res = trim_zeros(a) - assert_array_equal(res, np.array([1, 0, 2, 3, 0, 4])) + slc = np.s_[2:] + for arr in self.values(): + res = trim_zeros(arr, trim='F') + assert_array_equal(res, arr[slc]) + + def test_all_zero(self): + for _arr in self.values(): + arr = np.zeros_like(_arr, dtype=_arr.dtype) + ref = np.zeros((), dtype=_arr.dtype) + + res1 = trim_zeros(arr, trim='B') + assert_array_equal(res1, ref) + + res2 = trim_zeros(arr, trim='f') + assert_array_equal(res2, ref) class TestExtins: From 5f85a1cafc3e742c31f58a55044b0ad2f778c6a1 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 11 Jul 2020 19:18:54 +0200 Subject: [PATCH 04/25] Fixed a string-representation --- numpy/lib/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 4772feed60c9..52b5349f95dc 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1624,7 +1624,7 @@ def trim_zeros(filt, trim='fb'): >>> b = np.array(('', '', '' ,'a', 'b', 'c', '', 'b', 'a', '')) >>> np.trim_zeros(b) - array(['a', 'b', 'c', '', 'b', 'a']) + array(['a', 'b', 'c', '', 'b', 'a'], dtype=' Date: Mon, 20 Jul 2020 12:19:02 +0200 Subject: [PATCH 05/25] docstring update; fixed an issue with the try/except approach in 'B' --- numpy/lib/function_base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 52b5349f95dc..15a49172a0b7 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1594,7 +1594,10 @@ def _trim_zeros(filt, trim=None): @array_function_dispatch(_trim_zeros) def trim_zeros(filt, trim='fb'): """ - Trim the leading and/or trailing zeros from a 1-D array or sequence. + Trim all leading and/or trailing elements which evaluate to ``False``. + + .. versionchanged:: 1.20.0 + Trim any element which evaluates to ``False`` instead of just zeros. Parameters ---------- @@ -1622,7 +1625,7 @@ def trim_zeros(filt, trim='fb'): In practice all leading and/or trailing elements will be trimmed as long as they evaluate to ``False``. - >>> b = np.array(('', '', '' ,'a', 'b', 'c', '', 'b', 'a', '')) + >>> b = np.array(['', '', '' ,'a', 'b', 'c', '', 'b', 'a', '']) >>> np.trim_zeros(b) array(['a', 'b', 'c', '', 'b', 'a'], dtype=' Date: Mon, 20 Jul 2020 12:21:59 +0200 Subject: [PATCH 06/25] Removed a sentence which is redundant since 2a2d9d622e9bdeafd5cfafc9f3d20fd8d27c235f --- numpy/lib/function_base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 15a49172a0b7..1639a879a424 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1622,9 +1622,6 @@ def trim_zeros(filt, trim='fb'): >>> np.trim_zeros(a, 'b') array([0, 0, 0, ..., 0, 2, 1]) - In practice all leading and/or trailing elements will be trimmed as - long as they evaluate to ``False``. - >>> b = np.array(['', '', '' ,'a', 'b', 'c', '', 'b', 'a', '']) >>> np.trim_zeros(b) array(['a', 'b', 'c', '', 'b', 'a'], dtype=' Date: Mon, 20 Jul 2020 12:44:01 +0200 Subject: [PATCH 07/25] Reverted a redundant documentation change --- numpy/lib/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 1639a879a424..d851b394bb6d 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1601,7 +1601,7 @@ def trim_zeros(filt, trim='fb'): Parameters ---------- - filt : array_like, 1 dimension + filt : 1-D array or sequence Input array. trim : str, optional A string with 'f' representing trim from front and 'b' to trim from From ca5d4624d889f7572a9167dfae03701b455dd386 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 13:14:29 +0200 Subject: [PATCH 08/25] Improve readability: use ndarray.any() to check for empty arrays --- numpy/lib/function_base.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index d851b394bb6d..f77c5c1516c8 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1644,22 +1644,16 @@ def trim_zeros(filt, trim='fb'): if arr.ndim != 1: raise ValueError('trim_zeros requires an array of exactly one dimension') + elif not arr.any(): + return filt[len(arr):] trim_upper = trim.upper() - len_arr = len(arr) first = last = None if 'F' in trim_upper: first = arr.argmax() - # If `arr[first] is False` then so are all other elements - if not arr[first]: - return filt[len_arr:] - if 'B' in trim_upper: - last = len_arr - arr[::-1].argmax() - # If `last == 0 and arr[0] is False` then all elements are False - if not last and not arr[last]: - return filt[len_arr:] + last = len(arr) - arr[::-1].argmax() return filt[first:last] From f4ae5226cc2d4398130d0902c6974201580202e1 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 13:16:44 +0200 Subject: [PATCH 09/25] Revert "Improve readability: use ndarray.any() to check for empty arrays" This reverts commit 997bdc793234d6c1c75738035ba63730d686b356. --- numpy/lib/function_base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index f77c5c1516c8..d851b394bb6d 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1644,16 +1644,22 @@ def trim_zeros(filt, trim='fb'): if arr.ndim != 1: raise ValueError('trim_zeros requires an array of exactly one dimension') - elif not arr.any(): - return filt[len(arr):] trim_upper = trim.upper() + len_arr = len(arr) first = last = None if 'F' in trim_upper: first = arr.argmax() + # If `arr[first] is False` then so are all other elements + if not arr[first]: + return filt[len_arr:] + if 'B' in trim_upper: - last = len(arr) - arr[::-1].argmax() + last = len_arr - arr[::-1].argmax() + # If `last == 0 and arr[0] is False` then all elements are False + if not last and not arr[last]: + return filt[len_arr:] return filt[first:last] From 8aade1b4a5ab57ca27166ef02a919aa275f092a6 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 13:19:15 +0200 Subject: [PATCH 10/25] The performance increase of 997bdc793234d6c1c75738035ba63730d686b356 is significant for large arrays; revert it --- numpy/lib/function_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index d851b394bb6d..eba21f433179 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1657,8 +1657,8 @@ def trim_zeros(filt, trim='fb'): if 'B' in trim_upper: last = len_arr - arr[::-1].argmax() - # If `last == 0 and arr[0] is False` then all elements are False - if not last and not arr[last]: + # If `last == len(arr) and arr[-1] is False` then all elements are False + if last == len_arr and not arr[-1]: return filt[len_arr:] return filt[first:last] From 189e8084d2164edca864d6495146fe9732a63b6c Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 13:19:31 +0200 Subject: [PATCH 11/25] Increase the variety of the tests --- numpy/lib/tests/test_function_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 706b48c67670..daed56e3c935 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1169,7 +1169,7 @@ class TestTrimZeros: a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) b = a.astype(float) c = a.astype(complex) - d = a.astype([('a', int), ('b', float), ('c', bool)]) + d = a.astype([('a', int, (2, 3)), ('b', float), ('c', bool)]) e = np.array([None, [], 1, False, 'b', 3.0, range(4), b''], dtype=object) f = np.array(['', '', 'a', '', 'b', 'c', 'd', '']) @@ -1200,13 +1200,12 @@ def test_trailing_skip(self): def test_all_zero(self): for _arr in self.values(): arr = np.zeros_like(_arr, dtype=_arr.dtype) - ref = np.zeros((), dtype=_arr.dtype) res1 = trim_zeros(arr, trim='B') - assert_array_equal(res1, ref) + assert len(res1) == 0 res2 = trim_zeros(arr, trim='f') - assert_array_equal(res2, ref) + assert len(res2) == 0 class TestExtins: From 7a9d0e262462bf73fe15f7819f24b06e2e6c750a Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 14:02:19 +0200 Subject: [PATCH 12/25] BUG: check if array-length is non-zero before calling `argmax()` --- numpy/lib/function_base.py | 2 ++ numpy/lib/tests/test_function_base.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index eba21f433179..c05ea0954f5e 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1644,6 +1644,8 @@ def trim_zeros(filt, trim='fb'): if arr.ndim != 1: raise ValueError('trim_zeros requires an array of exactly one dimension') + elif not len(arr): + return filt trim_upper = trim.upper() len_arr = len(arr) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index daed56e3c935..7e88ea8cc01f 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1207,6 +1207,11 @@ def test_all_zero(self): res2 = trim_zeros(arr, trim='f') assert len(res2) == 0 + def test_size_zero(self): + arr = np.zeros(0) + res = trim_zeros(arr) + assert_array_equal(arr, res) + class TestExtins: From 75ad3053eaf752e8b407d171ae132458e9705161 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 15:04:48 +0200 Subject: [PATCH 13/25] Implement https://github.com/numpy/numpy/pull/16911#discussion_r457330789 --- benchmarks/benchmarks/bench_trim_zeros.py | 5 +---- numpy/lib/function_base.py | 19 ++----------------- numpy/lib/tests/test_function_base.py | 8 ++------ 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/benchmarks/benchmarks/bench_trim_zeros.py b/benchmarks/benchmarks/bench_trim_zeros.py index d72ed817bb1a..4e25a8b021b7 100644 --- a/benchmarks/benchmarks/bench_trim_zeros.py +++ b/benchmarks/benchmarks/bench_trim_zeros.py @@ -6,15 +6,12 @@ _COMPLEX = np.dtype('complex128') _INT = np.dtype('int64') _BOOL = np.dtype('bool') -_VOID = np.dtype([('a', 'int64'), ('b', 'float64'), ('c', bool)]) -_STR = np.dtype('U32') -_BYTES = np.dtype('S32') class TrimZeros(Benchmark): param_names = ["dtype", "size"] params = [ - [_INT, _FLOAT, _COMPLEX, _BOOL, _STR, _BYTES, _VOID], + [_INT, _FLOAT, _COMPLEX, _BOOL], [3000, 30_000, 300_000] ] diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index c05ea0954f5e..8115927f93b6 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1594,10 +1594,7 @@ def _trim_zeros(filt, trim=None): @array_function_dispatch(_trim_zeros) def trim_zeros(filt, trim='fb'): """ - Trim all leading and/or trailing elements which evaluate to ``False``. - - .. versionchanged:: 1.20.0 - Trim any element which evaluates to ``False`` instead of just zeros. + Trim the leading and/or trailing zeros from a 1-D array or sequence. Parameters ---------- @@ -1622,25 +1619,13 @@ def trim_zeros(filt, trim='fb'): >>> np.trim_zeros(a, 'b') array([0, 0, 0, ..., 0, 2, 1]) - >>> b = np.array(['', '', '' ,'a', 'b', 'c', '', 'b', 'a', '']) - >>> np.trim_zeros(b) - array(['a', 'b', 'c', '', 'b', 'a'], dtype='>> np.trim_zeros([0, 1, 2, 0]) [1, 2] """ - try: - arr = np.asanyarray(filt, dtype=bool) - - # str/bytes and structured arrays cannot be directly converted into bool arrays - except (TypeError, ValueError): - arr_any = np.asanyarray(filt) - _arr = arr_any.view(bool) - _arr.shape = arr_any.shape + (arr_any.dtype.itemsize,) - arr = _arr.any(axis=-1) + arr = np.asanyarray(filt).astype(bool, copy=False) if arr.ndim != 1: raise ValueError('trim_zeros requires an array of exactly one dimension') diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 7e88ea8cc01f..89c1a2d9bc01 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1169,14 +1169,10 @@ class TestTrimZeros: a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) b = a.astype(float) c = a.astype(complex) - d = a.astype([('a', int, (2, 3)), ('b', float), ('c', bool)]) - - e = np.array([None, [], 1, False, 'b', 3.0, range(4), b''], dtype=object) - f = np.array(['', '', 'a', '', 'b', 'c', 'd', '']) - g = f.astype(bytes) + d = np.array([None, [], 1, False, 'b', 3.0, range(4), b''], dtype=object) def values(self): - attr_names = ('a', 'b', 'c', 'd', 'e', 'f', 'g') + attr_names = ('a', 'b', 'c', 'd') return (getattr(self, name) for name in attr_names) def test_basic(self): From 9d1662bd3185699c1cb5212348662adda10164c2 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 15:11:18 +0200 Subject: [PATCH 14/25] Implement https://github.com/numpy/numpy/pull/16911#discussion_r457358202 --- numpy/lib/function_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 8115927f93b6..0dc3742afdf1 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1644,8 +1644,8 @@ def trim_zeros(filt, trim='fb'): if 'B' in trim_upper: last = len_arr - arr[::-1].argmax() - # If `last == len(arr) and arr[-1] is False` then all elements are False - if last == len_arr and not arr[-1]: + # If `arr[last - 1] is False` then so are all other elements + if not arr[last - 1]: return filt[len_arr:] return filt[first:last] From ccf1c21581e391cb8fbadef1ab2233c4bb91efd7 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 16:17:32 +0200 Subject: [PATCH 15/25] Implemented https://github.com/numpy/numpy/pull/16911#discussion_r457415065 --- numpy/lib/function_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 0dc3742afdf1..d5e5300ab08e 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1633,20 +1633,19 @@ def trim_zeros(filt, trim='fb'): return filt trim_upper = trim.upper() - len_arr = len(arr) first = last = None if 'F' in trim_upper: first = arr.argmax() # If `arr[first] is False` then so are all other elements if not arr[first]: - return filt[len_arr:] + return filt[:0] if 'B' in trim_upper: - last = len_arr - arr[::-1].argmax() + last = len(arr) - arr[::-1].argmax() # If `arr[last - 1] is False` then so are all other elements if not arr[last - 1]: - return filt[len_arr:] + return filt[:0] return filt[first:last] From 4d9131107eb797d6043b2c886951d7c8819a1953 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 20 Jul 2020 23:15:55 +0200 Subject: [PATCH 16/25] Implemented https://github.com/numpy/numpy/pull/16911#issuecomment-661231106 Fall back to the old `np.trim_zeros()` implementation if an exception is encountered. Emit a `DeprecationWarning` in such case. --- numpy/lib/function_base.py | 43 +++++++++++++++++++++++++++ numpy/lib/tests/test_function_base.py | 18 +++++++++++ 2 files changed, 61 insertions(+) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index d5e5300ab08e..efa640acd2fe 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1625,6 +1625,23 @@ def trim_zeros(filt, trim='fb'): [1, 2] """ + try: + return _trim_zeros_new(filt, trim) + except Exception as ex: + warning = DeprecationWarning( + "in the future trim_zeros will require a 1-D array as input" + "which is compatible with ndarray.astype(bool)" + ) + warning.__cause__ = ex + warnings.warn(warning, stacklevel=3) + + # Fall back to the old implementation if an exception is encountered + # Note that the same exception may or may not raised here as well + return _trim_zeros_old(filt, trim) + + +def _trim_zeros_new(filt, trim='fb'): + """Newer optimized implementation of ``trim_zeros()``.""" arr = np.asanyarray(filt).astype(bool, copy=False) if arr.ndim != 1: @@ -1650,6 +1667,32 @@ def trim_zeros(filt, trim='fb'): return filt[first:last] +def _trim_zeros_old(filt, trim='fb'): + """ + Older unoptimized implementation of ``trim_zeros()``. + + Used as fallback in case an exception is encountered + in ``_trim_zeros_new()``. + + """ + first = 0 + trim = trim.upper() + if 'F' in trim: + for i in filt: + if i != 0.: + break + else: + first = first + 1 + last = len(filt) + if 'B' in trim: + for i in filt[::-1]: + if i != 0.: + break + else: + last = last - 1 + return filt[first:last] + + def _extract_dispatcher(condition, arr): return (condition, arr) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 89c1a2d9bc01..4650d77bc7a1 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1208,6 +1208,24 @@ def test_size_zero(self): res = trim_zeros(arr) assert_array_equal(arr, res) + def test_deprecation(self): + arr = np.random.rand(10, 10).tolist() + + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + try: + trim_zeros(arr) + except Exception as ex: + assert isinstance(ex, DeprecationWarning) + assert isinstance(ex.__cause__, ValueError) + else: + raise Assertion('Failed to raise an exception') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + out = trim_zeros(arr) + assert_allclose(out, arr) + class TestExtins: From eae7275057df1f5d711c45ca835b15cce435501b Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 21 Jul 2020 11:32:36 +0200 Subject: [PATCH 17/25] TST: Avoid the use of `warnings.simplefilter('ignore', ...) --- numpy/lib/tests/test_function_base.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 4650d77bc7a1..f1c85880adef 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1221,10 +1221,8 @@ def test_deprecation(self): else: raise Assertion('Failed to raise an exception') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - out = trim_zeros(arr) - assert_allclose(out, arr) + out = nfb._trim_zeros_old(arr) + assert_allclose(out, arr) class TestExtins: From 20829cf1c130e4fc9c3a5cff92ad8e121d73a0b7 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 21 Jul 2020 11:33:38 +0200 Subject: [PATCH 18/25] MAINT: Gramar fix --- numpy/lib/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index efa640acd2fe..6194ccadd8d8 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1630,7 +1630,7 @@ def trim_zeros(filt, trim='fb'): except Exception as ex: warning = DeprecationWarning( "in the future trim_zeros will require a 1-D array as input" - "which is compatible with ndarray.astype(bool)" + "that is compatible with ndarray.astype(bool)" ) warning.__cause__ = ex warnings.warn(warning, stacklevel=3) From a25620e57421719ef5275e3f0074587f0f255f4b Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 21 Jul 2020 12:45:08 +0200 Subject: [PATCH 19/25] TST,MAINT: Fixed an incorrect exception name --- numpy/lib/tests/test_function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index f1c85880adef..9c55a571e7f1 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1219,7 +1219,7 @@ def test_deprecation(self): assert isinstance(ex, DeprecationWarning) assert isinstance(ex.__cause__, ValueError) else: - raise Assertion('Failed to raise an exception') + raise AssertionError('Failed to raise an exception') out = nfb._trim_zeros_old(arr) assert_allclose(out, arr) From 25d2d231b531370a096e34d0ed1b9a68fa7915b5 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 10:00:39 +0200 Subject: [PATCH 20/25] STY: remove trailing whitespaces --- numpy/lib/function_base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 6194ccadd8d8..5f842d5888b9 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -433,7 +433,7 @@ def asarray_chkfinite(a, dtype=None, order=None): By default, the data-type is inferred from the input data. order : {'C', 'F', 'A', 'K'}, optional Memory layout. 'A' and 'K' depend on the order of input array a. - 'C' row-major (C-style), + 'C' row-major (C-style), 'F' column-major (Fortran-style) memory representation. 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise 'K' (keep) preserve input order @@ -2596,11 +2596,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): for backwards compatibility with previous versions of this function. These arguments had no effect on the return values of the function and can be safely ignored in this and previous versions of numpy. - + Examples - -------- + -------- In this example we generate two random arrays, ``xarr`` and ``yarr``, and - compute the row-wise and column-wise Pearson correlation coefficients, + compute the row-wise and column-wise Pearson correlation coefficients, ``R``. Since ``rowvar`` is true by default, we first find the row-wise Pearson correlation coefficients between the variables of ``xarr``. @@ -2616,11 +2616,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): array([[ 1. , 0.99256089, -0.68080986], [ 0.99256089, 1. , -0.76492172], [-0.68080986, -0.76492172, 1. ]]) - - If we add another set of variables and observations ``yarr``, we can + + If we add another set of variables and observations ``yarr``, we can compute the row-wise Pearson correlation coefficients between the variables in ``xarr`` and ``yarr``. - + >>> yarr = rng.random((3, 3)) >>> yarr array([[0.45038594, 0.37079802, 0.92676499], @@ -2642,7 +2642,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): 1. ]]) Finally if we use the option ``rowvar=False``, the columns are now - being treated as the variables and we will find the column-wise Pearson + being treated as the variables and we will find the column-wise Pearson correlation coefficients between variables in ``xarr`` and ``yarr``. >>> R3 = np.corrcoef(xarr, yarr, rowvar=False) From 16d5072a9f382ef4a656928339eeb3e011ec1f7c Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 10:02:12 +0200 Subject: [PATCH 21/25] DEP: Added a date/version deprecation comment --- numpy/lib/function_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 5f842d5888b9..cd7513c25ac4 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1628,6 +1628,7 @@ def trim_zeros(filt, trim='fb'): try: return _trim_zeros_new(filt, trim) except Exception as ex: + # Numpy 1.20.0, 2020-07-31 warning = DeprecationWarning( "in the future trim_zeros will require a 1-D array as input" "that is compatible with ndarray.astype(bool)" @@ -1636,7 +1637,7 @@ def trim_zeros(filt, trim='fb'): warnings.warn(warning, stacklevel=3) # Fall back to the old implementation if an exception is encountered - # Note that the same exception may or may not raised here as well + # Note that the same exception may or may not be raised here as well return _trim_zeros_old(filt, trim) From ab313d5515c85e4f68cea346374448e077e0d625 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 10:35:34 +0200 Subject: [PATCH 22/25] MAINT: Typo fix; added a missing space --- numpy/lib/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index cd7513c25ac4..cd8862c94eeb 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1630,7 +1630,7 @@ def trim_zeros(filt, trim='fb'): except Exception as ex: # Numpy 1.20.0, 2020-07-31 warning = DeprecationWarning( - "in the future trim_zeros will require a 1-D array as input" + "in the future trim_zeros will require a 1-D array as input " "that is compatible with ndarray.astype(bool)" ) warning.__cause__ = ex From 8b99f60f670e193eaf41fe255f58eb73fb66a5f9 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 10:46:31 +0200 Subject: [PATCH 23/25] TST,DEP: Added a deprecation testcase --- numpy/core/tests/test_deprecations.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 47258ed08e21..9e13d7219de0 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -615,7 +615,7 @@ def test_deprecated(self): self.assert_deprecated(round, args=(scalar,)) self.assert_deprecated(round, args=(scalar, 0)) self.assert_deprecated(round, args=(scalar,), kwargs={'ndigits': 0}) - + def test_not_deprecated(self): for scalar_type in self.not_deprecated_types: scalar = scalar_type(0) @@ -706,3 +706,15 @@ def test_deprecated(self): # And when it is an assignment into a lower dimensional subarray: self.assert_deprecated(lambda: np.array([arr, [0]], dtype=np.float64)) self.assert_deprecated(lambda: np.array([[0], arr], dtype=np.float64)) + + +class TestTrimZeros(_DeprecationTestCase): + # Numpy 1.20.0, 2020-07-31 + def test_deprecated(self): + # Expects a 1D array-like objects + a = np.random.rand(10, 10).tolist() + self.assert_deprecated(np.trim_zeros, args=(a,)) + + # Must be compatible with ndarray.astype(str) + b = np.random.rand(10).astype(str) + self.assert_deprecated(np.trim_zeros, args=(b,)) From 2392fe3dd3de3cea4e98493af30bc3ad8d8db50f Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 11:07:10 +0200 Subject: [PATCH 24/25] DEP,REL: Added a deprecation release note --- doc/release/upcoming_changes/16911.deprecation.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release/upcoming_changes/16911.deprecation.rst diff --git a/doc/release/upcoming_changes/16911.deprecation.rst b/doc/release/upcoming_changes/16911.deprecation.rst new file mode 100644 index 000000000000..d4dcb629ceb2 --- /dev/null +++ b/doc/release/upcoming_changes/16911.deprecation.rst @@ -0,0 +1,7 @@ +``trim_zeros`` now requires a 1D array compatible with ``ndarray.astype(bool)`` +------------------------------------------------------------------------------- +The ``trim_zeros`` function will, in the future, require an array with the +following two properties: + +* It must be 1D. +* It must be convertable into a boolean array. From dea951f9dee11862b7da028296ef537e5eca7fe3 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 3 Aug 2020 15:29:09 +0200 Subject: [PATCH 25/25] TST,DEP: Moved leftovers from a deprecation test --- numpy/core/tests/test_deprecations.py | 22 ++++++++++++++-------- numpy/lib/tests/test_function_base.py | 16 ---------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 9e13d7219de0..9004bef30af3 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -710,11 +710,17 @@ def test_deprecated(self): class TestTrimZeros(_DeprecationTestCase): # Numpy 1.20.0, 2020-07-31 - def test_deprecated(self): - # Expects a 1D array-like objects - a = np.random.rand(10, 10).tolist() - self.assert_deprecated(np.trim_zeros, args=(a,)) - - # Must be compatible with ndarray.astype(str) - b = np.random.rand(10).astype(str) - self.assert_deprecated(np.trim_zeros, args=(b,)) + @pytest.mark.parametrize("arr", [np.random.rand(10, 10).tolist(), + np.random.rand(10).astype(str)]) + def test_deprecated(self, arr): + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + try: + np.trim_zeros(arr) + except DeprecationWarning as ex: + assert_(isinstance(ex.__cause__, ValueError)) + else: + raise AssertionError("No error raised during function call") + + out = np.lib.function_base._trim_zeros_old(arr) + assert_array_equal(arr, out) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 9c55a571e7f1..89c1a2d9bc01 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1208,22 +1208,6 @@ def test_size_zero(self): res = trim_zeros(arr) assert_array_equal(arr, res) - def test_deprecation(self): - arr = np.random.rand(10, 10).tolist() - - with warnings.catch_warnings(): - warnings.simplefilter('error', DeprecationWarning) - try: - trim_zeros(arr) - except Exception as ex: - assert isinstance(ex, DeprecationWarning) - assert isinstance(ex.__cause__, ValueError) - else: - raise AssertionError('Failed to raise an exception') - - out = nfb._trim_zeros_old(arr) - assert_allclose(out, arr) - class TestExtins: