From 6d8efc51d89e1c380130e695c8c8c6522f1570b6 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 4 Mar 2022 16:10:11 +0100 Subject: [PATCH 01/21] TST Add minimal setup to be able to run test suite on float32 Co-authored-by: Thomas J. Fan --- sklearn/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sklearn/conftest.py b/sklearn/conftest.py index a9cc21d1c6949..eb681de63284f 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -4,6 +4,7 @@ import sys import pytest +import numpy as np from threadpoolctl import threadpool_limits from _pytest.doctest import DoctestItem @@ -37,6 +38,17 @@ "fetch_rcv1_fxt": fetch_rcv1, } +_SKIP32_MARK = pytest.mark.skipif( + environ.get("SKLEARN_SKIP_FLOAT32", "1") != "0", + reason="Set SKLEARN_SKIP_FLOAT32=0 to run float32 dtype tests", +) + + +# Global fixture +@pytest.fixture(params=[pytest.param(np.float32, marks=_SKIP32_MARK), np.float64]) +def dtype(request): + yield request.param + def _fetch_fixture(f): """Fetch dataset (download if missing and requested by environment).""" From eb911dd03e7d098fa4abdf96b9e9559b8bfcdda4 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 9 Mar 2022 11:37:26 +0100 Subject: [PATCH 02/21] TST Use dtype fixture in one test --- sklearn/feature_selection/tests/test_mutual_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py index bb98dfaee4db9..c656488344d59 100644 --- a/sklearn/feature_selection/tests/test_mutual_info.py +++ b/sklearn/feature_selection/tests/test_mutual_info.py @@ -8,10 +8,10 @@ from sklearn.feature_selection import mutual_info_regression, mutual_info_classif -def test_compute_mi_dd(): +def test_compute_mi_dd(dtype): # In discrete case computations are straightforward and can be done # by hand on given vectors. - x = np.array([0, 1, 1, 0, 0]) + x = np.array([0, 1, 1, 0, 0], dtype=dtype) y = np.array([1, 0, 0, 0, 1]) H_x = H_y = -(3 / 5) * np.log(3 / 5) - (2 / 5) * np.log(2 / 5) From 86bf47f97cadcca42bd8972275d72e4ba7547c1f Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 9 Mar 2022 11:39:13 +0100 Subject: [PATCH 03/21] CI Do not skip 32bit test for py38_conda_defaults_openblas --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 42efc0a0c65b9..af02cb3a3d9fd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -201,6 +201,7 @@ jobs: MATPLOTLIB_VERSION: 'min' THREADPOOLCTL_VERSION: '2.2.0' SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1' + SKLEARN_SKIP_FLOAT32: '0' # Linux environment to test the latest available dependencies. # It runs tests requiring lightgbm, pandas and PyAMG. pylatest_pip_openblas_pandas: From 585e2787b6071944bbd2163fa94820adbfad6e51 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 9 Mar 2022 12:04:04 +0100 Subject: [PATCH 04/21] fixup! TST Use dtype fixture in one test --- sklearn/feature_selection/tests/test_mutual_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py index c656488344d59..0706454152008 100644 --- a/sklearn/feature_selection/tests/test_mutual_info.py +++ b/sklearn/feature_selection/tests/test_mutual_info.py @@ -8,10 +8,10 @@ from sklearn.feature_selection import mutual_info_regression, mutual_info_classif -def test_compute_mi_dd(dtype): +def test_compute_mi_dd(): # In discrete case computations are straightforward and can be done # by hand on given vectors. - x = np.array([0, 1, 1, 0, 0], dtype=dtype) + x = np.array([0, 1, 1, 0, 0]) y = np.array([1, 0, 0, 0, 1]) H_x = H_y = -(3 / 5) * np.log(3 / 5) - (2 / 5) * np.log(2 / 5) @@ -21,7 +21,7 @@ def test_compute_mi_dd(dtype): assert_almost_equal(_compute_mi(x, y, True, True), I_xy) -def test_compute_mi_cc(): +def test_compute_mi_cc(dtype): # For two continuous variables a good approach is to test on bivariate # normal distribution, where mutual information is known. @@ -43,7 +43,7 @@ def test_compute_mi_cc(): I_theory = np.log(sigma_1) + np.log(sigma_2) - 0.5 * np.log(np.linalg.det(cov)) rng = check_random_state(0) - Z = rng.multivariate_normal(mean, cov, size=1000) + Z = rng.multivariate_normal(mean, cov, size=1000).astype(dtype) x, y = Z[:, 0], Z[:, 1] From 09ccf3196c948532974fb40d67a1f14813f7b06d Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 10 Mar 2022 11:11:34 +0100 Subject: [PATCH 05/21] MAINT Apply reviews comments Co-authored-by: Thomas J. Fan --- azure-pipelines.yml | 2 +- sklearn/conftest.py | 8 ++++---- sklearn/feature_selection/tests/test_mutual_info.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index af02cb3a3d9fd..52712427cdba4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -201,7 +201,7 @@ jobs: MATPLOTLIB_VERSION: 'min' THREADPOOLCTL_VERSION: '2.2.0' SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1' - SKLEARN_SKIP_FLOAT32: '0' + SKLEARN_RUN_FLOAT32: '1' # Linux environment to test the latest available dependencies. # It runs tests requiring lightgbm, pandas and PyAMG. pylatest_pip_openblas_pandas: diff --git a/sklearn/conftest.py b/sklearn/conftest.py index eb681de63284f..6c5931fb715ff 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -39,14 +39,14 @@ } _SKIP32_MARK = pytest.mark.skipif( - environ.get("SKLEARN_SKIP_FLOAT32", "1") != "0", - reason="Set SKLEARN_SKIP_FLOAT32=0 to run float32 dtype tests", + environ.get("SKLEARN_RUN_FLOAT32", "0") != "1", + reason="Set SKLEARN_RUN_FLOAT32=1 to run float32 dtype tests", ) -# Global fixture +# Global fixtures @pytest.fixture(params=[pytest.param(np.float32, marks=_SKIP32_MARK), np.float64]) -def dtype(request): +def global_dtype(request): yield request.param diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py index 0706454152008..1e18035c5c7a6 100644 --- a/sklearn/feature_selection/tests/test_mutual_info.py +++ b/sklearn/feature_selection/tests/test_mutual_info.py @@ -21,7 +21,7 @@ def test_compute_mi_dd(): assert_almost_equal(_compute_mi(x, y, True, True), I_xy) -def test_compute_mi_cc(dtype): +def test_compute_mi_cc(global_dtype): # For two continuous variables a good approach is to test on bivariate # normal distribution, where mutual information is known. @@ -43,7 +43,7 @@ def test_compute_mi_cc(dtype): I_theory = np.log(sigma_1) + np.log(sigma_2) - 0.5 * np.log(np.linalg.det(cov)) rng = check_random_state(0) - Z = rng.multivariate_normal(mean, cov, size=1000).astype(dtype) + Z = rng.multivariate_normal(mean, cov, size=1000).astype(global_dtype) x, y = Z[:, 0], Z[:, 1] From f71db827a289dcc4120543d1f1ab1063d63c111b Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 10 Mar 2022 14:21:56 +0100 Subject: [PATCH 06/21] Use an more explicit name for the env variable Co-authored-by: Olivier Grisel --- azure-pipelines.yml | 2 +- sklearn/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 52712427cdba4..0a4ef6344ef16 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -201,7 +201,7 @@ jobs: MATPLOTLIB_VERSION: 'min' THREADPOOLCTL_VERSION: '2.2.0' SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1' - SKLEARN_RUN_FLOAT32: '1' + SKLEARN_TESTS_RUN_FLOAT32: '1' # Linux environment to test the latest available dependencies. # It runs tests requiring lightgbm, pandas and PyAMG. pylatest_pip_openblas_pandas: diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 6c5931fb715ff..2a63cd80944cf 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -39,8 +39,8 @@ } _SKIP32_MARK = pytest.mark.skipif( - environ.get("SKLEARN_RUN_FLOAT32", "0") != "1", - reason="Set SKLEARN_RUN_FLOAT32=1 to run float32 dtype tests", + environ.get("SKLEARN_TESTS_RUN_FLOAT32", "0") != "1", + reason="Set SKLEARN_TESTS_RUN_FLOAT32=1 to run float32 dtype tests", ) From 915a4c7800bba0d793a9063faac6c501df7892b6 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 11 Mar 2022 11:37:55 +0100 Subject: [PATCH 07/21] DOC Rename and document the environement variable So as to have a similar name to `SKLEARN_SKIP_NETWORK_TESTS`. Co-authored-by: Thomas J. Fan --- azure-pipelines.yml | 2 +- doc/computing/parallelism.rst | 7 +++++++ sklearn/conftest.py | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0a4ef6344ef16..1c927dd7ae526 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -201,7 +201,7 @@ jobs: MATPLOTLIB_VERSION: 'min' THREADPOOLCTL_VERSION: '2.2.0' SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1' - SKLEARN_TESTS_RUN_FLOAT32: '1' + SKLEARN_RUN_FLOAT32_TESTS: '1' # Linux environment to test the latest available dependencies. # It runs tests requiring lightgbm, pandas and PyAMG. pylatest_pip_openblas_pandas: diff --git a/doc/computing/parallelism.rst b/doc/computing/parallelism.rst index 2722324d6cbc2..70121521dc73a 100644 --- a/doc/computing/parallelism.rst +++ b/doc/computing/parallelism.rst @@ -200,6 +200,13 @@ These environment variables should be set before importing scikit-learn. that need network access are skipped. When this environment variable is not set then network tests are skipped. +:SKLEARN_TESTS_RUN_FLOAT32: + + When this environment variable is set to '1', the tests using the + `global_dtype` `pytest.fixture` are also run float32 data. + When this environment variable is not set, the tests are only run on + float64 data. + :SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: When this environment variable is set to a non zero value, the `Cython` diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 2a63cd80944cf..e213cedc30432 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -39,8 +39,8 @@ } _SKIP32_MARK = pytest.mark.skipif( - environ.get("SKLEARN_TESTS_RUN_FLOAT32", "0") != "1", - reason="Set SKLEARN_TESTS_RUN_FLOAT32=1 to run float32 dtype tests", + environ.get("SKLEARN_RUN_FLOAT32_TESTS", "0") != "1", + reason="Set SKLEARN_RUN_FLOAT32_TESTS=1 to run float32 dtype tests", ) From 53caeeb4afea7b5468c0920ae30bd7ad10630e4f Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 11 Mar 2022 11:39:41 +0100 Subject: [PATCH 08/21] fixup! DOC Rename and document the environement variable --- doc/computing/parallelism.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/computing/parallelism.rst b/doc/computing/parallelism.rst index 70121521dc73a..0134956b1728d 100644 --- a/doc/computing/parallelism.rst +++ b/doc/computing/parallelism.rst @@ -200,7 +200,7 @@ These environment variables should be set before importing scikit-learn. that need network access are skipped. When this environment variable is not set then network tests are skipped. -:SKLEARN_TESTS_RUN_FLOAT32: +:SKLEARN_RUN_FLOAT32_TESTS: When this environment variable is set to '1', the tests using the `global_dtype` `pytest.fixture` are also run float32 data. From 9f789418f33d2f8e0c6051d0b659e15161331b41 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 14 Mar 2022 15:47:17 +0100 Subject: [PATCH 09/21] TST Introduce custom assert_allclose This is heavily inspired by the one in PyTorch: https://github.com/pytorch/pytorch/blob/1f29b3130af218847a043e58fdc64511bbe072fe/torch/testing/_comparison.py Co-authored-by: Thomas J. Fan --- sklearn/utils/_testing.py | 79 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index ac84bf058df8c..4a53b4f8c4919 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -38,7 +38,7 @@ except NameError: WindowsError = None -from numpy.testing import assert_allclose +from numpy.testing import assert_allclose as np_assert_allclose from numpy.testing import assert_almost_equal from numpy.testing import assert_approx_equal from numpy.testing import assert_array_equal @@ -59,6 +59,7 @@ check_array, check_is_fitted, check_X_y, + _is_arraylike_not_scalar, ) @@ -87,6 +88,13 @@ assert_raises_regexp = assert_raises_regex +DTYPE_TOLERANCES = { + # rtol, atol + np.float32: (1e-6, 1e-5), + np.float64: (1e-7, 1e-7), +} + + # TODO: Remove in 1.2 @deprecated( # type: ignore "`assert_warns` is deprecated in 1.0 and will be removed in 1.2." @@ -387,6 +395,75 @@ def assert_raise_message(exceptions, message, function, *args, **kwargs): raise AssertionError("%s not raised by %s" % (names, function.__name__)) +def assert_allclose( + actual, desired, rtol=None, atol=None, equal_nan=True, err_msg="", verbose=True +): + """ + Adaptation of numpy.testing.assert_allclose to have tolerances + be set based on the input arrays' dtypes. + + Parameters + ---------- + actual : array_like + Array obtained. + desired : array_like + Array desired. + rtol : float, optional, default=None + Relative tolerance. + If None, it is set based on the provided arrays' dtypes. + atol : float, optional, default=None + Absolute tolerance. + If None, it is set based on the provided arrays' dtypes. + equal_nan : bool, optional, default=True + If True, NaNs will compare equal. + err_msg : str, optional, default='' + The error message to be printed in case of failure. + verbose : bool, optional, default=True + If True, the conflicting values are appended to the error message. + + Raises + ------ + AssertionError + If actual and desired are not equal up to specified precision. + + See Also + -------- + numpy.testing.assert_allclose + + Examples + -------- + >>> x = [1e-5, 1e-3, 1e-1] + >>> y = np.arccos(np.cos(x)) + >>> np.testing.assert_allclose(x, y, rtol=1e-5, atol=0) + + """ + dtypes = [] + + for input in (actual, desired): + if _is_arraylike_not_scalar(input): + dtypes.append(np.asarray(input).dtype) + elif isinstance(input, np.dtype): + dtypes.append(input) + elif np.isscalar(input): + dtypes.append(type(input)) + else: + raise TypeError( + f"Expected a np.array or a np.dtype, but got {type(input)} instead." + ) + + rtols, atols = zip( + *[DTYPE_TOLERANCES.get(dtype, (1e-10, 1e-10)) for dtype in dtypes] + ) + + if rtol is None: + rtol = max(rtols) + + if atol is None: + atol = max(atols) + + np_assert_allclose(actual, desired, rtol, atol, equal_nan, err_msg, verbose) + + def assert_allclose_dense_sparse(x, y, rtol=1e-07, atol=1e-9, err_msg=""): """Assert allclose for sparse and dense data. From 8826197bc21296dfb7bb51ffbf45b0e6760c47c5 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 16 Mar 2022 12:00:29 +0100 Subject: [PATCH 10/21] Review comments Co-authored-by: Olivier Grisel --- sklearn/utils/_testing.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index 4a53b4f8c4919..ec09f10b19c01 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -434,8 +434,10 @@ def assert_allclose( -------- >>> x = [1e-5, 1e-3, 1e-1] >>> y = np.arccos(np.cos(x)) - >>> np.testing.assert_allclose(x, y, rtol=1e-5, atol=0) + >>> assert_allclose(x, y, rtol=1e-5, atol=0) + >>> a = np.full(shape=10, fill_value=1e-5, dtype=np.float32) + >>> assert_allclose(a, 1e-5) """ dtypes = [] @@ -445,15 +447,14 @@ def assert_allclose( elif isinstance(input, np.dtype): dtypes.append(input) elif np.isscalar(input): - dtypes.append(type(input)) + dtypes.append(np.dtype(type(input))) else: raise TypeError( - f"Expected a np.array or a np.dtype, but got {type(input)} instead." + "Expected a scalar, a np.array or a np.dtype, but got" + f" {type(input)} instead." ) - rtols, atols = zip( - *[DTYPE_TOLERANCES.get(dtype, (1e-10, 1e-10)) for dtype in dtypes] - ) + rtols, atols = zip(*[DTYPE_TOLERANCES.get(dtype, (1e-9, 1e-9)) for dtype in dtypes]) if rtol is None: rtol = max(rtols) From 0d4260944c9c70f2e28d6253baa2171e747a1bac Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 16 Mar 2022 12:41:34 +0100 Subject: [PATCH 11/21] TST Add tests to testing tests Yes. --- sklearn/utils/_testing.py | 20 ++++------ sklearn/utils/tests/test_testing.py | 59 +++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index ec09f10b19c01..037cad32ed2bb 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -17,6 +17,7 @@ import sys import functools import tempfile +from numbers import Number from subprocess import check_output, STDOUT, CalledProcessError from subprocess import TimeoutExpired import re @@ -88,10 +89,9 @@ assert_raises_regexp = assert_raises_regex -DTYPE_TOLERANCES = { - # rtol, atol - np.float32: (1e-6, 1e-5), - np.float64: (1e-7, 1e-7), +DTYPE_RELATIVES_TOLERANCES = { + np.float32: 1e-4, + np.float64: 1e-7, } @@ -396,7 +396,7 @@ def assert_raise_message(exceptions, message, function, *args, **kwargs): def assert_allclose( - actual, desired, rtol=None, atol=None, equal_nan=True, err_msg="", verbose=True + actual, desired, rtol=None, atol=0.0, equal_nan=True, err_msg="", verbose=True ): """ Adaptation of numpy.testing.assert_allclose to have tolerances @@ -411,7 +411,7 @@ def assert_allclose( rtol : float, optional, default=None Relative tolerance. If None, it is set based on the provided arrays' dtypes. - atol : float, optional, default=None + atol : float, optional, default=0. Absolute tolerance. If None, it is set based on the provided arrays' dtypes. equal_nan : bool, optional, default=True @@ -446,7 +446,7 @@ def assert_allclose( dtypes.append(np.asarray(input).dtype) elif isinstance(input, np.dtype): dtypes.append(input) - elif np.isscalar(input): + elif isinstance(input, Number): dtypes.append(np.dtype(type(input))) else: raise TypeError( @@ -454,14 +454,10 @@ def assert_allclose( f" {type(input)} instead." ) - rtols, atols = zip(*[DTYPE_TOLERANCES.get(dtype, (1e-9, 1e-9)) for dtype in dtypes]) - if rtol is None: + rtols = [DTYPE_RELATIVES_TOLERANCES.get(dtype, 1e-9) for dtype in dtypes] rtol = max(rtols) - if atol is None: - atol = max(atols) - np_assert_allclose(actual, desired, rtol, atol, equal_nan, err_msg, verbose) diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py index ea4831fb02400..7543c796b791f 100644 --- a/sklearn/utils/tests/test_testing.py +++ b/sklearn/utils/tests/test_testing.py @@ -28,6 +28,7 @@ _delete_folder, _convert_container, raises, + assert_allclose, ) from sklearn.tree import DecisionTreeClassifier @@ -854,3 +855,61 @@ def test_raises(): with pytest.raises(AssertionError): with raises((TypeError, ValueError)): pass + + +@pytest.mark.parametrize( + "params, message", + [ + ({"actual": np.zeros(10), "desired": None}, "got instead."), + ({"actual": np.zeros(10), "desired": "astring"}, "got instead."), + ({"actual": np.zeros(10), "desired": float}, "got instead."), + ], +) +def test_assert_allclose_bad_params(params, message): + with pytest.raises(TypeError, match=message): + assert_allclose(**params) + + +def test_assert_allclose_zeros_elements(global_dtype): + # Arrays of quasi-zero elements must only be compared using + # an absolute tolerance + dtype_eps = np.finfo(global_dtype).eps + + a = np.zeros(10, dtype=global_dtype) + dtype_eps + b = np.zeros(10, dtype=global_dtype) + + msg = "Not equal to tolerance rtol=1e-09, atol=0" + with pytest.raises(AssertionError, match=msg): + assert_allclose(a, b) + + assert_allclose(a, b, atol=dtype_eps) + + +def test_assert_allclose(global_dtype): + dtype_eps = np.finfo(global_dtype).eps + + assert_allclose(6, 10, rtol=0.5) + msg = "Not equal to tolerance rtol=0.5, atol=0" + with pytest.raises(AssertionError, match=msg): + assert_allclose(10, 6, rtol=0.5) + + x = 1e-3 + y = 1e-9 + + assert_allclose(x, y, atol=1) + msg = "Not equal to tolerance rtol=1e-09, atol=0" + with pytest.raises(AssertionError, match=msg): + assert_allclose(x, y) + + a = np.array([x, y, x, y], dtype=global_dtype) + b = np.array([x, y, x, x], dtype=global_dtype) + + assert_allclose(a, b, atol=1) + msg = "Not equal to tolerance rtol=1e-09, atol=0" + with pytest.raises(AssertionError, match=msg): + assert_allclose(a, b) + + # Make a and b equal up to a `y * dtype_eps` absolute difference + b = a.copy() + b[-1] += dtype_eps + assert_allclose(a, b, atol=dtype_eps) From 0a4a1f9c2eba344a81fe376093cd301436662896 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 16 Mar 2022 15:37:01 +0100 Subject: [PATCH 12/21] TST Add more rtols --- sklearn/utils/_testing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index 037cad32ed2bb..05f6a55598d0b 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -91,7 +91,9 @@ DTYPE_RELATIVES_TOLERANCES = { np.float32: 1e-4, + np.dtype("float32"): 1e-4, np.float64: 1e-7, + np.dtype("float64"): 1e-7, } @@ -432,10 +434,10 @@ def assert_allclose( Examples -------- + >>> import numpy as np >>> x = [1e-5, 1e-3, 1e-1] >>> y = np.arccos(np.cos(x)) >>> assert_allclose(x, y, rtol=1e-5, atol=0) - >>> a = np.full(shape=10, fill_value=1e-5, dtype=np.float32) >>> assert_allclose(a, 1e-5) """ From 5701e8f768a2a68f7ea4efd53e26a70ea4923b01 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 16 Mar 2022 16:53:53 +0100 Subject: [PATCH 13/21] TST Adapt test_testing.py --- sklearn/conftest.py | 245 ---------------------------- sklearn/utils/tests/test_testing.py | 12 +- 2 files changed, 7 insertions(+), 250 deletions(-) delete mode 100644 sklearn/conftest.py diff --git a/sklearn/conftest.py b/sklearn/conftest.py deleted file mode 100644 index 4ecaac628d3fb..0000000000000 --- a/sklearn/conftest.py +++ /dev/null @@ -1,245 +0,0 @@ -from os import environ -from functools import wraps -import platform -import sys - -import pytest -import numpy as np -from threadpoolctl import threadpool_limits -from _pytest.doctest import DoctestItem - -from sklearn.utils import _IS_32BIT -from sklearn.utils._openmp_helpers import _openmp_effective_n_threads -from sklearn.externals import _pilutil -from sklearn._min_dependencies import PYTEST_MIN_VERSION -from sklearn.utils.fixes import parse_version -from sklearn.datasets import fetch_20newsgroups -from sklearn.datasets import fetch_20newsgroups_vectorized -from sklearn.datasets import fetch_california_housing -from sklearn.datasets import fetch_covtype -from sklearn.datasets import fetch_kddcup99 -from sklearn.datasets import fetch_olivetti_faces -from sklearn.datasets import fetch_rcv1 - - -# This plugin is necessary to define the random seed fixture -pytest_plugins = ("sklearn.tests.random_seed",) - - -if parse_version(pytest.__version__) < parse_version(PYTEST_MIN_VERSION): - raise ImportError( - "Your version of pytest is too old, you should have " - "at least pytest >= {} installed.".format(PYTEST_MIN_VERSION) - ) - -dataset_fetchers = { - "fetch_20newsgroups_fxt": fetch_20newsgroups, - "fetch_20newsgroups_vectorized_fxt": fetch_20newsgroups_vectorized, - "fetch_california_housing_fxt": fetch_california_housing, - "fetch_covtype_fxt": fetch_covtype, - "fetch_kddcup99_fxt": fetch_kddcup99, - "fetch_olivetti_faces_fxt": fetch_olivetti_faces, - "fetch_rcv1_fxt": fetch_rcv1, -} - -_SKIP32_MARK = pytest.mark.skipif( - environ.get("SKLEARN_RUN_FLOAT32_TESTS", "0") != "1", - reason="Set SKLEARN_RUN_FLOAT32_TESTS=1 to run float32 dtype tests", -) - - -# Global fixtures -@pytest.fixture(params=[pytest.param(np.float32, marks=_SKIP32_MARK), np.float64]) -def global_dtype(request): - yield request.param - - -def _fetch_fixture(f): - """Fetch dataset (download if missing and requested by environment).""" - download_if_missing = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" - - @wraps(f) - def wrapped(*args, **kwargs): - kwargs["download_if_missing"] = download_if_missing - try: - return f(*args, **kwargs) - except IOError as e: - if str(e) != "Data not found and `download_if_missing` is False": - raise - pytest.skip("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") - - return pytest.fixture(lambda: wrapped) - - -# Adds fixtures for fetching data -fetch_20newsgroups_fxt = _fetch_fixture(fetch_20newsgroups) -fetch_20newsgroups_vectorized_fxt = _fetch_fixture(fetch_20newsgroups_vectorized) -fetch_california_housing_fxt = _fetch_fixture(fetch_california_housing) -fetch_covtype_fxt = _fetch_fixture(fetch_covtype) -fetch_kddcup99_fxt = _fetch_fixture(fetch_kddcup99) -fetch_olivetti_faces_fxt = _fetch_fixture(fetch_olivetti_faces) -fetch_rcv1_fxt = _fetch_fixture(fetch_rcv1) - - -def pytest_collection_modifyitems(config, items): - """Called after collect is completed. - - Parameters - ---------- - config : pytest config - items : list of collected items - """ - run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" - skip_network = pytest.mark.skip( - reason="test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0" - ) - - # download datasets during collection to avoid thread unsafe behavior - # when running pytest in parallel with pytest-xdist - dataset_features_set = set(dataset_fetchers) - datasets_to_download = set() - - for item in items: - if not hasattr(item, "fixturenames"): - continue - item_fixtures = set(item.fixturenames) - dataset_to_fetch = item_fixtures & dataset_features_set - if not dataset_to_fetch: - continue - - if run_network_tests: - datasets_to_download |= dataset_to_fetch - else: - # network tests are skipped - item.add_marker(skip_network) - - # Only download datasets on the first worker spawned by pytest-xdist - # to avoid thread unsafe behavior. If pytest-xdist is not used, we still - # download before tests run. - worker_id = environ.get("PYTEST_XDIST_WORKER", "gw0") - if worker_id == "gw0" and run_network_tests: - for name in datasets_to_download: - dataset_fetchers[name]() - - for item in items: - # FeatureHasher is not compatible with PyPy - if ( - item.name.endswith(("_hash.FeatureHasher", "text.HashingVectorizer")) - and platform.python_implementation() == "PyPy" - ): - marker = pytest.mark.skip( - reason="FeatureHasher is not compatible with PyPy" - ) - item.add_marker(marker) - # Known failure on with GradientBoostingClassifier on ARM64 - elif ( - item.name.endswith("GradientBoostingClassifier") - and platform.machine() == "aarch64" - ): - - marker = pytest.mark.xfail( - reason=( - "know failure. See " - "https://github.com/scikit-learn/scikit-learn/issues/17797" # noqa - ) - ) - item.add_marker(marker) - - # numpy changed the str/repr formatting of numpy arrays in 1.14. We want to - # run doctests only for numpy >= 1.14. - skip_doctests = False - try: - import matplotlib # noqa - except ImportError: - skip_doctests = True - reason = "matplotlib is required to run the doctests" - - try: - if _IS_32BIT: - reason = "doctest are only run when the default numpy int is 64 bits." - skip_doctests = True - elif sys.platform.startswith("win32"): - reason = ( - "doctests are not run for Windows because numpy arrays " - "repr is inconsistent across platforms." - ) - skip_doctests = True - except ImportError: - pass - - # Normally doctest has the entire module's scope. Here we set globs to an empty dict - # to remove the module's scope: - # https://docs.python.org/3/library/doctest.html#what-s-the-execution-context - for item in items: - if isinstance(item, DoctestItem): - item.dtest.globs = {} - - if skip_doctests: - skip_marker = pytest.mark.skip(reason=reason) - - for item in items: - if isinstance(item, DoctestItem): - # work-around an internal error with pytest if adding a skip - # mark to a doctest in a contextmanager, see - # https://github.com/pytest-dev/pytest/issues/8796 for more - # details. - if item.name != "sklearn._config.config_context": - item.add_marker(skip_marker) - elif not _pilutil.pillow_installed: - skip_marker = pytest.mark.skip(reason="pillow (or PIL) not installed!") - for item in items: - if item.name in [ - "sklearn.feature_extraction.image.PatchExtractor", - "sklearn.feature_extraction.image.extract_patches_2d", - ]: - item.add_marker(skip_marker) - - -@pytest.fixture(scope="function") -def pyplot(): - """Setup and teardown fixture for matplotlib. - - This fixture checks if we can import matplotlib. If not, the tests will be - skipped. Otherwise, we close the figures before and after running the - functions. - - Returns - ------- - pyplot : module - The ``matplotlib.pyplot`` module. - """ - pyplot = pytest.importorskip("matplotlib.pyplot") - pyplot.close("all") - yield pyplot - pyplot.close("all") - - -def pytest_runtest_setup(item): - """Set the number of openmp threads based on the number of workers - xdist is using to prevent oversubscription. - - Parameters - ---------- - item : pytest item - item to be processed - """ - xdist_worker_count = environ.get("PYTEST_XDIST_WORKER_COUNT") - if xdist_worker_count is None: - # returns if pytest-xdist is not installed - return - else: - xdist_worker_count = int(xdist_worker_count) - - openmp_threads = _openmp_effective_n_threads() - threads_per_worker = max(openmp_threads // xdist_worker_count, 1) - threadpool_limits(threads_per_worker, user_api="openmp") - - -def pytest_configure(config): - # Use matplotlib agg backend during the tests including doctests - try: - import matplotlib - - matplotlib.use("agg") - except ImportError: - pass diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py index 7543c796b791f..4c516beca6458 100644 --- a/sklearn/utils/tests/test_testing.py +++ b/sklearn/utils/tests/test_testing.py @@ -874,11 +874,12 @@ def test_assert_allclose_zeros_elements(global_dtype): # Arrays of quasi-zero elements must only be compared using # an absolute tolerance dtype_eps = np.finfo(global_dtype).eps + rtol = 0.0001 if global_dtype is np.float32 else 1e-07 a = np.zeros(10, dtype=global_dtype) + dtype_eps b = np.zeros(10, dtype=global_dtype) - msg = "Not equal to tolerance rtol=1e-09, atol=0" + msg = f"Not equal to tolerance rtol={rtol}, atol=0" with pytest.raises(AssertionError, match=msg): assert_allclose(a, b) @@ -887,17 +888,18 @@ def test_assert_allclose_zeros_elements(global_dtype): def test_assert_allclose(global_dtype): dtype_eps = np.finfo(global_dtype).eps + rtol = 0.0001 if global_dtype is np.float32 else 1e-07 assert_allclose(6, 10, rtol=0.5) msg = "Not equal to tolerance rtol=0.5, atol=0" with pytest.raises(AssertionError, match=msg): assert_allclose(10, 6, rtol=0.5) - x = 1e-3 - y = 1e-9 + x = global_dtype(1e-3) + y = global_dtype(1e-9) assert_allclose(x, y, atol=1) - msg = "Not equal to tolerance rtol=1e-09, atol=0" + msg = f"Not equal to tolerance rtol={rtol}, atol=0" with pytest.raises(AssertionError, match=msg): assert_allclose(x, y) @@ -905,7 +907,7 @@ def test_assert_allclose(global_dtype): b = np.array([x, y, x, x], dtype=global_dtype) assert_allclose(a, b, atol=1) - msg = "Not equal to tolerance rtol=1e-09, atol=0" + msg = f"Not equal to tolerance rtol={rtol}, atol=0" with pytest.raises(AssertionError, match=msg): assert_allclose(a, b) From 15d53d8a08bc7f1ee8ca68d5148f4a53dce4254c Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 16 Mar 2022 18:55:10 +0100 Subject: [PATCH 14/21] Julien clearing his mess --- sklearn/conftest.py | 245 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 sklearn/conftest.py diff --git a/sklearn/conftest.py b/sklearn/conftest.py new file mode 100644 index 0000000000000..4ecaac628d3fb --- /dev/null +++ b/sklearn/conftest.py @@ -0,0 +1,245 @@ +from os import environ +from functools import wraps +import platform +import sys + +import pytest +import numpy as np +from threadpoolctl import threadpool_limits +from _pytest.doctest import DoctestItem + +from sklearn.utils import _IS_32BIT +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.externals import _pilutil +from sklearn._min_dependencies import PYTEST_MIN_VERSION +from sklearn.utils.fixes import parse_version +from sklearn.datasets import fetch_20newsgroups +from sklearn.datasets import fetch_20newsgroups_vectorized +from sklearn.datasets import fetch_california_housing +from sklearn.datasets import fetch_covtype +from sklearn.datasets import fetch_kddcup99 +from sklearn.datasets import fetch_olivetti_faces +from sklearn.datasets import fetch_rcv1 + + +# This plugin is necessary to define the random seed fixture +pytest_plugins = ("sklearn.tests.random_seed",) + + +if parse_version(pytest.__version__) < parse_version(PYTEST_MIN_VERSION): + raise ImportError( + "Your version of pytest is too old, you should have " + "at least pytest >= {} installed.".format(PYTEST_MIN_VERSION) + ) + +dataset_fetchers = { + "fetch_20newsgroups_fxt": fetch_20newsgroups, + "fetch_20newsgroups_vectorized_fxt": fetch_20newsgroups_vectorized, + "fetch_california_housing_fxt": fetch_california_housing, + "fetch_covtype_fxt": fetch_covtype, + "fetch_kddcup99_fxt": fetch_kddcup99, + "fetch_olivetti_faces_fxt": fetch_olivetti_faces, + "fetch_rcv1_fxt": fetch_rcv1, +} + +_SKIP32_MARK = pytest.mark.skipif( + environ.get("SKLEARN_RUN_FLOAT32_TESTS", "0") != "1", + reason="Set SKLEARN_RUN_FLOAT32_TESTS=1 to run float32 dtype tests", +) + + +# Global fixtures +@pytest.fixture(params=[pytest.param(np.float32, marks=_SKIP32_MARK), np.float64]) +def global_dtype(request): + yield request.param + + +def _fetch_fixture(f): + """Fetch dataset (download if missing and requested by environment).""" + download_if_missing = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" + + @wraps(f) + def wrapped(*args, **kwargs): + kwargs["download_if_missing"] = download_if_missing + try: + return f(*args, **kwargs) + except IOError as e: + if str(e) != "Data not found and `download_if_missing` is False": + raise + pytest.skip("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") + + return pytest.fixture(lambda: wrapped) + + +# Adds fixtures for fetching data +fetch_20newsgroups_fxt = _fetch_fixture(fetch_20newsgroups) +fetch_20newsgroups_vectorized_fxt = _fetch_fixture(fetch_20newsgroups_vectorized) +fetch_california_housing_fxt = _fetch_fixture(fetch_california_housing) +fetch_covtype_fxt = _fetch_fixture(fetch_covtype) +fetch_kddcup99_fxt = _fetch_fixture(fetch_kddcup99) +fetch_olivetti_faces_fxt = _fetch_fixture(fetch_olivetti_faces) +fetch_rcv1_fxt = _fetch_fixture(fetch_rcv1) + + +def pytest_collection_modifyitems(config, items): + """Called after collect is completed. + + Parameters + ---------- + config : pytest config + items : list of collected items + """ + run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" + skip_network = pytest.mark.skip( + reason="test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0" + ) + + # download datasets during collection to avoid thread unsafe behavior + # when running pytest in parallel with pytest-xdist + dataset_features_set = set(dataset_fetchers) + datasets_to_download = set() + + for item in items: + if not hasattr(item, "fixturenames"): + continue + item_fixtures = set(item.fixturenames) + dataset_to_fetch = item_fixtures & dataset_features_set + if not dataset_to_fetch: + continue + + if run_network_tests: + datasets_to_download |= dataset_to_fetch + else: + # network tests are skipped + item.add_marker(skip_network) + + # Only download datasets on the first worker spawned by pytest-xdist + # to avoid thread unsafe behavior. If pytest-xdist is not used, we still + # download before tests run. + worker_id = environ.get("PYTEST_XDIST_WORKER", "gw0") + if worker_id == "gw0" and run_network_tests: + for name in datasets_to_download: + dataset_fetchers[name]() + + for item in items: + # FeatureHasher is not compatible with PyPy + if ( + item.name.endswith(("_hash.FeatureHasher", "text.HashingVectorizer")) + and platform.python_implementation() == "PyPy" + ): + marker = pytest.mark.skip( + reason="FeatureHasher is not compatible with PyPy" + ) + item.add_marker(marker) + # Known failure on with GradientBoostingClassifier on ARM64 + elif ( + item.name.endswith("GradientBoostingClassifier") + and platform.machine() == "aarch64" + ): + + marker = pytest.mark.xfail( + reason=( + "know failure. See " + "https://github.com/scikit-learn/scikit-learn/issues/17797" # noqa + ) + ) + item.add_marker(marker) + + # numpy changed the str/repr formatting of numpy arrays in 1.14. We want to + # run doctests only for numpy >= 1.14. + skip_doctests = False + try: + import matplotlib # noqa + except ImportError: + skip_doctests = True + reason = "matplotlib is required to run the doctests" + + try: + if _IS_32BIT: + reason = "doctest are only run when the default numpy int is 64 bits." + skip_doctests = True + elif sys.platform.startswith("win32"): + reason = ( + "doctests are not run for Windows because numpy arrays " + "repr is inconsistent across platforms." + ) + skip_doctests = True + except ImportError: + pass + + # Normally doctest has the entire module's scope. Here we set globs to an empty dict + # to remove the module's scope: + # https://docs.python.org/3/library/doctest.html#what-s-the-execution-context + for item in items: + if isinstance(item, DoctestItem): + item.dtest.globs = {} + + if skip_doctests: + skip_marker = pytest.mark.skip(reason=reason) + + for item in items: + if isinstance(item, DoctestItem): + # work-around an internal error with pytest if adding a skip + # mark to a doctest in a contextmanager, see + # https://github.com/pytest-dev/pytest/issues/8796 for more + # details. + if item.name != "sklearn._config.config_context": + item.add_marker(skip_marker) + elif not _pilutil.pillow_installed: + skip_marker = pytest.mark.skip(reason="pillow (or PIL) not installed!") + for item in items: + if item.name in [ + "sklearn.feature_extraction.image.PatchExtractor", + "sklearn.feature_extraction.image.extract_patches_2d", + ]: + item.add_marker(skip_marker) + + +@pytest.fixture(scope="function") +def pyplot(): + """Setup and teardown fixture for matplotlib. + + This fixture checks if we can import matplotlib. If not, the tests will be + skipped. Otherwise, we close the figures before and after running the + functions. + + Returns + ------- + pyplot : module + The ``matplotlib.pyplot`` module. + """ + pyplot = pytest.importorskip("matplotlib.pyplot") + pyplot.close("all") + yield pyplot + pyplot.close("all") + + +def pytest_runtest_setup(item): + """Set the number of openmp threads based on the number of workers + xdist is using to prevent oversubscription. + + Parameters + ---------- + item : pytest item + item to be processed + """ + xdist_worker_count = environ.get("PYTEST_XDIST_WORKER_COUNT") + if xdist_worker_count is None: + # returns if pytest-xdist is not installed + return + else: + xdist_worker_count = int(xdist_worker_count) + + openmp_threads = _openmp_effective_n_threads() + threads_per_worker = max(openmp_threads // xdist_worker_count, 1) + threadpool_limits(threads_per_worker, user_api="openmp") + + +def pytest_configure(config): + # Use matplotlib agg backend during the tests including doctests + try: + import matplotlib + + matplotlib.use("agg") + except ImportError: + pass From 2ff567573078bb5df030d9a7ff571468fb1ec78b Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 16 Mar 2022 18:55:32 +0100 Subject: [PATCH 15/21] Simplify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérémie du Boisberranger --- sklearn/utils/_testing.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index 05f6a55598d0b..429c56707dcdc 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -17,7 +17,6 @@ import sys import functools import tempfile -from numbers import Number from subprocess import check_output, STDOUT, CalledProcessError from subprocess import TimeoutExpired import re @@ -60,7 +59,6 @@ check_array, check_is_fitted, check_X_y, - _is_arraylike_not_scalar, ) @@ -443,18 +441,8 @@ def assert_allclose( """ dtypes = [] - for input in (actual, desired): - if _is_arraylike_not_scalar(input): - dtypes.append(np.asarray(input).dtype) - elif isinstance(input, np.dtype): - dtypes.append(input) - elif isinstance(input, Number): - dtypes.append(np.dtype(type(input))) - else: - raise TypeError( - "Expected a scalar, a np.array or a np.dtype, but got" - f" {type(input)} instead." - ) + actual, desired = np.asanyarray(actual), np.asanyarray(desired) + dtypes = [actual.dtype, desired.dtype] if rtol is None: rtols = [DTYPE_RELATIVES_TOLERANCES.get(dtype, 1e-9) for dtype in dtypes] From f502f119ac5614b79dd506d5f45f8158eda5f540 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 17 Mar 2022 09:33:48 +0100 Subject: [PATCH 16/21] TST Trust numpy test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérémie du Boisberranger --- sklearn/utils/tests/test_testing.py | 61 ----------------------------- 1 file changed, 61 deletions(-) diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py index 4c516beca6458..ea4831fb02400 100644 --- a/sklearn/utils/tests/test_testing.py +++ b/sklearn/utils/tests/test_testing.py @@ -28,7 +28,6 @@ _delete_folder, _convert_container, raises, - assert_allclose, ) from sklearn.tree import DecisionTreeClassifier @@ -855,63 +854,3 @@ def test_raises(): with pytest.raises(AssertionError): with raises((TypeError, ValueError)): pass - - -@pytest.mark.parametrize( - "params, message", - [ - ({"actual": np.zeros(10), "desired": None}, "got instead."), - ({"actual": np.zeros(10), "desired": "astring"}, "got instead."), - ({"actual": np.zeros(10), "desired": float}, "got instead."), - ], -) -def test_assert_allclose_bad_params(params, message): - with pytest.raises(TypeError, match=message): - assert_allclose(**params) - - -def test_assert_allclose_zeros_elements(global_dtype): - # Arrays of quasi-zero elements must only be compared using - # an absolute tolerance - dtype_eps = np.finfo(global_dtype).eps - rtol = 0.0001 if global_dtype is np.float32 else 1e-07 - - a = np.zeros(10, dtype=global_dtype) + dtype_eps - b = np.zeros(10, dtype=global_dtype) - - msg = f"Not equal to tolerance rtol={rtol}, atol=0" - with pytest.raises(AssertionError, match=msg): - assert_allclose(a, b) - - assert_allclose(a, b, atol=dtype_eps) - - -def test_assert_allclose(global_dtype): - dtype_eps = np.finfo(global_dtype).eps - rtol = 0.0001 if global_dtype is np.float32 else 1e-07 - - assert_allclose(6, 10, rtol=0.5) - msg = "Not equal to tolerance rtol=0.5, atol=0" - with pytest.raises(AssertionError, match=msg): - assert_allclose(10, 6, rtol=0.5) - - x = global_dtype(1e-3) - y = global_dtype(1e-9) - - assert_allclose(x, y, atol=1) - msg = f"Not equal to tolerance rtol={rtol}, atol=0" - with pytest.raises(AssertionError, match=msg): - assert_allclose(x, y) - - a = np.array([x, y, x, y], dtype=global_dtype) - b = np.array([x, y, x, x], dtype=global_dtype) - - assert_allclose(a, b, atol=1) - msg = f"Not equal to tolerance rtol={rtol}, atol=0" - with pytest.raises(AssertionError, match=msg): - assert_allclose(a, b) - - # Make a and b equal up to a `y * dtype_eps` absolute difference - b = a.copy() - b[-1] += dtype_eps - assert_allclose(a, b, atol=dtype_eps) From 831b3e778b314e6f555beb2a99417b7efdcb3ec3 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 17 Mar 2022 09:52:42 +0100 Subject: [PATCH 17/21] C'mon, Julien --- sklearn/utils/_testing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index 429c56707dcdc..b19802c8cf890 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -433,6 +433,7 @@ def assert_allclose( Examples -------- >>> import numpy as np + >>> from sklearn.utils._testing import assert_allclose >>> x = [1e-5, 1e-3, 1e-1] >>> y = np.arccos(np.cos(x)) >>> assert_allclose(x, y, rtol=1e-5, atol=0) From 354e9c196dd7022c21026afd8d651767b3a8ba7c Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 17 Mar 2022 10:09:29 +0100 Subject: [PATCH 18/21] TST Actually use custom assert_allclose --- sklearn/feature_selection/tests/test_mutual_info.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py index 1e18035c5c7a6..b7442e045207e 100644 --- a/sklearn/feature_selection/tests/test_mutual_info.py +++ b/sklearn/feature_selection/tests/test_mutual_info.py @@ -3,7 +3,11 @@ from scipy.sparse import csr_matrix from sklearn.utils import check_random_state -from sklearn.utils._testing import assert_array_equal, assert_almost_equal +from sklearn.utils._testing import ( + assert_array_equal, + assert_almost_equal, + assert_allclose, +) from sklearn.feature_selection._mutual_info import _compute_mi from sklearn.feature_selection import mutual_info_regression, mutual_info_classif @@ -51,7 +55,7 @@ def test_compute_mi_cc(global_dtype): # first figures after decimal point match. for n_neighbors in [3, 5, 7]: I_computed = _compute_mi(x, y, False, False, n_neighbors) - assert_almost_equal(I_computed, I_theory, 1) + assert_allclose(I_computed, I_theory, 1) def test_compute_mi_cd(): From efb0d02419753f2b7bf9f2eccc1fdac25d8893cb Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 17 Mar 2022 11:41:04 +0100 Subject: [PATCH 19/21] Review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérémie du Boisberranger Co-authored-by: Olivier Grisel --- doc/computing/parallelism.rst | 2 +- doc/developers/develop.rst | 15 +++++++++++ .../tests/test_mutual_info.py | 4 +-- sklearn/utils/_testing.py | 25 ++++++++++--------- sklearn/utils/tests/test_testing.py | 19 ++++++++++++++ 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/doc/computing/parallelism.rst b/doc/computing/parallelism.rst index 05953f1d9ec7c..4d232b66125fc 100644 --- a/doc/computing/parallelism.rst +++ b/doc/computing/parallelism.rst @@ -267,7 +267,7 @@ network tests are skipped. ~~~~~~~~~~~~~~~~~~~~~~~~~~~ When this environment variable is set to '1', the tests using the -`global_dtype` `pytest.fixture` are also run float32 data. +`global_dtype` fixture are also run on float32 data. When this environment variable is not set, the tests are only run on float64 data. diff --git a/doc/developers/develop.rst b/doc/developers/develop.rst index d6955ee53a7cc..542fbf2bc4751 100644 --- a/doc/developers/develop.rst +++ b/doc/developers/develop.rst @@ -774,3 +774,18 @@ The reason for this setup is reproducibility: when an estimator is ``fit`` twice to the same data, it should produce an identical model both times, hence the validation in ``fit``, not ``__init__``. + +Numerical assertions in tests +----------------------------- + +When asserting the quasi-equality of arrays, do use +:func:`sklearn.utils._testing.assert_allclose`. + +The relative tolerance is automatically inferred from the provided arrays dtypes, +but you can override via ``rtol``. + +When comparing arrays of zero-elements, please do provide a non-zero value for +the absolute via ``atol``. + +For more information, please refer to the docstring of +:func:`sklearn.utils._testing.assert_allclose`. diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py index b7442e045207e..5782e012812b1 100644 --- a/sklearn/feature_selection/tests/test_mutual_info.py +++ b/sklearn/feature_selection/tests/test_mutual_info.py @@ -47,7 +47,7 @@ def test_compute_mi_cc(global_dtype): I_theory = np.log(sigma_1) + np.log(sigma_2) - 0.5 * np.log(np.linalg.det(cov)) rng = check_random_state(0) - Z = rng.multivariate_normal(mean, cov, size=1000).astype(global_dtype) + Z = rng.multivariate_normal(mean, cov, size=1000).astype(global_dtype, copy=False) x, y = Z[:, 0], Z[:, 1] @@ -55,7 +55,7 @@ def test_compute_mi_cc(global_dtype): # first figures after decimal point match. for n_neighbors in [3, 5, 7]: I_computed = _compute_mi(x, y, False, False, n_neighbors) - assert_allclose(I_computed, I_theory, 1) + assert_allclose(I_computed, I_theory, rtol=1e-1) def test_compute_mi_cd(): diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index b19802c8cf890..527061ac3e3b8 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -87,14 +87,6 @@ assert_raises_regexp = assert_raises_regex -DTYPE_RELATIVES_TOLERANCES = { - np.float32: 1e-4, - np.dtype("float32"): 1e-4, - np.float64: 1e-7, - np.dtype("float64"): 1e-7, -} - - # TODO: Remove in 1.2 @deprecated( # type: ignore "`assert_warns` is deprecated in 1.0 and will be removed in 1.2." @@ -398,9 +390,18 @@ def assert_raise_message(exceptions, message, function, *args, **kwargs): def assert_allclose( actual, desired, rtol=None, atol=0.0, equal_nan=True, err_msg="", verbose=True ): - """ - Adaptation of numpy.testing.assert_allclose to have tolerances - be set based on the input arrays' dtypes. + """dtype-aware variant of numpy.testing.assert_allclose + + This variant introspects the least precise floating point dtype + in the input argument and automatically sets the relative tolerance + parameter to 1e-4 float32 and use 1e-7 otherwise (typically float64 + in scikit-learn). + + `atol` is always left to 0. by default. It should be adjusted manually + to an assertion-specific value in case there are null values expected + in `desired`. + + The aggregate tolerance is `atol + rtol * abs(desired)`. Parameters ---------- @@ -446,7 +447,7 @@ def assert_allclose( dtypes = [actual.dtype, desired.dtype] if rtol is None: - rtols = [DTYPE_RELATIVES_TOLERANCES.get(dtype, 1e-9) for dtype in dtypes] + rtols = [1e-4 if dtype == np.float32 else 1e-7 for dtype in dtypes] rtol = max(rtols) np_assert_allclose(actual, desired, rtol, atol, equal_nan, err_msg, verbose) diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py index ea4831fb02400..f6d05d01e9237 100644 --- a/sklearn/utils/tests/test_testing.py +++ b/sklearn/utils/tests/test_testing.py @@ -28,6 +28,7 @@ _delete_folder, _convert_container, raises, + assert_allclose, ) from sklearn.tree import DecisionTreeClassifier @@ -854,3 +855,21 @@ def test_raises(): with pytest.raises(AssertionError): with raises((TypeError, ValueError)): pass + + +def test_float32_aware_assert_allclose(): + # The relative tolerance for float32 inputs is 1e-4 + assert_allclose(np.array([1.0 + 2e-5], dtype=np.float32), 1.0) + with pytest.raises(AssertionError): + assert_allclose(np.array([1.0 + 2e-4], dtype=np.float32), 1.0) + + # The relative tolerance for other inputs is left to 1e-7 as in + # the original numpy version. + assert_allclose(np.array([1.0 + 2e-8], dtype=np.float64), 1.0) + with pytest.raises(AssertionError): + assert_allclose(np.array([1.0 + 2e-7], dtype=np.float64), 1.0) + + # atol is left to 0.0 by default, even for float32 + with pytest.raises(AssertionError): + assert_allclose(np.array([1.0 + 1e-5], dtype=np.float32), 0.0) + assert_allclose(np.array([1.0 + 1e-5], dtype=np.float32), 1.0, atol=2e-5) From 1bd542e2e8215ee2a4b3f080612037321d2c6cc2 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 17 Mar 2022 12:03:43 +0100 Subject: [PATCH 20/21] TST Last changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérémie du Boisberranger --- doc/developers/develop.rst | 2 +- sklearn/feature_selection/tests/test_mutual_info.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developers/develop.rst b/doc/developers/develop.rst index 542fbf2bc4751..c8997d9709931 100644 --- a/doc/developers/develop.rst +++ b/doc/developers/develop.rst @@ -785,7 +785,7 @@ The relative tolerance is automatically inferred from the provided arrays dtypes but you can override via ``rtol``. When comparing arrays of zero-elements, please do provide a non-zero value for -the absolute via ``atol``. +the absolute tolerance via ``atol``. For more information, please refer to the docstring of :func:`sklearn.utils._testing.assert_allclose`. diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py index 5782e012812b1..7cc25c3ddd642 100644 --- a/sklearn/feature_selection/tests/test_mutual_info.py +++ b/sklearn/feature_selection/tests/test_mutual_info.py @@ -51,8 +51,8 @@ def test_compute_mi_cc(global_dtype): x, y = Z[:, 0], Z[:, 1] - # Theory and computed values won't be very close, assert that the - # first figures after decimal point match. + # Theory and computed values won't be very close + # We here check with a large relative tolerance for n_neighbors in [3, 5, 7]: I_computed = _compute_mi(x, y, False, False, n_neighbors) assert_allclose(I_computed, I_theory, rtol=1e-1) From fa673d2c7a9aacee1f199166c64ea640541dac89 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 17 Mar 2022 13:44:30 +0100 Subject: [PATCH 21/21] Apply suggestions from code review --- doc/developers/develop.rst | 9 +++++---- sklearn/utils/_testing.py | 10 +++++++++- sklearn/utils/tests/test_testing.py | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/doc/developers/develop.rst b/doc/developers/develop.rst index c8997d9709931..ae6b6e51307c6 100644 --- a/doc/developers/develop.rst +++ b/doc/developers/develop.rst @@ -778,11 +778,12 @@ hence the validation in ``fit``, not ``__init__``. Numerical assertions in tests ----------------------------- -When asserting the quasi-equality of arrays, do use -:func:`sklearn.utils._testing.assert_allclose`. +When asserting the quasi-equality of arrays of continuous values, +do use :func:`sklearn.utils._testing.assert_allclose`. -The relative tolerance is automatically inferred from the provided arrays dtypes, -but you can override via ``rtol``. +The relative tolerance is automatically inferred from the provided arrays +dtypes (for float32 and float64 dtypes in particular) but you can override +via ``rtol``. When comparing arrays of zero-elements, please do provide a non-zero value for the absolute tolerance via ``atol``. diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index 527061ac3e3b8..453f3437307a9 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -450,7 +450,15 @@ def assert_allclose( rtols = [1e-4 if dtype == np.float32 else 1e-7 for dtype in dtypes] rtol = max(rtols) - np_assert_allclose(actual, desired, rtol, atol, equal_nan, err_msg, verbose) + np_assert_allclose( + actual, + desired, + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + err_msg=err_msg, + verbose=verbose, + ) def assert_allclose_dense_sparse(x, y, rtol=1e-07, atol=1e-9, err_msg=""): diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py index f6d05d01e9237..9710272029b1e 100644 --- a/sklearn/utils/tests/test_testing.py +++ b/sklearn/utils/tests/test_testing.py @@ -871,5 +871,5 @@ def test_float32_aware_assert_allclose(): # atol is left to 0.0 by default, even for float32 with pytest.raises(AssertionError): - assert_allclose(np.array([1.0 + 1e-5], dtype=np.float32), 0.0) - assert_allclose(np.array([1.0 + 1e-5], dtype=np.float32), 1.0, atol=2e-5) + assert_allclose(np.array([1e-5], dtype=np.float32), 0.0) + assert_allclose(np.array([1e-5], dtype=np.float32), 0.0, atol=2e-5)