From d45fb579d594f9277ac773aff81f51fa891da56e Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 21:22:48 +0100 Subject: [PATCH 01/48] adding MAPE as a new regression loss --- sklearn/metrics/regression.py | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 0a1ec94e1dec4..a7f36a4c36e98 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -19,6 +19,7 @@ # Manoj Kumar # Michael Eickenberg # Konstantin Shmelkov +# Mohamed Ali Jamaoui # License: BSD 3 clause from __future__ import division @@ -32,6 +33,7 @@ __ALL__ = [ "mean_absolute_error", + "mean_absolute_percentage_error", "mean_squared_error", "mean_squared_log_error", "median_absolute_error", @@ -181,6 +183,41 @@ def mean_absolute_error(y_true, y_pred, return np.average(output_errors, weights=multioutput) +def mean_absolute_percentage_error(y_true, y_pred): + """Mean absolute percentage error regression loss + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + y_true : array-like of shape = (n_samples) + Ground truth (correct) target values. + + y_pred : array-like of shape = (n_samples) + Estimated target values. + + Returns + ------- + loss : float + A positive floating point value (the best value is 0.0). + + Examples + -------- + >>> from sklearn.metrics import median_absolute_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> mean_absolute_percentage_error(y_true, y_pred) + 32.738... + + """ + y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, + 'uniform_average') + if y_type == 'continuous-multioutput': + raise ValueError("Multioutput not supported " + "in mean_absolute_percentage_error") + return np.average(np.abs((y_true - y_pred)/y_true))*100 + + def mean_squared_error(y_true, y_pred, sample_weight=None, multioutput='uniform_average'): From 762fa194abd511321f1f39558e279c27e6ae5169 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 21:28:49 +0100 Subject: [PATCH 02/48] adding MAPE api reference --- doc/modules/classes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index a6174d282a36f..19283fa10a919 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -836,6 +836,7 @@ details. metrics.explained_variance_score metrics.mean_absolute_error + metrics.mean_absolute_percentage_error metrics.mean_squared_error metrics.mean_squared_log_error metrics.median_absolute_error From 9a6b8fdcb6449a39428edbf0efba06e833a89693 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 22:00:20 +0100 Subject: [PATCH 03/48] adding MAPE scorer --- sklearn/metrics/scorer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index 05231826a8998..630a52a93448f 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -24,7 +24,8 @@ import numpy as np from . import (r2_score, median_absolute_error, mean_absolute_error, - mean_squared_error, mean_squared_log_error, accuracy_score, + mean_squared_error, mean_absolute_percentage_error, + mean_squared_log_error, accuracy_score, f1_score, roc_auc_score, average_precision_score, precision_score, recall_score, log_loss, balanced_accuracy_score, explained_variance_score, brier_score_loss) @@ -486,6 +487,9 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, 'be removed in 0.20.') mean_absolute_error_scorer = make_scorer(mean_absolute_error, greater_is_better=False) +neg_mean_absolute_percentage_error_scorer = make_scorer( + mean_absolute_percentage_error, greater_is_better=False) + mean_absolute_error_scorer._deprecation_msg = deprecation_msg neg_median_absolute_error_scorer = make_scorer(median_absolute_error, greater_is_better=False) @@ -538,6 +542,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, r2=r2_scorer, neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, + neg_mean_absolute_percentage_error=neg_mean_absolute_percentage_error_scorer, neg_mean_squared_error=neg_mean_squared_error_scorer, neg_mean_squared_log_error=neg_mean_squared_log_error_scorer, median_absolute_error=median_absolute_error_scorer, From ea62611d85c3b6fcd7edeb139b688f92f45481e8 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 22:13:42 +0100 Subject: [PATCH 04/48] adding MAPE to metrics/__init__.py --- sklearn/metrics/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index c98b0e14493c6..aff6c8dd3e4db 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -54,6 +54,7 @@ from .regression import explained_variance_score from .regression import mean_absolute_error +from .regression import mean_absolute_percentage_error from .regression import mean_squared_error from .regression import mean_squared_log_error from .regression import median_absolute_error @@ -97,6 +98,7 @@ 'make_scorer', 'matthews_corrcoef', 'mean_absolute_error', + 'mean_absolute_percentage_error' 'mean_squared_error', 'mean_squared_log_error', 'median_absolute_error', From 70ab1e0856a9a9ab1d3e47c0d9a08afd6265afdc Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 22:14:37 +0100 Subject: [PATCH 05/48] configuring tests for MAPE scorer --- sklearn/metrics/tests/test_score_objects.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 6af6418635d59..8ee3ea830ae6b 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -41,9 +41,9 @@ from sklearn.externals import joblib -REGRESSION_SCORERS = ['explained_variance', 'r2', - 'neg_mean_absolute_error', 'neg_mean_squared_error', - 'neg_mean_squared_log_error', +REGRESSION_SCORERS = ['explained_variance', 'r2', 'neg_mean_absolute_error', + 'neg_mean_absolute_percentage_error', + 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error'] From 962a8e28d0a840a60ab6113b5ae0665c96eac65e Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 22:22:08 +0100 Subject: [PATCH 06/48] configuring common tests for MAPE metric --- sklearn/metrics/tests/test_common.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index ad323a5483621..983a3d7d1c1f3 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -42,6 +42,7 @@ from sklearn.metrics import log_loss from sklearn.metrics import matthews_corrcoef from sklearn.metrics import mean_absolute_error +from sklearn.metrics import mean_absolute_percentage_error from sklearn.metrics import mean_squared_error from sklearn.metrics import median_absolute_error from sklearn.metrics import precision_score @@ -93,6 +94,7 @@ REGRESSION_METRICS = { "mean_absolute_error": mean_absolute_error, + "mean_absolute_percentage_error": mean_absolute_percentage_error, "mean_squared_error": mean_squared_error, "median_absolute_error": median_absolute_error, "explained_variance_score": explained_variance_score, @@ -346,7 +348,8 @@ "micro_f0.5_score", "micro_f1_score", "micro_f2_score", "micro_precision_score", "micro_recall_score", - "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error", + "matthews_corrcoef_score", "mean_absolute_error", + "mean_absolute_percentage_error", "mean_squared_error", "median_absolute_error", "cohen_kappa_score", @@ -378,6 +381,7 @@ # confusion_matrix with sample_weight is in # test_classification.py "median_absolute_error", + "mean_absolute_percentage_error" ] From 4f16a45ee6f97e4ec0cc07c25cba864eb08fd957 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 22:50:04 +0100 Subject: [PATCH 07/48] correcting import in MAPE example --- sklearn/metrics/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index a7f36a4c36e98..0e14f24b4cba3 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -203,7 +203,7 @@ def mean_absolute_percentage_error(y_true, y_pred): Examples -------- - >>> from sklearn.metrics import median_absolute_error + >>> from sklearn.metrics import mean_absolute_percentage_error >>> y_true = [3, -0.5, 2, 7] >>> y_pred = [2.5, 0.0, 2, 8] >>> mean_absolute_percentage_error(y_true, y_pred) From e2e93b016544db85ccd86b706ae42836b970a614 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 22:51:04 +0100 Subject: [PATCH 08/48] adding documentation under model_evaluation --- doc/modules/model_evaluation.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 953648ef9c035..661196a6e610d 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -85,6 +85,7 @@ Scoring Function **Regression** 'explained_variance' :func:`metrics.explained_variance_score` 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` +'neg_mean_absolute_percentage_error' :func:`metrics.mean_absolute_percentage_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` 'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` 'neg_median_absolute_error' :func:`metrics.median_absolute_error` @@ -104,7 +105,7 @@ Usage examples: >>> model = svm.SVC() >>> cross_val_score(model, X, y, scoring='wrong_choice') Traceback (most recent call last): - ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'brier_score_loss', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'mutual_info_score', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score'] + ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'brier_score_loss', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'mutual_info_score', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_absolute_percentage_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score'] .. note:: @@ -1551,6 +1552,32 @@ Here is a small example of usage of the :func:`mean_absolute_error` function:: ... # doctest: +ELLIPSIS 0.849... +.. _mean_absolute_percentage_error: + +Mean absolute error +------------------- + +The :func:`mean_absolute_percentage_error` function computes `mean absolute +percentage error `_, a risk +metric corresponding to the expected value of the absolute percentage error loss or +:math:`l1`-norm of percentage loss. + +If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, +and :math:`y_i` is the corresponding true value, then the mean absolute percentage error +(MAPE) estimated over :math:`n_{\text{samples}}` is defined as + +.. math:: + + \text{MAPE}(y, \hat{y}) = \frac{100}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \left| \frac{y_i - \hat{y}_i}{y_i} \right|. + +Here is a small example of usage of the :func:`mean_absolute_percentage_error` function:: + + >>> from sklearn.metrics import mean_absolute_percentage_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> mean_absolute_percentage_error(y_true, y_pred) + 32.738... + .. _mean_squared_error: Mean squared error From 6df244646ca3fe95e000db1ab08438499bc038fa Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 23:20:20 +0100 Subject: [PATCH 09/48] fixing pep8 --- sklearn/metrics/regression.py | 2 +- sklearn/metrics/tests/test_common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 0e14f24b4cba3..e6ace01f1faa0 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -213,7 +213,7 @@ def mean_absolute_percentage_error(y_true, y_pred): y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, 'uniform_average') if y_type == 'continuous-multioutput': - raise ValueError("Multioutput not supported " + raise ValueError("Multioutput not supported " "in mean_absolute_percentage_error") return np.average(np.abs((y_true - y_pred)/y_true))*100 diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 983a3d7d1c1f3..0d0302ec1d388 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -348,7 +348,7 @@ "micro_f0.5_score", "micro_f1_score", "micro_f2_score", "micro_precision_score", "micro_recall_score", - "matthews_corrcoef_score", "mean_absolute_error", + "matthews_corrcoef_score", "mean_absolute_error", "mean_absolute_percentage_error", "mean_squared_error", "median_absolute_error", From 7c516ef643efc0a91fa74649cf18ffe0d58f2775 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Feb 2018 23:22:52 +0100 Subject: [PATCH 10/48] adding more MAPE regression tests --- sklearn/metrics/tests/test_regression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index 2faaaad3a39f2..a525c4ec076a4 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -11,6 +11,7 @@ from sklearn.metrics import explained_variance_score from sklearn.metrics import mean_absolute_error +from sklearn.metrics import mean_absolute_percentage_error from sklearn.metrics import mean_squared_error from sklearn.metrics import mean_squared_log_error from sklearn.metrics import median_absolute_error @@ -28,6 +29,7 @@ def test_regression_metrics(n_samples=50): mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred))) assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.) + assert_almost_equal(mean_absolute_percentage_error(y_true, y_pred), 98.0) assert_almost_equal(median_absolute_error(y_true, y_pred), 1.) assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2) assert_almost_equal(explained_variance_score(y_true, y_pred), 1.) @@ -72,7 +74,6 @@ def test_regression_metrics_at_limits(): mean_squared_log_error, [1., -2., 3.], [1., 2., 3.]) - def test__check_reg_targets(): # All of length 3 EXAMPLES = [ From cb7d8756a3dd5c25e78d992bdfdd919774e92146 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 00:30:23 +0100 Subject: [PATCH 11/48] add whitespace around operators --- sklearn/metrics/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index e6ace01f1faa0..b108058eaffdd 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -215,7 +215,7 @@ def mean_absolute_percentage_error(y_true, y_pred): if y_type == 'continuous-multioutput': raise ValueError("Multioutput not supported " "in mean_absolute_percentage_error") - return np.average(np.abs((y_true - y_pred)/y_true))*100 + return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 def mean_squared_error(y_true, y_pred, From 8194eb042902c70e1200d5082ed815ddd2b6fa77 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 00:35:50 +0100 Subject: [PATCH 12/48] fix bug: missing comma --- sklearn/metrics/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index aff6c8dd3e4db..dd353da7514c6 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -98,7 +98,7 @@ 'make_scorer', 'matthews_corrcoef', 'mean_absolute_error', - 'mean_absolute_percentage_error' + 'mean_absolute_percentage_error', 'mean_squared_error', 'mean_squared_log_error', 'median_absolute_error', From fd286451fbfbeb8f482f3165b7c697c0cd2c4810 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 15:59:23 +0100 Subject: [PATCH 13/48] avoiding division by zero in test scenario --- sklearn/metrics/tests/test_regression.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index a525c4ec076a4..eee0b819dff05 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -29,7 +29,11 @@ def test_regression_metrics(n_samples=50): mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred))) assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.) - assert_almost_equal(mean_absolute_percentage_error(y_true, y_pred), 98.0) + # comparing (y_true + 1) and (y_pred + 1) instead of + # y_true and y_pred to avoid division by zero + assert_almost_equal(mean_absolute_percentage_error(1 + y_true, + 1 + y_pred), + 8.998, 2) assert_almost_equal(median_absolute_error(y_true, y_pred), 1.) assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2) assert_almost_equal(explained_variance_score(y_true, y_pred), 1.) From d10af3c7d3d9da653edf459734ce173647fa0dc4 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 16:01:44 +0100 Subject: [PATCH 14/48] adding check to validate y_true contains no zeros --- sklearn/metrics/regression.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index b108058eaffdd..ee8fccd629822 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -212,9 +212,15 @@ def mean_absolute_percentage_error(y_true, y_pred): """ y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, 'uniform_average') + if y_type == 'continuous-multioutput': raise ValueError("Multioutput not supported " "in mean_absolute_percentage_error") + + if (y_true == 0).any(): + raise ValueError("mean_absolute_percentage_error requires" + " y_true to never be zero") + return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 From 61ea463c66287f25ece607d0f9662e6b8a321c6d Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 16:19:04 +0100 Subject: [PATCH 15/48] documenting the new feature in whats_new --- doc/whats_new/v0.20.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index c607174626171..c07cf73dc0672 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -80,6 +80,8 @@ Model evaluation :issue:`8066` by :user:`xyguo` and :user:`Aman Dalmia `. - Added :class:`multioutput.RegressorChain` for multi-target regression. :issue:`9257` by :user:`Kumar Ashutosh `. +- Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated + scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui ` Decomposition, manifold learning and clustering From 088467f8f2691dc3b8db7f7ec34c2ec1f2ea6bc1 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 16:25:02 +0100 Subject: [PATCH 16/48] precising that MAPE is a non symetric metric --- sklearn/metrics/tests/test_common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 0d0302ec1d388..3162de559b2c9 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -349,8 +349,7 @@ "micro_precision_score", "micro_recall_score", "matthews_corrcoef_score", "mean_absolute_error", - "mean_absolute_percentage_error", "mean_squared_error", - "median_absolute_error", + "mean_squared_error", "median_absolute_error", "cohen_kappa_score", ] @@ -369,7 +368,9 @@ "weighted_precision_score", "macro_f0.5_score", "macro_f2_score", "macro_precision_score", - "macro_recall_score", "log_loss", "hinge_loss" + "macro_recall_score", "log_loss", "hinge_loss", + + "mean_absolute_percentage_error" ] From f9ac4e4f67dc1309817fb8198ffab1552f95187e Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Tue, 27 Feb 2018 19:17:26 +0100 Subject: [PATCH 17/48] change scorer to neg_mape and fix pep8 --- doc/modules/model_evaluation.rst | 4 ++-- sklearn/metrics/regression.py | 3 +-- sklearn/metrics/scorer.py | 11 ++++++----- sklearn/metrics/tests/test_score_objects.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 661196a6e610d..163ad89d7bd75 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -85,7 +85,7 @@ Scoring Function **Regression** 'explained_variance' :func:`metrics.explained_variance_score` 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` -'neg_mean_absolute_percentage_error' :func:`metrics.mean_absolute_percentage_error` +'neg_mape' :func:`metrics.mean_absolute_percentage_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` 'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` 'neg_median_absolute_error' :func:`metrics.median_absolute_error` @@ -105,7 +105,7 @@ Usage examples: >>> model = svm.SVC() >>> cross_val_score(model, X, y, scoring='wrong_choice') Traceback (most recent call last): - ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'brier_score_loss', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'mutual_info_score', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_absolute_percentage_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score'] + ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'brier_score_loss', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'mutual_info_score', 'neg_log_loss', 'neg_mape', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score'] .. note:: diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index ee8fccd629822..d959752b68966 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -208,7 +208,6 @@ def mean_absolute_percentage_error(y_true, y_pred): >>> y_pred = [2.5, 0.0, 2, 8] >>> mean_absolute_percentage_error(y_true, y_pred) 32.738... - """ y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, 'uniform_average') @@ -219,7 +218,7 @@ def mean_absolute_percentage_error(y_true, y_pred): if (y_true == 0).any(): raise ValueError("mean_absolute_percentage_error requires" - " y_true to never be zero") + " y_true to not include zeros") return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index 630a52a93448f..ca491fcbde448 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -27,7 +27,8 @@ mean_squared_error, mean_absolute_percentage_error, mean_squared_log_error, accuracy_score, f1_score, roc_auc_score, average_precision_score, - precision_score, recall_score, log_loss, balanced_accuracy_score, + precision_score, recall_score, log_loss, + balanced_accuracy_score, explained_variance_score, brier_score_loss) from .cluster import adjusted_rand_score @@ -487,10 +488,10 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, 'be removed in 0.20.') mean_absolute_error_scorer = make_scorer(mean_absolute_error, greater_is_better=False) -neg_mean_absolute_percentage_error_scorer = make_scorer( - mean_absolute_percentage_error, greater_is_better=False) - mean_absolute_error_scorer._deprecation_msg = deprecation_msg +neg_mape_scorer = make_scorer(mean_absolute_percentage_error, + greater_is_better=False) + neg_median_absolute_error_scorer = make_scorer(median_absolute_error, greater_is_better=False) deprecation_msg = ('Scoring method median_absolute_error was renamed to ' @@ -540,9 +541,9 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, SCORERS = dict(explained_variance=explained_variance_scorer, r2=r2_scorer, + neg_mape=neg_mape_scorer, neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, - neg_mean_absolute_percentage_error=neg_mean_absolute_percentage_error_scorer, neg_mean_squared_error=neg_mean_squared_error_scorer, neg_mean_squared_log_error=neg_mean_squared_log_error_scorer, median_absolute_error=median_absolute_error_scorer, diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 8ee3ea830ae6b..bf1ed4fbc0dac 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -42,8 +42,8 @@ REGRESSION_SCORERS = ['explained_variance', 'r2', 'neg_mean_absolute_error', - 'neg_mean_absolute_percentage_error', - 'neg_mean_squared_error', 'neg_mean_squared_log_error', + 'neg_mape', 'neg_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error'] From 595bcf6cddace7f99f397cc8885b9b82422d34f1 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 01:05:45 +0100 Subject: [PATCH 18/48] adding a metrics category for non-zero y --- sklearn/metrics/tests/test_common.py | 91 +++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 15 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 3162de559b2c9..9bc6e1111085e 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -385,6 +385,11 @@ "mean_absolute_percentage_error" ] +# Metrics that only support non-zero y +METRICS_WITH_NON_ZERO_Y = [ + "mean_absolute_percentage_error" +] + @ignore_warnings def test_symmetry(): @@ -393,6 +398,11 @@ def test_symmetry(): y_true = random_state.randint(0, 2, size=(20, )) y_pred = random_state.randint(0, 2, size=(20, )) + # this used to test metrics that only support non zero y + # like mean_absolute_percentage_error + non_zero_y_true = random_state.randint(1, 3, size=(20, )) + non_zero_y_pred = random_state.randint(1, 3, size=(20, )) + # We shouldn't forget any metrics assert_equal(set(SYMMETRIC_METRICS).union( NOT_SYMMETRIC_METRICS, THRESHOLDED_METRICS, @@ -406,15 +416,25 @@ def test_symmetry(): # Symmetric metric for name in SYMMETRIC_METRICS: metric = ALL_METRICS[name] - assert_almost_equal(metric(y_true, y_pred), - metric(y_pred, y_true), - err_msg="%s is not symmetric" % name) + if name in METRICS_WITH_NON_ZERO_Y: + assert_almost_equal(metric(non_zero_y_true, non_zero_y_pred), + metric(non_zero_y_pred, non_zero_y_true), + err_msg="%s is not symmetric" % name) + else: + assert_almost_equal(metric(y_true, y_pred), + metric(y_pred, y_true), + err_msg="%s is not symmetric" % name) # Not symmetric metrics for name in NOT_SYMMETRIC_METRICS: metric = ALL_METRICS[name] - assert_true(np.any(metric(y_true, y_pred) != metric(y_pred, y_true)), - msg="%s seems to be symmetric" % name) + if name in METRICS_WITH_NON_ZERO_Y: + assert_true(np.any(metric(non_zero_y_true, + non_zero_y_pred) != metric(non_zero_y_pred, non_zero_y_true)), + msg="%s seems to be symmetric" % name) + else: + assert_true(np.any(metric(y_true, y_pred) != metric(y_pred, y_true)), + msg="%s seems to be symmetric" % name) @ignore_warnings @@ -423,15 +443,29 @@ def test_sample_order_invariance(): y_true = random_state.randint(0, 2, size=(20, )) y_pred = random_state.randint(0, 2, size=(20, )) y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) + # this used to test metrics that only support non zero y + # like mean_absolute_percentage_error + non_zero_y_true = random_state.randint(1, 3, size=(20, )) + non_zero_y_pred = random_state.randint(1, 3, size=(20, )) + non_zero_y_true_shuffle, non_zero_y_pred_shuffle = shuffle(non_zero_y_true, + non_zero_y_pred, + random_state=0) for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: continue - assert_almost_equal(metric(y_true, y_pred), - metric(y_true_shuffle, y_pred_shuffle), - err_msg="%s is not sample order invariant" - % name) + if name in METRICS_WITH_NON_ZERO_Y: + assert_almost_equal(metric(non_zero_y_true, non_zero_y_pred), + metric(non_zero_y_true_shuffle, + non_zero_y_pred_shuffle), + err_msg="%s is not sample order invariant" + % name) + else: + assert_almost_equal(metric(y_true, y_pred), + metric(y_true_shuffle, y_pred_shuffle), + err_msg="%s is not sample order invariant" + % name) @ignore_warnings @@ -474,11 +508,14 @@ def test_sample_order_invariance_multilabel_and_multioutput(): % name) -@ignore_warnings -def test_format_invariance_with_1d_vectors(): +def data_for_test_format_invariance_with_1d_vectors(non_zero_y=False): random_state = check_random_state(0) - y1 = random_state.randint(0, 2, size=(20, )) - y2 = random_state.randint(0, 2, size=(20, )) + if non_zero_y: + y1 = random_state.randint(1, 3, size=(20, )) + y2 = random_state.randint(1, 3, size=(20, )) + else: + y1 = random_state.randint(0, 2, size=(20, )) + y2 = random_state.randint(0, 2, size=(20, )) y1_list = list(y1) y2_list = list(y2) @@ -490,11 +527,31 @@ def test_format_invariance_with_1d_vectors(): y2_column = np.reshape(y2_1d, (-1, 1)) y1_row = np.reshape(y1_1d, (1, -1)) y2_row = np.reshape(y2_1d, (1, -1)) + return (y1, y2, y1_list, y2_list, y1_1d, y2_1d, + y1_column, y2_column, y1_row, y2_row) + + +@ignore_warnings +def test_format_invariance_with_1d_vectors(): + + data = data_for_test_format_invariance_with_1d_vectors( + non_zero_y=False) + # preparing the data + (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, + y2_column, y1_row, y2_row) = data for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: continue + if name in METRICS_WITH_NON_ZERO_Y: + # preparing the data for the special case of a metric + # that doesn't support y containing zeros. + data = data_for_test_format_invariance_with_1d_vectors( + non_zero_y=True) + (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, + y2_column, y1_row, y2_row) = data + measure = metric(y1, y2) assert_almost_equal(metric(y1_list, y2_list), measure, @@ -658,8 +715,12 @@ def check_single_sample(name): metric = ALL_METRICS[name] # assert that no exception is thrown - for i, j in product([0, 1], repeat=2): - metric([i], [j]) + if name in METRICS_WITH_NON_ZERO_Y: + for i, j in product([1, 2], repeat=2): + metric([i], [j]) + else: + for i, j in product([0, 1], repeat=2): + metric([i], [j]) @ignore_warnings From 7575d0e88272bef812a619268cf94a29efb42316 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 01:44:11 +0100 Subject: [PATCH 19/48] fix pep8 issues --- sklearn/metrics/tests/test_common.py | 55 +++++++++++++++------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 9bc6e1111085e..7d306e883fa96 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -376,11 +376,11 @@ # No Sample weight support METRICS_WITHOUT_SAMPLE_WEIGHT = [ - "confusion_matrix", # Left this one here because the tests in this file do - # not work for confusion_matrix, as its output is a - # matrix instead of a number. Testing of - # confusion_matrix with sample_weight is in - # test_classification.py + "confusion_matrix", # Left this one here because the tests in this file do + # not work for confusion_matrix, as its output is a + # matrix instead of a number. Testing of + # confusion_matrix with sample_weight is in + # test_classification.py "median_absolute_error", "mean_absolute_percentage_error" ] @@ -388,7 +388,7 @@ # Metrics that only support non-zero y METRICS_WITH_NON_ZERO_Y = [ "mean_absolute_percentage_error" -] +] @ignore_warnings @@ -399,7 +399,7 @@ def test_symmetry(): y_pred = random_state.randint(0, 2, size=(20, )) # this used to test metrics that only support non zero y - # like mean_absolute_percentage_error + # like mean_absolute_percentage_error non_zero_y_true = random_state.randint(1, 3, size=(20, )) non_zero_y_pred = random_state.randint(1, 3, size=(20, )) @@ -419,7 +419,7 @@ def test_symmetry(): if name in METRICS_WITH_NON_ZERO_Y: assert_almost_equal(metric(non_zero_y_true, non_zero_y_pred), metric(non_zero_y_pred, non_zero_y_true), - err_msg="%s is not symmetric" % name) + err_msg="%s is not symmetric" % name) else: assert_almost_equal(metric(y_true, y_pred), metric(y_pred, y_true), @@ -430,10 +430,13 @@ def test_symmetry(): metric = ALL_METRICS[name] if name in METRICS_WITH_NON_ZERO_Y: assert_true(np.any(metric(non_zero_y_true, - non_zero_y_pred) != metric(non_zero_y_pred, non_zero_y_true)), + non_zero_y_pred) != metric( + non_zero_y_pred, + non_zero_y_true)), msg="%s seems to be symmetric" % name) else: - assert_true(np.any(metric(y_true, y_pred) != metric(y_pred, y_true)), + assert_true(np.any(metric(y_true, + y_pred) != metric(y_pred, y_true)), msg="%s seems to be symmetric" % name) @@ -444,12 +447,12 @@ def test_sample_order_invariance(): y_pred = random_state.randint(0, 2, size=(20, )) y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) # this used to test metrics that only support non zero y - # like mean_absolute_percentage_error + # like mean_absolute_percentage_error non_zero_y_true = random_state.randint(1, 3, size=(20, )) non_zero_y_pred = random_state.randint(1, 3, size=(20, )) non_zero_y_true_shuffle, non_zero_y_pred_shuffle = shuffle(non_zero_y_true, non_zero_y_pred, - random_state=0) + random_state=0) for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: @@ -527,31 +530,31 @@ def data_for_test_format_invariance_with_1d_vectors(non_zero_y=False): y2_column = np.reshape(y2_1d, (-1, 1)) y1_row = np.reshape(y1_1d, (1, -1)) y2_row = np.reshape(y2_1d, (1, -1)) - return (y1, y2, y1_list, y2_list, y1_1d, y2_1d, - y1_column, y2_column, y1_row, y2_row) + return (y1, y2, y1_list, y2_list, y1_1d, y2_1d, + y1_column, y2_column, y1_row, y2_row) @ignore_warnings def test_format_invariance_with_1d_vectors(): data = data_for_test_format_invariance_with_1d_vectors( - non_zero_y=False) - # preparing the data - (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, - y2_column, y1_row, y2_row) = data + non_zero_y=False) + # preparing the data + (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, + y2_column, y1_row, y2_row) = data for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: continue if name in METRICS_WITH_NON_ZERO_Y: - # preparing the data for the special case of a metric - # that doesn't support y containing zeros. + # preparing the data for the special case of a metric + # that doesn't support y containing zeros. data = data_for_test_format_invariance_with_1d_vectors( non_zero_y=True) - (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, - y2_column, y1_row, y2_row) = data - + (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, + y2_column, y1_row, y2_row) = data + measure = metric(y1, y2) assert_almost_equal(metric(y1_list, y2_list), measure, @@ -683,9 +686,9 @@ def test_invariance_string_vs_numbers_labels(): def test_inf_nan_input(): - invalids =[([0, 1], [np.inf, np.inf]), - ([0, 1], [np.nan, np.nan]), - ([0, 1], [np.nan, np.inf])] + invalids = [([0, 1], [np.inf, np.inf]), + ([0, 1], [np.nan, np.nan]), + ([0, 1], [np.nan, np.inf])] METRICS = dict() METRICS.update(THRESHOLDED_METRICS) From a22811641d9c73ef515efc504a164a581f772b4b Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 08:57:29 +0100 Subject: [PATCH 20/48] undo unrelated pep8 changes --- sklearn/metrics/tests/test_common.py | 37 +++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 9bc6e1111085e..afbf41c17ac7a 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -388,7 +388,7 @@ # Metrics that only support non-zero y METRICS_WITH_NON_ZERO_Y = [ "mean_absolute_percentage_error" -] +] @ignore_warnings @@ -399,7 +399,7 @@ def test_symmetry(): y_pred = random_state.randint(0, 2, size=(20, )) # this used to test metrics that only support non zero y - # like mean_absolute_percentage_error + # like mean_absolute_percentage_error non_zero_y_true = random_state.randint(1, 3, size=(20, )) non_zero_y_pred = random_state.randint(1, 3, size=(20, )) @@ -419,7 +419,7 @@ def test_symmetry(): if name in METRICS_WITH_NON_ZERO_Y: assert_almost_equal(metric(non_zero_y_true, non_zero_y_pred), metric(non_zero_y_pred, non_zero_y_true), - err_msg="%s is not symmetric" % name) + err_msg="%s is not symmetric" % name) else: assert_almost_equal(metric(y_true, y_pred), metric(y_pred, y_true), @@ -430,10 +430,13 @@ def test_symmetry(): metric = ALL_METRICS[name] if name in METRICS_WITH_NON_ZERO_Y: assert_true(np.any(metric(non_zero_y_true, - non_zero_y_pred) != metric(non_zero_y_pred, non_zero_y_true)), + non_zero_y_pred) != metric( + non_zero_y_pred, + non_zero_y_true)), msg="%s seems to be symmetric" % name) else: - assert_true(np.any(metric(y_true, y_pred) != metric(y_pred, y_true)), + assert_true(np.any(metric(y_true, + y_pred) != metric(y_pred, y_true)), msg="%s seems to be symmetric" % name) @@ -444,12 +447,12 @@ def test_sample_order_invariance(): y_pred = random_state.randint(0, 2, size=(20, )) y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) # this used to test metrics that only support non zero y - # like mean_absolute_percentage_error + # like mean_absolute_percentage_error non_zero_y_true = random_state.randint(1, 3, size=(20, )) non_zero_y_pred = random_state.randint(1, 3, size=(20, )) non_zero_y_true_shuffle, non_zero_y_pred_shuffle = shuffle(non_zero_y_true, non_zero_y_pred, - random_state=0) + random_state=0) for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: @@ -527,8 +530,8 @@ def data_for_test_format_invariance_with_1d_vectors(non_zero_y=False): y2_column = np.reshape(y2_1d, (-1, 1)) y1_row = np.reshape(y1_1d, (1, -1)) y2_row = np.reshape(y2_1d, (1, -1)) - return (y1, y2, y1_list, y2_list, y1_1d, y2_1d, - y1_column, y2_column, y1_row, y2_row) + return (y1, y2, y1_list, y2_list, y1_1d, y2_1d, + y1_column, y2_column, y1_row, y2_row) @ignore_warnings @@ -536,22 +539,22 @@ def test_format_invariance_with_1d_vectors(): data = data_for_test_format_invariance_with_1d_vectors( non_zero_y=False) - # preparing the data - (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, - y2_column, y1_row, y2_row) = data + # preparing the data + (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, + y2_column, y1_row, y2_row) = data for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: continue if name in METRICS_WITH_NON_ZERO_Y: - # preparing the data for the special case of a metric - # that doesn't support y containing zeros. + # preparing the data for the special case of a metric + # that doesn't support y containing zeros. data = data_for_test_format_invariance_with_1d_vectors( non_zero_y=True) - (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, - y2_column, y1_row, y2_row) = data - + (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, + y2_column, y1_row, y2_row) = data + measure = metric(y1, y2) assert_almost_equal(metric(y1_list, y2_list), measure, From 7a7839649f0df0d3b4c1a9a5a92f91ad7af10fa8 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 16:31:52 +0100 Subject: [PATCH 21/48] fixing scorers tests in test_score_objects.py --- sklearn/metrics/tests/test_score_objects.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index bf1ed4fbc0dac..a90acbc643183 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -66,6 +66,7 @@ MULTILABEL_ONLY_SCORERS = ['precision_samples', 'recall_samples', 'f1_samples'] +NONZERO_Y_SCORERS = ['neg_mape'] def _make_estimators(X_train, y_train, y_ml_train): # Make estimators that make sense to test various scoring methods @@ -487,7 +488,10 @@ def check_scorer_memmap(scorer_name): if scorer_name in MULTILABEL_ONLY_SCORERS: score = scorer(estimator, X_mm, y_ml_mm) else: - score = scorer(estimator, X_mm, y_mm) + if scorer_name in NONZERO_Y_SCORERS: + score = scorer(estimator, X_mm, y_mm+1) + else: + score = scorer(estimator, X_mm, y_mm) assert isinstance(score, numbers.Number), scorer_name From 0ffc8249a21e36506d810638a00dcff73cc885b6 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 16:43:54 +0100 Subject: [PATCH 22/48] undo autopep8 irrelevant changes --- sklearn/metrics/tests/test_common.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 4bd95ac1fe60f..fc750e325fab8 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -377,10 +377,10 @@ # No Sample weight support METRICS_WITHOUT_SAMPLE_WEIGHT = [ "confusion_matrix", # Left this one here because the tests in this file do - # not work for confusion_matrix, as its output is a - # matrix instead of a number. Testing of - # confusion_matrix with sample_weight is in - # test_classification.py + # not work for confusion_matrix, as its output is a + # matrix instead of a number. Testing of + # confusion_matrix with sample_weight is in + # test_classification.py "median_absolute_error", "mean_absolute_percentage_error" ] @@ -686,9 +686,9 @@ def test_invariance_string_vs_numbers_labels(): def test_inf_nan_input(): - invalids = [([0, 1], [np.inf, np.inf]), - ([0, 1], [np.nan, np.nan]), - ([0, 1], [np.nan, np.inf])] + invalids =[([0, 1], [np.inf, np.inf]), + ([0, 1], [np.nan, np.nan]), + ([0, 1], [np.nan, np.inf])] METRICS = dict() METRICS.update(THRESHOLDED_METRICS) From 953a6e0efd1238a3de14833462726dba9a89cd8a Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 16:52:15 +0100 Subject: [PATCH 23/48] undo autopep8 changes --- sklearn/metrics/tests/test_common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index fc750e325fab8..afbf41c17ac7a 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -376,11 +376,11 @@ # No Sample weight support METRICS_WITHOUT_SAMPLE_WEIGHT = [ - "confusion_matrix", # Left this one here because the tests in this file do - # not work for confusion_matrix, as its output is a - # matrix instead of a number. Testing of - # confusion_matrix with sample_weight is in - # test_classification.py + "confusion_matrix", # Left this one here because the tests in this file do + # not work for confusion_matrix, as its output is a + # matrix instead of a number. Testing of + # confusion_matrix with sample_weight is in + # test_classification.py "median_absolute_error", "mean_absolute_percentage_error" ] From 064833bbc3dde6de68ae78f6ce0eda5632522595 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Feb 2018 16:56:51 +0100 Subject: [PATCH 24/48] undo autopep8 change --- sklearn/metrics/tests/test_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index afbf41c17ac7a..5a474df7f8c92 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -348,8 +348,8 @@ "micro_f0.5_score", "micro_f1_score", "micro_f2_score", "micro_precision_score", "micro_recall_score", - "matthews_corrcoef_score", "mean_absolute_error", - "mean_squared_error", "median_absolute_error", + "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error", + "median_absolute_error", "cohen_kappa_score", ] From e707fc86b01e7e93e877f6747a80e9e0f22560e5 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Thu, 1 Mar 2018 00:13:16 +0100 Subject: [PATCH 25/48] unduplicating tests by using randint(1,3) --- sklearn/metrics/tests/test_common.py | 98 +++++++--------------------- 1 file changed, 25 insertions(+), 73 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 5a474df7f8c92..9fa5ffcf87d38 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -395,13 +395,11 @@ def test_symmetry(): # Test the symmetry of score and loss functions random_state = check_random_state(0) - y_true = random_state.randint(0, 2, size=(20, )) - y_pred = random_state.randint(0, 2, size=(20, )) - - # this used to test metrics that only support non zero y + # Changing from randint(0, 2) to randint(1, 3) to generalize + # test data to metrics that only support non zero y # like mean_absolute_percentage_error - non_zero_y_true = random_state.randint(1, 3, size=(20, )) - non_zero_y_pred = random_state.randint(1, 3, size=(20, )) + y_true = random_state.randint(1, 3, size=(20, )) + y_pred = random_state.randint(1, 3, size=(20, )) # We shouldn't forget any metrics assert_equal(set(SYMMETRIC_METRICS).union( @@ -416,66 +414,40 @@ def test_symmetry(): # Symmetric metric for name in SYMMETRIC_METRICS: metric = ALL_METRICS[name] - if name in METRICS_WITH_NON_ZERO_Y: - assert_almost_equal(metric(non_zero_y_true, non_zero_y_pred), - metric(non_zero_y_pred, non_zero_y_true), - err_msg="%s is not symmetric" % name) - else: - assert_almost_equal(metric(y_true, y_pred), - metric(y_pred, y_true), - err_msg="%s is not symmetric" % name) + assert_almost_equal(metric(y_true, y_pred), + metric(y_pred, y_true), + err_msg="%s is not symmetric" % name) # Not symmetric metrics for name in NOT_SYMMETRIC_METRICS: metric = ALL_METRICS[name] - if name in METRICS_WITH_NON_ZERO_Y: - assert_true(np.any(metric(non_zero_y_true, - non_zero_y_pred) != metric( - non_zero_y_pred, - non_zero_y_true)), - msg="%s seems to be symmetric" % name) - else: - assert_true(np.any(metric(y_true, - y_pred) != metric(y_pred, y_true)), - msg="%s seems to be symmetric" % name) + assert_true(np.any(metric(y_true, y_pred) != metric(y_pred, y_true)), + msg="%s seems to be symmetric" % name) @ignore_warnings def test_sample_order_invariance(): random_state = check_random_state(0) - y_true = random_state.randint(0, 2, size=(20, )) - y_pred = random_state.randint(0, 2, size=(20, )) - y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) - # this used to test metrics that only support non zero y + # Changing from randint(0, 2) to randint(1, 3) to generalize + # test data to metrics that only support non zero y # like mean_absolute_percentage_error - non_zero_y_true = random_state.randint(1, 3, size=(20, )) - non_zero_y_pred = random_state.randint(1, 3, size=(20, )) - non_zero_y_true_shuffle, non_zero_y_pred_shuffle = shuffle(non_zero_y_true, - non_zero_y_pred, - random_state=0) + y_true = random_state.randint(1, 3, size=(20, )) + y_pred = random_state.randint(1, 3, size=(20, )) + y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: continue - if name in METRICS_WITH_NON_ZERO_Y: - assert_almost_equal(metric(non_zero_y_true, non_zero_y_pred), - metric(non_zero_y_true_shuffle, - non_zero_y_pred_shuffle), - err_msg="%s is not sample order invariant" - % name) - else: - assert_almost_equal(metric(y_true, y_pred), - metric(y_true_shuffle, y_pred_shuffle), - err_msg="%s is not sample order invariant" - % name) + assert_almost_equal(metric(y_true, y_pred), + metric(y_true_shuffle, y_pred_shuffle), + err_msg="%s is not sample order invariant" + % name) @ignore_warnings def test_sample_order_invariance_multilabel_and_multioutput(): random_state = check_random_state(0) - - # Generate some data y_true = random_state.randint(0, 2, size=(20, 25)) y_pred = random_state.randint(0, 2, size=(20, 25)) y_score = random_state.normal(size=y_true.shape) @@ -511,14 +483,14 @@ def test_sample_order_invariance_multilabel_and_multioutput(): % name) -def data_for_test_format_invariance_with_1d_vectors(non_zero_y=False): +@ignore_warnings +def test_format_invariance_with_1d_vectors(): random_state = check_random_state(0) - if non_zero_y: - y1 = random_state.randint(1, 3, size=(20, )) - y2 = random_state.randint(1, 3, size=(20, )) - else: - y1 = random_state.randint(0, 2, size=(20, )) - y2 = random_state.randint(0, 2, size=(20, )) + # Changing from randint(0, 2) to randint(1, 3) to generalize + # test data to metrics that only support non zero y + # like mean_absolute_percentage_error + y1 = random_state.randint(1, 3, size=(20, )) + y2 = random_state.randint(1, 3, size=(20, )) y1_list = list(y1) y2_list = list(y2) @@ -530,31 +502,11 @@ def data_for_test_format_invariance_with_1d_vectors(non_zero_y=False): y2_column = np.reshape(y2_1d, (-1, 1)) y1_row = np.reshape(y1_1d, (1, -1)) y2_row = np.reshape(y2_1d, (1, -1)) - return (y1, y2, y1_list, y2_list, y1_1d, y2_1d, - y1_column, y2_column, y1_row, y2_row) - - -@ignore_warnings -def test_format_invariance_with_1d_vectors(): - - data = data_for_test_format_invariance_with_1d_vectors( - non_zero_y=False) - # preparing the data - (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, - y2_column, y1_row, y2_row) = data for name, metric in ALL_METRICS.items(): if name in METRIC_UNDEFINED_BINARY_MULTICLASS: continue - if name in METRICS_WITH_NON_ZERO_Y: - # preparing the data for the special case of a metric - # that doesn't support y containing zeros. - data = data_for_test_format_invariance_with_1d_vectors( - non_zero_y=True) - (y1, y2, y1_list, y2_list, y1_1d, y2_1d, y1_column, - y2_column, y1_row, y2_row) = data - measure = metric(y1, y2) assert_almost_equal(metric(y1_list, y2_list), measure, From cc8cd10f46afbf40ddbe0da298cfc93f41a75a04 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Thu, 1 Mar 2018 00:58:30 +0100 Subject: [PATCH 26/48] use elif instead --- sklearn/metrics/tests/test_score_objects.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index a90acbc643183..b4cb74727b7d0 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -487,11 +487,10 @@ def check_scorer_memmap(scorer_name): scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name] if scorer_name in MULTILABEL_ONLY_SCORERS: score = scorer(estimator, X_mm, y_ml_mm) + elif scorer_name in NONZERO_Y_SCORERS: + score = scorer(estimator, X_mm, y_mm+1) else: - if scorer_name in NONZERO_Y_SCORERS: - score = scorer(estimator, X_mm, y_mm+1) - else: - score = scorer(estimator, X_mm, y_mm) + score = scorer(estimator, X_mm, y_mm) assert isinstance(score, numbers.Number), scorer_name From c780845b68c817c3fe203541426944dede1ef10c Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Thu, 1 Mar 2018 13:07:27 +0100 Subject: [PATCH 27/48] remove uncessary comments and add space to operator + --- sklearn/metrics/tests/test_common.py | 9 --------- sklearn/metrics/tests/test_score_objects.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 9fa5ffcf87d38..cebba95bfe8ee 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -395,9 +395,6 @@ def test_symmetry(): # Test the symmetry of score and loss functions random_state = check_random_state(0) - # Changing from randint(0, 2) to randint(1, 3) to generalize - # test data to metrics that only support non zero y - # like mean_absolute_percentage_error y_true = random_state.randint(1, 3, size=(20, )) y_pred = random_state.randint(1, 3, size=(20, )) @@ -428,9 +425,6 @@ def test_symmetry(): @ignore_warnings def test_sample_order_invariance(): random_state = check_random_state(0) - # Changing from randint(0, 2) to randint(1, 3) to generalize - # test data to metrics that only support non zero y - # like mean_absolute_percentage_error y_true = random_state.randint(1, 3, size=(20, )) y_pred = random_state.randint(1, 3, size=(20, )) y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) @@ -486,9 +480,6 @@ def test_sample_order_invariance_multilabel_and_multioutput(): @ignore_warnings def test_format_invariance_with_1d_vectors(): random_state = check_random_state(0) - # Changing from randint(0, 2) to randint(1, 3) to generalize - # test data to metrics that only support non zero y - # like mean_absolute_percentage_error y1 = random_state.randint(1, 3, size=(20, )) y2 = random_state.randint(1, 3, size=(20, )) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index b4cb74727b7d0..8473f52bf3517 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -488,7 +488,7 @@ def check_scorer_memmap(scorer_name): if scorer_name in MULTILABEL_ONLY_SCORERS: score = scorer(estimator, X_mm, y_ml_mm) elif scorer_name in NONZERO_Y_SCORERS: - score = scorer(estimator, X_mm, y_mm+1) + score = scorer(estimator, X_mm, y_mm + 1) else: score = scorer(estimator, X_mm, y_mm) assert isinstance(score, numbers.Number), scorer_name From 9dba98884e6c42c53cd928533e039c6edec014d8 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Fri, 2 Mar 2018 06:42:11 +0100 Subject: [PATCH 28/48] using [1, 2] as a single sample test --- sklearn/metrics/tests/test_common.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 9fa5ffcf87d38..f24d34ea5d807 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -670,11 +670,7 @@ def check_single_sample(name): metric = ALL_METRICS[name] # assert that no exception is thrown - if name in METRICS_WITH_NON_ZERO_Y: - for i, j in product([1, 2], repeat=2): - metric([i], [j]) - else: - for i, j in product([0, 1], repeat=2): + for i, j in product([1, 2], repeat=2): metric([i], [j]) From a48f9d7e2a3b72c787e10d505638534d2bdd05ca Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Fri, 2 Mar 2018 08:58:00 +0100 Subject: [PATCH 29/48] remove excessive indentation --- sklearn/metrics/tests/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index bfc8a57761f7c..2295863d34409 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -662,7 +662,7 @@ def check_single_sample(name): # assert that no exception is thrown for i, j in product([1, 2], repeat=2): - metric([i], [j]) + metric([i], [j]) @ignore_warnings From a0f2cd038243ef12ebb93cbc86086a7807179f4f Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Fri, 2 Mar 2018 09:31:06 +0100 Subject: [PATCH 30/48] test raise error when y contains zeros --- sklearn/metrics/tests/test_common.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 2295863d34409..c0f430f8d5f7c 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -1121,3 +1121,16 @@ def test_no_averaging_labels(): score_labels = metric(y_true, y_pred, labels=labels, average=None) score = metric(y_true, y_pred, average=None) assert_array_equal(score_labels, score[inverse_labels]) + + +def test_raise_value_error_y_with_zeros(): + random_state = check_random_state(0) + y_true = random_state.randint(0, 2, size=(20, )) + y_pred = random_state.randint(0, 2, size=(20, )) + + for name in METRICS_WITH_NON_ZERO_Y: + metric = ALL_METRICS[name] + assert_raise_message(ValueError, + "mean_absolute_percentage_error requires" + " y_true to not include zeros", + metric, y_true, y_pred) From bfe2143779d0b1d12a2bf291adbffa827a90db9a Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Sun, 4 Mar 2018 09:07:31 +0100 Subject: [PATCH 31/48] use memmap type for test scenario --- sklearn/metrics/tests/test_score_objects.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 8473f52bf3517..88cda82a5230a 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -84,21 +84,22 @@ def _make_estimators(X_train, y_train, y_ml_train): ) -X_mm, y_mm, y_ml_mm = None, None, None +X_mm, y_mm, y_ml_mm, nonzero_y_mm = None, None, None, None ESTIMATORS = None TEMP_FOLDER = None def setup_module(): # Create some memory mapped data - global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS + global X_mm, y_mm, y_ml_mm, nonzero_y_mm, TEMP_FOLDER, ESTIMATORS TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_') X, y = make_classification(n_samples=30, n_features=5, random_state=0) _, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0) + nonzero_y = y + 1 filename = os.path.join(TEMP_FOLDER, 'test_data.pkl') - joblib.dump((X, y, y_ml), filename) - X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode='r') + joblib.dump((X, y, y_ml, nonzero_y), filename) + X_mm, y_mm, y_ml_mm, nonzero_y_mm = joblib.load(filename, mmap_mode='r') ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm) @@ -488,7 +489,7 @@ def check_scorer_memmap(scorer_name): if scorer_name in MULTILABEL_ONLY_SCORERS: score = scorer(estimator, X_mm, y_ml_mm) elif scorer_name in NONZERO_Y_SCORERS: - score = scorer(estimator, X_mm, y_mm + 1) + score = scorer(estimator, X_mm, nonzero_y_mm) else: score = scorer(estimator, X_mm, y_mm) assert isinstance(score, numbers.Number), scorer_name From e4cb1401e61c5b2e3ab3b7b7bc9f990498fe1dae Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Sun, 4 Mar 2018 09:48:14 +0100 Subject: [PATCH 32/48] tear down nonzero_y_mm --- sklearn/metrics/tests/test_score_objects.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 88cda82a5230a..050249789786e 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -68,6 +68,7 @@ NONZERO_Y_SCORERS = ['neg_mape'] + def _make_estimators(X_train, y_train, y_ml_train): # Make estimators that make sense to test various scoring methods sensible_regr = DecisionTreeRegressor(random_state=0) @@ -104,9 +105,10 @@ def setup_module(): def teardown_module(): - global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS + global X_mm, y_mm, y_ml_mm, nonzero_y_mm, TEMP_FOLDER, ESTIMATORS # GC closes the mmap file descriptors X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None + nonzero_y_mm = None shutil.rmtree(TEMP_FOLDER) From 731299860ca335b23b64eea93c2148d277903b01 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Sun, 4 Mar 2018 22:11:26 +0100 Subject: [PATCH 33/48] keep naming consistency --- sklearn/metrics/tests/test_score_objects.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 050249789786e..4800a511e6a2e 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -66,7 +66,7 @@ MULTILABEL_ONLY_SCORERS = ['precision_samples', 'recall_samples', 'f1_samples'] -NONZERO_Y_SCORERS = ['neg_mape'] +y_nozero_SCORERS = ['neg_mape'] def _make_estimators(X_train, y_train, y_ml_train): @@ -85,30 +85,30 @@ def _make_estimators(X_train, y_train, y_ml_train): ) -X_mm, y_mm, y_ml_mm, nonzero_y_mm = None, None, None, None +X_mm, y_mm, y_ml_mm, y_nozero_mm = None, None, None, None ESTIMATORS = None TEMP_FOLDER = None def setup_module(): # Create some memory mapped data - global X_mm, y_mm, y_ml_mm, nonzero_y_mm, TEMP_FOLDER, ESTIMATORS + global X_mm, y_mm, y_ml_mm, y_nozero_mm, TEMP_FOLDER, ESTIMATORS TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_') X, y = make_classification(n_samples=30, n_features=5, random_state=0) _, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0) - nonzero_y = y + 1 + y_nozero = y + 1 filename = os.path.join(TEMP_FOLDER, 'test_data.pkl') - joblib.dump((X, y, y_ml, nonzero_y), filename) - X_mm, y_mm, y_ml_mm, nonzero_y_mm = joblib.load(filename, mmap_mode='r') + joblib.dump((X, y, y_ml, y_nozero), filename) + X_mm, y_mm, y_ml_mm, y_nozero_mm = joblib.load(filename, mmap_mode='r') ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm) def teardown_module(): - global X_mm, y_mm, y_ml_mm, nonzero_y_mm, TEMP_FOLDER, ESTIMATORS + global X_mm, y_mm, y_ml_mm, y_nozero_mm, TEMP_FOLDER, ESTIMATORS # GC closes the mmap file descriptors X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None - nonzero_y_mm = None + y_nozero_mm = None shutil.rmtree(TEMP_FOLDER) @@ -490,8 +490,8 @@ def check_scorer_memmap(scorer_name): scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name] if scorer_name in MULTILABEL_ONLY_SCORERS: score = scorer(estimator, X_mm, y_ml_mm) - elif scorer_name in NONZERO_Y_SCORERS: - score = scorer(estimator, X_mm, nonzero_y_mm) + elif scorer_name in y_nozero_SCORERS: + score = scorer(estimator, X_mm, y_nozero_mm) else: score = scorer(estimator, X_mm, y_mm) assert isinstance(score, numbers.Number), scorer_name From 9c3a776cd9415e176ed3771e2b2635ebf29a05ab Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Sun, 4 Mar 2018 23:22:09 +0100 Subject: [PATCH 34/48] Trigger travis after timeout From 9ea397551eb0c45a8f05f9d8bb7860d53e0f5c2c Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 14:17:10 +0200 Subject: [PATCH 35/48] add comma aster n_samples --- sklearn/metrics/regression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index d959752b68966..dcb60d4804a25 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -190,10 +190,10 @@ def mean_absolute_percentage_error(y_true, y_pred): Parameters ---------- - y_true : array-like of shape = (n_samples) + y_true : array-like of shape = (n_samples,) Ground truth (correct) target values. - y_pred : array-like of shape = (n_samples) + y_pred : array-like of shape = (n_samples,) Estimated target values. Returns From 407332a4321788e11b04968ca7daecf0fe2d2946 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 14:18:43 +0200 Subject: [PATCH 36/48] specify that mape is between 0 and 100 --- sklearn/metrics/regression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index dcb60d4804a25..63c5087796329 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -199,7 +199,8 @@ def mean_absolute_percentage_error(y_true, y_pred): Returns ------- loss : float - A positive floating point value (the best value is 0.0). + A positive floating point value between 0.0 and 100.0, + the best value is 0.0. Examples -------- From c74eabb95bd3570dd6eeb6425da5489dbc971dd0 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 14:22:58 +0200 Subject: [PATCH 37/48] update MAPE description in model_evalution --- doc/modules/model_evaluation.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 9cdb758d34260..1fbf4f643991a 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1558,13 +1558,15 @@ Here is a small example of usage of the :func:`mean_absolute_error` function:: .. _mean_absolute_percentage_error: -Mean absolute error -------------------- +Mean absolute percentage error +------------------------------ -The :func:`mean_absolute_percentage_error` function computes `mean absolute +The :func:`mean_absolute_percentage_error` function, also known as **MAPE**, computes `mean absolute percentage error `_, a risk metric corresponding to the expected value of the absolute percentage error loss or -:math:`l1`-norm of percentage loss. +:math:`l1`-norm of percentage loss. MAPE computes the error relative to the true value. +Therefore the same absolute distance between prediction and ground truth will lead to a smaller +error if the true value is larger. In particular the metrics is not shift-invariant. If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, and :math:`y_i` is the corresponding true value, then the mean absolute percentage error From 60f370da42fffdc00109060f6a0ee8d68c221342 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 16:39:08 +0200 Subject: [PATCH 38/48] updated mape description and add not shift-invariant demo --- doc/modules/model_evaluation.rst | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 1fbf4f643991a..20dafa6e7b192 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1564,9 +1564,7 @@ Mean absolute percentage error The :func:`mean_absolute_percentage_error` function, also known as **MAPE**, computes `mean absolute percentage error `_, a risk metric corresponding to the expected value of the absolute percentage error loss or -:math:`l1`-norm of percentage loss. MAPE computes the error relative to the true value. -Therefore the same absolute distance between prediction and ground truth will lead to a smaller -error if the true value is larger. In particular the metrics is not shift-invariant. +:math:`l1`-norm of percentage loss. If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, and :math:`y_i` is the corresponding true value, then the mean absolute percentage error @@ -1584,6 +1582,23 @@ Here is a small example of usage of the :func:`mean_absolute_percentage_error` f >>> mean_absolute_percentage_error(y_true, y_pred) 32.738... +MAPE computes the error relative to the true value. Therefore the same absolute distance between +prediction and ground truth will lead to a smaller error if the true value is larger. +In particular the metric is not shift-invariant. For example, if :math:`y_{true}` and :math:`y_{pred}` +in the example above are shifted by adding 10, the error becomes smaller: + + >>> import numpy as np + >>> y_true = np.array([3, -0.5, 2, 7]) + >>> y_pred = np.array([2.5, 0.0, 2, 8]) + >>> y_true = y_true + 10 + >>> y_pred = y_pred + 10 + >>> y_true + array([13. , 9.5, 12. , 17. ]) + >>> y_pred + array([12.5, 10. , 12. , 18. ]) + >>> mean_absolute_percentage_error(y_true, y_pred) + 3.747916170516789 + .. _mean_squared_error: Mean squared error From e04fe376dfc39b8379a695026a41b1638947b501 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 17:08:18 +0200 Subject: [PATCH 39/48] fix travis error --- doc/modules/model_evaluation.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 20dafa6e7b192..f98452e20d256 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1592,10 +1592,6 @@ in the example above are shifted by adding 10, the error becomes smaller: >>> y_pred = np.array([2.5, 0.0, 2, 8]) >>> y_true = y_true + 10 >>> y_pred = y_pred + 10 - >>> y_true - array([13. , 9.5, 12. , 17. ]) - >>> y_pred - array([12.5, 10. , 12. , 18. ]) >>> mean_absolute_percentage_error(y_true, y_pred) 3.747916170516789 From 2e20a483d57aaf003b95393a6ff60996160667f4 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 17:47:37 +0200 Subject: [PATCH 40/48] fix failing doctests --- doc/modules/model_evaluation.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index f98452e20d256..fd08d86cda12b 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1587,13 +1587,18 @@ prediction and ground truth will lead to a smaller error if the true value is la In particular the metric is not shift-invariant. For example, if :math:`y_{true}` and :math:`y_{pred}` in the example above are shifted by adding 10, the error becomes smaller: + >>> from sklearn.metrics import mean_absolute_percentage_error >>> import numpy as np >>> y_true = np.array([3, -0.5, 2, 7]) >>> y_pred = np.array([2.5, 0.0, 2, 8]) >>> y_true = y_true + 10 >>> y_pred = y_pred + 10 + >>> y_true + array([ 13. , 9.5 , 12. , 17. ]) + >>> y_pred + array([ 12.5 , 10. , 12. , 18. ]) >>> mean_absolute_percentage_error(y_true, y_pred) - 3.747916170516789 + 3.747... .. _mean_squared_error: From 203aed1e0947e223537b271a16fff4fea39df8b5 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Mon, 26 Mar 2018 18:23:39 +0200 Subject: [PATCH 41/48] fix travis error --- doc/modules/model_evaluation.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index fd08d86cda12b..885e2e294882d 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1593,10 +1593,6 @@ in the example above are shifted by adding 10, the error becomes smaller: >>> y_pred = np.array([2.5, 0.0, 2, 8]) >>> y_true = y_true + 10 >>> y_pred = y_pred + 10 - >>> y_true - array([ 13. , 9.5 , 12. , 17. ]) - >>> y_pred - array([ 12.5 , 10. , 12. , 18. ]) >>> mean_absolute_percentage_error(y_true, y_pred) 3.747... From 7aac4c07fc0c63be12d08e97e4b7312b51513b6d Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Mar 2018 22:07:35 +0200 Subject: [PATCH 42/48] remove line to keep code consistent --- sklearn/metrics/scorer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index ca491fcbde448..4cf8636b14221 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -491,7 +491,6 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, mean_absolute_error_scorer._deprecation_msg = deprecation_msg neg_mape_scorer = make_scorer(mean_absolute_percentage_error, greater_is_better=False) - neg_median_absolute_error_scorer = make_scorer(median_absolute_error, greater_is_better=False) deprecation_msg = ('Scoring method median_absolute_error was renamed to ' From 6f1ab55a819aa80913e8b9456695e227e78176b9 Mon Sep 17 00:00:00 2001 From: Mohamed Ali Jamaoui Date: Wed, 28 Mar 2018 22:11:05 +0200 Subject: [PATCH 43/48] put mape in metrics section --- doc/whats_new/v0.20.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index f2bbe2e9c57e6..4d77a9242bdbf 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -91,8 +91,6 @@ Model evaluation - Added the :func:`metrics.balanced_accuracy_score` metric and a corresponding ``'balanced_accuracy'`` scorer for binary classification. :issue:`8066` by :user:`xyguo` and :user:`Aman Dalmia `. -- Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated - scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui ` Decomposition, manifold learning and clustering @@ -105,6 +103,8 @@ Metrics - Partial AUC is available via ``max_fpr`` parameter in :func:`metrics.roc_auc_score`. :issue:`3273` by :user:`Alexander Niederbühl `. +- Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated + scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui ` Enhancements ............ From e2184f38ad97ebb7881d4a2ebcd3ec64b9b5d4d3 Mon Sep 17 00:00:00 2001 From: mohamed-ali Date: Sun, 18 Aug 2019 14:13:05 +0200 Subject: [PATCH 44/48] fix syntax error --- sklearn/metrics/regression.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index a089692e0180e..cb6fc09cae032 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -19,11 +19,8 @@ # Manoj Kumar # Michael Eickenberg # Konstantin Shmelkov -<<<<<<< HEAD -# Mohamed Ali Jamaoui -======= # Christian Lorentzen ->>>>>>> master +# Mohamed Ali Jamaoui # License: BSD 3 clause From 3dbe7635efba7bda03dda03d41e1e858a893e7c5 Mon Sep 17 00:00:00 2001 From: mohamed-ali Date: Sun, 18 Aug 2019 14:26:50 +0200 Subject: [PATCH 45/48] fix merge conflicts in metrics.scorer --- sklearn/metrics/scorer.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index e3164d8419bfa..d545d42f8a968 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -22,24 +22,13 @@ from collections.abc import Iterable import numpy as np - -<<<<<<< HEAD -from . import (r2_score, median_absolute_error, mean_absolute_error, - mean_squared_error, mean_absolute_percentage_error, - mean_squared_log_error, accuracy_score, - f1_score, roc_auc_score, average_precision_score, - precision_score, recall_score, log_loss, - balanced_accuracy_score, - explained_variance_score, brier_score_loss) -======= from . import (r2_score, median_absolute_error, max_error, mean_absolute_error, mean_squared_error, mean_squared_log_error, mean_tweedie_deviance, accuracy_score, f1_score, roc_auc_score, average_precision_score, precision_score, recall_score, log_loss, balanced_accuracy_score, explained_variance_score, - brier_score_loss, jaccard_score) ->>>>>>> master + brier_score_loss, jaccard_score, mean_absolute_percentage_error) from .cluster import adjusted_rand_score from .cluster import homogeneity_score @@ -503,17 +492,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, greater_is_better=False) neg_mean_absolute_error_scorer = make_scorer(mean_absolute_error, greater_is_better=False) -<<<<<<< HEAD -deprecation_msg = ('Scoring method mean_absolute_error was renamed to ' - 'neg_mean_absolute_error in version 0.18 and will ' - 'be removed in 0.20.') -mean_absolute_error_scorer = make_scorer(mean_absolute_error, - greater_is_better=False) -mean_absolute_error_scorer._deprecation_msg = deprecation_msg neg_mape_scorer = make_scorer(mean_absolute_percentage_error, greater_is_better=False) -======= ->>>>>>> master neg_median_absolute_error_scorer = make_scorer(median_absolute_error, greater_is_better=False) neg_root_mean_squared_error_scorer = make_scorer(mean_squared_error, @@ -568,11 +548,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, SCORERS = dict(explained_variance=explained_variance_scorer, r2=r2_scorer, -<<<<<<< HEAD neg_mape=neg_mape_scorer, -======= max_error=max_error_scorer, ->>>>>>> master neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, neg_mean_squared_error=neg_mean_squared_error_scorer, From 83c4aa5c4ae5c3b4a90ad1cfa34baf164a5abce9 Mon Sep 17 00:00:00 2001 From: mohamed-ali Date: Sun, 18 Aug 2019 14:36:00 +0200 Subject: [PATCH 46/48] fix merge conflicts --- sklearn/metrics/tests/test_common.py | 39 +++++++--------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 76f2d2224b5e9..62ca53d0f35e7 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -477,34 +477,24 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs): "macro_f0.5_score", "macro_f2_score", "macro_precision_score", "macro_recall_score", "log_loss", "hinge_loss", -<<<<<<< HEAD - - "mean_absolute_percentage_error" -] -======= "mean_gamma_deviance", "mean_poisson_deviance", - "mean_compound_poisson_deviance" + "mean_compound_poisson_deviance", "mean_absolute_percentage_error" } ->>>>>>> master - # No Sample weight support METRICS_WITHOUT_SAMPLE_WEIGHT = { "median_absolute_error", -<<<<<<< HEAD - "mean_absolute_percentage_error" -] + "mean_absolute_percentage_error", + "max_error", + "ovo_roc_auc", + "weighted_ovo_roc_auc" +} # Metrics that only support non-zero y METRICS_WITH_NON_ZERO_Y = [ + "mean_absolute_percentage_error", "mean_absolute_percentage_error" ] -======= - "max_error", - "ovo_roc_auc", - "weighted_ovo_roc_auc" -} ->>>>>>> master METRICS_REQUIRE_POSITIVE_Y = { "mean_poisson_deviance", @@ -584,16 +574,10 @@ def test_not_symmetric_metric(name): sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS)) def test_sample_order_invariance(name): random_state = check_random_state(0) -<<<<<<< HEAD - y_true = random_state.randint(1, 3, size=(20, )) - y_pred = random_state.randint(1, 3, size=(20, )) - y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) -======= y_true = random_state.randint(0, 2, size=(20, )) y_pred = random_state.randint(0, 2, size=(20, )) if name in METRICS_REQUIRE_POSITIVE_Y: y_true, y_pred = _require_positive_targets(y_true, y_pred) ->>>>>>> master y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0) @@ -837,15 +821,11 @@ def check_single_sample(name): metric = ALL_METRICS[name] # assert that no exception is thrown -<<<<<<< HEAD - for i, j in product([1, 2], repeat=2): -======= if name in METRICS_REQUIRE_POSITIVE_Y: values = [1, 2] else: values = [0, 1] for i, j in product(values, repeat=2): ->>>>>>> master metric([i], [j]) @@ -1331,7 +1311,6 @@ def test_no_averaging_labels(): assert_array_equal(score_labels, score[inverse_labels]) -<<<<<<< HEAD def test_raise_value_error_y_with_zeros(): random_state = check_random_state(0) y_true = random_state.randint(0, 2, size=(20, )) @@ -1343,7 +1322,8 @@ def test_raise_value_error_y_with_zeros(): "mean_absolute_percentage_error requires" " y_true to not include zeros", metric, y_true, y_pred) -======= + + @pytest.mark.parametrize( 'name', sorted(MULTILABELS_METRICS - {"unnormalized_multilabel_confusion_matrix"})) @@ -1411,4 +1391,3 @@ def test_thresholded_metric_permutation_invariance(name): current_score = metric(y_true_perm, y_score_perm) assert_almost_equal(score, current_score) ->>>>>>> master From 6b350a5185e367e239121ae60dd2639583ceddf8 Mon Sep 17 00:00:00 2001 From: mohamed-ali Date: Sun, 18 Aug 2019 14:40:23 +0200 Subject: [PATCH 47/48] fix merge conflicts --- sklearn/metrics/tests/test_score_objects.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 3734c0becb13d..d2d2d3f981a73 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -513,7 +513,6 @@ def test_scorer_sample_weight(): ignored)) except TypeError as e: -<<<<<<< HEAD assert_true("sample_weight" in str(e), "scorer {0} raises unhelpful exception when called " "with sample weights: {1}".format(name, str(e))) @@ -529,11 +528,6 @@ def check_scorer_memmap(scorer_name): else: score = scorer(estimator, X_mm, y_mm) assert isinstance(score, numbers.Number), scorer_name -======= - assert "sample_weight" in str(e), ( - "scorer {0} raises unhelpful exception when called " - "with sample weights: {1}".format(name, str(e))) ->>>>>>> master @pytest.mark.parametrize('name', SCORERS) From bb36550df4c9f7080c428068025bf71c8843f883 Mon Sep 17 00:00:00 2001 From: mohamed-ali Date: Tue, 20 Aug 2019 09:19:33 +0200 Subject: [PATCH 48/48] add what's new --- doc/whats_new/v0.20.rst | 1 - doc/whats_new/v0.22.rst | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index 4e3a4891b70e2..0688635af406a 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -647,7 +647,6 @@ Support for Python 3.3 has been officially dropped. by `Andreas Müller`_ and :user:`Guillaume Lemaitre `. - :mod:`sklearn.covariance` ......................... diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst index 4ac7afe644e89..484fa1116bb42 100644 --- a/doc/whats_new/v0.22.rst +++ b/doc/whats_new/v0.22.rst @@ -245,6 +245,8 @@ Changelog precomputed distance matrix contains non-zero diagonal entries. :pr:`12258` by :user:`Stephen Tierney `. +- |Feature| Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui ` + :mod:`sklearn.model_selection` ..............................