diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index f6bd3d995c099..5534bb88f53be 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -910,6 +910,7 @@ details. metrics.explained_variance_score metrics.max_error metrics.mean_absolute_error + metrics.mean_absolute_percentage_error metrics.mean_squared_error metrics.mean_squared_log_error metrics.median_absolute_error diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 6cea0b62bee5e..434a450a9de4f 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -87,6 +87,7 @@ Scoring Function 'explained_variance' :func:`metrics.explained_variance_score` 'max_error' :func:`metrics.max_error` 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` +'neg_mape' :func:`metrics.mean_absolute_percentage_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` 'neg_root_mean_squared_error' :func:`metrics.mean_squared_error` 'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` @@ -1859,6 +1860,46 @@ Here is a small example of usage of the :func:`mean_absolute_error` function:: >>> mean_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7]) 0.85... +.. _mean_absolute_percentage_error: + +Mean absolute percentage error +------------------------------ + +The :func:`mean_absolute_percentage_error` function, also known as **MAPE**, computes `mean absolute +percentage error `_, a risk +metric corresponding to the expected value of the absolute percentage error loss or +:math:`l1`-norm of percentage loss. + +If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, +and :math:`y_i` is the corresponding true value, then the mean absolute percentage error +(MAPE) estimated over :math:`n_{\text{samples}}` is defined as + +.. math:: + + \text{MAPE}(y, \hat{y}) = \frac{100}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \left| \frac{y_i - \hat{y}_i}{y_i} \right|. + +Here is a small example of usage of the :func:`mean_absolute_percentage_error` function:: + + >>> from sklearn.metrics import mean_absolute_percentage_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> mean_absolute_percentage_error(y_true, y_pred) + 32.738... + +MAPE computes the error relative to the true value. Therefore the same absolute distance between +prediction and ground truth will lead to a smaller error if the true value is larger. +In particular the metric is not shift-invariant. For example, if :math:`y_{true}` and :math:`y_{pred}` +in the example above are shifted by adding 10, the error becomes smaller: + + >>> from sklearn.metrics import mean_absolute_percentage_error + >>> import numpy as np + >>> y_true = np.array([3, -0.5, 2, 7]) + >>> y_pred = np.array([2.5, 0.0, 2, 8]) + >>> y_true = y_true + 10 + >>> y_pred = y_pred + 10 + >>> mean_absolute_percentage_error(y_true, y_pred) + 3.747... + .. _mean_squared_error: Mean squared error diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index 4e3a4891b70e2..0688635af406a 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -647,7 +647,6 @@ Support for Python 3.3 has been officially dropped. by `Andreas Müller`_ and :user:`Guillaume Lemaitre `. - :mod:`sklearn.covariance` ......................... diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst index 4ac7afe644e89..484fa1116bb42 100644 --- a/doc/whats_new/v0.22.rst +++ b/doc/whats_new/v0.22.rst @@ -245,6 +245,8 @@ Changelog precomputed distance matrix contains non-zero diagonal entries. :pr:`12258` by :user:`Stephen Tierney `. +- |Feature| Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui ` + :mod:`sklearn.model_selection` .............................. diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index d0b65ad1f4cfa..9202ad8c512c3 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -62,6 +62,7 @@ from .regression import explained_variance_score from .regression import max_error from .regression import mean_absolute_error +from .regression import mean_absolute_percentage_error from .regression import mean_squared_error from .regression import mean_squared_log_error from .regression import median_absolute_error @@ -118,6 +119,7 @@ 'matthews_corrcoef', 'max_error', 'mean_absolute_error', + 'mean_absolute_percentage_error', 'mean_squared_error', 'mean_squared_log_error', 'mean_poisson_deviance', diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index f19a1c9474164..cb6fc09cae032 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -20,6 +20,7 @@ # Michael Eickenberg # Konstantin Shmelkov # Christian Lorentzen +# Mohamed Ali Jamaoui # License: BSD 3 clause @@ -36,6 +37,7 @@ __ALL__ = [ "max_error", "mean_absolute_error", + "mean_absolute_percentage_error", "mean_squared_error", "mean_squared_log_error", "median_absolute_error", @@ -189,6 +191,47 @@ def mean_absolute_error(y_true, y_pred, return np.average(output_errors, weights=multioutput) +def mean_absolute_percentage_error(y_true, y_pred): + """Mean absolute percentage error regression loss + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + y_true : array-like of shape = (n_samples,) + Ground truth (correct) target values. + + y_pred : array-like of shape = (n_samples,) + Estimated target values. + + Returns + ------- + loss : float + A positive floating point value between 0.0 and 100.0, + the best value is 0.0. + + Examples + -------- + >>> from sklearn.metrics import mean_absolute_percentage_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> mean_absolute_percentage_error(y_true, y_pred) + 32.738... + """ + y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, + 'uniform_average') + + if y_type == 'continuous-multioutput': + raise ValueError("Multioutput not supported " + "in mean_absolute_percentage_error") + + if (y_true == 0).any(): + raise ValueError("mean_absolute_percentage_error requires" + " y_true to not include zeros") + + return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 + + def mean_squared_error(y_true, y_pred, sample_weight=None, multioutput='uniform_average', squared=True): diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index bf2892bdf83a2..d545d42f8a968 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -22,14 +22,13 @@ from collections.abc import Iterable import numpy as np - from . import (r2_score, median_absolute_error, max_error, mean_absolute_error, mean_squared_error, mean_squared_log_error, mean_tweedie_deviance, accuracy_score, f1_score, roc_auc_score, average_precision_score, precision_score, recall_score, log_loss, balanced_accuracy_score, explained_variance_score, - brier_score_loss, jaccard_score) + brier_score_loss, jaccard_score, mean_absolute_percentage_error) from .cluster import adjusted_rand_score from .cluster import homogeneity_score @@ -493,6 +492,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, greater_is_better=False) neg_mean_absolute_error_scorer = make_scorer(mean_absolute_error, greater_is_better=False) +neg_mape_scorer = make_scorer(mean_absolute_percentage_error, + greater_is_better=False) neg_median_absolute_error_scorer = make_scorer(median_absolute_error, greater_is_better=False) neg_root_mean_squared_error_scorer = make_scorer(mean_squared_error, @@ -547,6 +548,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, SCORERS = dict(explained_variance=explained_variance_scorer, r2=r2_scorer, + neg_mape=neg_mape_scorer, max_error=max_error_scorer, neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 6459f93c68449..62ca53d0f35e7 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -43,6 +43,7 @@ from sklearn.metrics import max_error from sklearn.metrics import matthews_corrcoef from sklearn.metrics import mean_absolute_error +from sklearn.metrics import mean_absolute_percentage_error from sklearn.metrics import mean_squared_error from sklearn.metrics import mean_tweedie_deviance from sklearn.metrics import mean_poisson_deviance @@ -98,6 +99,7 @@ REGRESSION_METRICS = { "max_error": max_error, "mean_absolute_error": mean_absolute_error, + "mean_absolute_percentage_error": mean_absolute_percentage_error, "mean_squared_error": mean_squared_error, "median_absolute_error": median_absolute_error, "explained_variance_score": explained_variance_score, @@ -476,18 +478,24 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs): "macro_f0.5_score", "macro_f2_score", "macro_precision_score", "macro_recall_score", "log_loss", "hinge_loss", "mean_gamma_deviance", "mean_poisson_deviance", - "mean_compound_poisson_deviance" + "mean_compound_poisson_deviance", "mean_absolute_percentage_error" } - # No Sample weight support METRICS_WITHOUT_SAMPLE_WEIGHT = { "median_absolute_error", + "mean_absolute_percentage_error", "max_error", "ovo_roc_auc", "weighted_ovo_roc_auc" } +# Metrics that only support non-zero y +METRICS_WITH_NON_ZERO_Y = [ + "mean_absolute_percentage_error", + "mean_absolute_percentage_error" +] + METRICS_REQUIRE_POSITIVE_Y = { "mean_poisson_deviance", "mean_gamma_deviance", @@ -520,8 +528,8 @@ def test_symmetry_consistency(): def test_symmetric_metric(name): # Test the symmetry of score and loss functions random_state = check_random_state(0) - y_true = random_state.randint(0, 2, size=(20, )) - y_pred = random_state.randint(0, 2, size=(20, )) + y_true = random_state.randint(1, 3, size=(20, )) + y_pred = random_state.randint(1, 3, size=(20, )) if name in METRICS_REQUIRE_POSITIVE_Y: y_true, y_pred = _require_positive_targets(y_true, y_pred) @@ -583,8 +591,6 @@ def test_sample_order_invariance(name): @ignore_warnings def test_sample_order_invariance_multilabel_and_multioutput(): random_state = check_random_state(0) - - # Generate some data y_true = random_state.randint(0, 2, size=(20, 25)) y_pred = random_state.randint(0, 2, size=(20, 25)) y_score = random_state.normal(size=y_true.shape) @@ -621,8 +627,8 @@ def test_sample_order_invariance_multilabel_and_multioutput(): sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS)) def test_format_invariance_with_1d_vectors(name): random_state = check_random_state(0) - y1 = random_state.randint(0, 2, size=(20, )) - y2 = random_state.randint(0, 2, size=(20, )) + y1 = random_state.randint(1, 3, size=(20, )) + y2 = random_state.randint(1, 3, size=(20, )) if name in METRICS_REQUIRE_POSITIVE_Y: y1, y2 = _require_positive_targets(y1, y2) @@ -1305,6 +1311,19 @@ def test_no_averaging_labels(): assert_array_equal(score_labels, score[inverse_labels]) +def test_raise_value_error_y_with_zeros(): + random_state = check_random_state(0) + y_true = random_state.randint(0, 2, size=(20, )) + y_pred = random_state.randint(0, 2, size=(20, )) + + for name in METRICS_WITH_NON_ZERO_Y: + metric = ALL_METRICS[name] + assert_raise_message(ValueError, + "mean_absolute_percentage_error requires" + " y_true to not include zeros", + metric, y_true, y_pred) + + @pytest.mark.parametrize( 'name', sorted(MULTILABELS_METRICS - {"unnormalized_multilabel_confusion_matrix"})) diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index 01ec8727aa330..af70816714503 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -11,6 +11,7 @@ from sklearn.metrics import explained_variance_score from sklearn.metrics import mean_absolute_error +from sklearn.metrics import mean_absolute_percentage_error from sklearn.metrics import mean_squared_error from sklearn.metrics import mean_squared_log_error from sklearn.metrics import median_absolute_error @@ -32,6 +33,11 @@ def test_regression_metrics(n_samples=50): mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred))) assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.) + # comparing (y_true + 1) and (y_pred + 1) instead of + # y_true and y_pred to avoid division by zero + assert_almost_equal(mean_absolute_percentage_error(1 + y_true, + 1 + y_pred), + 8.998, 2) assert_almost_equal(median_absolute_error(y_true, y_pred), 1.) assert_almost_equal(max_error(y_true, y_pred), 1.) assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2) @@ -136,7 +142,6 @@ def test_regression_metrics_at_limits(): match="deviance is only defined for p<=0 and p>=1."): mean_tweedie_deviance([0.], [0.], p=0.5) - def test__check_reg_targets(): # All of length 3 EXAMPLES = [ diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index a6ebaadcc1f01..d2d2d3f981a73 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -38,8 +38,8 @@ from sklearn.multiclass import OneVsRestClassifier -REGRESSION_SCORERS = ['explained_variance', 'r2', - 'neg_mean_absolute_error', 'neg_mean_squared_error', +REGRESSION_SCORERS = ['explained_variance', 'r2', 'neg_mean_absolute_error', + 'neg_mape', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'neg_root_mean_squared_error', @@ -80,6 +80,8 @@ def _require_positive_y(y): y = y + offset return y +y_nozero_SCORERS = ['neg_mape'] + def _make_estimators(X_train, y_train, y_ml_train): # Make estimators that make sense to test various scoring methods @@ -98,28 +100,30 @@ def _make_estimators(X_train, y_train, y_ml_train): ) -X_mm, y_mm, y_ml_mm = None, None, None +X_mm, y_mm, y_ml_mm, y_nozero_mm = None, None, None, None ESTIMATORS = None TEMP_FOLDER = None def setup_module(): # Create some memory mapped data - global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS + global X_mm, y_mm, y_ml_mm, y_nozero_mm, TEMP_FOLDER, ESTIMATORS TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_') X, y = make_classification(n_samples=30, n_features=5, random_state=0) _, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0) + y_nozero = y + 1 filename = os.path.join(TEMP_FOLDER, 'test_data.pkl') - joblib.dump((X, y, y_ml), filename) - X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode='r') + joblib.dump((X, y, y_ml, y_nozero), filename) + X_mm, y_mm, y_ml_mm, y_nozero_mm = joblib.load(filename, mmap_mode='r') ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm) def teardown_module(): - global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS + global X_mm, y_mm, y_ml_mm, y_nozero_mm, TEMP_FOLDER, ESTIMATORS # GC closes the mmap file descriptors X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None + y_nozero_mm = None shutil.rmtree(TEMP_FOLDER) @@ -509,9 +513,21 @@ def test_scorer_sample_weight(): ignored)) except TypeError as e: - assert "sample_weight" in str(e), ( - "scorer {0} raises unhelpful exception when called " - "with sample weights: {1}".format(name, str(e))) + assert_true("sample_weight" in str(e), + "scorer {0} raises unhelpful exception when called " + "with sample weights: {1}".format(name, str(e))) + + +@ignore_warnings # UndefinedMetricWarning for P / R scores +def check_scorer_memmap(scorer_name): + scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name] + if scorer_name in MULTILABEL_ONLY_SCORERS: + score = scorer(estimator, X_mm, y_ml_mm) + elif scorer_name in y_nozero_SCORERS: + score = scorer(estimator, X_mm, y_nozero_mm) + else: + score = scorer(estimator, X_mm, y_mm) + assert isinstance(score, numbers.Number), scorer_name @pytest.mark.parametrize('name', SCORERS)