scikit-learn · mohamed-ali · Feb 26, 2018 · Feb 26, 2018 · Feb 26, 2018 · Feb 26, 2018
diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst
@@ -910,6 +910,7 @@ details.
    metrics.explained_variance_score
    metrics.max_error
    metrics.mean_absolute_error
+   metrics.mean_absolute_percentage_error
    metrics.mean_squared_error
    metrics.mean_squared_log_error
    metrics.median_absolute_error

diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst
@@ -87,6 +87,7 @@ Scoring                           Function
 'explained_variance'              :func:`metrics.explained_variance_score`
 'max_error'                       :func:`metrics.max_error`
 'neg_mean_absolute_error'         :func:`metrics.mean_absolute_error`
+'neg_mape'                        :func:`metrics.mean_absolute_percentage_error`
 'neg_mean_squared_error'          :func:`metrics.mean_squared_error`
 'neg_root_mean_squared_error'     :func:`metrics.mean_squared_error`
 'neg_mean_squared_log_error'      :func:`metrics.mean_squared_log_error`
@@ -1859,6 +1860,46 @@ Here is a small example of usage of the :func:`mean_absolute_error` function::
   >>> mean_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7])
   0.85...
 
+.. _mean_absolute_percentage_error:
+
+Mean absolute percentage error
+------------------------------
+
+The :func:`mean_absolute_percentage_error` function, also known as **MAPE**, computes `mean absolute
+percentage error <https://en.wikipedia.org/wiki/Mean_absolute_percentage_error>`_, a risk
+metric corresponding to the expected value of the absolute percentage error loss or
+:math:`l1`-norm of percentage loss.
+
+If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample,
+and :math:`y_i` is the corresponding true value, then the mean absolute percentage error
+(MAPE) estimated over :math:`n_{\text{samples}}` is defined as
+
+.. math::
+
+  \text{MAPE}(y, \hat{y}) = \frac{100}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \left| \frac{y_i - \hat{y}_i}{y_i} \right|.
+
+Here is a small example of usage of the :func:`mean_absolute_percentage_error` function::
+
+  >>> from sklearn.metrics import mean_absolute_percentage_error
+  >>> y_true = [3, -0.5, 2, 7]
+  >>> y_pred = [2.5, 0.0, 2, 8]
+  >>> mean_absolute_percentage_error(y_true, y_pred)
+  32.738...
+
+MAPE computes the error relative to the true value. Therefore the same absolute distance between
+prediction and ground truth will lead to a smaller error if the true value is larger.
+In particular the metric is not shift-invariant. For example, if :math:`y_{true}` and :math:`y_{pred}`
+in the example above are shifted by adding 10, the error becomes smaller:
+
+   >>> from sklearn.metrics import mean_absolute_percentage_error
+   >>> import numpy as np
+   >>> y_true = np.array([3, -0.5, 2, 7])
+   >>> y_pred = np.array([2.5, 0.0, 2, 8])
+   >>> y_true = y_true + 10
+   >>> y_pred = y_pred + 10
+   >>> mean_absolute_percentage_error(y_true, y_pred)
+   3.747...
+
 .. _mean_squared_error:
 
 Mean squared error

diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst
@@ -647,7 +647,6 @@ Support for Python 3.3 has been officially dropped.
   by `Andreas Müller`_ and :user:`Guillaume Lemaitre <glemaitre>`.
 
 
-
 :mod:`sklearn.covariance`
 .........................
 

diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst
@@ -245,6 +245,8 @@ Changelog
   precomputed distance matrix contains non-zero diagonal entries.
   :pr:`12258` by :user:`Stephen Tierney <sjtrny>`.
 
+- |Feature| Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui <mohamed-ali>`
+
 :mod:`sklearn.model_selection`
 ..............................
 

diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py
@@ -62,6 +62,7 @@
 from .regression import explained_variance_score
 from .regression import max_error
 from .regression import mean_absolute_error
+from .regression import mean_absolute_percentage_error
 from .regression import mean_squared_error
 from .regression import mean_squared_log_error
 from .regression import median_absolute_error
@@ -118,6 +119,7 @@
     'matthews_corrcoef',
     'max_error',
     'mean_absolute_error',
+    'mean_absolute_percentage_error',
     'mean_squared_error',
     'mean_squared_log_error',
     'mean_poisson_deviance',

diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py
@@ -20,6 +20,7 @@
 #          Michael Eickenberg <[email protected]>
 #          Konstantin Shmelkov <[email protected]>
 #          Christian Lorentzen <[email protected]>
+#          Mohamed Ali Jamaoui <[email protected]>
 # License: BSD 3 clause
 
 
@@ -36,6 +37,7 @@
 __ALL__ = [
     "max_error",
     "mean_absolute_error",
+    "mean_absolute_percentage_error",
     "mean_squared_error",
     "mean_squared_log_error",
     "median_absolute_error",
@@ -189,6 +191,47 @@ def mean_absolute_error(y_true, y_pred,
     return np.average(output_errors, weights=multioutput)
 
 
+def mean_absolute_percentage_error(y_true, y_pred):
+    """Mean absolute percentage error regression loss
+
+    Read more in the :ref:`User Guide <mean_absolute_percentage_error>`.
+
+    Parameters
+    ----------
+    y_true : array-like of shape = (n_samples,)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape = (n_samples,)
+        Estimated target values.
+
+    Returns
+    -------
+    loss : float
+        A positive floating point value between 0.0 and 100.0,
+        the best value is 0.0.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import mean_absolute_percentage_error
+    >>> y_true = [3, -0.5, 2, 7]
+    >>> y_pred = [2.5, 0.0, 2, 8]
+    >>> mean_absolute_percentage_error(y_true, y_pred)
+    32.738...
+    """
+    y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred,
+                                                   'uniform_average')
+
+    if y_type == 'continuous-multioutput':
+        raise ValueError("Multioutput not supported "
+                         "in mean_absolute_percentage_error")
+
+    if (y_true == 0).any():
+        raise ValueError("mean_absolute_percentage_error requires"
+                         " y_true to not include zeros")
+
+    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+
+
 def mean_squared_error(y_true, y_pred,
                        sample_weight=None,
                        multioutput='uniform_average', squared=True):

diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py
@@ -22,14 +22,13 @@
 from collections.abc import Iterable
 
 import numpy as np
-
 from . import (r2_score, median_absolute_error, max_error, mean_absolute_error,
                mean_squared_error, mean_squared_log_error,
                mean_tweedie_deviance, accuracy_score,
                f1_score, roc_auc_score, average_precision_score,
                precision_score, recall_score, log_loss,
                balanced_accuracy_score, explained_variance_score,
-               brier_score_loss, jaccard_score)
+               brier_score_loss, jaccard_score, mean_absolute_percentage_error)
 
 from .cluster import adjusted_rand_score
 from .cluster import homogeneity_score
@@ -493,6 +492,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False,
                                                 greater_is_better=False)
 neg_mean_absolute_error_scorer = make_scorer(mean_absolute_error,
                                              greater_is_better=False)
+neg_mape_scorer = make_scorer(mean_absolute_percentage_error,
+                              greater_is_better=False)
 neg_median_absolute_error_scorer = make_scorer(median_absolute_error,
                                                greater_is_better=False)
 neg_root_mean_squared_error_scorer = make_scorer(mean_squared_error,
@@ -547,6 +548,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False,
 
 SCORERS = dict(explained_variance=explained_variance_scorer,
                r2=r2_scorer,
+               neg_mape=neg_mape_scorer,
                max_error=max_error_scorer,
                neg_median_absolute_error=neg_median_absolute_error_scorer,
                neg_mean_absolute_error=neg_mean_absolute_error_scorer,

diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py
@@ -43,6 +43,7 @@
 from sklearn.metrics import max_error
 from sklearn.metrics import matthews_corrcoef
 from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import mean_absolute_percentage_error
 from sklearn.metrics import mean_squared_error
 from sklearn.metrics import mean_tweedie_deviance
 from sklearn.metrics import mean_poisson_deviance
@@ -98,6 +99,7 @@
 REGRESSION_METRICS = {
     "max_error": max_error,
     "mean_absolute_error": mean_absolute_error,
+    "mean_absolute_percentage_error": mean_absolute_percentage_error,
     "mean_squared_error": mean_squared_error,
     "median_absolute_error": median_absolute_error,
     "explained_variance_score": explained_variance_score,
@@ -476,18 +478,24 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "macro_f0.5_score", "macro_f2_score", "macro_precision_score",
     "macro_recall_score", "log_loss", "hinge_loss",
     "mean_gamma_deviance", "mean_poisson_deviance",
-    "mean_compound_poisson_deviance"
+    "mean_compound_poisson_deviance", "mean_absolute_percentage_error"
 }
 
-
 # No Sample weight support
 METRICS_WITHOUT_SAMPLE_WEIGHT = {
     "median_absolute_error",
+    "mean_absolute_percentage_error",
     "max_error",
     "ovo_roc_auc",
     "weighted_ovo_roc_auc"
 }
 
+# Metrics that only support non-zero y
+METRICS_WITH_NON_ZERO_Y = [
+    "mean_absolute_percentage_error",
+    "mean_absolute_percentage_error"
+]
+
 METRICS_REQUIRE_POSITIVE_Y = {
     "mean_poisson_deviance",
     "mean_gamma_deviance",
@@ -520,8 +528,8 @@ def test_symmetry_consistency():
 def test_symmetric_metric(name):
     # Test the symmetry of score and loss functions
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(20, ))
-    y_pred = random_state.randint(0, 2, size=(20, ))
+    y_true = random_state.randint(1, 3, size=(20, ))
+    y_pred = random_state.randint(1, 3, size=(20, ))
 
     if name in METRICS_REQUIRE_POSITIVE_Y:
         y_true, y_pred = _require_positive_targets(y_true, y_pred)
@@ -583,8 +591,6 @@ def test_sample_order_invariance(name):
 @ignore_warnings
 def test_sample_order_invariance_multilabel_and_multioutput():
     random_state = check_random_state(0)
-
-    # Generate some data
     y_true = random_state.randint(0, 2, size=(20, 25))
     y_pred = random_state.randint(0, 2, size=(20, 25))
     y_score = random_state.normal(size=y_true.shape)
@@ -621,8 +627,8 @@ def test_sample_order_invariance_multilabel_and_multioutput():
         sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS))
 def test_format_invariance_with_1d_vectors(name):
     random_state = check_random_state(0)
-    y1 = random_state.randint(0, 2, size=(20, ))
-    y2 = random_state.randint(0, 2, size=(20, ))
+    y1 = random_state.randint(1, 3, size=(20, ))
+    y2 = random_state.randint(1, 3, size=(20, ))
 
     if name in METRICS_REQUIRE_POSITIVE_Y:
         y1, y2 = _require_positive_targets(y1, y2)
@@ -1305,6 +1311,19 @@ def test_no_averaging_labels():
             assert_array_equal(score_labels, score[inverse_labels])
 
 
+def test_raise_value_error_y_with_zeros():
+    random_state = check_random_state(0)
+    y_true = random_state.randint(0, 2, size=(20, ))
+    y_pred = random_state.randint(0, 2, size=(20, ))
+
+    for name in METRICS_WITH_NON_ZERO_Y:
+        metric = ALL_METRICS[name]
+        assert_raise_message(ValueError,
+                             "mean_absolute_percentage_error requires"
+                             " y_true to not include zeros",
+                             metric, y_true, y_pred)
+
+
 @pytest.mark.parametrize(
     'name',
     sorted(MULTILABELS_METRICS - {"unnormalized_multilabel_confusion_matrix"}))

diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py
@@ -11,6 +11,7 @@
 
 from sklearn.metrics import explained_variance_score
 from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import mean_absolute_percentage_error
 from sklearn.metrics import mean_squared_error
 from sklearn.metrics import mean_squared_log_error
 from sklearn.metrics import median_absolute_error
@@ -32,6 +33,11 @@ def test_regression_metrics(n_samples=50):
                         mean_squared_error(np.log(1 + y_true),
                                            np.log(1 + y_pred)))
     assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
+    # comparing (y_true + 1) and (y_pred + 1) instead of
+    # y_true and y_pred to avoid division by zero
+    assert_almost_equal(mean_absolute_percentage_error(1 + y_true,
+                                                       1 + y_pred),
+                        8.998, 2)
     assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
     assert_almost_equal(max_error(y_true, y_pred), 1.)
     assert_almost_equal(r2_score(y_true, y_pred),  0.995, 2)
@@ -136,7 +142,6 @@ def test_regression_metrics_at_limits():
                        match="deviance is only defined for p<=0 and p>=1."):
         mean_tweedie_deviance([0.], [0.], p=0.5)
 
-
 def test__check_reg_targets():
     # All of length 3
     EXAMPLES = [

diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py
@@ -38,8 +38,8 @@
 from sklearn.multiclass import OneVsRestClassifier
 
 
-REGRESSION_SCORERS = ['explained_variance', 'r2',
-                      'neg_mean_absolute_error', 'neg_mean_squared_error',
+REGRESSION_SCORERS = ['explained_variance', 'r2', 'neg_mean_absolute_error',
+                      'neg_mape', 'neg_mean_squared_error',
                       'neg_mean_squared_log_error',
                       'neg_median_absolute_error',
                       'neg_root_mean_squared_error',
@@ -80,6 +80,8 @@ def _require_positive_y(y):
     y = y + offset
     return y
 
+y_nozero_SCORERS = ['neg_mape']
+
 
 def _make_estimators(X_train, y_train, y_ml_train):
     # Make estimators that make sense to test various scoring methods
@@ -98,28 +100,30 @@ def _make_estimators(X_train, y_train, y_ml_train):
     )
 
 
-X_mm, y_mm, y_ml_mm = None, None, None
+X_mm, y_mm, y_ml_mm, y_nozero_mm = None, None, None, None
 ESTIMATORS = None
 TEMP_FOLDER = None
 
 
 def setup_module():
     # Create some memory mapped data
-    global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
+    global X_mm, y_mm, y_ml_mm, y_nozero_mm, TEMP_FOLDER, ESTIMATORS
     TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_')
     X, y = make_classification(n_samples=30, n_features=5, random_state=0)
     _, y_ml = make_multilabel_classification(n_samples=X.shape[0],
                                              random_state=0)
+    y_nozero = y + 1
     filename = os.path.join(TEMP_FOLDER, 'test_data.pkl')
-    joblib.dump((X, y, y_ml), filename)
-    X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode='r')
+    joblib.dump((X, y, y_ml, y_nozero), filename)
+    X_mm, y_mm, y_ml_mm, y_nozero_mm = joblib.load(filename, mmap_mode='r')
     ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm)
 
 
 def teardown_module():
-    global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
+    global X_mm, y_mm, y_ml_mm, y_nozero_mm, TEMP_FOLDER, ESTIMATORS
     # GC closes the mmap file descriptors
     X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None
+    y_nozero_mm = None
     shutil.rmtree(TEMP_FOLDER)
 
 
@@ -509,9 +513,21 @@ def test_scorer_sample_weight():
                                                         ignored))
 
         except TypeError as e:
-            assert "sample_weight" in str(e), (
-                "scorer {0} raises unhelpful exception when called "
-                "with sample weights: {1}".format(name, str(e)))
+            assert_true("sample_weight" in str(e),
+                        "scorer {0} raises unhelpful exception when called "
+                        "with sample weights: {1}".format(name, str(e)))
+
+
+@ignore_warnings  # UndefinedMetricWarning for P / R scores
+def check_scorer_memmap(scorer_name):
+    scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name]
+    if scorer_name in MULTILABEL_ONLY_SCORERS:
+        score = scorer(estimator, X_mm, y_ml_mm)
+    elif scorer_name in y_nozero_SCORERS:
+        score = scorer(estimator, X_mm, y_nozero_mm)
+    else:
+        score = scorer(estimator, X_mm, y_mm)
+    assert isinstance(score, numbers.Number), scorer_name
 
 
 @pytest.mark.parametrize('name', SCORERS)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -647,7 +647,6 @@ Support for Python 3.3 has been officially dropped.
		by `Andreas Müller`_ and :user:`Guillaume Lemaitre <glemaitre>`.



		:mod:`sklearn.covariance`
		.........................

Expand Down