diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index e7585823cd2dc..5b44889bfae2f 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -844,6 +844,7 @@ details. metrics.explained_variance_score metrics.mean_absolute_error metrics.mean_squared_error + metrics.mean_squared_log_error metrics.median_absolute_error metrics.r2_score @@ -1418,4 +1419,4 @@ To be removed in 0.20 cross_validation.cross_val_score cross_validation.check_cv cross_validation.permutation_test_score - cross_validation.train_test_split \ No newline at end of file + cross_validation.train_test_split diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index ee79f062d447d..5b13f824280b4 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -77,6 +77,7 @@ Scoring Function Co **Regression** 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` +'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` 'neg_median_absolute_error' :func:`metrics.median_absolute_error` 'r2' :func:`metrics.r2_score` =========================== ========================================= ================================== @@ -93,7 +94,7 @@ Usage examples: >>> model = svm.SVC() >>> cross_val_score(model, X, y, scoring='wrong_choice') Traceback (most recent call last): - ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'] + ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'] .. note:: @@ -1360,7 +1361,7 @@ Mean squared error The :func:`mean_squared_error` function computes `mean square error `_, a risk -metric corresponding to the expected value of the squared (quadratic) error loss or +metric corresponding to the expected value of the squared (quadratic) error or loss. If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, @@ -1390,6 +1391,43 @@ function:: for an example of mean squared error usage to evaluate gradient boosting regression. +.. _mean_squared_log_error: + +Mean squared logarithmic error +------------------------------ + +The :func:`mean_squared_log_error` function computes a risk metric +corresponding to the expected value of the squared logarithmic (quadratic) +error or loss. + +If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, +and :math:`y_i` is the corresponding true value, then the mean squared +logarithmic error (MSLE) estimated over :math:`n_{\text{samples}}` is +defined as + +.. math:: + + \text{MSLE}(y, \hat{y}) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples} - 1} (\log_e (1 + y_i) - \log_e (1 + \hat{y}_i) )^2. + +Where :math:`\log_e (x)` means the natural logarithm of :math:`x`. This metric +is best to use when targets having exponential growth, such as population +counts, average sales of a commodity over a span of years etc. Note that this +metric penalizes an under-predicted estimate greater than an over-predicted +estimate. + +Here is a small example of usage of the :func:`mean_squared_log_error` +function:: + + >>> from sklearn.metrics import mean_squared_log_error + >>> y_true = [3, 5, 2.5, 7] + >>> y_pred = [2.5, 5, 4, 8] + >>> mean_squared_log_error(y_true, y_pred) # doctest: +ELLIPSIS + 0.039... + >>> y_true = [[0.5, 1], [1, 2], [7, 6]] + >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]] + >>> mean_squared_log_error(y_true, y_pred) # doctest: +ELLIPSIS + 0.044... + .. _median_absolute_error: Median absolute error diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index 413831939fbbc..cae8f9b6c7d03 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -54,6 +54,7 @@ from .regression import explained_variance_score from .regression import mean_absolute_error from .regression import mean_squared_error +from .regression import mean_squared_log_error from .regression import median_absolute_error from .regression import r2_score @@ -90,6 +91,7 @@ 'matthews_corrcoef', 'mean_absolute_error', 'mean_squared_error', + 'mean_squared_log_error', 'median_absolute_error', 'mutual_info_score', 'normalized_mutual_info_score', diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index beeae240b4ea1..e4af5b292183f 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -14,6 +14,7 @@ # Jochen Wersdorfer # Lars Buitinck # Joel Nothman +# Karan Desai # Noel Dawe # Manoj Kumar # Michael Eickenberg @@ -33,6 +34,7 @@ __ALL__ = [ "mean_absolute_error", "mean_squared_error", + "mean_squared_log_error", "median_absolute_error", "r2_score", "explained_variance_score" @@ -241,6 +243,73 @@ def mean_squared_error(y_true, y_pred, return np.average(output_errors, weights=multioutput) +def mean_squared_log_error(y_true, y_pred, + sample_weight=None, + multioutput='uniform_average'): + """Mean squared logarithmic error regression loss + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + y_true : array-like of shape = (n_samples) or (n_samples, n_outputs) + Ground truth (correct) target values. + + y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs) + Estimated target values. + + sample_weight : array-like of shape = (n_samples), optional + Sample weights. + + multioutput : string in ['raw_values', 'uniform_average'] \ + or array-like of shape = (n_outputs) + + Defines aggregating of multiple output values. + Array-like value defines weights used to average errors. + + 'raw_values' : + Returns a full set of errors when the input is of multioutput + format. + + 'uniform_average' : + Errors of all outputs are averaged with uniform weight. + + Returns + ------- + loss : float or ndarray of floats + A non-negative floating point value (the best value is 0.0), or an + array of floating point values, one for each individual target. + + Examples + -------- + >>> from sklearn.metrics import mean_squared_log_error + >>> y_true = [3, 5, 2.5, 7] + >>> y_pred = [2.5, 5, 4, 8] + >>> mean_squared_log_error(y_true, y_pred) # doctest: +ELLIPSIS + 0.039... + >>> y_true = [[0.5, 1], [1, 2], [7, 6]] + >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]] + >>> mean_squared_log_error(y_true, y_pred) # doctest: +ELLIPSIS + 0.044... + >>> mean_squared_log_error(y_true, y_pred, multioutput='raw_values') + ... # doctest: +ELLIPSIS + array([ 0.004..., 0.083...]) + >>> mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7]) + ... # doctest: +ELLIPSIS + 0.060... + + """ + y_type, y_true, y_pred, multioutput = _check_reg_targets( + y_true, y_pred, multioutput) + + if not (y_true >= 0).all() and not (y_pred >= 0).all(): + raise ValueError("Mean Squared Logarithmic Error cannot be used when " + "targets contain negative values.") + + return mean_squared_error(np.log(y_true + 1), np.log(y_pred + 1), + sample_weight, multioutput) + + def median_absolute_error(y_true, y_pred): """Median absolute error regression loss diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index c0ad2cd7d9587..4aeea1710d018 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -24,8 +24,8 @@ import numpy as np from . import (r2_score, median_absolute_error, mean_absolute_error, - mean_squared_error, accuracy_score, f1_score, - roc_auc_score, average_precision_score, + mean_squared_error, mean_squared_log_error, accuracy_score, + f1_score, roc_auc_score, average_precision_score, precision_score, recall_score, log_loss) from .cluster import adjusted_rand_score from ..utils.multiclass import type_of_target @@ -349,6 +349,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, mean_squared_error_scorer = make_scorer(mean_squared_error, greater_is_better=False) mean_squared_error_scorer._deprecation_msg = deprecation_msg +neg_mean_squared_log_error_scorer = make_scorer(mean_squared_log_error, + greater_is_better=False) neg_mean_absolute_error_scorer = make_scorer(mean_absolute_error, greater_is_better=False) deprecation_msg = ('Scoring method mean_absolute_error was renamed to ' @@ -396,6 +398,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, neg_mean_squared_error=neg_mean_squared_error_scorer, + neg_mean_squared_log_error=neg_mean_squared_log_error_scorer, median_absolute_error=median_absolute_error_scorer, mean_absolute_error=mean_absolute_error_scorer, mean_squared_error=mean_squared_error_scorer, diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index 600bcc135a202..bb842caca47df 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -3,7 +3,7 @@ import numpy as np from itertools import product -from sklearn.utils.testing import assert_raises +from sklearn.utils.testing import assert_raises, assert_raises_regex from sklearn.utils.testing import assert_equal from sklearn.utils.testing import assert_almost_equal from sklearn.utils.testing import assert_array_equal @@ -12,6 +12,7 @@ from sklearn.metrics import explained_variance_score from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error +from sklearn.metrics import mean_squared_log_error from sklearn.metrics import median_absolute_error from sklearn.metrics import r2_score @@ -23,6 +24,9 @@ def test_regression_metrics(n_samples=50): y_pred = y_true + 1 assert_almost_equal(mean_squared_error(y_true, y_pred), 1.) + assert_almost_equal(mean_squared_log_error(y_true, y_pred), + mean_squared_error(np.log(1 + y_true), + np.log(1 + y_pred))) assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.) assert_almost_equal(median_absolute_error(y_true, y_pred), 1.) assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2) @@ -36,6 +40,9 @@ def test_multioutput_regression(): error = mean_squared_error(y_true, y_pred) assert_almost_equal(error, (1. / 3 + 2. / 3 + 2. / 3) / 4.) + error = mean_squared_log_error(y_true, y_pred) + assert_almost_equal(error, 0.200, decimal=2) + # mean_absolute_error and mean_squared_error are equal because # it is a binary problem. error = mean_absolute_error(y_true, y_pred) @@ -49,10 +56,14 @@ def test_multioutput_regression(): def test_regression_metrics_at_limits(): assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2) + assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.00, 2) assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2) assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2) assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2) assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2) + assert_raises_regex(ValueError, "Mean Squared Logarithmic Error cannot be " + "used when targets contain negative values.", + mean_squared_log_error, [-1.], [-1.]) def test__check_reg_targets(): @@ -127,6 +138,14 @@ def test_regression_multioutput_array(): assert_array_almost_equal(evs, [1., -3.], decimal=2) assert_equal(np.mean(evs), explained_variance_score(y_true, y_pred)) + # Handling msle separately as it does not accept negative inputs. + y_true = np.array([[0.5, 1], [1, 2], [7, 6]]) + y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]]) + msle = mean_squared_log_error(y_true, y_pred, multioutput='raw_values') + msle2 = mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred), + multioutput='raw_values') + assert_array_almost_equal(msle, msle2, decimal=2) + def test_regression_custom_weights(): y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]] @@ -141,3 +160,11 @@ def test_regression_custom_weights(): assert_almost_equal(maew, 0.475, decimal=3) assert_almost_equal(rw, 0.94, decimal=2) assert_almost_equal(evsw, 0.94, decimal=2) + + # Handling msle separately as it does not accept negative inputs. + y_true = np.array([[0.5, 1], [1, 2], [7, 6]]) + y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]]) + msle = mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7]) + msle2 = mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred), + multioutput=[0.3, 0.7]) + assert_almost_equal(msle, msle2, decimal=2) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 32834f5e69e8e..17a4811f52653 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -39,8 +39,8 @@ REGRESSION_SCORERS = ['r2', 'neg_mean_absolute_error', - 'neg_mean_squared_error', 'neg_median_absolute_error', - 'mean_absolute_error', + 'neg_mean_squared_error', 'neg_mean_squared_log_error', + 'neg_median_absolute_error', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error'] CLF_SCORERS = ['accuracy', 'f1', 'f1_weighted', 'f1_macro', 'f1_micro',