From e40258f507ea6bcf6b059f5017367dbe7cb7c4ac Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 1 Oct 2018 16:04:10 +0530 Subject: [PATCH 01/23] Add max error metric --- sklearn/metrics/regression.py | 37 ++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index e9084a4276e18..1da51443fc5b6 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -36,7 +36,8 @@ "mean_squared_log_error", "median_absolute_error", "r2_score", - "explained_variance_score" + "explained_variance_score", + "max_error" ] @@ -573,3 +574,37 @@ def r2_score(y_true, y_pred, sample_weight=None, avg_weights = multioutput return np.average(output_scores, weights=avg_weights) + + +def max_error(y_true, y_pred): + """ + The Max-Error metric is the worst case error + between the predicted value and the true value. + + Parameters + ---------- + y_true : array-like of shape = (n_samples) + Ground truth (correct) target values. + + y_pred : array-like of shape = (n_samples) + Estimated target values. + + Returns + ------- + max_error : float + A positive floating point value (the best value is 0.0). + + Examples + -------- + >>> from sklearn.metrics import max_error + >>> y_true = [3.1, 2.4, 7.6, 1.9] + >>> y_pred = [4.1, 2.3, 7.4, 1.7] + >>> median_absolute_error(y_true, y_pred) + 1.0 + """ + y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, + 'uniform_average') + if y_type == 'continuous-multioutput': + raise ValueError("Multioutput not supported in max_error") + max_error = np.around(np.max(np.abs(y_true - y_pred)), decimals=3) + return max_error From 6905ee40c30b3186af8c36e108866b33e76f56f1 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 1 Oct 2018 16:09:28 +0530 Subject: [PATCH 02/23] Fix typo --- sklearn/metrics/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 1da51443fc5b6..bce878dc306fc 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -599,7 +599,7 @@ def max_error(y_true, y_pred): >>> from sklearn.metrics import max_error >>> y_true = [3.1, 2.4, 7.6, 1.9] >>> y_pred = [4.1, 2.3, 7.4, 1.7] - >>> median_absolute_error(y_true, y_pred) + >>> max_error(y_true, y_pred) 1.0 """ y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, From 0e92ee3858da0217a3d5812cf7226ccbf6ce91b5 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 1 Oct 2018 17:57:32 +0530 Subject: [PATCH 03/23] Fix some imports --- sklearn/metrics/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index 2d19e24db8683..5f0c5cd293115 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -60,6 +60,7 @@ from .regression import mean_squared_log_error from .regression import median_absolute_error from .regression import r2_score +from .regression import max_error from .scorer import check_scoring from .scorer import make_scorer @@ -99,6 +100,7 @@ 'log_loss', 'make_scorer', 'matthews_corrcoef', + 'max_error', 'mean_absolute_error', 'mean_squared_error', 'mean_squared_log_error', From 06f14d8a0bf7de0ae554fe58a6e9e746b436a3a1 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 1 Oct 2018 18:43:24 +0530 Subject: [PATCH 04/23] Adding some test for max error --- sklearn/metrics/tests/test_regression.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index 2faaaad3a39f2..86849bd159358 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -14,6 +14,7 @@ from sklearn.metrics import mean_squared_error from sklearn.metrics import mean_squared_log_error from sklearn.metrics import median_absolute_error +from sklearn.metrics import max_error from sklearn.metrics import r2_score from sklearn.metrics.regression import _check_reg_targets @@ -29,6 +30,7 @@ def test_regression_metrics(n_samples=50): np.log(1 + y_pred))) assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.) assert_almost_equal(median_absolute_error(y_true, y_pred), 1.) + assert_almost_equal(max_error(y_true, y_pred), 1.) assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2) assert_almost_equal(explained_variance_score(y_true, y_pred), 1.) @@ -59,6 +61,7 @@ def test_regression_metrics_at_limits(): assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.00, 2) assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2) assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2) + assert_almost_equal(max_error([0.], [0.]), 0.00, 2) assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2) assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2) assert_raises_regex(ValueError, "Mean Squared Logarithmic Error cannot be " From ea9d3f07f4916b4179903dbe7afd48d65ed81bd9 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 1 Oct 2018 19:04:29 +0530 Subject: [PATCH 05/23] Add scorer for max error --- sklearn/metrics/scorer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index 2661a379b4e53..84e5daa4f2307 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -22,7 +22,7 @@ import numpy as np -from . import (r2_score, median_absolute_error, mean_absolute_error, +from . import (r2_score, median_absolute_error, max_error, mean_absolute_error, mean_squared_error, mean_squared_log_error, accuracy_score, f1_score, roc_auc_score, average_precision_score, precision_score, recall_score, log_loss, @@ -454,6 +454,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, # Standard regression scores explained_variance_scorer = make_scorer(explained_variance_score) r2_scorer = make_scorer(r2_score) +max_error_scorer = make_scorer(max_error) neg_mean_squared_error_scorer = make_scorer(mean_squared_error, greater_is_better=False) neg_mean_squared_log_error_scorer = make_scorer(mean_squared_log_error, @@ -498,6 +499,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, SCORERS = dict(explained_variance=explained_variance_scorer, r2=r2_scorer, + max_error=max_error_scorer, neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, neg_mean_squared_error=neg_mean_squared_error_scorer, From 7b8222f456dc42530f5f6243823f607317e62302 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Wed, 3 Oct 2018 14:20:21 +0530 Subject: [PATCH 06/23] Add max_error_scorer to fix the failing test --- sklearn/metrics/tests/test_score_objects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index da04b4215dce0..85360151a1655 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -47,7 +47,8 @@ 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'mean_absolute_error', - 'mean_squared_error', 'median_absolute_error'] + 'mean_squared_error', 'median_absolute_error', + 'max_error_scorer'] CLF_SCORERS = ['accuracy', 'balanced_accuracy', 'f1', 'f1_weighted', 'f1_macro', 'f1_micro', From 237759afaaa2bd4930175bf9245b6018651209c4 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Wed, 3 Oct 2018 15:11:19 +0530 Subject: [PATCH 07/23] correct typo in scorer name --- sklearn/metrics/tests/test_score_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 85360151a1655..633caf9a9b339 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -48,7 +48,7 @@ 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', - 'max_error_scorer'] + 'max_error'] CLF_SCORERS = ['accuracy', 'balanced_accuracy', 'f1', 'f1_weighted', 'f1_macro', 'f1_micro', From 9f0f6de2802ae091820a11f4665df311c14e4c13 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 13:39:44 +0530 Subject: [PATCH 08/23] Changes based on review comments --- sklearn/metrics/__init__.py | 3 ++- sklearn/metrics/regression.py | 25 +++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index 5f0c5cd293115..6aa885a9d5315 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -55,12 +55,13 @@ from .pairwise import pairwise_distances_chunked from .regression import explained_variance_score +from .regression import max_error from .regression import mean_absolute_error from .regression import mean_squared_error from .regression import mean_squared_log_error from .regression import median_absolute_error from .regression import r2_score -from .regression import max_error + from .scorer import check_scoring from .scorer import make_scorer diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index bce878dc306fc..9d66a71641bad 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -31,13 +31,13 @@ __ALL__ = [ + "max_error", "mean_absolute_error", "mean_squared_error", "mean_squared_log_error", "median_absolute_error", "r2_score", - "explained_variance_score", - "max_error" + "explained_variance_score" ] @@ -576,7 +576,7 @@ def r2_score(y_true, y_pred, sample_weight=None, return np.average(output_scores, weights=avg_weights) -def max_error(y_true, y_pred): +def max_error(y_true, y_pred, sample_weight=None): """ The Max-Error metric is the worst case error between the predicted value and the true value. @@ -589,6 +589,9 @@ def max_error(y_true, y_pred): y_pred : array-like of shape = (n_samples) Estimated target values. + sample_weight : array-like of shape = (n_samples), optional + Sample weights. + Returns ------- max_error : float @@ -597,14 +600,20 @@ def max_error(y_true, y_pred): Examples -------- >>> from sklearn.metrics import max_error - >>> y_true = [3.1, 2.4, 7.6, 1.9] - >>> y_pred = [4.1, 2.3, 7.4, 1.7] + >>> y_true = [3, 2, 7, 1] + >>> y_pred = [4, 2, 7, 1] >>> max_error(y_true, y_pred) 1.0 """ y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, - 'uniform_average') + None) + check_consistent_length(y_true, y_pred, sample_weight) if y_type == 'continuous-multioutput': raise ValueError("Multioutput not supported in max_error") - max_error = np.around(np.max(np.abs(y_true - y_pred)), decimals=3) - return max_error + + if sample_weight is not None: + sample_weight = column_or_1d(sample_weight) + weight = sample_weight[:, np.newaxis] + else: + weight = 1. + return np.max(np.abs(weight * (y_true - y_pred))) From 495d6f0ccd2b3184224007b45b68ef274f01987f Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 15:01:33 +0530 Subject: [PATCH 09/23] Sample weight initialization needs to be tweaked to make the test pass --- sklearn/metrics/tests/test_score_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 633caf9a9b339..34a3546c994ca 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -458,7 +458,7 @@ def test_scorer_sample_weight(): X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split sample_weight = np.ones_like(y_test) - sample_weight[:10] = 0 + sample_weight[:10] = 2 # get sensible estimators for each metric estimator = _make_estimators(X_train, y_train, y_ml_train) From 0ab351346d297fc939c06949e498b1426bc311ed Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 15:27:52 +0530 Subject: [PATCH 10/23] Update sample weight only for the max_error scorer --- sklearn/metrics/tests/test_score_objects.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 34a3546c994ca..23f4581fbd1ec 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -458,12 +458,14 @@ def test_scorer_sample_weight(): X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split sample_weight = np.ones_like(y_test) - sample_weight[:10] = 2 + sample_weight[:10] = 0 # get sensible estimators for each metric estimator = _make_estimators(X_train, y_train, y_ml_train) for name, scorer in SCORERS.items(): + if name == "max_error": + sample_weight[:10] = 2 if name in MULTILABEL_ONLY_SCORERS: target = y_ml_test else: From bb567610964c409d6a09d1acc5634cb704a4651a Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 16:26:04 +0530 Subject: [PATCH 11/23] Change the sample weight slightly --- sklearn/metrics/tests/test_score_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 23f4581fbd1ec..7f36345645195 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -465,7 +465,7 @@ def test_scorer_sample_weight(): for name, scorer in SCORERS.items(): if name == "max_error": - sample_weight[:10] = 2 + sample_weight[:10] = 1.01 if name in MULTILABEL_ONLY_SCORERS: target = y_ml_test else: From 7cfdb71cafd39639b139e3b4faa31553cacbea5c Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 16:48:56 +0530 Subject: [PATCH 12/23] Remove support for sample weight --- sklearn/metrics/regression.py | 15 ++++----------- sklearn/metrics/scorer.py | 3 ++- sklearn/metrics/tests/test_score_objects.py | 2 -- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 9d66a71641bad..0037b9988cc26 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -576,10 +576,9 @@ def r2_score(y_true, y_pred, sample_weight=None, return np.average(output_scores, weights=avg_weights) -def max_error(y_true, y_pred, sample_weight=None): +def max_error(y_true, y_pred): """ - The Max-Error metric is the worst case error - between the predicted value and the true value. + max_error metric calculates the maximum residual error. Parameters ---------- @@ -607,13 +606,7 @@ def max_error(y_true, y_pred, sample_weight=None): """ y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, None) - check_consistent_length(y_true, y_pred, sample_weight) + check_consistent_length(y_true, y_pred) if y_type == 'continuous-multioutput': raise ValueError("Multioutput not supported in max_error") - - if sample_weight is not None: - sample_weight = column_or_1d(sample_weight) - weight = sample_weight[:, np.newaxis] - else: - weight = 1. - return np.max(np.abs(weight * (y_true - y_pred))) + return np.max(np.abs(y_true - y_pred)) diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index 84e5daa4f2307..f596d41637155 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -454,7 +454,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, # Standard regression scores explained_variance_scorer = make_scorer(explained_variance_score) r2_scorer = make_scorer(r2_score) -max_error_scorer = make_scorer(max_error) +max_error_scorer = make_scorer(max_error, + greater_is_better=False) neg_mean_squared_error_scorer = make_scorer(mean_squared_error, greater_is_better=False) neg_mean_squared_log_error_scorer = make_scorer(mean_squared_log_error, diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 7f36345645195..633caf9a9b339 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -464,8 +464,6 @@ def test_scorer_sample_weight(): estimator = _make_estimators(X_train, y_train, y_ml_train) for name, scorer in SCORERS.items(): - if name == "max_error": - sample_weight[:10] = 1.01 if name in MULTILABEL_ONLY_SCORERS: target = y_ml_test else: From ae86ca4a55d6e91ea61ca80aaa1a4b41c0c45cd9 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 16:59:41 +0530 Subject: [PATCH 13/23] Changes based on code review --- doc/modules/classes.rst | 1 + sklearn/metrics/tests/test_common.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 57ccfb5cff704..0084bb3f76b85 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -865,6 +865,7 @@ details. :template: function.rst metrics.explained_variance_score + metrics.max_error metrics.mean_absolute_error metrics.mean_squared_error metrics.mean_squared_log_error diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 16e4f5d4c76da..21c5e97444db6 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -40,6 +40,7 @@ from sklearn.metrics import label_ranking_average_precision_score from sklearn.metrics import label_ranking_loss from sklearn.metrics import log_loss +from sklearn.metrics import max_error from sklearn.metrics import matthews_corrcoef from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error @@ -89,6 +90,7 @@ # REGRESSION_METRICS = { + "max_error": max_error, "mean_absolute_error": mean_absolute_error, "mean_squared_error": mean_squared_error, "median_absolute_error": median_absolute_error, @@ -399,7 +401,7 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs): "micro_precision_score", "micro_recall_score", "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error", - "median_absolute_error", + "median_absolute_error", "max_error", "cohen_kappa_score", } @@ -429,6 +431,7 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs): # No Sample weight support METRICS_WITHOUT_SAMPLE_WEIGHT = { "median_absolute_error", + "max_error" } From d85f997b73c91e059726298099cf90e326eb27ef Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 4 Oct 2018 17:28:09 +0530 Subject: [PATCH 14/23] Fix the doctest --- sklearn/metrics/regression.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 0037b9988cc26..3718b8f2c95b9 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -588,9 +588,6 @@ def max_error(y_true, y_pred): y_pred : array-like of shape = (n_samples) Estimated target values. - sample_weight : array-like of shape = (n_samples), optional - Sample weights. - Returns ------- max_error : float @@ -602,7 +599,7 @@ def max_error(y_true, y_pred): >>> y_true = [3, 2, 7, 1] >>> y_pred = [4, 2, 7, 1] >>> max_error(y_true, y_pred) - 1.0 + 1 """ y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, None) From 361675af298574c4a1e39473ca8ddd594f8385a9 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Sun, 7 Oct 2018 11:54:54 +0530 Subject: [PATCH 15/23] update what's new --- doc/whats_new/v0.21.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index 27a756d9eefe5..f7234f59aa60a 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -52,6 +52,14 @@ Support for Python 3.4 and below has been officially dropped. graph, which would add explicitly zeros on the diagonal even when already present. :issue:`12105` by `Tom Dupre la Tour`_. + +:mod:`sklearn.metrics` +...................... + +- |Feature| A new regression metric: :class:`metrics.max_error`: a + metric that captures the max residual error, by :user:`Krishna Sangeeth `. + + Multiple modules ................ From 5b82f0966a8a26a9b54680e8256d76e72c1e71ba Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Sun, 7 Oct 2018 11:57:27 +0530 Subject: [PATCH 16/23] Update the PR number also --- doc/whats_new/v0.21.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index f7234f59aa60a..ed916a902f1e4 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -57,7 +57,8 @@ Support for Python 3.4 and below has been officially dropped. ...................... - |Feature| A new regression metric: :class:`metrics.max_error`: a - metric that captures the max residual error, by :user:`Krishna Sangeeth `. + metric that captures the max residual error. + :issue:`12232` by :user:`Krishna Sangeeth `. Multiple modules From e296cdda15c53944a4580df3e0eeca194455a6c3 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 8 Oct 2018 11:02:40 +0530 Subject: [PATCH 17/23] Update the user guide --- doc/modules/model_evaluation.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 89bc3bb4a84e9..f20ccf303ebb1 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -84,6 +84,7 @@ Scoring Function **Regression** 'explained_variance' :func:`metrics.explained_variance_score` +'max_error' :func:`metrics.max_error` 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` 'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` @@ -1530,6 +1531,35 @@ function:: ... # doctest: +ELLIPSIS 0.990... + + +Max error +------------------- + +The :func:`max_error` function computes the maximum `residual error >> from sklearn.metrics import max_error + >>> y_true = [3, 2, 7, 1] + >>> y_pred = [4, 2, 7, 1] + >>> max_error(y_true, y_pred) + 1 + +The :func:`max_error` does not support multioutput. + .. _mean_absolute_error: Mean absolute error From ff3550448b0d78b257759857319e40a422dda0db Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 8 Oct 2018 14:10:56 +0530 Subject: [PATCH 18/23] update the documentation based on review comments --- doc/modules/model_evaluation.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index f20ccf303ebb1..c19faefd6820b 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1538,8 +1538,8 @@ Max error The :func:`max_error` function computes the maximum `residual error >> from sklearn.metrics import max_error >>> y_true = [3, 2, 7, 1] - >>> y_pred = [4, 2, 7, 1] + >>> y_pred = [9, 2, 7, 1] >>> max_error(y_true, y_pred) - 1 + 6 The :func:`max_error` does not support multioutput. From cb50f2304d85b14a66f742b234f2446ab0a3d8c5 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Mon, 8 Oct 2018 20:00:08 +0530 Subject: [PATCH 19/23] Add reference and style fix for doc to have 80 char limit --- doc/modules/model_evaluation.rst | 17 ++++++++++------- sklearn/metrics/regression.py | 2 ++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index c19faefd6820b..ef1d5096cf32a 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1531,20 +1531,23 @@ function:: ... # doctest: +ELLIPSIS 0.990... - +.. _max_error: Max error ------------------- -The :func:`max_error` function computes the maximum `residual error `_ ,a metric +that captures the worst case error between the predicted value and +the true value. In a perfectly fitted single output regression +model, `max_error` would be `0` on the training set and though this +would be highly unlikely in the real world, this metric shows the +extent of error that the model had when it was fitted. If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, -and :math:`y_i` is the corresponding true value, then the max error is defined as +and :math:`y_i` is the corresponding true value, then the max error is +defined as .. math:: diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 3718b8f2c95b9..fbee56306f131 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -580,6 +580,8 @@ def max_error(y_true, y_pred): """ max_error metric calculates the maximum residual error. + Read more in the :ref:`User Guide `. + Parameters ---------- y_true : array-like of shape = (n_samples) From f156d5974e30b48ea573c1ac2e76b969eaa042b9 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 11 Oct 2018 00:11:46 +0530 Subject: [PATCH 20/23] Update review comments --- doc/modules/model_evaluation.rst | 4 ++-- sklearn/metrics/regression.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index ef1d5096cf32a..d56356c2f1949 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1537,10 +1537,10 @@ Max error ------------------- The :func:`max_error` function computes the maximum `residual error -`_ ,a metric +`_ , a metric that captures the worst case error between the predicted value and the true value. In a perfectly fitted single output regression -model, `max_error` would be `0` on the training set and though this +model, ``max_error`` would be ``0`` on the training set and though this would be highly unlikely in the real world, this metric shows the extent of error that the model had when it was fitted. diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index fbee56306f131..f4854ff244bc4 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -603,8 +603,7 @@ def max_error(y_true, y_pred): >>> max_error(y_true, y_pred) 1 """ - y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, - None) + y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, None) check_consistent_length(y_true, y_pred) if y_type == 'continuous-multioutput': raise ValueError("Multioutput not supported in max_error") From 3e0dfcf26dbc2f4a9883798778acc125c495c3c4 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 11 Oct 2018 11:00:13 +0530 Subject: [PATCH 21/23] Doc updates --- doc/modules/model_evaluation.rst | 2 +- doc/whats_new/v0.21.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index d56356c2f1949..9c4e7efdacf6f 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1536,7 +1536,7 @@ function:: Max error ------------------- -The :func:`max_error` function computes the maximum `residual error +The :func:`max_error` function computes the maximum `residual error `_ , a metric that captures the worst case error between the predicted value and the true value. In a perfectly fitted single output regression diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index ed916a902f1e4..97d66a2161098 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -57,7 +57,8 @@ Support for Python 3.4 and below has been officially dropped. ...................... - |Feature| A new regression metric: :class:`metrics.max_error`: a - metric that captures the max residual error. + metric that captures the maximum residual error and a corresponding + ``'max_error'`` scorer for single output regression . :issue:`12232` by :user:`Krishna Sangeeth `. From a088a76129ad5980867a7ad7db1976441c2f9f46 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 11 Oct 2018 11:13:07 +0530 Subject: [PATCH 22/23] Better structuring of sentence --- doc/whats_new/v0.21.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index 97d66a2161098..e969a0420dfb2 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -56,9 +56,8 @@ Support for Python 3.4 and below has been officially dropped. :mod:`sklearn.metrics` ...................... -- |Feature| A new regression metric: :class:`metrics.max_error`: a - metric that captures the maximum residual error and a corresponding - ``'max_error'`` scorer for single output regression . +- |Feature| Added the :func:`metrics.max_error` and a corresponding + ``'max_error'`` scorer for single output regression. :issue:`12232` by :user:`Krishna Sangeeth `. From 90d69dbf944194c8aa0cb8942ca65fb1832b79f3 Mon Sep 17 00:00:00 2001 From: Krishna Sangeeth Date: Thu, 11 Oct 2018 11:14:24 +0530 Subject: [PATCH 23/23] Missed to add the word metric --- doc/whats_new/v0.21.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index e969a0420dfb2..d560f980beb91 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -56,7 +56,7 @@ Support for Python 3.4 and below has been officially dropped. :mod:`sklearn.metrics` ...................... -- |Feature| Added the :func:`metrics.max_error` and a corresponding +- |Feature| Added the :func:`metrics.max_error` metric and a corresponding ``'max_error'`` scorer for single output regression. :issue:`12232` by :user:`Krishna Sangeeth `.