From 11b77ec08cbdf7f55c03642d0b333b2449ba89c4 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Tue, 11 Aug 2020 18:16:20 +0200 Subject: [PATCH 1/6] ENH allow to pass str or scorer to make_scorer --- doc/whats_new/v0.24.rst | 4 ++ sklearn/metrics/_scorer.py | 56 +++++++++++++++------ sklearn/metrics/tests/test_score_objects.py | 31 ++++++++++++ 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/doc/whats_new/v0.24.rst b/doc/whats_new/v0.24.rst index 2682902a20983..3ffe2727f1cd1 100644 --- a/doc/whats_new/v0.24.rst +++ b/doc/whats_new/v0.24.rst @@ -280,6 +280,10 @@ Changelog class to be used when computing the roc auc statistics. :pr:`17651` by :user:`Clara Matos `. +- |Enhancement| Allow to pass a scorer or a string to create a new scorer in + :func:`metrics.make_scorer`. + :pr:`xxx` by :user:`Guillaume Lemaitre `. + :mod:`sklearn.model_selection` .............................. diff --git a/sklearn/metrics/_scorer.py b/sklearn/metrics/_scorer.py index 9ad57f4611e52..b0de1a8918f6a 100644 --- a/sklearn/metrics/_scorer.py +++ b/sklearn/metrics/_scorer.py @@ -18,9 +18,9 @@ # Arnaud Joly # License: Simplified BSD +from collections import Counter from collections.abc import Iterable from functools import partial -from collections import Counter import numpy as np @@ -507,9 +507,21 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, Parameters ---------- - score_func : callable, - Score function (or loss function) with signature - ``score_func(y, y_pred, **kwargs)``. + scoring : str or callable + This parameter can be: + + * a string (see model evaluation documentation). The parameters + `greater_is_better`, `needs_proba`, and `needs_threshold` will be + ignored and inferred from the base scorers. However, you can pass any + additional parameters required by the scoring function as `**kwargs`; + * a scorer callable object originally constructed with + :func:`make_scorer` or returned by :func:`get_scorer`. In this case, + the parameters `greater_is_better`, `needs_proba`, and + `needs_threshold` will be ignored and inferred from the base scorers. + However, you can pass any additional parameters required by the + scoring function as `**kwargs`; + * a scorer callable object / function with signature + `scorer(estimator, X, y)`. greater_is_better : bool, default=True Whether score_func is a score function (default), meaning high is good, @@ -565,17 +577,33 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, `needs_threshold=True`, the score function is supposed to accept the output of :term:`decision_function`. """ - sign = 1 if greater_is_better else -1 - if needs_proba and needs_threshold: - raise ValueError("Set either needs_proba or needs_threshold to True," - " but not both.") - if needs_proba: - cls = _ProbaScorer - elif needs_threshold: - cls = _ThresholdScorer + if isinstance(score_func, str): + base_scorer = get_scorer(score_func) + return base_scorer.__class__( + score_func=base_scorer._score_func, + sign=base_scorer._sign, + kwargs=kwargs + ) + elif isinstance(score_func, _BaseScorer): + return score_func.__class__( + score_func=score_func._score_func, + sign=score_func._sign, + kwargs=kwargs + ) else: - cls = _PredictScorer - return cls(score_func, sign, kwargs) + sign = 1 if greater_is_better else -1 + if needs_proba and needs_threshold: + raise ValueError( + "Set either needs_proba or needs_threshold to True, but not " + "both." + ) + if needs_proba: + cls = _ProbaScorer + elif needs_threshold: + cls = _ThresholdScorer + else: + cls = _PredictScorer + return cls(score_func, sign, kwargs) # Standard regression scores diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 67900b7cb77c3..336a9d669eedb 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -747,3 +747,34 @@ def test_multiclass_roc_no_proba_scorer_errors(scorer_name): msg = "'Perceptron' object has no attribute 'predict_proba'" with pytest.raises(AttributeError, match=msg): scorer(lr, X, y) + + +@pytest.mark.parametrize( + "scoring", + ["roc_auc", get_scorer("roc_auc")], + ids=["str", "scorer_instance"], +) +def test_make_scorer_from_str_or_base_scorer(scoring): + # check that we can create a scorer from a string or a previous scorer + base_scorer = get_scorer(scoring) if isinstance(scoring, str) else scoring + scorer = make_scorer(scoring) + + # check that we have a different object but with the same parameter values + assert scorer is not base_scorer + assert scorer._score_func == base_scorer._score_func + assert scorer._sign == base_scorer._sign + assert scorer._kwargs == base_scorer._kwargs + + # check that the parameters of `make_scorer` do not have any effect when + # passing a string. The following would have raised an error because a + # scorer cannot be a _ProbaScorer and a _ThresholdScorer at the same time. + scorer = make_scorer( + scoring, + greater_is_better=False, + needs_threshold=True, + needs_proba=True, + ) + + # check that we can overwrite the scoring function parameters + scorer = make_scorer(scoring, multi_class="ovo") + assert scorer._kwargs == {"multi_class": "ovo"} From 2a3de1245627caab5b7cf78688a2676ec0bed50e Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Tue, 11 Aug 2020 18:22:29 +0200 Subject: [PATCH 2/6] update whats new --- doc/whats_new/v0.24.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats_new/v0.24.rst b/doc/whats_new/v0.24.rst index 3ffe2727f1cd1..774bf93f92d9f 100644 --- a/doc/whats_new/v0.24.rst +++ b/doc/whats_new/v0.24.rst @@ -282,7 +282,7 @@ Changelog - |Enhancement| Allow to pass a scorer or a string to create a new scorer in :func:`metrics.make_scorer`. - :pr:`xxx` by :user:`Guillaume Lemaitre `. + :pr:`18141` by :user:`Guillaume Lemaitre `. :mod:`sklearn.model_selection` .............................. From d0e4d2ad7d6e19a3ee79bc9c93e6656961b52722 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Tue, 11 Aug 2020 18:25:58 +0200 Subject: [PATCH 3/6] STY --- sklearn/metrics/_scorer.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/sklearn/metrics/_scorer.py b/sklearn/metrics/_scorer.py index b0de1a8918f6a..80a65bd8210c1 100644 --- a/sklearn/metrics/_scorer.py +++ b/sklearn/metrics/_scorer.py @@ -577,19 +577,15 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, `needs_threshold=True`, the score function is supposed to accept the output of :term:`decision_function`. """ - if isinstance(score_func, str): - base_scorer = get_scorer(score_func) - return base_scorer.__class__( - score_func=base_scorer._score_func, - sign=base_scorer._sign, - kwargs=kwargs - ) - elif isinstance(score_func, _BaseScorer): - return score_func.__class__( - score_func=score_func._score_func, - sign=score_func._sign, - kwargs=kwargs + if isinstance(score_func, (str, _BaseScorer)): + base_scorer = ( + get_scorer(score_func) + if isinstance(score_func, str) + else score_func ) + cls = base_scorer.__class__ + score_func = base_scorer._score_func + sign = base_scorer._sign else: sign = 1 if greater_is_better else -1 if needs_proba and needs_threshold: @@ -603,7 +599,7 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, cls = _ThresholdScorer else: cls = _PredictScorer - return cls(score_func, sign, kwargs) + return cls(score_func, sign, kwargs) # Standard regression scores From 7ee4b23cd960384ce3e90dc33c613a06f49703a2 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Tue, 11 Aug 2020 18:35:34 +0200 Subject: [PATCH 4/6] DOC improve example docstring --- sklearn/metrics/_scorer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sklearn/metrics/_scorer.py b/sklearn/metrics/_scorer.py index 80a65bd8210c1..4680d51afc60c 100644 --- a/sklearn/metrics/_scorer.py +++ b/sklearn/metrics/_scorer.py @@ -558,6 +558,8 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, Examples -------- + You can create a scorer from a callable function: + >>> from sklearn.metrics import fbeta_score, make_scorer >>> ftwo_scorer = make_scorer(fbeta_score, beta=2) >>> ftwo_scorer @@ -567,6 +569,23 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, >>> grid = GridSearchCV(LinearSVC(), param_grid={'C': [1, 10]}, ... scoring=ftwo_scorer) + Otherwise, you can use a string avoiding to pass the parameters required + by `make_scorer`: + + >>> from sklearn.datasets import load_breast_cancer + >>> X, y = load_breast_cancer(return_X_y=True) + >>> roc_auc_scorer = make_scorer("roc_auc") + >>> clf = LinearSVC().fit(X, y) + >>> roc_auc_scorer(clf, X, y) + 0.98... + + Similarly, you can use a scorer obtained with :func:`get_scorer`: + + >>> from sklearn.metrics import get_scorer + >>> roc_auc_scorer = get_scorer("roc_auc") + >>> roc_auc_scorer(clf, X, y) + 0.98... + Notes ----- If `needs_proba=False` and `needs_threshold=False`, the score From cea5ad8a9b14592c73d2b872185a104745356f1f Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 12 Aug 2020 08:08:09 +0200 Subject: [PATCH 5/6] DOC improve user guide --- doc/modules/model_evaluation.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index f8874869a0274..8a09b20cd76ad 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -202,6 +202,26 @@ Here is an example of building custom scorers, and of using the >>> score(clf, X, y) -0.69... +You can as well used predefined metrics, shown in the table above, where the +parameters `greater_is_better`, `needs_proba`, and `needs_threshold` will not +be required. Only the additional scoring function parameters should be given if +there is any:: + + >>> precision_scorer = make_scorer("precision", average="micro") + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification( + ... n_classes=3, n_informative=3, random_state=0 + ... ) + >>> clf.fit(X, y) + DummyClassifier(random_state=0, strategy='most_frequent') + >>> precision_scorer(clf, X, y) + 0.35... + +Similarly, you can use a scorer to create a new scorer:: + + >>> new_scorer = make_scorer(precision_scorer, average="macro") + >>> new_scorer(clf, X, y) + 0.11... .. _diy_scoring: From f7f8746d6a037e1bdac19a25da3d5857dfd8b8a4 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 12 Aug 2020 08:10:22 +0200 Subject: [PATCH 6/6] FIX set random_state --- sklearn/metrics/_scorer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/_scorer.py b/sklearn/metrics/_scorer.py index 4680d51afc60c..e3e8a6ab699bf 100644 --- a/sklearn/metrics/_scorer.py +++ b/sklearn/metrics/_scorer.py @@ -575,7 +575,7 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False, >>> from sklearn.datasets import load_breast_cancer >>> X, y = load_breast_cancer(return_X_y=True) >>> roc_auc_scorer = make_scorer("roc_auc") - >>> clf = LinearSVC().fit(X, y) + >>> clf = LinearSVC(random_state=0).fit(X, y) >>> roc_auc_scorer(clf, X, y) 0.98...