From 8c918e318b903758ceecd6ff62416e3781a6893b Mon Sep 17 00:00:00 2001 From: KaikeWesleyReis Date: Sat, 5 Dec 2020 16:26:35 -0300 Subject: [PATCH 1/3] add spherical payoff score for classification --- sklearn/metrics/_classification.py | 67 ++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/sklearn/metrics/_classification.py b/sklearn/metrics/_classification.py index 44a84e125727b..0ffbfcbd5051a 100644 --- a/sklearn/metrics/_classification.py +++ b/sklearn/metrics/_classification.py @@ -19,6 +19,7 @@ # Saurabh Jha # Bernardo Stein # Shangwu Yao +# Kaike Wesley Reis # License: BSD 3 clause @@ -2504,3 +2505,69 @@ def brier_score_loss(y_true, y_prob, *, sample_weight=None, pos_label=None): raise y_true = np.array(y_true == pos_label, int) return np.average((y_true - y_prob) ** 2, weights=sample_weight) + +def spherical_payoff_score(y_true, y_prob, *, sample_weight=None): + """Compute the Spherical Payoff. + The Spherical Payoff works for binary and multiclass classification + and meausures the model's confidence to predict the correct category. + The Spherical Payoff have a defined interval: [0, 1]. Best possible + score is 1.0, and the worst is 0.0. It's calculated as the average + over all samples. + + Parameters + ---------- + y_true : array, shape (n_samples,) + True targets. + y_prob : array, shape (n_samples,) + Probabilities of the positive class. + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + Returns + ------- + score : float + Spherical Payoff + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import load_iris + >>> from sklearn.metrics import spherical_payoff_score + >>> from sklearn.ensemble import RandomForestClassifier + >>> from sklearn.model_selection import train_test_split + >>> X, y = load_iris(return_X_y=True) + >>> x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1206) + >>> rfc = RandomForestClassifier(random_state=1206).fit(x_train, y_train) + >>> y_prob = rfc.predict_proba(x_test) + >>> spherical_payoff_score(y_test, y_prob) + 0.983... + + References + ---------- + .. [1] `Wikipedia entry for Scoring Rule, including spherical payoff + ` + .. [2] `Netica Tutorial for scoring rules + ` + .. [3] `Guideline for developing and updating Bayesian Belief Networks + applied to ecological modelling and conservation + ` + """ + y_true = column_or_1d(y_true) + assert_all_finite(y_true) + assert_all_finite(y_prob) + check_consistent_length(y_true, y_prob, sample_weight) + + if y_prob.max() > 1: + raise ValueError("y_prob contains values greater than 1.") + if y_prob.min() < 0: + raise ValueError("y_prob contains values less than 0.") + + # Loop to change the category values to default index + categories = np.unique(y_true) + for category, idx in zip(categories, np.arange(0, len(categories))): + y_true[y_true==category] = idx + + # Spherical Payoff + correct_prob = y_prob[np.arange(y_prob.shape[0]), y_test] + sqrt_all_prob = np.sqrt(np.power(y_prob, 2).sum(axis=1)) + + return np.average(correct_prob/sqrt_all_prob, weights=sample_weight) From e40fe24fb9f5e7fde0d2aee94dde8d48fc46be63 Mon Sep 17 00:00:00 2001 From: "Kaike W. Reis" Date: Sat, 5 Dec 2020 19:01:52 -0300 Subject: [PATCH 2/3] add spherical payoff in init --- sklearn/metrics/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index 84e7c98e29324..78aa1faa759ac 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -35,6 +35,7 @@ from ._classification import zero_one_loss from ._classification import brier_score_loss from ._classification import multilabel_confusion_matrix +from ._classification import spherical_payoff_score from . import cluster from .cluster import adjusted_mutual_info_score From f60696d1d0573a00a31aebd1d24f3d84d51f07ae Mon Sep 17 00:00:00 2001 From: "Kaike W. Reis" Date: Sat, 5 Dec 2020 19:23:21 -0300 Subject: [PATCH 3/3] fix spherical_payoff bug --- sklearn/metrics/_classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/metrics/_classification.py b/sklearn/metrics/_classification.py index 0ffbfcbd5051a..4b9836136e516 100644 --- a/sklearn/metrics/_classification.py +++ b/sklearn/metrics/_classification.py @@ -2567,7 +2567,7 @@ def spherical_payoff_score(y_true, y_prob, *, sample_weight=None): y_true[y_true==category] = idx # Spherical Payoff - correct_prob = y_prob[np.arange(y_prob.shape[0]), y_test] + correct_prob = y_prob[np.arange(y_prob.shape[0]), y_true] sqrt_all_prob = np.sqrt(np.power(y_prob, 2).sum(axis=1)) return np.average(correct_prob/sqrt_all_prob, weights=sample_weight)