From 9f5a360abd3a182c808e3bacf0b68128ed763c5e Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 12 Nov 2017 17:08:03 +0100 Subject: [PATCH 001/100] add rough implementation of threshold calibrator --- sklearn/calibration.py | 129 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 3c09d5c02f13d..53bfe93a0e2f3 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -4,6 +4,7 @@ # Balazs Kegl # Jan Hendrik Metzen # Mathieu Blondel +# Prokopis Gryllos # # License: BSD 3 clause @@ -25,6 +26,134 @@ from .svm import LinearSVC from .model_selection import check_cv from .metrics.classification import _check_binary_probabilistic_predictions +from .metrics.pairwise import euclidean_distances +from .metrics.ranking import roc_curve + + +class CalibratedThresholdClassifier(BaseEstimator, ClassifierMixin): + """Decision threshold calibration for binary classification using the + point on the roc curve that is closest to the ideal corner. + + If cv="prefit" the base estimator is assumed to be fitted and all data will + be used for the calibration of the decision threshold that determines the + output of predict. Otherwise predict will use the average of the thresholds + of the calibrated classifiers resulting from the cross-validation loop. + + Parameters + ---------- + base_estimator : instance BaseEstimator + The classifier whose prediction threshold will be calibrated + + pos_label : 0 or 1 (optional) + Label considered as positive. + (default value: 1) + + cv : int, cross-validation generator, iterable or "prefit" (optional) + Determines the cross-validation splitting strategy. If cv="prefit" the + base estimator is assumed to be fitted and all data will be used for the + calibration of the probability threshold + (default value: "prefit") + + Attributes + ---------- + calibrated_threshold : float + Decision threshold for the positive class that determines the output + of predict + """ + def __init__(self, base_estimator=None, pos_label=1, cv=3): + self.base_estimator = base_estimator + self.pos_label = pos_label + self.cv = cv + self.calibrated_threshold = None + + def fit(self, X, y): + """Fit model + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + Training data. + + y : array-like, shape (n_samples,) + Target values. + + Returns + ------- + self : object + Instance of self. + """ + if self.cv == 'prefit': + self.calibrated_threshold = _CalibratedThresholdClassifier( + self.base_estimator, self.pos_label + ).fit(X, y).threshold + else: + cv = check_cv(self.cv, y, classifier=True) + calibrated_thresholds = [] + + for train, test in cv.split(X, y): + estimator = clone(self.base_estimator).fit(X[train], y[train]) + calibrated_thresholds.append(_CalibratedThresholdClassifier( + estimator, self.pos_label + ).fit(X[test], y[test]).threshold + ) + self.calibrated_threshold = sum(calibrated_thresholds) /\ + len(calibrated_thresholds) + return self + + def predict(self, X): + return (self.base_estimator.predict_proba(X)[:, self.pos_label] > + self.calibrated_threshold).astype(int) + + +class _CalibratedThresholdClassifier(object): + """Decision threshold calibration using the optimal point of the roc curve + + It assumes that base_estimator has already been fit, and trains the + calibration on the input set of the fit function. Note that this class + should not be used as an estimator directly. Use CalibratedClassifierCV + with cv="prefit" instead. + + Parameters + ---------- + base_estimator : instance BaseEstimator + The classifier whose prediction threshold will be calibrated + + pos_label : 0 or 1 + Label considered as positive during the roc_curve construction. + + Attributes + ---------- + threshold : float + Calibrated decision threshold for the positive class + """ + def __init__(self, base_estimator, pos_label): + self.base_estimator = base_estimator + self.pos_label = pos_label + self.threshold = None + + def fit(self, X, y): + """Calibrate the decision threshold for the fitted model's positive + class + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + Training data. + + y : array-like, shape (n_samples,) + Target values. + + Returns + ------- + self : object + Instance of self. + """ + y_score = self.base_estimator.predict_proba(X)[:, self.pos_label] + fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) + self.threshold = thresholds[np.argmin( + euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) + )] + return self class CalibratedClassifierCV(BaseEstimator, ClassifierMixin): From 132864b36154949466a1bdedd06cb443ffdf7d80 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 12 Nov 2017 17:51:45 +0100 Subject: [PATCH 002/100] fit base estimator after calibration --- sklearn/calibration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 53bfe93a0e2f3..c8db50c992dcb 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -98,6 +98,7 @@ def fit(self, X, y): ) self.calibrated_threshold = sum(calibrated_thresholds) /\ len(calibrated_thresholds) + self.base_estimator.fit(X, y) return self def predict(self, X): From 6477392074ea19827106c74216ffdb35edc4d7f6 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 12 Nov 2017 17:08:03 +0100 Subject: [PATCH 003/100] add rough implementation of threshold calibrator --- sklearn/calibration.py | 129 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 3c09d5c02f13d..53bfe93a0e2f3 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -4,6 +4,7 @@ # Balazs Kegl # Jan Hendrik Metzen # Mathieu Blondel +# Prokopis Gryllos # # License: BSD 3 clause @@ -25,6 +26,134 @@ from .svm import LinearSVC from .model_selection import check_cv from .metrics.classification import _check_binary_probabilistic_predictions +from .metrics.pairwise import euclidean_distances +from .metrics.ranking import roc_curve + + +class CalibratedThresholdClassifier(BaseEstimator, ClassifierMixin): + """Decision threshold calibration for binary classification using the + point on the roc curve that is closest to the ideal corner. + + If cv="prefit" the base estimator is assumed to be fitted and all data will + be used for the calibration of the decision threshold that determines the + output of predict. Otherwise predict will use the average of the thresholds + of the calibrated classifiers resulting from the cross-validation loop. + + Parameters + ---------- + base_estimator : instance BaseEstimator + The classifier whose prediction threshold will be calibrated + + pos_label : 0 or 1 (optional) + Label considered as positive. + (default value: 1) + + cv : int, cross-validation generator, iterable or "prefit" (optional) + Determines the cross-validation splitting strategy. If cv="prefit" the + base estimator is assumed to be fitted and all data will be used for the + calibration of the probability threshold + (default value: "prefit") + + Attributes + ---------- + calibrated_threshold : float + Decision threshold for the positive class that determines the output + of predict + """ + def __init__(self, base_estimator=None, pos_label=1, cv=3): + self.base_estimator = base_estimator + self.pos_label = pos_label + self.cv = cv + self.calibrated_threshold = None + + def fit(self, X, y): + """Fit model + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + Training data. + + y : array-like, shape (n_samples,) + Target values. + + Returns + ------- + self : object + Instance of self. + """ + if self.cv == 'prefit': + self.calibrated_threshold = _CalibratedThresholdClassifier( + self.base_estimator, self.pos_label + ).fit(X, y).threshold + else: + cv = check_cv(self.cv, y, classifier=True) + calibrated_thresholds = [] + + for train, test in cv.split(X, y): + estimator = clone(self.base_estimator).fit(X[train], y[train]) + calibrated_thresholds.append(_CalibratedThresholdClassifier( + estimator, self.pos_label + ).fit(X[test], y[test]).threshold + ) + self.calibrated_threshold = sum(calibrated_thresholds) /\ + len(calibrated_thresholds) + return self + + def predict(self, X): + return (self.base_estimator.predict_proba(X)[:, self.pos_label] > + self.calibrated_threshold).astype(int) + + +class _CalibratedThresholdClassifier(object): + """Decision threshold calibration using the optimal point of the roc curve + + It assumes that base_estimator has already been fit, and trains the + calibration on the input set of the fit function. Note that this class + should not be used as an estimator directly. Use CalibratedClassifierCV + with cv="prefit" instead. + + Parameters + ---------- + base_estimator : instance BaseEstimator + The classifier whose prediction threshold will be calibrated + + pos_label : 0 or 1 + Label considered as positive during the roc_curve construction. + + Attributes + ---------- + threshold : float + Calibrated decision threshold for the positive class + """ + def __init__(self, base_estimator, pos_label): + self.base_estimator = base_estimator + self.pos_label = pos_label + self.threshold = None + + def fit(self, X, y): + """Calibrate the decision threshold for the fitted model's positive + class + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + Training data. + + y : array-like, shape (n_samples,) + Target values. + + Returns + ------- + self : object + Instance of self. + """ + y_score = self.base_estimator.predict_proba(X)[:, self.pos_label] + fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) + self.threshold = thresholds[np.argmin( + euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) + )] + return self class CalibratedClassifierCV(BaseEstimator, ClassifierMixin): From f1f91125b7dafe047043da987f691807ed0d51d0 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 12 Nov 2017 17:51:45 +0100 Subject: [PATCH 004/100] fit base estimator after calibration --- sklearn/calibration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 53bfe93a0e2f3..c8db50c992dcb 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -98,6 +98,7 @@ def fit(self, X, y): ) self.calibrated_threshold = sum(calibrated_thresholds) /\ len(calibrated_thresholds) + self.base_estimator.fit(X, y) return self def predict(self, X): From 97f1fa7163478a0e4eda49916e45b030d7f87cfb Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 17 Dec 2017 21:33:46 +0100 Subject: [PATCH 005/100] change name to OptimalCutoffClassifier --- sklearn/calibration.py | 62 ++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index c8db50c992dcb..7d2f20bb3652c 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -30,22 +30,22 @@ from .metrics.ranking import roc_curve -class CalibratedThresholdClassifier(BaseEstimator, ClassifierMixin): - """Decision threshold calibration for binary classification using the - point on the roc curve that is closest to the ideal corner. +class OptimalCutoffClassifier(BaseEstimator, ClassifierMixin): + """Optimal cutoff point selection. If cv="prefit" the base estimator is assumed to be fitted and all data will - be used for the calibration of the decision threshold that determines the + be used for the selection of the cutoff point that determines the output of predict. Otherwise predict will use the average of the thresholds - of the calibrated classifiers resulting from the cross-validation loop. + resulting from the cross-validation loop. Parameters ---------- base_estimator : instance BaseEstimator - The classifier whose prediction threshold will be calibrated + The classifier whose decision threshold will be adapted according to the + acquired optimal cutoff point pos_label : 0 or 1 (optional) - Label considered as positive. + Label considered as positive (default value: 1) cv : int, cross-validation generator, iterable or "prefit" (optional) @@ -56,15 +56,15 @@ class CalibratedThresholdClassifier(BaseEstimator, ClassifierMixin): Attributes ---------- - calibrated_threshold : float - Decision threshold for the positive class that determines the output - of predict + threshold : float + Decision threshold for the positive class. Determines the output of + predict """ def __init__(self, base_estimator=None, pos_label=1, cv=3): self.base_estimator = base_estimator self.pos_label = pos_label self.cv = cv - self.calibrated_threshold = None + self.threshold = None def fit(self, X, y): """Fit model @@ -72,10 +72,10 @@ def fit(self, X, y): Parameters ---------- X : array-like, shape (n_samples, n_features) - Training data. + Training data y : array-like, shape (n_samples,) - Target values. + Target values Returns ------- @@ -83,41 +83,43 @@ def fit(self, X, y): Instance of self. """ if self.cv == 'prefit': - self.calibrated_threshold = _CalibratedThresholdClassifier( + self.threshold = _OptimalCutoffClassifier( self.base_estimator, self.pos_label ).fit(X, y).threshold else: cv = check_cv(self.cv, y, classifier=True) - calibrated_thresholds = [] + thresholds = [] for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) - calibrated_thresholds.append(_CalibratedThresholdClassifier( + thresholds.append(_OptimalCutoffClassifier( estimator, self.pos_label ).fit(X[test], y[test]).threshold - ) - self.calibrated_threshold = sum(calibrated_thresholds) /\ - len(calibrated_thresholds) + ) + self.threshold = sum(thresholds) / \ + len(thresholds) self.base_estimator.fit(X, y) return self def predict(self, X): return (self.base_estimator.predict_proba(X)[:, self.pos_label] > - self.calibrated_threshold).astype(int) + self.threshold).astype(int) -class _CalibratedThresholdClassifier(object): - """Decision threshold calibration using the optimal point of the roc curve +class _OptimalCutoffClassifier(object): + """Optimal cutoff point selection based on diagnostic test accuracy + measures (Sensitivity / Specificity). - It assumes that base_estimator has already been fit, and trains the - calibration on the input set of the fit function. Note that this class - should not be used as an estimator directly. Use CalibratedClassifierCV - with cv="prefit" instead. + It assumes that base_estimator has already been fit, and uses the input set + of the fit function to select an optimal cutoff point. Note that this + class should not be used as an estimator directly. Use the + OptimalCutoffClassifier with cv="prefit" instead. Parameters ---------- base_estimator : instance BaseEstimator - The classifier whose prediction threshold will be calibrated + The classifier whose decision threshold will be adapted according to the + acquired optimal cutoff point pos_label : 0 or 1 Label considered as positive during the roc_curve construction. @@ -125,7 +127,7 @@ class _CalibratedThresholdClassifier(object): Attributes ---------- threshold : float - Calibrated decision threshold for the positive class + Acquired optimal decision threshold for the positive class """ def __init__(self, base_estimator, pos_label): self.base_estimator = base_estimator @@ -133,8 +135,8 @@ def __init__(self, base_estimator, pos_label): self.threshold = None def fit(self, X, y): - """Calibrate the decision threshold for the fitted model's positive - class + """Select a decision threshold based on an optimal cutoff point for the + fitted model's positive class Parameters ---------- From 13ee903721056df5f26903b33f7d85e1ac162cd6 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 17 Dec 2017 22:34:24 +0100 Subject: [PATCH 006/100] support arbitrary target values --- sklearn/calibration.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 7d2f20bb3652c..cf323caa9d4bc 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -44,8 +44,8 @@ class OptimalCutoffClassifier(BaseEstimator, ClassifierMixin): The classifier whose decision threshold will be adapted according to the acquired optimal cutoff point - pos_label : 0 or 1 (optional) - Label considered as positive + pos_label : object + Object representing the positive label (default value: 1) cv : int, cross-validation generator, iterable or "prefit" (optional) @@ -75,13 +75,19 @@ def fit(self, X, y): Training data y : array-like, shape (n_samples,) - Target values + Target values. There must be two 2 distinct values. Returns ------- self : object Instance of self. """ + self.label_encoder = LabelEncoder().fit(y) + if len(self.label_encoder.classes_) != 2: + raise ValueError('Target must contain two distinct values') + y = self.label_encoder.transform(y) + self.pos_label = self.label_encoder.transform(self.pos_label) + if self.cv == 'prefit': self.threshold = _OptimalCutoffClassifier( self.base_estimator, self.pos_label @@ -102,8 +108,10 @@ def fit(self, X, y): return self def predict(self, X): - return (self.base_estimator.predict_proba(X)[:, self.pos_label] > - self.threshold).astype(int) + return self.label_encoder.inverse_transform( + (self.base_estimator.predict_proba(X)[:, self.pos_label] > + self.threshold).astype(int) + ) class _OptimalCutoffClassifier(object): @@ -121,7 +129,7 @@ class should not be used as an estimator directly. Use the The classifier whose decision threshold will be adapted according to the acquired optimal cutoff point - pos_label : 0 or 1 + pos_label : object Label considered as positive during the roc_curve construction. Attributes From 6b8fd243cf23d946e52748d01c2cc8d29d10838e Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 18 Dec 2017 00:25:11 +0100 Subject: [PATCH 007/100] add methods max_sp and max_se --- sklearn/calibration.py | 72 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index cf323caa9d4bc..0015380fe0782 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -44,6 +44,18 @@ class OptimalCutoffClassifier(BaseEstimator, ClassifierMixin): The classifier whose decision threshold will be adapted according to the acquired optimal cutoff point + method : str + The method to use for choosing the optimal cutoff point. + + - 'roc', selects the point on the roc_curve that is closer to the ideal + corner (0, 1) + + - 'max_se', selects the point that yields the highest sensitivity with + specificity at least equal to the value of the parameter min_val_sp + + - 'max_sp', selects the point that yields the highest specificity with + sensitivity at least equal to the value of the parameter min_val_se + pos_label : object Object representing the positive label (default value: 1) @@ -54,16 +66,28 @@ class OptimalCutoffClassifier(BaseEstimator, ClassifierMixin): calibration of the probability threshold (default value: "prefit") + min_val_sp : float in [0, 1] + In case method = 'max_se' this value must be set to specify the minimum + required value for the specificity + + min_val_se : float in [0, 1] + In case method = 'max_sp' this value must be set to specify the minimum + required value for the sensitivity + Attributes ---------- threshold : float Decision threshold for the positive class. Determines the output of predict """ - def __init__(self, base_estimator=None, pos_label=1, cv=3): + def __init__(self, base_estimator=None, method='roc', pos_label=1, cv=3, + min_val_sp=None, min_val_se=None): self.base_estimator = base_estimator + self.method = method self.pos_label = pos_label self.cv = cv + self.min_val_sp = min_val_sp + self.min_val_se = min_val_se self.threshold = None def fit(self, X, y): @@ -90,7 +114,8 @@ def fit(self, X, y): if self.cv == 'prefit': self.threshold = _OptimalCutoffClassifier( - self.base_estimator, self.pos_label + self.base_estimator, self.method, self.pos_label, + self.min_val_sp, self.min_val_se ).fit(X, y).threshold else: cv = check_cv(self.cv, y, classifier=True) @@ -98,12 +123,14 @@ def fit(self, X, y): for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) - thresholds.append(_OptimalCutoffClassifier( - estimator, self.pos_label + thresholds.append( + _OptimalCutoffClassifier( + estimator, self.method, self.pos_label, self.min_val_sp, + self.min_val_se ).fit(X[test], y[test]).threshold - ) + ) self.threshold = sum(thresholds) / \ - len(thresholds) + len(thresholds) self.base_estimator.fit(X, y) return self @@ -129,17 +156,34 @@ class should not be used as an estimator directly. Use the The classifier whose decision threshold will be adapted according to the acquired optimal cutoff point + method : 'roc' or 'max_se' or 'max_sp' + The method to use for choosing the optimal cutoff point. + pos_label : object Label considered as positive during the roc_curve construction. + min_val_sp : float in [0, 1] + minimum required value for specificity in case method 'max_se' is used + + min_val_se : float in [0, 1] + minimum required value for sensitivity in case method 'max_sp' is used + + min_val_se : float in [0, 1] + In case method = 'max_sp' this value must be set to specify the minimum + required value for the sensitivity + Attributes ---------- threshold : float Acquired optimal decision threshold for the positive class """ - def __init__(self, base_estimator, pos_label): + def __init__(self, base_estimator, method, pos_label, min_val_sp, + min_val_se): self.base_estimator = base_estimator + self.method = method self.pos_label = pos_label + self.min_val_sp = min_val_sp + self.min_val_se = min_val_se self.threshold = None def fit(self, X, y): @@ -161,9 +205,17 @@ def fit(self, X, y): """ y_score = self.base_estimator.predict_proba(X)[:, self.pos_label] fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) - self.threshold = thresholds[np.argmin( - euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) - )] + + if self.method == 'roc': + self.threshold = thresholds[np.argmin( + euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) + )] + elif self.method == 'max_se': + indices = np.where(1 - fpr >= self.min_val_sp) + self.threshold = thresholds[indices[np.argmax(tpr[indices])]] + elif self.method == 'max_sp': + indices = np.where(tpr >= self.min_val_se) + self.threshold = thresholds[indices[np.argmax(1 - fpr[indices])]] return self From 60f641fff72734e2ef3a49a2c96741b728d73a36 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Fri, 22 Dec 2017 22:12:43 +0200 Subject: [PATCH 008/100] rename to CutoffClassifier --- sklearn/calibration.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 0015380fe0782..7e37cc4cd3edc 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -30,8 +30,13 @@ from .metrics.ranking import roc_curve -class OptimalCutoffClassifier(BaseEstimator, ClassifierMixin): - """Optimal cutoff point selection. +class CutoffClassifier(BaseEstimator, ClassifierMixin): + """Uses a base estimator and finds a cutoff point (decision threshold) to be + used by predict. Applicable only on binary classification problems. + + The methods for picking "optimal" cutoff points are inferred from ROC + analysis; making use of true positive and true negative rates and their + corresponding thresholds. If cv="prefit" the base estimator is assumed to be fitted and all data will be used for the selection of the cutoff point that determines the @@ -42,10 +47,10 @@ class OptimalCutoffClassifier(BaseEstimator, ClassifierMixin): ---------- base_estimator : instance BaseEstimator The classifier whose decision threshold will be adapted according to the - acquired optimal cutoff point + acquired cutoff point method : str - The method to use for choosing the optimal cutoff point. + The method to use for choosing the cutoff point. - 'roc', selects the point on the roc_curve that is closer to the ideal corner (0, 1) @@ -113,7 +118,7 @@ def fit(self, X, y): self.pos_label = self.label_encoder.transform(self.pos_label) if self.cv == 'prefit': - self.threshold = _OptimalCutoffClassifier( + self.threshold = _CutoffClassifier( self.base_estimator, self.method, self.pos_label, self.min_val_sp, self.min_val_se ).fit(X, y).threshold @@ -124,7 +129,7 @@ def fit(self, X, y): for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) thresholds.append( - _OptimalCutoffClassifier( + _CutoffClassifier( estimator, self.method, self.pos_label, self.min_val_sp, self.min_val_se ).fit(X[test], y[test]).threshold @@ -141,9 +146,8 @@ def predict(self, X): ) -class _OptimalCutoffClassifier(object): - """Optimal cutoff point selection based on diagnostic test accuracy - measures (Sensitivity / Specificity). +class _CutoffClassifier(object): + """Optimal cutoff point selection. It assumes that base_estimator has already been fit, and uses the input set of the fit function to select an optimal cutoff point. Note that this @@ -157,7 +161,7 @@ class should not be used as an estimator directly. Use the acquired optimal cutoff point method : 'roc' or 'max_se' or 'max_sp' - The method to use for choosing the optimal cutoff point. + The method to use for choosing the cutoff point. pos_label : object Label considered as positive during the roc_curve construction. From 4e5a0189fe012c4bd98ec5ffc0cf7f098dc7a784 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Fri, 22 Dec 2017 23:42:09 +0200 Subject: [PATCH 009/100] rename sensitivity / specificity to tpr / tnr --- sklearn/calibration.py | 50 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 7e37cc4cd3edc..ed059fde3a96c 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -55,11 +55,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin): - 'roc', selects the point on the roc_curve that is closer to the ideal corner (0, 1) - - 'max_se', selects the point that yields the highest sensitivity with - specificity at least equal to the value of the parameter min_val_sp + - 'max_tpr', selects the point that yields the highest true positive + rate with true negative rate at least equal to the value of the + parameter min_val_tnr - - 'max_sp', selects the point that yields the highest specificity with - sensitivity at least equal to the value of the parameter min_val_se + - 'max_tnr', selects the point that yields the highest true negative + rate with true positive rate at least equal to the value of the + parameter min_val_tpr pos_label : object Object representing the positive label @@ -71,13 +73,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin): calibration of the probability threshold (default value: "prefit") - min_val_sp : float in [0, 1] - In case method = 'max_se' this value must be set to specify the minimum - required value for the specificity + min_val_tnr : float in [0, 1] + In case method = 'max_tpr' this value must be set to specify the minimum + required value for the true negative rate - min_val_se : float in [0, 1] - In case method = 'max_sp' this value must be set to specify the minimum - required value for the sensitivity + min_val_tpr : float in [0, 1] + In case method = 'max_tnr' this value must be set to specify the minimum + required value for the true positive rate Attributes ---------- @@ -86,13 +88,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin): predict """ def __init__(self, base_estimator=None, method='roc', pos_label=1, cv=3, - min_val_sp=None, min_val_se=None): + min_val_tnr=None, min_val_tpr=None): self.base_estimator = base_estimator self.method = method self.pos_label = pos_label self.cv = cv - self.min_val_sp = min_val_sp - self.min_val_se = min_val_se + self.min_val_tnr = min_val_tnr + self.min_val_tpr = min_val_tpr self.threshold = None def fit(self, X, y): @@ -120,7 +122,7 @@ def fit(self, X, y): if self.cv == 'prefit': self.threshold = _CutoffClassifier( self.base_estimator, self.method, self.pos_label, - self.min_val_sp, self.min_val_se + self.min_val_tnr, self.min_val_tpr ).fit(X, y).threshold else: cv = check_cv(self.cv, y, classifier=True) @@ -130,8 +132,8 @@ def fit(self, X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) thresholds.append( _CutoffClassifier( - estimator, self.method, self.pos_label, self.min_val_sp, - self.min_val_se + estimator, self.method, self.pos_label, self.min_val_tnr, + self.min_val_tpr ).fit(X[test], y[test]).threshold ) self.threshold = sum(thresholds) / \ @@ -181,13 +183,13 @@ class should not be used as an estimator directly. Use the threshold : float Acquired optimal decision threshold for the positive class """ - def __init__(self, base_estimator, method, pos_label, min_val_sp, - min_val_se): + def __init__(self, base_estimator, method, pos_label, min_val_tnr, + min_val_tpr): self.base_estimator = base_estimator self.method = method self.pos_label = pos_label - self.min_val_sp = min_val_sp - self.min_val_se = min_val_se + self.min_val_tnr = min_val_tnr + self.min_val_tpr = min_val_tpr self.threshold = None def fit(self, X, y): @@ -214,11 +216,11 @@ def fit(self, X, y): self.threshold = thresholds[np.argmin( euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) )] - elif self.method == 'max_se': - indices = np.where(1 - fpr >= self.min_val_sp) + elif self.method == 'max_tpr': + indices = np.where(1 - fpr >= self.min_val_tnr) self.threshold = thresholds[indices[np.argmax(tpr[indices])]] - elif self.method == 'max_sp': - indices = np.where(tpr >= self.min_val_se) + elif self.method == 'max_tnr': + indices = np.where(tpr >= self.min_val_tpr) self.threshold = thresholds[indices[np.argmax(1 - fpr[indices])]] return self From 125a9b2fae1fa061897252cfb2689e4b7899fb70 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 00:32:11 +0200 Subject: [PATCH 010/100] remove attribute set in __init__ --- sklearn/calibration.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index ed059fde3a96c..122c40102bc34 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -95,7 +95,6 @@ def __init__(self, base_estimator=None, method='roc', pos_label=1, cv=3, self.cv = cv self.min_val_tnr = min_val_tnr self.min_val_tpr = min_val_tpr - self.threshold = None def fit(self, X, y): """Fit model @@ -190,7 +189,6 @@ def __init__(self, base_estimator, method, pos_label, min_val_tnr, self.pos_label = pos_label self.min_val_tnr = min_val_tnr self.min_val_tpr = min_val_tpr - self.threshold = None def fit(self, X, y): """Select a decision threshold based on an optimal cutoff point for the From 6a822c60fdcbd2990810419d8570f4df92f23eb1 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 00:32:43 +0200 Subject: [PATCH 011/100] remove target check for binary values --- sklearn/calibration.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 122c40102bc34..7e26c92fd65d0 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -113,8 +113,6 @@ def fit(self, X, y): Instance of self. """ self.label_encoder = LabelEncoder().fit(y) - if len(self.label_encoder.classes_) != 2: - raise ValueError('Target must contain two distinct values') y = self.label_encoder.transform(y) self.pos_label = self.label_encoder.transform(self.pos_label) From c42564db33fcc6b320bc8283d1241976c2e83d15 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 00:37:31 +0200 Subject: [PATCH 012/100] fix pep8 --- sklearn/calibration.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 7e26c92fd65d0..e7074389e3ce7 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -39,8 +39,8 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin): corresponding thresholds. If cv="prefit" the base estimator is assumed to be fitted and all data will - be used for the selection of the cutoff point that determines the - output of predict. Otherwise predict will use the average of the thresholds + be used for the selection of the cutoff point that determines the output of + predict. Otherwise predict will use the average of the thresholds resulting from the cross-validation loop. Parameters @@ -128,10 +128,13 @@ def fit(self, X, y): for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) thresholds.append( - _CutoffClassifier( - estimator, self.method, self.pos_label, self.min_val_tnr, - self.min_val_tpr - ).fit(X[test], y[test]).threshold + _CutoffClassifier(estimator, + self.method, + self.pos_label, + self.min_val_tnr, + self.min_val_tpr).fit( + X[test], y[test] + ).threshold ) self.threshold = sum(thresholds) / \ len(thresholds) From 31da085d8b2e85a8bf346f3202ac67e5c4f79377 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 12:20:10 +0200 Subject: [PATCH 013/100] fix input to label encoder --- sklearn/calibration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index e7074389e3ce7..f6a459b8aca4a 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -114,7 +114,7 @@ def fit(self, X, y): """ self.label_encoder = LabelEncoder().fit(y) y = self.label_encoder.transform(y) - self.pos_label = self.label_encoder.transform(self.pos_label) + self.pos_label = self.label_encoder.transform([self.pos_label])[0] if self.cv == 'prefit': self.threshold = _CutoffClassifier( @@ -143,9 +143,9 @@ def fit(self, X, y): def predict(self, X): return self.label_encoder.inverse_transform( - (self.base_estimator.predict_proba(X)[:, self.pos_label] > - self.threshold).astype(int) - ) + [(self.base_estimator.predict_proba(X)[:, self.pos_label] > + self.threshold).astype(int)] + )[0] class _CutoffClassifier(object): From 19285fc02dda77b06c9e7240815ed53aed62c997 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 14:34:10 +0200 Subject: [PATCH 014/100] use LinearSVC if base estimator not provided --- sklearn/calibration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index f6a459b8aca4a..be28be6ee378e 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -116,6 +116,9 @@ def fit(self, X, y): y = self.label_encoder.transform(y) self.pos_label = self.label_encoder.transform([self.pos_label])[0] + if not self.base_estimator: + self.base_estimator = LinearSVC(random_state=0) + if self.cv == 'prefit': self.threshold = _CutoffClassifier( self.base_estimator, self.method, self.pos_label, From 84ed3bc3153a81e9607f3323fac9411441126141 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 14:52:21 +0200 Subject: [PATCH 015/100] add check_is_fitted check --- sklearn/calibration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index be28be6ee378e..32f6ad02c98ee 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -145,6 +145,8 @@ def fit(self, X, y): return self def predict(self, X): + check_is_fitted(self, ["label_encoder", "threshold"]) + return self.label_encoder.inverse_transform( [(self.base_estimator.predict_proba(X)[:, self.pos_label] > self.threshold).astype(int)] From 2e5a9bb4759aab7a391153d8536f4ae67cf73627 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 14:54:14 +0200 Subject: [PATCH 016/100] add input validation checks --- sklearn/calibration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 32f6ad02c98ee..d835f43169970 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -112,6 +112,8 @@ def fit(self, X, y): self : object Instance of self. """ + X, y = check_X_y(X, y) + self.label_encoder = LabelEncoder().fit(y) y = self.label_encoder.transform(y) self.pos_label = self.label_encoder.transform([self.pos_label])[0] @@ -145,6 +147,7 @@ def fit(self, X, y): return self def predict(self, X): + X = check_array(X) check_is_fitted(self, ["label_encoder", "threshold"]) return self.label_encoder.inverse_transform( From e63657c0624d3ad43b7d9e03ab580e732f0b67f3 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 15:38:37 +0200 Subject: [PATCH 017/100] Not allow None base estimator --- sklearn/calibration.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index d835f43169970..7912fb76135d5 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -87,7 +87,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin): Decision threshold for the positive class. Determines the output of predict """ - def __init__(self, base_estimator=None, method='roc', pos_label=1, cv=3, + def __init__(self, base_estimator, method='roc', pos_label=1, cv=3, min_val_tnr=None, min_val_tpr=None): self.base_estimator = base_estimator self.method = method @@ -112,15 +112,16 @@ def fit(self, X, y): self : object Instance of self. """ + if not isinstance(self.base_estimator, BaseEstimator): + raise AttributeError('Base estimator must be of type BaseEstimator;' + 'got %s instead' % type(self.base_estimator)) + X, y = check_X_y(X, y) self.label_encoder = LabelEncoder().fit(y) y = self.label_encoder.transform(y) self.pos_label = self.label_encoder.transform([self.pos_label])[0] - if not self.base_estimator: - self.base_estimator = LinearSVC(random_state=0) - if self.cv == 'prefit': self.threshold = _CutoffClassifier( self.base_estimator, self.method, self.pos_label, From a145a16f0672954b6b1838ca460df2032779d2ec Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 15:50:41 +0200 Subject: [PATCH 018/100] readd target validation check for binary values --- sklearn/calibration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 7912fb76135d5..402edc7019fb5 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -119,6 +119,9 @@ def fit(self, X, y): X, y = check_X_y(X, y) self.label_encoder = LabelEncoder().fit(y) + if len(self.label_encoder.classes_) > 2: + raise ValueError('Found more than two distinct values in target y') + y = self.label_encoder.transform(y) self.pos_label = self.label_encoder.transform([self.pos_label])[0] From 3abd1fee7dcb0a3f8230de65245778fd5d3f2790 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 23 Dec 2017 15:58:55 +0200 Subject: [PATCH 019/100] add trailing underscores to attributes --- sklearn/calibration.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 402edc7019fb5..4a3eb6a037d47 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -83,7 +83,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin): Attributes ---------- - threshold : float + threshold_ : float Decision threshold for the positive class. Determines the output of predict """ @@ -118,18 +118,18 @@ def fit(self, X, y): X, y = check_X_y(X, y) - self.label_encoder = LabelEncoder().fit(y) - if len(self.label_encoder.classes_) > 2: + self.label_encoder_ = LabelEncoder().fit(y) + if len(self.label_encoder_.classes_) > 2: raise ValueError('Found more than two distinct values in target y') - y = self.label_encoder.transform(y) - self.pos_label = self.label_encoder.transform([self.pos_label])[0] + y = self.label_encoder_.transform(y) + self.pos_label = self.label_encoder_.transform([self.pos_label])[0] if self.cv == 'prefit': - self.threshold = _CutoffClassifier( + self.threshold_ = _CutoffClassifier( self.base_estimator, self.method, self.pos_label, self.min_val_tnr, self.min_val_tpr - ).fit(X, y).threshold + ).fit(X, y).threshold_ else: cv = check_cv(self.cv, y, classifier=True) thresholds = [] @@ -143,20 +143,20 @@ def fit(self, X, y): self.min_val_tnr, self.min_val_tpr).fit( X[test], y[test] - ).threshold + ).threshold_ ) - self.threshold = sum(thresholds) / \ - len(thresholds) + self.threshold_ = sum(thresholds) / \ + len(thresholds) self.base_estimator.fit(X, y) return self def predict(self, X): X = check_array(X) - check_is_fitted(self, ["label_encoder", "threshold"]) + check_is_fitted(self, ["label_encoder_", "threshold_"]) - return self.label_encoder.inverse_transform( + return self.label_encoder_.inverse_transform( [(self.base_estimator.predict_proba(X)[:, self.pos_label] > - self.threshold).astype(int)] + self.threshold_).astype(int)] )[0] @@ -192,7 +192,7 @@ class should not be used as an estimator directly. Use the Attributes ---------- - threshold : float + threshold_ : float Acquired optimal decision threshold for the positive class """ def __init__(self, base_estimator, method, pos_label, min_val_tnr, @@ -224,15 +224,15 @@ def fit(self, X, y): fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) if self.method == 'roc': - self.threshold = thresholds[np.argmin( + self.threshold_ = thresholds[np.argmin( euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) )] elif self.method == 'max_tpr': indices = np.where(1 - fpr >= self.min_val_tnr) - self.threshold = thresholds[indices[np.argmax(tpr[indices])]] + self.threshold_ = thresholds[indices[np.argmax(tpr[indices])]] elif self.method == 'max_tnr': indices = np.where(tpr >= self.min_val_tpr) - self.threshold = thresholds[indices[np.argmax(1 - fpr[indices])]] + self.threshold_ = thresholds[indices[np.argmax(1 - fpr[indices])]] return self From 2f778fb7d43e324812673e18926f63c6cb4743e9 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 26 Dec 2017 19:42:12 +0200 Subject: [PATCH 020/100] make cutoffclassifier meta --- sklearn/calibration.py | 10 ++++++---- sklearn/utils/testing.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 4a3eb6a037d47..ff9bec53f86ef 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -17,7 +17,8 @@ from scipy.optimize import fmin_bfgs from sklearn.preprocessing import LabelEncoder -from .base import BaseEstimator, ClassifierMixin, RegressorMixin, clone +from .base import BaseEstimator, ClassifierMixin, RegressorMixin,\ + MetaEstimatorMixin, clone from .preprocessing import label_binarize, LabelBinarizer from .utils import check_X_y, check_array, indexable, column_or_1d from .utils.validation import check_is_fitted, check_consistent_length @@ -30,9 +31,10 @@ from .metrics.ranking import roc_curve -class CutoffClassifier(BaseEstimator, ClassifierMixin): - """Uses a base estimator and finds a cutoff point (decision threshold) to be - used by predict. Applicable only on binary classification problems. +class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): + """Meta estimator that uses a base estimator and finds a cutoff point + (decision threshold) to be used by predict. Applicable only on binary + classification problems. The methods for picking "optimal" cutoff points are inferred from ROC analysis; making use of true positive and true negative rates and their diff --git a/sklearn/utils/testing.py b/sklearn/utils/testing.py index 6e2d9d5902add..c9f3791e2de05 100644 --- a/sklearn/utils/testing.py +++ b/sklearn/utils/testing.py @@ -520,7 +520,7 @@ def uninstall_mldata_mock(): "MultiOutputRegressor", "MultiOutputClassifier", "OutputCodeClassifier", "OneVsRestClassifier", "RFE", "RFECV", "BaseEnsemble", "ClassifierChain", - "RegressorChain"] + "RegressorChain", "CutoffClassifier"] # estimators that there is no way to default-construct sensibly OTHER = ["Pipeline", "FeatureUnion", "GridSearchCV", "RandomizedSearchCV", "SelectFromModel"] From 971c0ee7ef40b5f49b3ece9383caad01f468f69b Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Wed, 27 Dec 2017 03:42:16 +0200 Subject: [PATCH 021/100] fix pep8 --- sklearn/calibration.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index ff9bec53f86ef..2bf650e2b0c15 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -48,8 +48,8 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Parameters ---------- base_estimator : instance BaseEstimator - The classifier whose decision threshold will be adapted according to the - acquired cutoff point + The classifier whose decision threshold will be adapted according to + the acquired cutoff point method : str The method to use for choosing the cutoff point. @@ -71,17 +71,17 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): cv : int, cross-validation generator, iterable or "prefit" (optional) Determines the cross-validation splitting strategy. If cv="prefit" the - base estimator is assumed to be fitted and all data will be used for the - calibration of the probability threshold + base estimator is assumed to be fitted and all data will be used for + the calibration of the probability threshold (default value: "prefit") min_val_tnr : float in [0, 1] - In case method = 'max_tpr' this value must be set to specify the minimum - required value for the true negative rate + In case method = 'max_tpr' this value must be set to specify the + minimum required value for the true negative rate min_val_tpr : float in [0, 1] - In case method = 'max_tnr' this value must be set to specify the minimum - required value for the true positive rate + In case method = 'max_tnr' this value must be set to specify the + minimum required value for the true positive rate Attributes ---------- @@ -115,8 +115,8 @@ def fit(self, X, y): Instance of self. """ if not isinstance(self.base_estimator, BaseEstimator): - raise AttributeError('Base estimator must be of type BaseEstimator;' - 'got %s instead' % type(self.base_estimator)) + raise AttributeError('Base estimator must be of type BaseEstimator' + ' got %s instead' % type(self.base_estimator)) X, y = check_X_y(X, y) @@ -147,8 +147,7 @@ def fit(self, X, y): X[test], y[test] ).threshold_ ) - self.threshold_ = sum(thresholds) / \ - len(thresholds) + self.threshold_ = sum(thresholds) / len(thresholds) self.base_estimator.fit(X, y) return self @@ -173,8 +172,8 @@ class should not be used as an estimator directly. Use the Parameters ---------- base_estimator : instance BaseEstimator - The classifier whose decision threshold will be adapted according to the - acquired optimal cutoff point + The classifier whose decision threshold will be adapted according to + the acquired optimal cutoff point method : 'roc' or 'max_se' or 'max_sp' The method to use for choosing the cutoff point. From e3d6a156b0f7c171d4487f9e3b570887630fe951 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 00:41:00 +0100 Subject: [PATCH 022/100] fix docstring --- sklearn/calibration.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 2bf650e2b0c15..5f428d58565e4 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -175,21 +175,19 @@ class should not be used as an estimator directly. Use the The classifier whose decision threshold will be adapted according to the acquired optimal cutoff point - method : 'roc' or 'max_se' or 'max_sp' + method : 'roc' or 'max_tpr' or 'max_tnr' The method to use for choosing the cutoff point. pos_label : object Label considered as positive during the roc_curve construction. - min_val_sp : float in [0, 1] - minimum required value for specificity in case method 'max_se' is used - - min_val_se : float in [0, 1] - minimum required value for sensitivity in case method 'max_sp' is used + min_val_tnr : float in [0, 1] + minimum required value for true negative rate (specificity) in case + method 'max_tpr' is used - min_val_se : float in [0, 1] - In case method = 'max_sp' this value must be set to specify the minimum - required value for the sensitivity + min_val_tpr : float in [0, 1] + minimum required value for true positive rate (sensitivity) in case + method 'max_tnr' is used Attributes ---------- From 7a99622ecc0ac5740b79a55e590236b377874825 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 00:41:36 +0100 Subject: [PATCH 023/100] add value validation for min_val_tpr and min_val_tnr --- sklearn/calibration.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 5f428d58565e4..dd6b85f4e62fd 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -227,11 +227,22 @@ def fit(self, X, y): euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) )] elif self.method == 'max_tpr': - indices = np.where(1 - fpr >= self.min_val_tnr) + if not self.min_val_tnr or not isinstance(self.min_val_tnr, float)\ + or not self.min_val_tnr >= 0 or not self.min_val_tnr <= 1: + raise ValueError('max_tnr must be a number in [1, 0]. ' + 'Got %s instead' % repr(self.min_val_tnr)) + indices = np.where(1 - fpr >= self.min_val_tnr)[0] self.threshold_ = thresholds[indices[np.argmax(tpr[indices])]] elif self.method == 'max_tnr': - indices = np.where(tpr >= self.min_val_tpr) + if not self.min_val_tpr or not isinstance(self.min_val_tpr, float)\ + or not self.min_val_tpr >= 0 or not self.min_val_tpr <= 1: + raise ValueError('max_tpr must be a number in [1, 0]. ' + 'Got %s instead' % repr(self.min_val_tnr)) + indices = np.where(tpr >= self.min_val_tpr)[0] self.threshold_ = thresholds[indices[np.argmax(1 - fpr[indices])]] + else: + raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' + 'Got %s instead' % self.method) return self From 773a78ee9efc9971fe2af9bea347fe75bd405593 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 00:42:16 +0100 Subject: [PATCH 024/100] add test for cutoff classifier --- sklearn/tests/test_calibration.py | 74 +++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index e4499e35d5a67..9d6aa68c8f636 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -1,10 +1,11 @@ # Authors: Alexandre Gramfort +# Prokopios Gryllos # License: BSD 3 clause from __future__ import division import numpy as np from scipy import sparse -from sklearn.model_selection import LeaveOneOut +from sklearn.model_selection import LeaveOneOut, train_test_split from sklearn.utils.testing import (assert_array_almost_equal, assert_equal, assert_greater, assert_almost_equal, @@ -13,17 +14,84 @@ assert_raises, ignore_warnings) from sklearn.datasets import make_classification, make_blobs +from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import MultinomialNB from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from sklearn.svm import LinearSVC from sklearn.pipeline import Pipeline from sklearn.preprocessing import Imputer -from sklearn.metrics import brier_score_loss, log_loss -from sklearn.calibration import CalibratedClassifierCV +from sklearn.metrics import brier_score_loss, log_loss, confusion_matrix +from sklearn.calibration import CalibratedClassifierCV, CutoffClassifier from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration from sklearn.calibration import calibration_curve +def test_cutoff_prefit(): + calibration_samples = 200 + X, y = make_classification(n_samples=1000, n_features=6, random_state=42, + n_classes=2) + + X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.6, + random_state=42) + lr = LogisticRegression().fit(X_train, y_train) + + clf = CutoffClassifier(lr, method='roc', cv='prefit').fit( + X_test[:calibration_samples], y_train[:calibration_samples] + ) + + y_pred = lr.predict(X_test[calibration_samples:]) + y_pred_clf = clf.predict(X_test[calibration_samples:]) + + tn, fp, fn, tp = confusion_matrix( + y_test[calibration_samples:], y_pred).ravel() + tn_clf, fp_clf, fn_clf, tp_clf = confusion_matrix( + y_test[calibration_samples:], y_pred_clf).ravel() + + tpr = tp / (tp + fn) + tnr = tn / (tn + fp) + + tpr_clf_roc = tp_clf / (tp_clf + fn_clf) + tnr_clf_roc = tn_clf / (tn_clf + fp_clf) + + # check that the sum of tpr + tnr has improved + assert_greater(tpr_clf_roc + tnr_clf_roc, tpr + tnr) + + clf = CutoffClassifier( + lr, method='max_tpr', cv='prefit', min_val_tnr=0.3 + ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) + + y_pred_clf = clf.predict(X_test[calibration_samples:]) + + tn_clf, fp_clf, fn_clf, tp_clf = confusion_matrix( + y_test[calibration_samples:], y_pred_clf).ravel() + + tpr_clf_max_tpr = tp_clf / (tp_clf + fn_clf) + tnr_clf_max_tpr = tn_clf / (tn_clf + fp_clf) + + # check that the tpr increases with tnr >= min_val_tnr + assert_greater(tpr_clf_max_tpr, tpr) + assert_greater(tpr_clf_max_tpr, tpr_clf_roc) + assert_greater_equal(tnr_clf_max_tpr, 0.3) + + clf = CutoffClassifier( + lr, method='max_tnr', cv='prefit', min_val_tpr=0.3 + ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) + + y_pred_clf = clf.predict(X_test[calibration_samples:]) + + tn_clf, fp_clf, fn_clf, tp_clf = confusion_matrix( + y_test[calibration_samples:], y_pred_clf).ravel() + + tnr_clf_max_tnr = tn_clf / (tn_clf + fp_clf) + tpr_clf_max_tnr = tp_clf / (tp_clf + fn_clf) + + # check that the tnr increases with tpr >= min_val_tpr + assert_greater(tnr_clf_max_tnr, tnr) + assert_greater(tnr_clf_max_tnr, tnr_clf_roc) + assert_greater_equal(tpr_clf_max_tnr, 0.3) + + @ignore_warnings def test_calibration(): """Test calibration objects with isotonic and sigmoid""" From e7506ca57ebb2bc47bcc57278d7f7062ab7f186e Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 20:18:10 +0100 Subject: [PATCH 025/100] fix inverse transforming of predictions --- sklearn/calibration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index dd6b85f4e62fd..717460d7604a4 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -156,9 +156,9 @@ def predict(self, X): check_is_fitted(self, ["label_encoder_", "threshold_"]) return self.label_encoder_.inverse_transform( - [(self.base_estimator.predict_proba(X)[:, self.pos_label] > - self.threshold_).astype(int)] - )[0] + (self.base_estimator.predict_proba(X)[:, self.pos_label] > + self.threshold_).astype(int) + ) class _CutoffClassifier(object): From bba7003aa434e905e56a612571dd14741c3ad32e Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 22:50:09 +0100 Subject: [PATCH 026/100] extend cutoff_prefit test --- sklearn/tests/test_calibration.py | 56 ++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 9d6aa68c8f636..53a9d333c7a92 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -36,49 +36,49 @@ def test_cutoff_prefit(): random_state=42) lr = LogisticRegression().fit(X_train, y_train) - clf = CutoffClassifier(lr, method='roc', cv='prefit').fit( + clf_roc = CutoffClassifier(lr, method='roc', cv='prefit').fit( X_test[:calibration_samples], y_train[:calibration_samples] ) y_pred = lr.predict(X_test[calibration_samples:]) - y_pred_clf = clf.predict(X_test[calibration_samples:]) + y_pred_roc = clf_roc.predict(X_test[calibration_samples:]) tn, fp, fn, tp = confusion_matrix( y_test[calibration_samples:], y_pred).ravel() - tn_clf, fp_clf, fn_clf, tp_clf = confusion_matrix( - y_test[calibration_samples:], y_pred_clf).ravel() + tn_roc, fp_roc, fn_roc, tp_roc = confusion_matrix( + y_test[calibration_samples:], y_pred_roc).ravel() tpr = tp / (tp + fn) tnr = tn / (tn + fp) - tpr_clf_roc = tp_clf / (tp_clf + fn_clf) - tnr_clf_roc = tn_clf / (tn_clf + fp_clf) + tpr_roc = tp_roc / (tp_roc + fn_roc) + tnr_roc = tn_roc / (tn_roc + fp_roc) # check that the sum of tpr + tnr has improved - assert_greater(tpr_clf_roc + tnr_clf_roc, tpr + tnr) + assert_greater(tpr_roc + tnr_roc, tpr + tnr) - clf = CutoffClassifier( + clf_max_tpr = CutoffClassifier( lr, method='max_tpr', cv='prefit', min_val_tnr=0.3 ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) - y_pred_clf = clf.predict(X_test[calibration_samples:]) + y_pred_max_tpr = clf_max_tpr.predict(X_test[calibration_samples:]) - tn_clf, fp_clf, fn_clf, tp_clf = confusion_matrix( - y_test[calibration_samples:], y_pred_clf).ravel() + tn_max_tpr, fp_max_tpr, fn_max_tpr, tp_max_tpr = confusion_matrix( + y_test[calibration_samples:], y_pred_max_tpr).ravel() - tpr_clf_max_tpr = tp_clf / (tp_clf + fn_clf) - tnr_clf_max_tpr = tn_clf / (tn_clf + fp_clf) + tpr_max_tpr = tp_max_tpr / (tp_max_tpr + fn_max_tpr) + tnr_max_tpr = tn_max_tpr / (tn_max_tpr + fp_max_tpr) # check that the tpr increases with tnr >= min_val_tnr - assert_greater(tpr_clf_max_tpr, tpr) - assert_greater(tpr_clf_max_tpr, tpr_clf_roc) - assert_greater_equal(tnr_clf_max_tpr, 0.3) + assert_greater(tpr_max_tpr, tpr) + assert_greater(tpr_max_tpr, tpr_roc) + assert_greater_equal(tnr_max_tpr, 0.3) - clf = CutoffClassifier( + clf_max_tnr = CutoffClassifier( lr, method='max_tnr', cv='prefit', min_val_tpr=0.3 ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) - y_pred_clf = clf.predict(X_test[calibration_samples:]) + y_pred_clf = clf_max_tnr.predict(X_test[calibration_samples:]) tn_clf, fp_clf, fn_clf, tp_clf = confusion_matrix( y_test[calibration_samples:], y_pred_clf).ravel() @@ -88,9 +88,27 @@ def test_cutoff_prefit(): # check that the tnr increases with tpr >= min_val_tpr assert_greater(tnr_clf_max_tnr, tnr) - assert_greater(tnr_clf_max_tnr, tnr_clf_roc) + assert_greater(tnr_clf_max_tnr, tnr_roc) assert_greater_equal(tpr_clf_max_tnr, 0.3) + # check error cases + clf_non_base_estimator = CutoffClassifier([]) + assert_raises(AttributeError, clf_non_base_estimator.fit, X_train, y_train) + + X_non_binary, y_non_binary = make_classification( + n_samples=20, n_features=6, random_state=42, n_classes=4, + n_informative=4 + ) + assert_raises(ValueError, clf_roc.fit, X_non_binary, y_non_binary) + + clf_foo = CutoffClassifier(lr, method='foo') + assert_raises(ValueError, clf_foo.fit, X_train, y_train) + + for method in ['max_tpr', 'max_tnr']: + clf_missing_info = CutoffClassifier(lr, method=method) + assert_raises(ValueError, clf_missing_info.fit, X_train, y_train) + + @ignore_warnings def test_calibration(): From ba56ed8988ae1ddc63d2d554e671be745e36f1f1 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 23:08:40 +0100 Subject: [PATCH 027/100] ignore warning from train_test_split --- sklearn/tests/test_calibration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 53a9d333c7a92..96de2ce4f7155 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -26,6 +26,7 @@ from sklearn.calibration import calibration_curve +@ignore_warnings def test_cutoff_prefit(): calibration_samples = 200 X, y = make_classification(n_samples=1000, n_features=6, random_state=42, From a5807fa0d491b613d19893ce71ff714feae2a54a Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Jan 2018 23:09:19 +0100 Subject: [PATCH 028/100] add cutoff cv test --- sklearn/tests/test_calibration.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 96de2ce4f7155..3ccb3940d123e 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -110,6 +110,36 @@ def test_cutoff_prefit(): assert_raises(ValueError, clf_missing_info.fit, X_train, y_train) +@ignore_warnings +def test_cutoff_cv(): + X, y = make_classification(n_samples=1000, n_features=6, random_state=42, + n_classes=2) + + X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.6, + random_state=42) + lr = LogisticRegression().fit(X_train, y_train) + clf_roc = CutoffClassifier(LogisticRegression(), method='roc', cv=3).fit( + X_train, y_train + ) + + y_pred = lr.predict(X_test) + y_pred_roc = clf_roc.predict(X_test) + + tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel() + tn_roc, fp_roc, fn_roc, tp_roc = confusion_matrix( + y_test, y_pred_roc + ).ravel() + + tpr = tp / (tp + fn) + tnr = tn / (tn + fp) + + tpr_roc = tp_roc / (tp_roc + fn_roc) + tnr_roc = tn_roc / (tn_roc + fp_roc) + + # check that the sum of tpr + tnr has improved + assert_greater(tpr_roc + tnr_roc, tpr + tnr) + @ignore_warnings def test_calibration(): From 3058bc0474530049adcebdeaafa24927a588fcde Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Fri, 12 Jan 2018 18:00:59 +0100 Subject: [PATCH 029/100] change affiliation --- sklearn/calibration.py | 2 +- sklearn/tests/test_calibration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 717460d7604a4..90a44bcc19a8e 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -4,7 +4,7 @@ # Balazs Kegl # Jan Hendrik Metzen # Mathieu Blondel -# Prokopis Gryllos +# Prokopis Gryllos # # License: BSD 3 clause diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 3ccb3940d123e..31f038bb8d40c 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -1,5 +1,5 @@ # Authors: Alexandre Gramfort -# Prokopios Gryllos +# Prokopios Gryllos # License: BSD 3 clause from __future__ import division From 24d074c5ad3c7a4941631e7b33dfe09499483540 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 16 Jan 2018 00:08:43 +0100 Subject: [PATCH 030/100] add citation --- sklearn/calibration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 90a44bcc19a8e..aa4dc5a5b4f18 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -88,6 +88,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): threshold_ : float Decision threshold for the positive class. Determines the output of predict + + References + ---------- + .. [1] Receiver-operating characteristic (ROC) plots: a fundamental + evaluation tool in clinical medicine, MH Zweig, G Campbell - + Clinical chemistry, 1993 + """ def __init__(self, base_estimator, method='roc', pos_label=1, cv=3, min_val_tnr=None, min_val_tpr=None): From 6f9ce4ac7337e448c6619a45c6136ceb1c6297d3 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 16 Jan 2018 00:13:41 +0100 Subject: [PATCH 031/100] fix docstring --- sklearn/calibration.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index aa4dc5a5b4f18..a4ba5654d386e 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -32,18 +32,18 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): - """Meta estimator that uses a base estimator and finds a cutoff point - (decision threshold) to be used by predict. Applicable only on binary + """Meta estimator that calibrates the decision threshold (cutoff point) + that is used for prediction by a base estimator. Applicable only on binary classification problems. - The methods for picking "optimal" cutoff points are inferred from ROC - analysis; making use of true positive and true negative rates and their - corresponding thresholds. + The methods for picking cutoff points are inferred from ROC analysis; + making use of true positive and true negative rates and their corresponding + thresholds. If cv="prefit" the base estimator is assumed to be fitted and all data will - be used for the selection of the cutoff point that determines the output of - predict. Otherwise predict will use the average of the thresholds - resulting from the cross-validation loop. + be used for the selection of the cutoff point. Otherwise the decision + threshold is calculated as the average of the thresholds resulting from the + cross-validation loop. Parameters ---------- @@ -210,8 +210,8 @@ def __init__(self, base_estimator, method, pos_label, min_val_tnr, self.min_val_tpr = min_val_tpr def fit(self, X, y): - """Select a decision threshold based on an optimal cutoff point for the - fitted model's positive class + """Select a decision threshold for the fitted model's positive class + using one of the available methods Parameters ---------- From 8fda0c48f2839d7e957118e869e1536e10e4f1b8 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Wed, 17 Jan 2018 23:41:50 +0100 Subject: [PATCH 032/100] re-fix affiliation --- sklearn/calibration.py | 2 +- sklearn/tests/test_calibration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index a4ba5654d386e..6766726b2e6b4 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -4,7 +4,7 @@ # Balazs Kegl # Jan Hendrik Metzen # Mathieu Blondel -# Prokopis Gryllos +# Prokopios Gryllos # # License: BSD 3 clause diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 31f038bb8d40c..10a883dc704e2 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -1,5 +1,5 @@ # Authors: Alexandre Gramfort -# Prokopios Gryllos +# Prokopios Gryllos # License: BSD 3 clause from __future__ import division From cf0e8006bf103f61983146fabe55c3d6f20db123 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Wed, 17 Jan 2018 23:44:05 +0100 Subject: [PATCH 033/100] add example for roc decision threshold calibration method --- examples/calibration/README.txt | 2 +- .../calibration/plot_cutoff_calibration.py | 154 ++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 examples/calibration/plot_cutoff_calibration.py diff --git a/examples/calibration/README.txt b/examples/calibration/README.txt index 5e4a31b966b50..a820b63654f98 100644 --- a/examples/calibration/README.txt +++ b/examples/calibration/README.txt @@ -3,4 +3,4 @@ Calibration ----------------------- -Examples illustrating the calibration of predicted probabilities of classifiers. +Examples concerning the :mod:`sklearn.calibration` module. diff --git a/examples/calibration/plot_cutoff_calibration.py b/examples/calibration/plot_cutoff_calibration.py new file mode 100644 index 0000000000000..47dc6c18916d6 --- /dev/null +++ b/examples/calibration/plot_cutoff_calibration.py @@ -0,0 +1,154 @@ +""" +======================================================================= +Decision threshold (cutoff point) calibration for binary classification +======================================================================= + +Most machine learning classifiers that offer probability estimation do so by +optimizing for accuracy (minimizing the classification error). The class with +the highest probability, which can also be interpreted as confidence or score, +is the predicted one. For a binary classification task that sets the decision +threshold arbitrarily to 0.5. + +Depending on the classification task and the cost of error per class using an +arbitrary decision threshold of 0.5 can be elusive. Calibrating the decision +threshold to achieve better true positive rate or better true negative rate +or both can be a valid way for increasing the classifiers trustworthiness. + +This example illustrates how the decision threshold calibration can be used on +a binary classification task with imbalanced classes for finding a decision +threshold for a logistic regression and an AdaBoost classifier with the goal +to improve the sum each their respective true positive and true negative rates. +""" + +# Author: Prokopios Gryllos +# +# License: BSD 3 clause + +from __future__ import division + +import numpy as np + +from sklearn.ensemble import AdaBoostClassifier +from sklearn.metrics import confusion_matrix +from sklearn.calibration import CutoffClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.datasets import make_classification +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split + + +print(__doc__) + + +n_samples = 20000 +calibration_samples = 2000 + +X, y = make_classification(n_samples=n_samples, n_features=30, random_state=42, + n_classes=2, shuffle=True, flip_y=0.17, + n_informative=6) + +# unbalance dataset by removing 50% of the samples that belong to class 0 +indexes_to_delete = np.random.choice( + np.where(y == 0)[0], size=int((n_samples / 2) * 0.5) +) + +X = np.delete(X, indexes_to_delete, axis=0) +y = np.delete(y, indexes_to_delete, axis=0) + +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=0.4, random_state=42 +) + +# we hold out a part of the training dataset to use for calibration +clf_lr = LogisticRegression().fit( + X_train[:-calibration_samples], y_train[:-calibration_samples] +) + +clf_ada = AdaBoostClassifier().fit( + X_train[:-calibration_samples], y_train[:-calibration_samples] +) + +clf_lr_roc = CutoffClassifier(clf_lr, method='roc', cv='prefit').fit( + X_train[calibration_samples:], y_train[calibration_samples:] +) + +clf_ada_roc = CutoffClassifier(clf_ada, method='roc', cv='prefit').fit( + X_train[calibration_samples:], y_train[calibration_samples:] +) + +y_pred_lr = clf_lr.predict(X_test) +y_pred_ada = clf_ada.predict(X_test) +y_pred_lr_roc = clf_lr_roc.predict(X_test) +y_pred_ada_roc = clf_ada_roc.predict(X_test) + +tn_lr, fp_lr, fn_lr, tp_lr = confusion_matrix(y_test, y_pred_lr).ravel() +tn_ada, fp_ada, fn_ada, tp_ada = confusion_matrix(y_test, y_pred_ada).ravel() +tn_lr_roc, fp_lr_roc, fn_lr_roc, tp_lr_roc = \ + confusion_matrix(y_test, y_pred_lr_roc).ravel() +tn_ada_roc, fp_ada_roc, fn_ada_roc, tp_ada_roc = \ + confusion_matrix(y_test, y_pred_ada_roc).ravel() + +print('\n') +print('Calibrated threshold') +print('Logistic Regression classifier: {}'.format(clf_lr_roc.threshold_)) +print('AdaBoost classifier: {}'.format(clf_ada_roc.threshold_)) + +print('\n') +print('Sum of true positive and true negative rate before calibration') + +tpr_lr = tp_lr / (tp_lr + fn_lr) +tnr_lr = tn_lr / (tn_lr + fp_lr) + +print('Logistic Regression classifier: tpr + tnr = {} + {} = {}'.format( + tpr_lr, tnr_lr, tpr_lr + tnr_lr +)) + +tpr_ada = tp_ada / (tp_ada + fn_ada) +tnr_ada = tn_ada / (tn_ada + fp_ada) + +print('AdaBoost classifier: tpr + tnr = {} + {} = {}'.format( + tpr_ada, tnr_ada, tpr_ada + tnr_ada +)) + +print('\n') +print('Sum of true positive and true negative rate after calibration') + +tpr_lr_roc = tp_lr_roc / (tp_lr_roc + fn_lr_roc) +tnr_lr_roc = tn_lr_roc / (tn_lr_roc + fp_lr_roc) + +print('Logistic Regression classifier: tpr + tnr = {} + {} = {}'.format( + tpr_lr_roc, tnr_lr_roc, tpr_lr_roc + tnr_lr_roc +)) + +tpr_ada_roc = tp_ada_roc / (tp_ada_roc + fn_ada_roc) +tnr_ada_roc = tn_ada_roc / (tn_ada_roc + fp_ada_roc) + +print('AdaBoost classifier: tpr + tnr = {} + {} = {}'.format( + tpr_ada_roc, tnr_ada_roc, tpr_ada_roc + tnr_ada_roc +)) + +####### +# plots +####### +bar_width = 0.2 +opacity = 0.35 + +index = np.asarray([1, 2, 3, 4]) +plt.bar(index, [tpr_lr, tnr_lr, tpr_ada, tnr_ada], + bar_width, alpha=opacity, color='b', label='Before') + +plt.bar(index + bar_width, [tpr_lr_roc, tnr_lr_roc, tpr_ada_roc, tnr_ada_roc], + bar_width, alpha=opacity, color='g', label='After') + +plt.xticks( + index + bar_width / 2, + ('true positive rate logistic regression', + 'true negative rate logistic regression', + 'true positive rate adaboost', + 'true negative rate adaboost') +) +plt.ylabel('scores') +plt.title('Classifiers tpr and tnr before and after calibration') +plt.legend() +plt.show() + From 071ed09ed1267ab904c063a566b325de59dec768 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 18 Jan 2018 00:16:25 +0100 Subject: [PATCH 034/100] fix flake8 --- examples/calibration/plot_cutoff_calibration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/calibration/plot_cutoff_calibration.py b/examples/calibration/plot_cutoff_calibration.py index 47dc6c18916d6..6730eab7a9ca4 100644 --- a/examples/calibration/plot_cutoff_calibration.py +++ b/examples/calibration/plot_cutoff_calibration.py @@ -151,4 +151,3 @@ plt.title('Classifiers tpr and tnr before and after calibration') plt.legend() plt.show() - From fd648a5f968e6a1c9a109a501aae860b21cea389 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 28 Jan 2018 18:45:44 +0100 Subject: [PATCH 035/100] update plot title --- examples/calibration/plot_cutoff_calibration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/calibration/plot_cutoff_calibration.py b/examples/calibration/plot_cutoff_calibration.py index 6730eab7a9ca4..93114b76f47f1 100644 --- a/examples/calibration/plot_cutoff_calibration.py +++ b/examples/calibration/plot_cutoff_calibration.py @@ -148,6 +148,7 @@ 'true negative rate adaboost') ) plt.ylabel('scores') -plt.title('Classifiers tpr and tnr before and after calibration') +plt.title('tpr and tnr before and after calibration on Logistic Regression and' + 'Adaboost') plt.legend() plt.show() From 5110b3e13319cdd4374ba01b5372e3ce993c75c9 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 28 Jan 2018 18:46:45 +0100 Subject: [PATCH 036/100] add decision threshold calibration example on breast cancer dataset --- .../plot_cutoff_calibration_breast_cancer.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 examples/calibration/plot_cutoff_calibration_breast_cancer.py diff --git a/examples/calibration/plot_cutoff_calibration_breast_cancer.py b/examples/calibration/plot_cutoff_calibration_breast_cancer.py new file mode 100644 index 0000000000000..7cf4f94333d88 --- /dev/null +++ b/examples/calibration/plot_cutoff_calibration_breast_cancer.py @@ -0,0 +1,144 @@ +""" +====================================================================== +Decision threshold (cutoff point) calibration on breast cancer dataset +====================================================================== + +Most machine learning classifiers that offer probability estimation do so by +optimizing for accuracy (minimizing the classification error). The class with +the highest probability, which can also be interpreted as confidence or score, +is the predicted one. For a binary classification task that sets the decision +threshold arbitrarily to 0.5. + +Depending on the classification task and the cost of error per class using an +arbitrary decision threshold of 0.5 can be elusive. Calibrating the decision +threshold to achieve better true positive rate or better true negative rate +or both can be a valid way for increasing the classifiers trustworthiness. + +In this example the decision threshold calibration is applied on the breast +cancer dataset to maximize the true positive and true negative rate +respectively +""" + +# Author: Prokopios Gryllos +# +# License: BSD 3 clause + +from __future__ import division + +import numpy as np + +from sklearn.ensemble import AdaBoostClassifier +from sklearn.metrics import confusion_matrix +from sklearn.calibration import CutoffClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.datasets import load_breast_cancer +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split + + +print(__doc__) + +# percentage of the training set that will be used for calibration +calibration_samples_percentage = 0.3 + +X, y = load_breast_cancer(return_X_y=True) + +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=0.6, random_state=42 +) + +calibration_samples = int(len(X_train) * calibration_samples_percentage) + +clf_lr = LogisticRegression().fit( + X_train[:-calibration_samples], y_train[:-calibration_samples] +) + +clf_ada = AdaBoostClassifier().fit( + X_train[:-calibration_samples], y_train[:-calibration_samples] +) + +# we want to maximize the true positive rate while the true negative rate is at +# least 0.5 +clf_lr_max_tpr = CutoffClassifier( + clf_lr, method='max_tpr', cv=3, min_val_tnr=0.5 +).fit(X_train[calibration_samples:], y_train[calibration_samples:]) + +clf_ada_max_tpr = CutoffClassifier( + clf_ada, method='max_tpr', cv=3, min_val_tnr=0.5 +).fit(X_train[calibration_samples:], y_train[calibration_samples:]) + +y_pred_lr = clf_lr.predict(X_test) +y_pred_ada = clf_ada.predict(X_test) +y_pred_lr_max_tpr = clf_lr_max_tpr.predict(X_test) +y_pred_ada_max_tpr = clf_ada_max_tpr.predict(X_test) + +tn_lr, fp_lr, fn_lr, tp_lr = confusion_matrix(y_test, y_pred_lr).ravel() +tn_ada, fp_ada, fn_ada, tp_ada = confusion_matrix(y_test, y_pred_ada).ravel() + +tn_lr_max_tpr, fp_lr_max_tpr, fn_lr_max_tpr, tp_lr_max_tpr = \ + confusion_matrix(y_test, y_pred_lr_max_tpr).ravel() +tn_ada_max_tpr, fp_ada_max_tpr, fn_ada_max_tpr, tp_ada_max_tpr = \ + confusion_matrix(y_test, y_pred_ada_max_tpr).ravel() + +print('\n') +print('Calibrated threshold') +print('Logistic Regression classifier: {}'.format(clf_lr_max_tpr.threshold_)) +print('AdaBoost classifier: {}'.format(clf_ada_max_tpr.threshold_)) + +print('\n') +print('true positive and true negative rates before calibration') + +tpr_lr = tp_lr / (tp_lr + fn_lr) +tnr_lr = tn_lr / (tn_lr + fp_lr) + +print('Logistic Regression classifier: tpr = {}, tnr = {}'.format( + tpr_lr, tnr_lr +)) + +tpr_ada = tp_ada / (tp_ada + fn_ada) +tnr_ada = tn_ada / (tn_ada + fp_ada) + +print('AdaBoost classifier: tpr = {}, tpn = {}'.format(tpr_ada, tnr_ada)) + +print('\n') +print('true positive and true negative rates after calibration') + +tpr_lr_max_tpr = tp_lr_max_tpr / (tp_lr_max_tpr + fn_lr_max_tpr) +tnr_lr_max_tpr = tn_lr_max_tpr / (tn_lr_max_tpr + fp_lr_max_tpr) + +print('Logistic Regression classifier: tpr = {}, tnr = {}'.format( + tpr_lr_max_tpr, tnr_lr_max_tpr +)) + +tpr_ada_max_tpr = tp_ada_max_tpr / (tp_ada_max_tpr + fn_ada_max_tpr) +tnr_ada_max_tpr = tn_ada_max_tpr / (tn_ada_max_tpr + fp_ada_max_tpr) + +print('AdaBoost classifier: tpr = {}, tnr = {}'.format( + tpr_ada_max_tpr, tnr_ada_max_tpr +)) + +####### +# plots +####### +bar_width = 0.2 +opacity = 0.35 + +index = np.asarray([1, 2, 3, 4]) +plt.bar(index, [tpr_lr, tnr_lr, tpr_ada, tnr_ada], + bar_width, alpha=opacity, color='b', label='Before') + +plt.bar(index + bar_width, + [tpr_lr_max_tpr, tnr_lr_max_tpr, tpr_ada_max_tpr, tnr_ada_max_tpr], + bar_width, alpha=opacity, color='g', label='After') + +plt.xticks( + index + bar_width / 2, + ('true positive rate logistic regression', + 'true negative rate logistic regression', + 'true positive rate adaboost', + 'true negative rate adaboost') +) +plt.ylabel('scores') +plt.title('Classifiers tpr and tnr before and after calibration') +plt.legend() +plt.show() From 091dd37a51cd9f0daa9a040c81971f39a0340363 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 30 Jan 2018 23:33:09 +0100 Subject: [PATCH 037/100] fix docstring --- sklearn/calibration.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 6766726b2e6b4..6be4daa31fcfe 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -32,13 +32,12 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): - """Meta estimator that calibrates the decision threshold (cutoff point) - that is used for prediction by a base estimator. Applicable only on binary - classification problems. + """Decision threshold calibration for binary classification - The methods for picking cutoff points are inferred from ROC analysis; - making use of true positive and true negative rates and their corresponding - thresholds. + Meta estimator that calibrates the decision threshold (cutoff point) + that is used for prediction. The methods for picking cutoff points are + inferred from ROC analysis; making use of true positive and true negative + rates and their corresponding thresholds. If cv="prefit" the base estimator is assumed to be fitted and all data will be used for the selection of the cutoff point. Otherwise the decision @@ -54,8 +53,8 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): method : str The method to use for choosing the cutoff point. - - 'roc', selects the point on the roc_curve that is closer to the ideal - corner (0, 1) + - 'roc', selects the point on the roc_curve that is closest to the + ideal corner (0, 1) - 'max_tpr', selects the point that yields the highest true positive rate with true negative rate at least equal to the value of the From dc94b5285825c2c48ff466a7f30b2030d850f7ee Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 30 Jan 2018 23:39:10 +0100 Subject: [PATCH 038/100] rename min_val_tpr/tnr to min_tpr/tnr --- .../plot_cutoff_calibration_breast_cancer.py | 4 +- sklearn/calibration.py | 47 +++++++++---------- sklearn/tests/test_calibration.py | 4 +- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/examples/calibration/plot_cutoff_calibration_breast_cancer.py b/examples/calibration/plot_cutoff_calibration_breast_cancer.py index 7cf4f94333d88..c6315c1fa1f33 100644 --- a/examples/calibration/plot_cutoff_calibration_breast_cancer.py +++ b/examples/calibration/plot_cutoff_calibration_breast_cancer.py @@ -60,11 +60,11 @@ # we want to maximize the true positive rate while the true negative rate is at # least 0.5 clf_lr_max_tpr = CutoffClassifier( - clf_lr, method='max_tpr', cv=3, min_val_tnr=0.5 + clf_lr, method='max_tpr', cv=3, min_tnr=0.5 ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) clf_ada_max_tpr = CutoffClassifier( - clf_ada, method='max_tpr', cv=3, min_val_tnr=0.5 + clf_ada, method='max_tpr', cv=3, min_tnr=0.5 ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_lr = clf_lr.predict(X_test) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 6be4daa31fcfe..a2523743464bc 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -58,11 +58,11 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): - 'max_tpr', selects the point that yields the highest true positive rate with true negative rate at least equal to the value of the - parameter min_val_tnr + parameter min_tnr - 'max_tnr', selects the point that yields the highest true negative rate with true positive rate at least equal to the value of the - parameter min_val_tpr + parameter min_tpr pos_label : object Object representing the positive label @@ -74,11 +74,11 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): the calibration of the probability threshold (default value: "prefit") - min_val_tnr : float in [0, 1] + min_tnr : float in [0, 1] In case method = 'max_tpr' this value must be set to specify the minimum required value for the true negative rate - min_val_tpr : float in [0, 1] + min_tpr : float in [0, 1] In case method = 'max_tnr' this value must be set to specify the minimum required value for the true positive rate @@ -96,13 +96,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): """ def __init__(self, base_estimator, method='roc', pos_label=1, cv=3, - min_val_tnr=None, min_val_tpr=None): + min_tnr=None, min_tpr=None): self.base_estimator = base_estimator self.method = method self.pos_label = pos_label self.cv = cv - self.min_val_tnr = min_val_tnr - self.min_val_tpr = min_val_tpr + self.min_tnr = min_tnr + self.min_tpr = min_tpr def fit(self, X, y): """Fit model @@ -136,7 +136,7 @@ def fit(self, X, y): if self.cv == 'prefit': self.threshold_ = _CutoffClassifier( self.base_estimator, self.method, self.pos_label, - self.min_val_tnr, self.min_val_tpr + self.min_tnr, self.min_tpr ).fit(X, y).threshold_ else: cv = check_cv(self.cv, y, classifier=True) @@ -148,8 +148,8 @@ def fit(self, X, y): _CutoffClassifier(estimator, self.method, self.pos_label, - self.min_val_tnr, - self.min_val_tpr).fit( + self.min_tnr, + self.min_tpr).fit( X[test], y[test] ).threshold_ ) @@ -187,11 +187,11 @@ class should not be used as an estimator directly. Use the pos_label : object Label considered as positive during the roc_curve construction. - min_val_tnr : float in [0, 1] + min_tnr : float in [0, 1] minimum required value for true negative rate (specificity) in case method 'max_tpr' is used - min_val_tpr : float in [0, 1] + min_tpr : float in [0, 1] minimum required value for true positive rate (sensitivity) in case method 'max_tnr' is used @@ -200,13 +200,12 @@ class should not be used as an estimator directly. Use the threshold_ : float Acquired optimal decision threshold for the positive class """ - def __init__(self, base_estimator, method, pos_label, min_val_tnr, - min_val_tpr): + def __init__(self, base_estimator, method, pos_label, min_tnr, min_tpr): self.base_estimator = base_estimator self.method = method self.pos_label = pos_label - self.min_val_tnr = min_val_tnr - self.min_val_tpr = min_val_tpr + self.min_tnr = min_tnr + self.min_tpr = min_tpr def fit(self, X, y): """Select a decision threshold for the fitted model's positive class @@ -233,18 +232,18 @@ def fit(self, X, y): euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) )] elif self.method == 'max_tpr': - if not self.min_val_tnr or not isinstance(self.min_val_tnr, float)\ - or not self.min_val_tnr >= 0 or not self.min_val_tnr <= 1: + if not self.min_tnr or not isinstance(self.min_tnr, float)\ + or not self.min_tnr >= 0 or not self.min_tnr <= 1: raise ValueError('max_tnr must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.min_val_tnr)) - indices = np.where(1 - fpr >= self.min_val_tnr)[0] + 'Got %s instead' % repr(self.min_tnr)) + indices = np.where(1 - fpr >= self.min_tnr)[0] self.threshold_ = thresholds[indices[np.argmax(tpr[indices])]] elif self.method == 'max_tnr': - if not self.min_val_tpr or not isinstance(self.min_val_tpr, float)\ - or not self.min_val_tpr >= 0 or not self.min_val_tpr <= 1: + if not self.min_tpr or not isinstance(self.min_tpr, float)\ + or not self.min_tpr >= 0 or not self.min_tpr <= 1: raise ValueError('max_tpr must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.min_val_tnr)) - indices = np.where(tpr >= self.min_val_tpr)[0] + 'Got %s instead' % repr(self.min_tnr)) + indices = np.where(tpr >= self.min_tpr)[0] self.threshold_ = thresholds[indices[np.argmax(1 - fpr[indices])]] else: raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 10a883dc704e2..c123b5ac57b36 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -59,7 +59,7 @@ def test_cutoff_prefit(): assert_greater(tpr_roc + tnr_roc, tpr + tnr) clf_max_tpr = CutoffClassifier( - lr, method='max_tpr', cv='prefit', min_val_tnr=0.3 + lr, method='max_tpr', cv='prefit', min_tnr=0.3 ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) y_pred_max_tpr = clf_max_tpr.predict(X_test[calibration_samples:]) @@ -76,7 +76,7 @@ def test_cutoff_prefit(): assert_greater_equal(tnr_max_tpr, 0.3) clf_max_tnr = CutoffClassifier( - lr, method='max_tnr', cv='prefit', min_val_tpr=0.3 + lr, method='max_tnr', cv='prefit', min_tpr=0.3 ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) y_pred_clf = clf_max_tnr.predict(X_test[calibration_samples:]) From c3df7ae3a883c9e577727e630869ae8e20c50c65 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 12 Feb 2018 17:03:32 +0100 Subject: [PATCH 039/100] update example doc --- .../plot_cutoff_calibration_breast_cancer.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/examples/calibration/plot_cutoff_calibration_breast_cancer.py b/examples/calibration/plot_cutoff_calibration_breast_cancer.py index c6315c1fa1f33..fa50b3616a776 100644 --- a/examples/calibration/plot_cutoff_calibration_breast_cancer.py +++ b/examples/calibration/plot_cutoff_calibration_breast_cancer.py @@ -3,16 +3,14 @@ Decision threshold (cutoff point) calibration on breast cancer dataset ====================================================================== -Most machine learning classifiers that offer probability estimation do so by -optimizing for accuracy (minimizing the classification error). The class with -the highest probability, which can also be interpreted as confidence or score, -is the predicted one. For a binary classification task that sets the decision -threshold arbitrarily to 0.5. - -Depending on the classification task and the cost of error per class using an -arbitrary decision threshold of 0.5 can be elusive. Calibrating the decision -threshold to achieve better true positive rate or better true negative rate -or both can be a valid way for increasing the classifiers trustworthiness. +Machine learning classifiers often base their predictions on real-valued +decision functions that don't always have accuracy as their objective. Moreover +the learning objective of a model can differ from the user's needs hence using +an arbitrary decision threshold as defined by the model can be not ideal. + +The CutoffClassifier can be used to calibrate the decision threshold of a model +in order to increase the true positive rate or the true negative rate or their +sum as a way to improve the classifiers trustworthiness. In this example the decision threshold calibration is applied on the breast cancer dataset to maximize the true positive and true negative rate From 4f1b936058b4bc920c9a2be5d3d1a810c39760d6 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 12 Feb 2018 17:03:54 +0100 Subject: [PATCH 040/100] rm redundant example --- .../calibration/plot_cutoff_calibration.py | 154 ------------------ 1 file changed, 154 deletions(-) delete mode 100644 examples/calibration/plot_cutoff_calibration.py diff --git a/examples/calibration/plot_cutoff_calibration.py b/examples/calibration/plot_cutoff_calibration.py deleted file mode 100644 index 93114b76f47f1..0000000000000 --- a/examples/calibration/plot_cutoff_calibration.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -======================================================================= -Decision threshold (cutoff point) calibration for binary classification -======================================================================= - -Most machine learning classifiers that offer probability estimation do so by -optimizing for accuracy (minimizing the classification error). The class with -the highest probability, which can also be interpreted as confidence or score, -is the predicted one. For a binary classification task that sets the decision -threshold arbitrarily to 0.5. - -Depending on the classification task and the cost of error per class using an -arbitrary decision threshold of 0.5 can be elusive. Calibrating the decision -threshold to achieve better true positive rate or better true negative rate -or both can be a valid way for increasing the classifiers trustworthiness. - -This example illustrates how the decision threshold calibration can be used on -a binary classification task with imbalanced classes for finding a decision -threshold for a logistic regression and an AdaBoost classifier with the goal -to improve the sum each their respective true positive and true negative rates. -""" - -# Author: Prokopios Gryllos -# -# License: BSD 3 clause - -from __future__ import division - -import numpy as np - -from sklearn.ensemble import AdaBoostClassifier -from sklearn.metrics import confusion_matrix -from sklearn.calibration import CutoffClassifier -from sklearn.linear_model import LogisticRegression -from sklearn.datasets import make_classification -import matplotlib.pyplot as plt -from sklearn.model_selection import train_test_split - - -print(__doc__) - - -n_samples = 20000 -calibration_samples = 2000 - -X, y = make_classification(n_samples=n_samples, n_features=30, random_state=42, - n_classes=2, shuffle=True, flip_y=0.17, - n_informative=6) - -# unbalance dataset by removing 50% of the samples that belong to class 0 -indexes_to_delete = np.random.choice( - np.where(y == 0)[0], size=int((n_samples / 2) * 0.5) -) - -X = np.delete(X, indexes_to_delete, axis=0) -y = np.delete(y, indexes_to_delete, axis=0) - -X_train, X_test, y_train, y_test = train_test_split( - X, y, train_size=0.4, random_state=42 -) - -# we hold out a part of the training dataset to use for calibration -clf_lr = LogisticRegression().fit( - X_train[:-calibration_samples], y_train[:-calibration_samples] -) - -clf_ada = AdaBoostClassifier().fit( - X_train[:-calibration_samples], y_train[:-calibration_samples] -) - -clf_lr_roc = CutoffClassifier(clf_lr, method='roc', cv='prefit').fit( - X_train[calibration_samples:], y_train[calibration_samples:] -) - -clf_ada_roc = CutoffClassifier(clf_ada, method='roc', cv='prefit').fit( - X_train[calibration_samples:], y_train[calibration_samples:] -) - -y_pred_lr = clf_lr.predict(X_test) -y_pred_ada = clf_ada.predict(X_test) -y_pred_lr_roc = clf_lr_roc.predict(X_test) -y_pred_ada_roc = clf_ada_roc.predict(X_test) - -tn_lr, fp_lr, fn_lr, tp_lr = confusion_matrix(y_test, y_pred_lr).ravel() -tn_ada, fp_ada, fn_ada, tp_ada = confusion_matrix(y_test, y_pred_ada).ravel() -tn_lr_roc, fp_lr_roc, fn_lr_roc, tp_lr_roc = \ - confusion_matrix(y_test, y_pred_lr_roc).ravel() -tn_ada_roc, fp_ada_roc, fn_ada_roc, tp_ada_roc = \ - confusion_matrix(y_test, y_pred_ada_roc).ravel() - -print('\n') -print('Calibrated threshold') -print('Logistic Regression classifier: {}'.format(clf_lr_roc.threshold_)) -print('AdaBoost classifier: {}'.format(clf_ada_roc.threshold_)) - -print('\n') -print('Sum of true positive and true negative rate before calibration') - -tpr_lr = tp_lr / (tp_lr + fn_lr) -tnr_lr = tn_lr / (tn_lr + fp_lr) - -print('Logistic Regression classifier: tpr + tnr = {} + {} = {}'.format( - tpr_lr, tnr_lr, tpr_lr + tnr_lr -)) - -tpr_ada = tp_ada / (tp_ada + fn_ada) -tnr_ada = tn_ada / (tn_ada + fp_ada) - -print('AdaBoost classifier: tpr + tnr = {} + {} = {}'.format( - tpr_ada, tnr_ada, tpr_ada + tnr_ada -)) - -print('\n') -print('Sum of true positive and true negative rate after calibration') - -tpr_lr_roc = tp_lr_roc / (tp_lr_roc + fn_lr_roc) -tnr_lr_roc = tn_lr_roc / (tn_lr_roc + fp_lr_roc) - -print('Logistic Regression classifier: tpr + tnr = {} + {} = {}'.format( - tpr_lr_roc, tnr_lr_roc, tpr_lr_roc + tnr_lr_roc -)) - -tpr_ada_roc = tp_ada_roc / (tp_ada_roc + fn_ada_roc) -tnr_ada_roc = tn_ada_roc / (tn_ada_roc + fp_ada_roc) - -print('AdaBoost classifier: tpr + tnr = {} + {} = {}'.format( - tpr_ada_roc, tnr_ada_roc, tpr_ada_roc + tnr_ada_roc -)) - -####### -# plots -####### -bar_width = 0.2 -opacity = 0.35 - -index = np.asarray([1, 2, 3, 4]) -plt.bar(index, [tpr_lr, tnr_lr, tpr_ada, tnr_ada], - bar_width, alpha=opacity, color='b', label='Before') - -plt.bar(index + bar_width, [tpr_lr_roc, tnr_lr_roc, tpr_ada_roc, tnr_ada_roc], - bar_width, alpha=opacity, color='g', label='After') - -plt.xticks( - index + bar_width / 2, - ('true positive rate logistic regression', - 'true negative rate logistic regression', - 'true positive rate adaboost', - 'true negative rate adaboost') -) -plt.ylabel('scores') -plt.title('tpr and tnr before and after calibration on Logistic Regression and' - 'Adaboost') -plt.legend() -plt.show() From 09af8ae4ed97efe7802e70df0e4d7aea672db688 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 12 Feb 2018 17:08:00 +0100 Subject: [PATCH 041/100] remove @ignore_warning from tests --- sklearn/tests/test_calibration.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index c123b5ac57b36..bc861394634c1 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -26,7 +26,6 @@ from sklearn.calibration import calibration_curve -@ignore_warnings def test_cutoff_prefit(): calibration_samples = 200 X, y = make_classification(n_samples=1000, n_features=6, random_state=42, @@ -110,7 +109,6 @@ def test_cutoff_prefit(): assert_raises(ValueError, clf_missing_info.fit, X_train, y_train) -@ignore_warnings def test_cutoff_cv(): X, y = make_classification(n_samples=1000, n_features=6, random_state=42, n_classes=2) From cb35eb9cce8104f16fb3c26c2373c1bc9f73ebcb Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 12 Feb 2018 17:57:51 +0100 Subject: [PATCH 042/100] simplify min distance point calculation --- sklearn/calibration.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index cd527fd841787..350fd43cf6f72 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -228,23 +228,25 @@ def fit(self, X, y): fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) if self.method == 'roc': - self.threshold_ = thresholds[np.argmin( - euclidean_distances(np.column_stack((fpr, tpr)), [[0, 1]]) - )] + # we find the threshold of the point (fpr, tpr) with the smallest + # euclidean distance from the "ideal" corner (0, 1) + self.threshold_ = thresholds[np.argmin(fpr**2 + (tpr - 1)**2)] elif self.method == 'max_tpr': if not self.min_tnr or not isinstance(self.min_tnr, float)\ or not self.min_tnr >= 0 or not self.min_tnr <= 1: raise ValueError('max_tnr must be a number in [1, 0]. ' 'Got %s instead' % repr(self.min_tnr)) indices = np.where(1 - fpr >= self.min_tnr)[0] - self.threshold_ = thresholds[indices[np.argmax(tpr[indices])]] + max_tpr_index = np.argmax(tpr[indices]) + self.threshold_ = thresholds[indices[max_tpr_index]] elif self.method == 'max_tnr': if not self.min_tpr or not isinstance(self.min_tpr, float)\ or not self.min_tpr >= 0 or not self.min_tpr <= 1: raise ValueError('max_tpr must be a number in [1, 0]. ' 'Got %s instead' % repr(self.min_tnr)) indices = np.where(tpr >= self.min_tpr)[0] - self.threshold_ = thresholds[indices[np.argmax(1 - fpr[indices])]] + max_tnr_index = np.argmax(1 - fpr[indices]) + self.threshold_ = thresholds[indices[max_tnr_index]] else: raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' 'Got %s instead' % self.method) From 8214fcd8b595ea42ca5bd9c2a5bd5246ff1a4d9f Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 12 Feb 2018 21:48:15 +0100 Subject: [PATCH 043/100] add docstring for predict --- sklearn/calibration.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 350fd43cf6f72..5c0902a1ee637 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -158,6 +158,18 @@ def fit(self, X, y): return self def predict(self, X): + """Predict using the calibrated decision threshold + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + The samples. + + Returns + ------- + C : array, shape (n_samples,) + The predicted class. + """ X = check_array(X) check_is_fitted(self, ["label_encoder_", "threshold_"]) From 73ab4c9460f2877ae9d860cafaee1c2d37540473 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 12 Feb 2018 22:51:51 +0100 Subject: [PATCH 044/100] remove unused import --- sklearn/calibration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 5c0902a1ee637..758294ce142ea 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -27,7 +27,6 @@ from .svm import LinearSVC from .model_selection import check_cv from .metrics.classification import _check_binary_probabilistic_predictions -from .metrics.pairwise import euclidean_distances from .metrics.ranking import roc_curve From 11697e9f82eb30eebd509e4cfc477d7b496430d2 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 13 Feb 2018 22:35:58 +0100 Subject: [PATCH 045/100] move validation in the beginning of fit - base estimator does not have to be instance of BaseEstimator --- sklearn/calibration.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 758294ce142ea..3427c4d02e697 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -28,6 +28,7 @@ from .model_selection import check_cv from .metrics.classification import _check_binary_probabilistic_predictions from .metrics.ranking import roc_curve +from .utils.multiclass import type_of_target class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): @@ -45,7 +46,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Parameters ---------- - base_estimator : instance BaseEstimator + base_estimator : obj The classifier whose decision threshold will be adapted according to the acquired cutoff point @@ -119,15 +120,29 @@ def fit(self, X, y): self : object Instance of self. """ - if not isinstance(self.base_estimator, BaseEstimator): - raise AttributeError('Base estimator must be of type BaseEstimator' - ' got %s instead' % type(self.base_estimator)) + if self.method not in ['roc', 'max_tpr', 'max_tnr']: + raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' + 'Got %s instead' % self.method) + + if self.method == 'max_tpr': + if not self.min_tnr or not isinstance(self.min_tnr, float) \ + or not self.min_tnr >= 0 or not self.min_tnr <= 1: + raise ValueError('max_tnr must be a number in [1, 0]. ' + 'Got %s instead' % repr(self.min_tnr)) + + elif self.method == 'max_tnr': + if not self.min_tpr or not isinstance(self.min_tpr, float) \ + or not self.min_tpr >= 0 or not self.min_tpr <= 1: + raise ValueError('max_tpr must be a number in [1, 0]. ' + 'Got %s instead' % repr(self.min_tnr)) X, y = check_X_y(X, y) + y_type = type_of_target(y) + if y_type is not 'binary': + raise ValueError('Expected target of binary type. Got %s ' % y_type) + self.label_encoder_ = LabelEncoder().fit(y) - if len(self.label_encoder_.classes_) > 2: - raise ValueError('Found more than two distinct values in target y') y = self.label_encoder_.transform(y) self.pos_label = self.label_encoder_.transform([self.pos_label])[0] @@ -188,7 +203,7 @@ class should not be used as an estimator directly. Use the Parameters ---------- - base_estimator : instance BaseEstimator + base_estimator : obj The classifier whose decision threshold will be adapted according to the acquired optimal cutoff point @@ -243,24 +258,13 @@ def fit(self, X, y): # euclidean distance from the "ideal" corner (0, 1) self.threshold_ = thresholds[np.argmin(fpr**2 + (tpr - 1)**2)] elif self.method == 'max_tpr': - if not self.min_tnr or not isinstance(self.min_tnr, float)\ - or not self.min_tnr >= 0 or not self.min_tnr <= 1: - raise ValueError('max_tnr must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.min_tnr)) indices = np.where(1 - fpr >= self.min_tnr)[0] max_tpr_index = np.argmax(tpr[indices]) self.threshold_ = thresholds[indices[max_tpr_index]] - elif self.method == 'max_tnr': - if not self.min_tpr or not isinstance(self.min_tpr, float)\ - or not self.min_tpr >= 0 or not self.min_tpr <= 1: - raise ValueError('max_tpr must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.min_tnr)) + else: indices = np.where(tpr >= self.min_tpr)[0] max_tnr_index = np.argmax(1 - fpr[indices]) self.threshold_ = thresholds[indices[max_tnr_index]] - else: - raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' - 'Got %s instead' % self.method) return self From d91aa354c5ea7ff411f96cc0f95bfed834a36862 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 17 Feb 2018 22:03:56 +0100 Subject: [PATCH 046/100] enable cutoff point estimation on decision_function --- sklearn/calibration.py | 89 +++++++++++++++++++++++++++---- sklearn/tests/test_calibration.py | 45 ++++++++++++++++ 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 3427c4d02e697..a068e98c9112d 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -48,7 +48,8 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): ---------- base_estimator : obj The classifier whose decision threshold will be adapted according to - the acquired cutoff point + the acquired cutoff point. The estimator must have a decision_function + or a predict_proba. method : str The method to use for choosing the cutoff point. @@ -64,6 +65,18 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): rate with true positive rate at least equal to the value of the parameter min_tpr + scoring : str or None, optional (default=None) + The method to be used for acquiring the score. + + - 'decision_function'. base_estimator.decision_function will be used + for scoring. + + - 'predict_proba'. base_estimator.predict_proba will be used for + scoring + + - None. base_estimator.decision_function will be used first and if not + available base_estimator.predict_proba. + pos_label : object Object representing the positive label (default value: 1) @@ -95,10 +108,11 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Clinical chemistry, 1993 """ - def __init__(self, base_estimator, method='roc', pos_label=1, cv=3, - min_tnr=None, min_tpr=None): + def __init__(self, base_estimator, method='roc', scoring=None, pos_label=1, + cv=3, min_tnr=None, min_tpr=None): self.base_estimator = base_estimator self.method = method + self.scoring = scoring self.pos_label = pos_label self.cv = cv self.min_tnr = min_tnr @@ -120,6 +134,11 @@ def fit(self, X, y): self : object Instance of self. """ + if not hasattr(self.base_estimator, 'decision_function') and \ + not hasattr(self.base_estimator, 'predict_proba'): + raise TypeError('The base_estimator needs to have either a ' + 'decision_function or a predict_proba method') + if self.method not in ['roc', 'max_tpr', 'max_tnr']: raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' 'Got %s instead' % self.method) @@ -149,7 +168,7 @@ def fit(self, X, y): if self.cv == 'prefit': self.threshold_ = _CutoffClassifier( - self.base_estimator, self.method, self.pos_label, + self.base_estimator, self.method, self.scoring, self.pos_label, self.min_tnr, self.min_tpr ).fit(X, y).threshold_ else: @@ -161,6 +180,7 @@ def fit(self, X, y): thresholds.append( _CutoffClassifier(estimator, self.method, + self.scoring, self.pos_label, self.min_tnr, self.min_tpr).fit( @@ -187,9 +207,10 @@ def predict(self, X): X = check_array(X) check_is_fitted(self, ["label_encoder_", "threshold_"]) + y_score = _get_binary_score(self.base_estimator, X, self.pos_label, + self.scoring) return self.label_encoder_.inverse_transform( - (self.base_estimator.predict_proba(X)[:, self.pos_label] > - self.threshold_).astype(int) + (y_score > self.threshold_).astype(int) ) @@ -205,11 +226,18 @@ class should not be used as an estimator directly. Use the ---------- base_estimator : obj The classifier whose decision threshold will be adapted according to - the acquired optimal cutoff point + the acquired cutoff point. The estimator must have a decision_function + or a predict_proba. method : 'roc' or 'max_tpr' or 'max_tnr' The method to use for choosing the cutoff point. + scoring : str or None, optional (default=None) + The method to be used for acquiring the score. Can either be + "decision_function" or "predict_proba" or None. If None then + decision_function will be used first and if not available + predict_proba. + pos_label : object Label considered as positive during the roc_curve construction. @@ -226,9 +254,11 @@ class should not be used as an estimator directly. Use the threshold_ : float Acquired optimal decision threshold for the positive class """ - def __init__(self, base_estimator, method, pos_label, min_tnr, min_tpr): + def __init__(self, base_estimator, method, scoring, pos_label, min_tnr, + min_tpr): self.base_estimator = base_estimator self.method = method + self.scoring = scoring self.pos_label = pos_label self.min_tnr = min_tnr self.min_tpr = min_tpr @@ -250,7 +280,8 @@ def fit(self, X, y): self : object Instance of self. """ - y_score = self.base_estimator.predict_proba(X)[:, self.pos_label] + y_score = _get_binary_score(self.base_estimator, X, self.pos_label, + self.scoring) fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) if self.method == 'roc': @@ -268,6 +299,46 @@ def fit(self, X, y): return self +def _get_binary_score(clf, X, pos_label=1, scoring=None): + """Binary classification score for the positive label (0 or 1) + + Returns the score that a binary classifier for the positive label acquired + either from decision_function or predict_proba + + Parameters + ---------- + clf : object + Classifier object to be used for acquiring the scores. Needs to have + a decision_function or a predict_proba method. + + X : array-like, shape (n_samples, n_features) + The samples. + + pos_label : int, optional (default=1) + The positive label. Can either be 0 or 1. + + scoring : str or None, optional (default=None) + The method to be used for acquiring the score. Can either be + "decision_function" or "predict_proba" or None. If None then + decision_function will be used first and if not available + predict_proba. + """ + if not scoring: + try: + y_score = clf.decision_function(X) + if pos_label == 0: + y_score = - y_score + except (NotImplementedError, AttributeError): + y_score = clf.predict_proba(X)[:, pos_label] + elif scoring == 'decision_function': + y_score = clf.decision_function(X) + if pos_label == 0: + y_score = - y_score + else: + y_score = clf.predict_proba(X)[:, pos_label] + return y_score + + class CalibratedClassifierCV(BaseEstimator, ClassifierMixin): """Probability calibration with isotonic regression or sigmoid. diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index bc861394634c1..01eeddfdb86c4 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -22,6 +22,7 @@ from sklearn.preprocessing import Imputer from sklearn.metrics import brier_score_loss, log_loss, confusion_matrix from sklearn.calibration import CalibratedClassifierCV, CutoffClassifier +from sklearn.calibration import _get_binary_score from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration from sklearn.calibration import calibration_curve @@ -139,6 +140,50 @@ def test_cutoff_cv(): assert_greater(tpr_roc + tnr_roc, tpr + tnr) +def test_get_binary_score(): + X, y = make_classification(n_samples=200, n_features=6, random_state=42, + n_classes=2) + + X_train, X_test, y_train, _ = train_test_split(X, y, train_size=0.6, + random_state=42) + lr = LogisticRegression().fit(X_train, y_train) + y_pred_proba = lr.predict_proba(X_test) + y_pred_score = lr.decision_function(X_test) + + assert_array_equal( + y_pred_score, + _get_binary_score(lr, X_test, pos_label=1, scoring='decision_function') + ) + + assert_array_equal( + - y_pred_score, + _get_binary_score(lr, X_test, pos_label=0, scoring='decision_function') + ) + + assert_array_equal( + y_pred_proba[:, 1], + _get_binary_score(lr, X_test, pos_label=1, scoring='predict_proba') + ) + + assert_array_equal( + y_pred_proba[:, 0], + _get_binary_score(lr, X_test, pos_label=0, scoring='predict_proba') + ) + + assert_array_equal( + y_pred_score, + _get_binary_score(lr, X_test, pos_label=1, scoring=None) + ) + + # classifier that does not have a decision_function + rf = RandomForestClassifier().fit(X_train, y_train) + y_pred_proba_rf = rf.predict_proba(X_test) + assert_array_equal( + y_pred_proba_rf[:, 1], + _get_binary_score(rf, X_test, pos_label=1, scoring=None) + ) + + @ignore_warnings def test_calibration(): """Test calibration objects with isotonic and sigmoid""" From 4d36f4e47e98bcbae67a11865aa659edaf9179a5 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 17 Feb 2018 22:06:53 +0100 Subject: [PATCH 047/100] fix docstring --- sklearn/calibration.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index a068e98c9112d..089a671a7bfa4 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -77,15 +77,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): - None. base_estimator.decision_function will be used first and if not available base_estimator.predict_proba. - pos_label : object + pos_label : object, optional (default=1) Object representing the positive label - (default value: 1) - cv : int, cross-validation generator, iterable or "prefit" (optional) - Determines the cross-validation splitting strategy. If cv="prefit" the - base estimator is assumed to be fitted and all data will be used for - the calibration of the probability threshold - (default value: "prefit") + cv : int, cross-validation generator, iterable or 'prefit', optional + (default='prefit'). Determines the cross-validation splitting strategy. + If cv='prefit' the base estimator is assumed to be fitted and all data + will be used for the calibration of the probability threshold. min_tnr : float in [0, 1] In case method = 'max_tpr' this value must be set to specify the From 45c2d4f34142d84ef6801852b9bd854f5f3595d2 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 17 Feb 2018 22:10:26 +0100 Subject: [PATCH 048/100] make naming consistent --- sklearn/tests/test_calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 01eeddfdb86c4..f868390b0b9bf 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -93,8 +93,8 @@ def test_cutoff_prefit(): assert_greater_equal(tpr_clf_max_tnr, 0.3) # check error cases - clf_non_base_estimator = CutoffClassifier([]) - assert_raises(AttributeError, clf_non_base_estimator.fit, X_train, y_train) + clf_bad_base_estimator = CutoffClassifier([]) + assert_raises(TypeError, clf_bad_base_estimator.fit, X_train, y_train) X_non_binary, y_non_binary = make_classification( n_samples=20, n_features=6, random_state=42, n_classes=4, From d4d406b3c40e0cf83778717c9d7a43706dd19b41 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 17 Feb 2018 22:10:47 +0100 Subject: [PATCH 049/100] fix flake8 --- sklearn/calibration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 089a671a7bfa4..6457056f952fd 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -157,7 +157,8 @@ def fit(self, X, y): y_type = type_of_target(y) if y_type is not 'binary': - raise ValueError('Expected target of binary type. Got %s ' % y_type) + raise ValueError('Expected target of binary type. Got %s ' % + y_type) self.label_encoder_ = LabelEncoder().fit(y) From c862de4d7e27b939f918a573e17fb8e2d08ce3fe Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sat, 17 Feb 2018 22:52:06 +0100 Subject: [PATCH 050/100] fix lgtm --- sklearn/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 6457056f952fd..902b9f504e406 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -156,7 +156,7 @@ def fit(self, X, y): X, y = check_X_y(X, y) y_type = type_of_target(y) - if y_type is not 'binary': + if y_type != 'binary': raise ValueError('Expected target of binary type. Got %s ' % y_type) From 69377145807fff18af7c06c3acbd63b3e25ffe93 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Feb 2018 01:24:51 +0100 Subject: [PATCH 051/100] extend validation checks for scoring param --- sklearn/calibration.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 902b9f504e406..f630cc68ad8fd 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -134,12 +134,17 @@ def fit(self, X, y): """ if not hasattr(self.base_estimator, 'decision_function') and \ not hasattr(self.base_estimator, 'predict_proba'): - raise TypeError('The base_estimator needs to have either a ' + raise TypeError('The base_estimator needs to implement either a ' 'decision_function or a predict_proba method') if self.method not in ['roc', 'max_tpr', 'max_tnr']: - raise ValueError('method must be "roc" or "max_tpr" or "max_tnr.' - 'Got %s instead' % self.method) + raise ValueError('method can either be "roc" or "max_tpr" or ' + '"max_tnr. Got %s instead' % self.method) + + if self.scoring not in [None, 'decision_function', 'predict_proba']: + raise ValueError('scoring param can either be "decision_function" ' + 'or "predict_proba" or None. Got %s instead' % + self.scoring) if self.method == 'max_tpr': if not self.min_tnr or not isinstance(self.min_tnr, float) \ From 59888aa4163e3cb54601197ac57d0c01c76823b3 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Feb 2018 01:26:43 +0100 Subject: [PATCH 052/100] fix docstring --- sklearn/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index f630cc68ad8fd..a2b76bb5c405d 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -306,8 +306,8 @@ def fit(self, X, y): def _get_binary_score(clf, X, pos_label=1, scoring=None): """Binary classification score for the positive label (0 or 1) - Returns the score that a binary classifier for the positive label acquired - either from decision_function or predict_proba + Returns the score that a binary classifier outputs for the positive label + acquired either from decision_function or predict_proba Parameters ---------- From f9eaa66772eed68f26f376ee1b88d06e53be260e Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Feb 2018 01:28:01 +0100 Subject: [PATCH 053/100] change signature of _get_binary_score to be consistent --- sklearn/calibration.py | 10 +++++----- sklearn/tests/test_calibration.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index a2b76bb5c405d..287dc6f7f4649 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -211,8 +211,8 @@ def predict(self, X): X = check_array(X) check_is_fitted(self, ["label_encoder_", "threshold_"]) - y_score = _get_binary_score(self.base_estimator, X, self.pos_label, - self.scoring) + y_score = _get_binary_score(self.base_estimator, X, self.scoring, + self.pos_label) return self.label_encoder_.inverse_transform( (y_score > self.threshold_).astype(int) ) @@ -284,8 +284,8 @@ def fit(self, X, y): self : object Instance of self. """ - y_score = _get_binary_score(self.base_estimator, X, self.pos_label, - self.scoring) + y_score = _get_binary_score(self.base_estimator, X, self.scoring, + self.pos_label) fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) if self.method == 'roc': @@ -303,7 +303,7 @@ def fit(self, X, y): return self -def _get_binary_score(clf, X, pos_label=1, scoring=None): +def _get_binary_score(clf, X, scoring=None, pos_label=1): """Binary classification score for the positive label (0 or 1) Returns the score that a binary classifier outputs for the positive label diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index bd1c57b7a1112..0168ad27f5ad5 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -152,27 +152,27 @@ def test_get_binary_score(): assert_array_equal( y_pred_score, - _get_binary_score(lr, X_test, pos_label=1, scoring='decision_function') + _get_binary_score(lr, X_test, scoring='decision_function', pos_label=1) ) assert_array_equal( - y_pred_score, - _get_binary_score(lr, X_test, pos_label=0, scoring='decision_function') + _get_binary_score(lr, X_test, scoring='decision_function', pos_label=0) ) assert_array_equal( y_pred_proba[:, 1], - _get_binary_score(lr, X_test, pos_label=1, scoring='predict_proba') + _get_binary_score(lr, X_test, scoring='predict_proba', pos_label=1) ) assert_array_equal( y_pred_proba[:, 0], - _get_binary_score(lr, X_test, pos_label=0, scoring='predict_proba') + _get_binary_score(lr, X_test, scoring='predict_proba', pos_label=0) ) assert_array_equal( y_pred_score, - _get_binary_score(lr, X_test, pos_label=1, scoring=None) + _get_binary_score(lr, X_test, scoring=None, pos_label=1) ) # classifier that does not have a decision_function @@ -180,7 +180,7 @@ def test_get_binary_score(): y_pred_proba_rf = rf.predict_proba(X_test) assert_array_equal( y_pred_proba_rf[:, 1], - _get_binary_score(rf, X_test, pos_label=1, scoring=None) + _get_binary_score(rf, X_test, scoring=None, pos_label=1) ) From 47d8d2c47c081ed5106014a371b6b3af91a9e05e Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Mar 2018 21:16:38 +0100 Subject: [PATCH 054/100] fix docstring --- sklearn/calibration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 287dc6f7f4649..8a35b37c29b90 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -219,12 +219,12 @@ def predict(self, X): class _CutoffClassifier(object): - """Optimal cutoff point selection. + """Cutoff point selection. It assumes that base_estimator has already been fit, and uses the input set - of the fit function to select an optimal cutoff point. Note that this - class should not be used as an estimator directly. Use the - OptimalCutoffClassifier with cv="prefit" instead. + of the fit function to select a cutoff point. Note that this class should + not be used as an estimator directly. Use the CutoffClassifier with + cv="prefit" instead. Parameters ---------- From 59e97cfc66a726dfcf4f92f91daa0be445c86986 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Mar 2018 21:36:21 +0100 Subject: [PATCH 055/100] rename threshold_ to decision_threshold_ --- sklearn/calibration.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 8a35b37c29b90..1efb7c60f1792 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -95,7 +95,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Attributes ---------- - threshold_ : float + decision_threshold_ : float Decision threshold for the positive class. Determines the output of predict @@ -171,17 +171,17 @@ def fit(self, X, y): self.pos_label = self.label_encoder_.transform([self.pos_label])[0] if self.cv == 'prefit': - self.threshold_ = _CutoffClassifier( + self.decision_threshold_ = _CutoffClassifier( self.base_estimator, self.method, self.scoring, self.pos_label, self.min_tnr, self.min_tpr - ).fit(X, y).threshold_ + ).fit(X, y).decision_threshold_ else: cv = check_cv(self.cv, y, classifier=True) - thresholds = [] + decision_thresholds = [] for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) - thresholds.append( + decision_thresholds.append( _CutoffClassifier(estimator, self.method, self.scoring, @@ -189,9 +189,10 @@ def fit(self, X, y): self.min_tnr, self.min_tpr).fit( X[test], y[test] - ).threshold_ + ).decision_threshold_ ) - self.threshold_ = sum(thresholds) / len(thresholds) + self.decision_threshold_ = sum(decision_thresholds) /\ + len(decision_thresholds) self.base_estimator.fit(X, y) return self @@ -209,12 +210,12 @@ def predict(self, X): The predicted class. """ X = check_array(X) - check_is_fitted(self, ["label_encoder_", "threshold_"]) + check_is_fitted(self, ["label_encoder_", "decision_threshold_"]) y_score = _get_binary_score(self.base_estimator, X, self.scoring, self.pos_label) return self.label_encoder_.inverse_transform( - (y_score > self.threshold_).astype(int) + (y_score > self.decision_threshold_).astype(int) ) @@ -255,8 +256,8 @@ class _CutoffClassifier(object): Attributes ---------- - threshold_ : float - Acquired optimal decision threshold for the positive class + decision_threshold_ : float + Acquired decision threshold for the positive class """ def __init__(self, base_estimator, method, scoring, pos_label, min_tnr, min_tpr): @@ -291,15 +292,17 @@ def fit(self, X, y): if self.method == 'roc': # we find the threshold of the point (fpr, tpr) with the smallest # euclidean distance from the "ideal" corner (0, 1) - self.threshold_ = thresholds[np.argmin(fpr**2 + (tpr - 1)**2)] + self.decision_threshold_ = thresholds[ + np.argmin(fpr ** 2 + (tpr - 1) ** 2) + ] elif self.method == 'max_tpr': indices = np.where(1 - fpr >= self.min_tnr)[0] max_tpr_index = np.argmax(tpr[indices]) - self.threshold_ = thresholds[indices[max_tpr_index]] + self.decision_threshold_ = thresholds[indices[max_tpr_index]] else: indices = np.where(tpr >= self.min_tpr)[0] max_tnr_index = np.argmax(1 - fpr[indices]) - self.threshold_ = thresholds[indices[max_tnr_index]] + self.decision_threshold_ = thresholds[indices[max_tnr_index]] return self From 93ee09189864575ea9ec174e6d0eff4cc17d5031 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Mar 2018 22:21:27 +0100 Subject: [PATCH 056/100] replace params min_tnr, min_tpr with param threshold --- sklearn/calibration.py | 58 +++++++++++-------------------- sklearn/tests/test_calibration.py | 4 +-- 2 files changed, 23 insertions(+), 39 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 1efb7c60f1792..6583f63acb66e 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -85,13 +85,10 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): If cv='prefit' the base estimator is assumed to be fitted and all data will be used for the calibration of the probability threshold. - min_tnr : float in [0, 1] - In case method = 'max_tpr' this value must be set to specify the - minimum required value for the true negative rate - - min_tpr : float in [0, 1] - In case method = 'max_tnr' this value must be set to specify the - minimum required value for the true positive rate + threshold : float in [0, 1] or None, (default=None) + In case method is 'max_tpr' or 'max_tnr' this parameter must be set to + specify the threshold for the true negative rate or true positive rate + respectively that needs to be achieved Attributes ---------- @@ -107,14 +104,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): """ def __init__(self, base_estimator, method='roc', scoring=None, pos_label=1, - cv=3, min_tnr=None, min_tpr=None): + cv=3, threshold=None): self.base_estimator = base_estimator self.method = method self.scoring = scoring self.pos_label = pos_label self.cv = cv - self.min_tnr = min_tnr - self.min_tpr = min_tpr + self.threshold = threshold def fit(self, X, y): """Fit model @@ -146,17 +142,11 @@ def fit(self, X, y): 'or "predict_proba" or None. Got %s instead' % self.scoring) - if self.method == 'max_tpr': - if not self.min_tnr or not isinstance(self.min_tnr, float) \ - or not self.min_tnr >= 0 or not self.min_tnr <= 1: - raise ValueError('max_tnr must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.min_tnr)) - - elif self.method == 'max_tnr': - if not self.min_tpr or not isinstance(self.min_tpr, float) \ - or not self.min_tpr >= 0 or not self.min_tpr <= 1: - raise ValueError('max_tpr must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.min_tnr)) + if self.method == 'max_tpr' or self.method == 'max_tnr': + if not self.threshold or not isinstance(self.threshold, float) \ + or not self.threshold >= 0 or not self.threshold <= 1: + raise ValueError('threshold must be a number in [1, 0]. ' + 'Got %s instead' % repr(self.threshold)) X, y = check_X_y(X, y) @@ -173,7 +163,7 @@ def fit(self, X, y): if self.cv == 'prefit': self.decision_threshold_ = _CutoffClassifier( self.base_estimator, self.method, self.scoring, self.pos_label, - self.min_tnr, self.min_tpr + self.threshold ).fit(X, y).decision_threshold_ else: cv = check_cv(self.cv, y, classifier=True) @@ -186,8 +176,7 @@ def fit(self, X, y): self.method, self.scoring, self.pos_label, - self.min_tnr, - self.min_tpr).fit( + self.threshold).fit( X[test], y[test] ).decision_threshold_ ) @@ -246,27 +235,22 @@ class _CutoffClassifier(object): pos_label : object Label considered as positive during the roc_curve construction. - min_tnr : float in [0, 1] - minimum required value for true negative rate (specificity) in case - method 'max_tpr' is used - - min_tpr : float in [0, 1] - minimum required value for true positive rate (sensitivity) in case - method 'max_tnr' is used + threshold : float in [0, 1] + minimum required value for the true negative rate (specificity) in case + method 'max_tpr' is used or for the true positive rate (sensitivity) in + case method 'max_tnr' is used Attributes ---------- decision_threshold_ : float Acquired decision threshold for the positive class """ - def __init__(self, base_estimator, method, scoring, pos_label, min_tnr, - min_tpr): + def __init__(self, base_estimator, method, scoring, pos_label, threshold): self.base_estimator = base_estimator self.method = method self.scoring = scoring self.pos_label = pos_label - self.min_tnr = min_tnr - self.min_tpr = min_tpr + self.threshold = threshold def fit(self, X, y): """Select a decision threshold for the fitted model's positive class @@ -296,11 +280,11 @@ def fit(self, X, y): np.argmin(fpr ** 2 + (tpr - 1) ** 2) ] elif self.method == 'max_tpr': - indices = np.where(1 - fpr >= self.min_tnr)[0] + indices = np.where(1 - fpr >= self.threshold)[0] max_tpr_index = np.argmax(tpr[indices]) self.decision_threshold_ = thresholds[indices[max_tpr_index]] else: - indices = np.where(tpr >= self.min_tpr)[0] + indices = np.where(tpr >= self.threshold)[0] max_tnr_index = np.argmax(1 - fpr[indices]) self.decision_threshold_ = thresholds[indices[max_tnr_index]] return self diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 0168ad27f5ad5..806d2fbe129e1 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -59,7 +59,7 @@ def test_cutoff_prefit(): assert_greater(tpr_roc + tnr_roc, tpr + tnr) clf_max_tpr = CutoffClassifier( - lr, method='max_tpr', cv='prefit', min_tnr=0.3 + lr, method='max_tpr', cv='prefit', threshold=0.3 ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) y_pred_max_tpr = clf_max_tpr.predict(X_test[calibration_samples:]) @@ -76,7 +76,7 @@ def test_cutoff_prefit(): assert_greater_equal(tnr_max_tpr, 0.3) clf_max_tnr = CutoffClassifier( - lr, method='max_tnr', cv='prefit', min_tpr=0.3 + lr, method='max_tnr', cv='prefit', threshold=0.3 ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) y_pred_clf = clf_max_tnr.predict(X_test[calibration_samples:]) From 26b934cc928d9582048d4554bba29dac1cf94137 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 18 Mar 2018 22:21:55 +0100 Subject: [PATCH 057/100] fix example --- .../plot_cutoff_calibration_breast_cancer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/calibration/plot_cutoff_calibration_breast_cancer.py b/examples/calibration/plot_cutoff_calibration_breast_cancer.py index fa50b3616a776..1fc77d4ebc70f 100644 --- a/examples/calibration/plot_cutoff_calibration_breast_cancer.py +++ b/examples/calibration/plot_cutoff_calibration_breast_cancer.py @@ -12,9 +12,9 @@ in order to increase the true positive rate or the true negative rate or their sum as a way to improve the classifiers trustworthiness. -In this example the decision threshold calibration is applied on the breast -cancer dataset to maximize the true positive and true negative rate -respectively +In this example the decision threshold calibration is applied on two +classifiers trained on the breast cancer dataset. The goal is to maximize the +true positive rate while maintaining a minimum true negative rate. """ # Author: Prokopios Gryllos @@ -58,11 +58,11 @@ # we want to maximize the true positive rate while the true negative rate is at # least 0.5 clf_lr_max_tpr = CutoffClassifier( - clf_lr, method='max_tpr', cv=3, min_tnr=0.5 + clf_lr, method='max_tpr', cv=3, threshold=0.7, scoring='predict_proba' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) clf_ada_max_tpr = CutoffClassifier( - clf_ada, method='max_tpr', cv=3, min_tnr=0.5 + clf_ada, method='max_tpr', cv=3, threshold=0.7, scoring='predict_proba' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_lr = clf_lr.predict(X_test) @@ -80,8 +80,10 @@ print('\n') print('Calibrated threshold') -print('Logistic Regression classifier: {}'.format(clf_lr_max_tpr.threshold_)) -print('AdaBoost classifier: {}'.format(clf_ada_max_tpr.threshold_)) +print('Logistic Regression classifier: {}'.format( + clf_lr_max_tpr.decision_threshold_ +)) +print('AdaBoost classifier: {}'.format(clf_ada_max_tpr.decision_threshold_)) print('\n') print('true positive and true negative rates before calibration') From 7c1326c078ae24a0db9c613d79c3616697f26340 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 8 Apr 2018 16:12:33 +0200 Subject: [PATCH 058/100] add support for f_beta --- sklearn/calibration.py | 46 +++++++++++++++++++++++-------- sklearn/tests/test_calibration.py | 31 +++++++++++++++------ 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 6583f63acb66e..bc7f931c58529 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -27,7 +27,7 @@ from .svm import LinearSVC from .model_selection import check_cv from .metrics.classification import _check_binary_probabilistic_predictions -from .metrics.ranking import roc_curve +from .metrics.ranking import precision_recall_curve, roc_curve from .utils.multiclass import type_of_target @@ -103,10 +103,11 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Clinical chemistry, 1993 """ - def __init__(self, base_estimator, method='roc', scoring=None, pos_label=1, - cv=3, threshold=None): + def __init__(self, base_estimator, method='roc', beta=None, scoring=None, + pos_label=1, cv=3, threshold=None): self.base_estimator = base_estimator self.method = method + self.beta = beta self.scoring = scoring self.pos_label = pos_label self.cv = cv @@ -133,7 +134,7 @@ def fit(self, X, y): raise TypeError('The base_estimator needs to implement either a ' 'decision_function or a predict_proba method') - if self.method not in ['roc', 'max_tpr', 'max_tnr']: + if self.method not in ['roc', 'f_beta', 'max_tpr', 'max_tnr']: raise ValueError('method can either be "roc" or "max_tpr" or ' '"max_tnr. Got %s instead' % self.method) @@ -143,10 +144,18 @@ def fit(self, X, y): self.scoring) if self.method == 'max_tpr' or self.method == 'max_tnr': - if not self.threshold or not isinstance(self.threshold, float) \ + if not self.threshold or not \ + isinstance(self.threshold, (int, float)) \ or not self.threshold >= 0 or not self.threshold <= 1: - raise ValueError('threshold must be a number in [1, 0]. ' - 'Got %s instead' % repr(self.threshold)) + raise ValueError('parameter threshold must be a number in' + '[0, 1]. Got %s instead' % + repr(self.threshold)) + + if self.method == 'f_beta': + if not self.beta or not isinstance(self.beta, (int, float)) \ + or not self.beta >= 0 or not self.beta <= 1: + raise ValueError('parameter beta must be a number in [0, 1]. ' + 'Got %s instead' % repr(self.beta)) X, y = check_X_y(X, y) @@ -162,8 +171,8 @@ def fit(self, X, y): if self.cv == 'prefit': self.decision_threshold_ = _CutoffClassifier( - self.base_estimator, self.method, self.scoring, self.pos_label, - self.threshold + self.base_estimator, self.method, self.beta, self.scoring, + self.pos_label, self.threshold ).fit(X, y).decision_threshold_ else: cv = check_cv(self.cv, y, classifier=True) @@ -174,6 +183,7 @@ def fit(self, X, y): decision_thresholds.append( _CutoffClassifier(estimator, self.method, + self.beta, self.scoring, self.pos_label, self.threshold).fit( @@ -223,9 +233,12 @@ class _CutoffClassifier(object): the acquired cutoff point. The estimator must have a decision_function or a predict_proba. - method : 'roc' or 'max_tpr' or 'max_tnr' + method : 'roc' or 'f_beta' or 'max_tpr' or 'max_tnr' The method to use for choosing the cutoff point. + beta : float in [0, 1] + beta value to be used in case method == 'f_beta' + scoring : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then @@ -245,9 +258,11 @@ class _CutoffClassifier(object): decision_threshold_ : float Acquired decision threshold for the positive class """ - def __init__(self, base_estimator, method, scoring, pos_label, threshold): + def __init__(self, base_estimator, method, beta, scoring, pos_label, + threshold): self.base_estimator = base_estimator self.method = method + self.beta = beta self.scoring = scoring self.pos_label = pos_label self.threshold = threshold @@ -271,6 +286,15 @@ def fit(self, X, y): """ y_score = _get_binary_score(self.base_estimator, X, self.scoring, self.pos_label) + if self.method == 'f_beta': + precision, recall, thresholds = precision_recall_curve( + y, y_score, self.pos_label + ) + f_beta = (1 + self.beta**2) * (precision * recall) /\ + (self.beta**2 * precision + recall) + self.decision_threshold_ = thresholds[np.argmax(f_beta)] + return self + fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) if self.method == 'roc': diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 806d2fbe129e1..5945c79f88053 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -20,7 +20,8 @@ from sklearn.svm import LinearSVC from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer -from sklearn.metrics import brier_score_loss, log_loss, confusion_matrix +from sklearn.metrics import brier_score_loss, log_loss, confusion_matrix,\ + f1_score from sklearn.calibration import CalibratedClassifierCV, CutoffClassifier from sklearn.calibration import _get_binary_score from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration @@ -38,7 +39,7 @@ def test_cutoff_prefit(): lr = LogisticRegression().fit(X_train, y_train) clf_roc = CutoffClassifier(lr, method='roc', cv='prefit').fit( - X_test[:calibration_samples], y_train[:calibration_samples] + X_test[:calibration_samples], y_test[:calibration_samples] ) y_pred = lr.predict(X_test[calibration_samples:]) @@ -55,12 +56,21 @@ def test_cutoff_prefit(): tpr_roc = tp_roc / (tp_roc + fn_roc) tnr_roc = tn_roc / (tn_roc + fp_roc) - # check that the sum of tpr + tnr has improved + # check that the sum of tpr and tnr has improved assert_greater(tpr_roc + tnr_roc, tpr + tnr) + clf_f1 = CutoffClassifier( + lr, method='f_beta', beta=1, cv='prefit', scoring='predict_proba').fit( + X_test[:calibration_samples], y_test[:calibration_samples] + ) + + y_pred_f1 = clf_f1.predict(X_test[calibration_samples:]) + assert_greater(f1_score(y_test[calibration_samples:], y_pred_f1), + f1_score(y_test[calibration_samples:], y_pred)) + clf_max_tpr = CutoffClassifier( - lr, method='max_tpr', cv='prefit', threshold=0.3 - ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) + lr, method='max_tpr', cv='prefit', threshold=0.7 + ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) y_pred_max_tpr = clf_max_tpr.predict(X_test[calibration_samples:]) @@ -73,11 +83,11 @@ def test_cutoff_prefit(): # check that the tpr increases with tnr >= min_val_tnr assert_greater(tpr_max_tpr, tpr) assert_greater(tpr_max_tpr, tpr_roc) - assert_greater_equal(tnr_max_tpr, 0.3) + assert_greater_equal(tnr_max_tpr, 0.7) clf_max_tnr = CutoffClassifier( - lr, method='max_tnr', cv='prefit', threshold=0.3 - ).fit(X_test[:calibration_samples], y_train[:calibration_samples]) + lr, method='max_tnr', cv='prefit', threshold=0.7 + ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) y_pred_clf = clf_max_tnr.predict(X_test[calibration_samples:]) @@ -90,7 +100,7 @@ def test_cutoff_prefit(): # check that the tnr increases with tpr >= min_val_tpr assert_greater(tnr_clf_max_tnr, tnr) assert_greater(tnr_clf_max_tnr, tnr_roc) - assert_greater_equal(tpr_clf_max_tnr, 0.3) + assert_greater_equal(tpr_clf_max_tnr, 0.7) # check error cases clf_bad_base_estimator = CutoffClassifier([]) @@ -102,6 +112,9 @@ def test_cutoff_prefit(): ) assert_raises(ValueError, clf_roc.fit, X_non_binary, y_non_binary) + clf_foo = CutoffClassifier(lr, method='f_beta', beta='foo') + assert_raises(ValueError, clf_foo.fit, X_train, y_train) + clf_foo = CutoffClassifier(lr, method='foo') assert_raises(ValueError, clf_foo.fit, X_train, y_train) From 6ff615e372a1451f93d1eeb06d21c72ce3c00c89 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 8 Apr 2018 16:12:49 +0200 Subject: [PATCH 059/100] update docstring --- sklearn/calibration.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index bc7f931c58529..25f33e7c3f506 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -35,9 +35,9 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): """Decision threshold calibration for binary classification Meta estimator that calibrates the decision threshold (cutoff point) - that is used for prediction. The methods for picking cutoff points are - inferred from ROC analysis; making use of true positive and true negative - rates and their corresponding thresholds. + that is used for prediction. The methods for picking cutoff points make use + of traditional binary classification evaluation statistic such as the + true positive and true negative rates and F-scores. If cv="prefit" the base estimator is assumed to be fitted and all data will be used for the selection of the cutoff point. Otherwise the decision @@ -54,16 +54,22 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): method : str The method to use for choosing the cutoff point. - - 'roc', selects the point on the roc_curve that is closest to the + - 'roc', selects the point on the roc curve that is closest to the ideal corner (0, 1) + - 'f_beta', selects a decision threshold that maximizes the f_beta + score. + - 'max_tpr', selects the point that yields the highest true positive rate with true negative rate at least equal to the value of the - parameter min_tnr + parameter threshold - 'max_tnr', selects the point that yields the highest true negative rate with true positive rate at least equal to the value of the - parameter min_tpr + parameter threshold + + beta : float in [0, 1], optional (default=None) + beta value to be used in case method == 'f_beta' scoring : str or None, optional (default=None) The method to be used for acquiring the score. From 52fdb9137b337ff6762f49f3a2ae20a09a807ef2 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 17:28:28 +0200 Subject: [PATCH 060/100] update example --- ...=> plot_decision_threshold_calibration.py} | 140 ++++++++++-------- 1 file changed, 76 insertions(+), 64 deletions(-) rename examples/calibration/{plot_cutoff_calibration_breast_cancer.py => plot_decision_threshold_calibration.py} (53%) diff --git a/examples/calibration/plot_cutoff_calibration_breast_cancer.py b/examples/calibration/plot_decision_threshold_calibration.py similarity index 53% rename from examples/calibration/plot_cutoff_calibration_breast_cancer.py rename to examples/calibration/plot_decision_threshold_calibration.py index 1fc77d4ebc70f..55341630a02f0 100644 --- a/examples/calibration/plot_cutoff_calibration_breast_cancer.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -9,8 +9,9 @@ an arbitrary decision threshold as defined by the model can be not ideal. The CutoffClassifier can be used to calibrate the decision threshold of a model -in order to increase the true positive rate or the true negative rate or their -sum as a way to improve the classifiers trustworthiness. +in order to increase the classifiers trustworthiness. Optimization objectives +during the decision threshold calibration can be the true positive and / or +the true negative rate as well as the f beta score. In this example the decision threshold calibration is applied on two classifiers trained on the breast cancer dataset. The goal is to maximize the @@ -26,7 +27,7 @@ import numpy as np from sklearn.ensemble import AdaBoostClassifier -from sklearn.metrics import confusion_matrix +from sklearn.metrics import confusion_matrix, f1_score from sklearn.calibration import CutoffClassifier from sklearn.linear_model import LogisticRegression from sklearn.datasets import load_breast_cancer @@ -37,99 +38,109 @@ print(__doc__) # percentage of the training set that will be used for calibration -calibration_samples_percentage = 0.3 +calibration_samples_percentage = 0.2 X, y = load_breast_cancer(return_X_y=True) -X_train, X_test, y_train, y_test = train_test_split( - X, y, train_size=0.6, random_state=42 -) +X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.6, + random_state=42) calibration_samples = int(len(X_train) * calibration_samples_percentage) -clf_lr = LogisticRegression().fit( - X_train[:-calibration_samples], y_train[:-calibration_samples] -) - -clf_ada = AdaBoostClassifier().fit( - X_train[:-calibration_samples], y_train[:-calibration_samples] -) - -# we want to maximize the true positive rate while the true negative rate is at -# least 0.5 -clf_lr_max_tpr = CutoffClassifier( - clf_lr, method='max_tpr', cv=3, threshold=0.7, scoring='predict_proba' -).fit(X_train[calibration_samples:], y_train[calibration_samples:]) - -clf_ada_max_tpr = CutoffClassifier( - clf_ada, method='max_tpr', cv=3, threshold=0.7, scoring='predict_proba' -).fit(X_train[calibration_samples:], y_train[calibration_samples:]) - -y_pred_lr = clf_lr.predict(X_test) -y_pred_ada = clf_ada.predict(X_test) -y_pred_lr_max_tpr = clf_lr_max_tpr.predict(X_test) -y_pred_ada_max_tpr = clf_ada_max_tpr.predict(X_test) - +lr = LogisticRegression().fit( + X_train[:-calibration_samples], y_train[:-calibration_samples]) +y_pred_lr = lr.predict(X_test) tn_lr, fp_lr, fn_lr, tp_lr = confusion_matrix(y_test, y_pred_lr).ravel() -tn_ada, fp_ada, fn_ada, tp_ada = confusion_matrix(y_test, y_pred_ada).ravel() - -tn_lr_max_tpr, fp_lr_max_tpr, fn_lr_max_tpr, tp_lr_max_tpr = \ - confusion_matrix(y_test, y_pred_lr_max_tpr).ravel() -tn_ada_max_tpr, fp_ada_max_tpr, fn_ada_max_tpr, tp_ada_max_tpr = \ - confusion_matrix(y_test, y_pred_ada_max_tpr).ravel() - -print('\n') -print('Calibrated threshold') -print('Logistic Regression classifier: {}'.format( - clf_lr_max_tpr.decision_threshold_ -)) -print('AdaBoost classifier: {}'.format(clf_ada_max_tpr.decision_threshold_)) - -print('\n') -print('true positive and true negative rates before calibration') - tpr_lr = tp_lr / (tp_lr + fn_lr) tnr_lr = tn_lr / (tn_lr + fp_lr) +f_one_lr = f1_score(y_test, y_pred_lr) -print('Logistic Regression classifier: tpr = {}, tnr = {}'.format( - tpr_lr, tnr_lr -)) - +ada = AdaBoostClassifier().fit( + X_train[:-calibration_samples], y_train[:-calibration_samples]) +y_pred_ada = ada.predict(X_test) +tn_ada, fp_ada, fn_ada, tp_ada = confusion_matrix(y_test, y_pred_ada).ravel() tpr_ada = tp_ada / (tp_ada + fn_ada) tnr_ada = tn_ada / (tn_ada + fp_ada) +f_one_ada = f1_score(y_test, y_pred_ada) -print('AdaBoost classifier: tpr = {}, tpn = {}'.format(tpr_ada, tnr_ada)) +# objective 1: we want to calibrate the decision threshold in order to achieve +# better f1 score +lr_f_beta = CutoffClassifier( + lr, method='f_beta', beta=1, cv='prefit', scoring='predict_proba' +).fit(X_train[calibration_samples:], y_train[calibration_samples:]) +y_pred_lr_f_beta = lr_f_beta.predict(X_test) +f_one_lr_f_beta = f1_score(y_test, y_pred_lr_f_beta) -print('\n') -print('true positive and true negative rates after calibration') +ada_f_beta = CutoffClassifier( + ada, method='f_beta', beta=1, cv='prefit', scoring='predict_proba' +).fit(X_train[calibration_samples:], y_train[calibration_samples:]) +y_pred_ada_f_beta = ada_f_beta.predict(X_test) +f_one_ada_f_beta = f1_score(y_test, y_pred_ada_f_beta) +# objective 2: we want to maximize the true positive rate while the true +# negative rate is at least 0.7 +lr_max_tpr = CutoffClassifier( + lr, method='max_tpr', cv='prefit', threshold=0.7, scoring='predict_proba' +).fit(X_train[calibration_samples:], y_train[calibration_samples:]) +y_pred_lr_max_tpr = lr_max_tpr.predict(X_test) +tn_lr_max_tpr, fp_lr_max_tpr, fn_lr_max_tpr, tp_lr_max_tpr = \ + confusion_matrix(y_test, y_pred_lr_max_tpr).ravel() tpr_lr_max_tpr = tp_lr_max_tpr / (tp_lr_max_tpr + fn_lr_max_tpr) tnr_lr_max_tpr = tn_lr_max_tpr / (tn_lr_max_tpr + fp_lr_max_tpr) -print('Logistic Regression classifier: tpr = {}, tnr = {}'.format( - tpr_lr_max_tpr, tnr_lr_max_tpr -)) - +ada_max_tpr = CutoffClassifier( + ada, method='max_tpr', cv='prefit', threshold=0.7, scoring='predict_proba' +).fit(X_train[calibration_samples:], y_train[calibration_samples:]) +y_pred_ada_max_tpr = ada_max_tpr.predict(X_test) +tn_ada_max_tpr, fp_ada_max_tpr, fn_ada_max_tpr, tp_ada_max_tpr = \ + confusion_matrix(y_test, y_pred_ada_max_tpr).ravel() tpr_ada_max_tpr = tp_ada_max_tpr / (tp_ada_max_tpr + fn_ada_max_tpr) tnr_ada_max_tpr = tn_ada_max_tpr / (tn_ada_max_tpr + fp_ada_max_tpr) -print('AdaBoost classifier: tpr = {}, tnr = {}'.format( - tpr_ada_max_tpr, tnr_ada_max_tpr -)) +print('Calibrated threshold') +print('Logistic Regression classifier: {}'.format( + lr_max_tpr.decision_threshold_)) +print('AdaBoost classifier: {}'.format(ada_max_tpr.decision_threshold_)) +print('before calibration') +print('Logistic Regression classifier: tpr = {}, tnr = {}, f1 = {}'.format( + tpr_lr, tnr_lr, f_one_lr)) +print('AdaBoost classifier: tpr = {}, tpn = {}, f1 = {}'.format( + tpr_ada, tnr_ada, f_one_ada)) + +print('true positive and true negative rates after calibration') +print('Logistic Regression classifier: tpr = {}, tnr = {}, f1 = {}'.format( + tpr_lr_max_tpr, tnr_lr_max_tpr, f_one_lr_f_beta)) +print('AdaBoost classifier: tpr = {}, tnr = {}, f1 = {}'.format( + tpr_ada_max_tpr, tnr_ada_max_tpr, f_one_ada_f_beta)) ####### # plots ####### bar_width = 0.2 -opacity = 0.35 +plt.subplot(2, 1, 1) +index = np.asarray([1, 2]) +plt.bar(index, [f_one_lr, f_one_ada], bar_width, color='r', label='Before') + +plt.bar(index + bar_width, [f_one_lr_f_beta, f_one_ada_f_beta], bar_width, + color='b', label='After') + +plt.xticks( + index + bar_width / 2, + ('f1 score logistic regression', 'f1 score adaboost')) + +plt.ylabel('scores') +plt.title('Classifiers f1 score before and after calibration') +plt.legend() + +plt.subplot(2, 1, 2) index = np.asarray([1, 2, 3, 4]) plt.bar(index, [tpr_lr, tnr_lr, tpr_ada, tnr_ada], - bar_width, alpha=opacity, color='b', label='Before') + bar_width, color='r', label='Before') plt.bar(index + bar_width, [tpr_lr_max_tpr, tnr_lr_max_tpr, tpr_ada_max_tpr, tnr_ada_max_tpr], - bar_width, alpha=opacity, color='g', label='After') + bar_width, color='b', label='After') plt.xticks( index + bar_width / 2, @@ -141,4 +152,5 @@ plt.ylabel('scores') plt.title('Classifiers tpr and tnr before and after calibration') plt.legend() + plt.show() From 02b61ea3e0daf4cf39c16a504bed7aea12a4920f Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 17:28:50 +0200 Subject: [PATCH 061/100] fix docstring --- sklearn/calibration.py | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 25f33e7c3f506..0c00b676a5602 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -36,7 +36,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Meta estimator that calibrates the decision threshold (cutoff point) that is used for prediction. The methods for picking cutoff points make use - of traditional binary classification evaluation statistic such as the + of traditional binary classification evaluation statistics such as the true positive and true negative rates and F-scores. If cv="prefit" the base estimator is assumed to be fitted and all data will @@ -49,16 +49,16 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): base_estimator : obj The classifier whose decision threshold will be adapted according to the acquired cutoff point. The estimator must have a decision_function - or a predict_proba. + or a predict_proba method : str - The method to use for choosing the cutoff point. + The method to use for choosing the cutoff point - 'roc', selects the point on the roc curve that is closest to the ideal corner (0, 1) - 'f_beta', selects a decision threshold that maximizes the f_beta - score. + score - 'max_tpr', selects the point that yields the highest true positive rate with true negative rate at least equal to the value of the @@ -72,16 +72,16 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): beta value to be used in case method == 'f_beta' scoring : str or None, optional (default=None) - The method to be used for acquiring the score. + The method to be used for acquiring the score - 'decision_function'. base_estimator.decision_function will be used - for scoring. + for scoring - 'predict_proba'. base_estimator.predict_proba will be used for scoring - None. base_estimator.decision_function will be used first and if not - available base_estimator.predict_proba. + available base_estimator.predict_proba pos_label : object, optional (default=1) Object representing the positive label @@ -89,7 +89,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): cv : int, cross-validation generator, iterable or 'prefit', optional (default='prefit'). Determines the cross-validation splitting strategy. If cv='prefit' the base estimator is assumed to be fitted and all data - will be used for the calibration of the probability threshold. + will be used for the calibration of the probability threshold threshold : float in [0, 1] or None, (default=None) In case method is 'max_tpr' or 'max_tnr' this parameter must be set to @@ -128,12 +128,12 @@ def fit(self, X, y): Training data y : array-like, shape (n_samples,) - Target values. There must be two 2 distinct values. + Target values. There must be two 2 distinct values Returns ------- self : object - Instance of self. + Instance of self """ if not hasattr(self.base_estimator, 'decision_function') and \ not hasattr(self.base_estimator, 'predict_proba'): @@ -207,12 +207,12 @@ def predict(self, X): Parameters ---------- X : array-like, shape (n_samples, n_features) - The samples. + The samples Returns ------- C : array, shape (n_samples,) - The predicted class. + The predicted class """ X = check_array(X) check_is_fitted(self, ["label_encoder_", "decision_threshold_"]) @@ -237,10 +237,10 @@ class _CutoffClassifier(object): base_estimator : obj The classifier whose decision threshold will be adapted according to the acquired cutoff point. The estimator must have a decision_function - or a predict_proba. + or a predict_proba method : 'roc' or 'f_beta' or 'max_tpr' or 'max_tnr' - The method to use for choosing the cutoff point. + The method to use for choosing the cutoff point beta : float in [0, 1] beta value to be used in case method == 'f_beta' @@ -249,10 +249,10 @@ class _CutoffClassifier(object): The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available - predict_proba. + predict_proba pos_label : object - Label considered as positive during the roc_curve construction. + Label considered as positive during the roc_curve construction threshold : float in [0, 1] minimum required value for the true negative rate (specificity) in case @@ -280,15 +280,15 @@ def fit(self, X, y): Parameters ---------- X : array-like, shape (n_samples, n_features) - Training data. + Training data y : array-like, shape (n_samples,) - Target values. + Target values Returns ------- self : object - Instance of self. + Instance of self """ y_score = _get_binary_score(self.base_estimator, X, self.scoring, self.pos_label) @@ -330,19 +330,19 @@ def _get_binary_score(clf, X, scoring=None, pos_label=1): ---------- clf : object Classifier object to be used for acquiring the scores. Needs to have - a decision_function or a predict_proba method. + a decision_function or a predict_proba method X : array-like, shape (n_samples, n_features) - The samples. + The samples pos_label : int, optional (default=1) - The positive label. Can either be 0 or 1. + The positive label. Can either be 0 or 1 scoring : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available - predict_proba. + predict_proba """ if not scoring: try: From 73a3609042b4194aae63465703c043747310acd5 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 17:29:07 +0200 Subject: [PATCH 062/100] add user guide --- doc/modules/calibration.rst | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index d7bb10479ce63..8bdd04bc24d26 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -205,3 +205,109 @@ a similar decrease in log-loss. .. [5] On the combination of forecast probabilities for consecutive precipitation periods. Wea. Forecasting, 5, 640–650., Wilks, D. S., 1990a + + +Decision Threshold calibration +============================== + +Often Machine Learning classifiers base their predictions on real-valued decision +functions or probability estimates that carry the inherited biases of their models. +Additionally when using a machine learning model the evaluation criteria can differ +from the optimisation objectives used by the model during training. + +When predicting between two classes it is commonly advised that an appropriate +decision threshold is estimated based on some cutoff criteria rather than arbitrarily +using the midpoint of the space of possible values. Estimating a decision threshold +for a specific use case can help to increase the overall accuracy of the model and +provide better handling for sensitive classes. + +.. currentmodule:: sklearn.calibration + +For example the :class:`LogisticRegression` classifier is predicting the class +for which the :func:`decision_function` returns the highest value. For a binary +classification task that sets decision threshold to ``0``. + +:class:`CutoffClassifier` can be used as a wrapper around a model for binary +classification to help obtain a more appropriate decision threshold and use it +for predicting new samples. + +Usage +----- + +To use the :class:`CutoffClassifier` you need to provide an estimator that has +a ``decision_function`` or a ``predict_proba`` method. The ``scoring`` parameter +controls whether the first will be preferred over the second if both are available. + +The wrapped estimator can be pre-trained, in which case ``cv='prefit'``, or not. If +the classifier is not trained then a cross-validation loop specified by the parameter +``cv`` can be used to obtain a decision threshold by averaging all decision thresholds +calculated on the hold-out parts of each cross validation iteration. Finally the model +is trained on all the provided data. When using ``cv='prefit'`` you need to make sure +to use a hold-out part of your data for calibration. + +The methods for finding appropriate decision thresholds are based either on precision +recall estimates or true positive and true negative rates. Specifically: + +* ``f_beta`` + selects a decision threshold that maximizes the f_beta score. The value of + beta is specified by the parameter ``beta`` + +* ``roc`` + selects the decision threshold for the point on the roc curve that is + closest to the ideal corner (0, 1) + +* ``max_tpr`` + selects the decision threshold for the point that yields the highest true positive + rate while maintaining a minimum, specified by the parameter ``threshold``, for the + true negative rate + +* ``max_tnr`` + selects the decision threshold for the point that yields the highest true + negative rate while maintaining a minimum, specified by the parameter ``threshold``, + for the true positive rate + +Here is a simple usage example:: + + >>> from sklearn.calibration import CutoffClassifier + >>> from sklearn.datasets import load_breast_cancer + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.model_selection import train_test_split + >>> + >>> X, y = load_breast_cancer(return_X_y=True) + >>> X_train, X_test, y_train, y_test = train_test_split( + >>> X, y, train_size=0.6, random_state=42) + >>> n_calibration_samples = int(len(X_train) * 0.2) + >>> clf = CutoffClassifier(LogisticRegression(), cv=3).fit( + >>> X_train[n_calibration_samples:], y_train[n_calibration_samples:] + >>> ) + >>> clf.decision_threshold_ + 1.3422651585209107 + +.. topic:: Examples: + * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py` + +The following image shows the results of using the :class:`CutoffClassifier` +for finding a decision threshold for a :class:`LogisticRegression` classifier +and an :class:`AdaBoostClassifier` for two use cases. + +In the first one we want to increase the overall accuracy of the classifiers on +the breast cancer dataset. As you can see after calibration the `f1 score` of +:class:`LogisticRegression` has increased slightly whereas the accuracy of +:class:`AdaBoostClassifier` has stayed the same. + +In the second case we want to find a decision threshold that yields maximum +true positive rate while maintaining a minimum value of ``0.7`` for the true negative +rate. As seen after calibration both classifiers achieve better true positive rate +while their respective true negative rates have decreased slightly or remained +stable. + +.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_decision_threshold_calibration_000.png + :target: ../auto_examples/calibration/plot_decision_threshold_calibration.html + :align: center + + +Notes +----- + +Calibrating the decision threshold of a classifier does not guarantee increased performance. +The generalisation ability of the obtained decision threshold has to be evaluated. \ No newline at end of file From e274c747f194f07dd41ba8ad423843c56b130c9d Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 19:22:03 +0200 Subject: [PATCH 063/100] fix doc --- doc/modules/calibration.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 8bdd04bc24d26..c313ab0adbc03 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -275,13 +275,10 @@ Here is a simple usage example:: >>> >>> X, y = load_breast_cancer(return_X_y=True) >>> X_train, X_test, y_train, y_test = train_test_split( - >>> X, y, train_size=0.6, random_state=42) - >>> n_calibration_samples = int(len(X_train) * 0.2) - >>> clf = CutoffClassifier(LogisticRegression(), cv=3).fit( - >>> X_train[n_calibration_samples:], y_train[n_calibration_samples:] - >>> ) + ... X, y, train_size=0.6, random_state=42) + >>> clf = CutoffClassifier(LogisticRegression(), cv=3).fit(X_train, y_train) >>> clf.decision_threshold_ - 1.3422651585209107 + 0.93181185424849922 .. topic:: Examples: * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py` From 193d7c7d2653a07b8ebfd9594fe0cbef201f75bf Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 20:10:24 +0200 Subject: [PATCH 064/100] fix example image --- doc/modules/calibration.rst | 2 +- .../plot_decision_threshold_calibration.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index c313ab0adbc03..ca29ce0829023 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -298,7 +298,7 @@ rate. As seen after calibration both classifiers achieve better true positive ra while their respective true negative rates have decreased slightly or remained stable. -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_decision_threshold_calibration_000.png +.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_decision_threshold_calibration_001.png :target: ../auto_examples/calibration/plot_decision_threshold_calibration.html :align: center diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index 55341630a02f0..39dd93fe411fc 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -127,10 +127,10 @@ plt.xticks( index + bar_width / 2, - ('f1 score logistic regression', 'f1 score adaboost')) + ('f1 logistic regression', 'f1 adaboost')) plt.ylabel('scores') -plt.title('Classifiers f1 score before and after calibration') +plt.title('f1 score before and after calibration') plt.legend() plt.subplot(2, 1, 2) @@ -144,13 +144,13 @@ plt.xticks( index + bar_width / 2, - ('true positive rate logistic regression', - 'true negative rate logistic regression', - 'true positive rate adaboost', - 'true negative rate adaboost') + ('tpr logistic regression', + 'tnr logistic regression', + 'tpr adaboost', + 'tnr adaboost') ) plt.ylabel('scores') -plt.title('Classifiers tpr and tnr before and after calibration') +plt.title('tpr and tnr before and after calibration') plt.legend() plt.show() From ef37050b4a19dbf1bb52327ceb0b21fbf7a270fc Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 21:17:14 +0200 Subject: [PATCH 065/100] fix doc example --- doc/modules/calibration.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index ca29ce0829023..0257ff46cb827 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -270,15 +270,15 @@ Here is a simple usage example:: >>> from sklearn.calibration import CutoffClassifier >>> from sklearn.datasets import load_breast_cancer - >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.model_selection import train_test_split >>> >>> X, y = load_breast_cancer(return_X_y=True) >>> X_train, X_test, y_train, y_test = train_test_split( ... X, y, train_size=0.6, random_state=42) - >>> clf = CutoffClassifier(LogisticRegression(), cv=3).fit(X_train, y_train) + >>> clf = CutoffClassifier(GaussianNB(), cv=3).fit(X_train, y_train) >>> clf.decision_threshold_ - 0.93181185424849922 + 0.93244970838859154 .. topic:: Examples: * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py` From 1896a226d1637652076e843689c258e35abdbfb4 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 21:51:54 +0200 Subject: [PATCH 066/100] fix image --- .../plot_decision_threshold_calibration.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index 39dd93fe411fc..891207568a445 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -120,37 +120,32 @@ plt.subplot(2, 1, 1) index = np.asarray([1, 2]) -plt.bar(index, [f_one_lr, f_one_ada], bar_width, color='r', label='Before') +plt.bar(index, [f_one_lr, f_one_ada], bar_width, color='r', + label='Before calibration') plt.bar(index + bar_width, [f_one_lr_f_beta, f_one_ada_f_beta], bar_width, - color='b', label='After') + color='b', label='After calibration') -plt.xticks( - index + bar_width / 2, - ('f1 logistic regression', 'f1 adaboost')) +plt.xticks(index + bar_width / 2, ('f1 logistic', 'f1 adaboost')) plt.ylabel('scores') -plt.title('f1 score before and after calibration') -plt.legend() +plt.title('f1 score') +plt.legend(bbox_to_anchor=(.5, -.2), loc='center', borderaxespad=0.) plt.subplot(2, 1, 2) index = np.asarray([1, 2, 3, 4]) plt.bar(index, [tpr_lr, tnr_lr, tpr_ada, tnr_ada], - bar_width, color='r', label='Before') + bar_width, color='r', label='Before calibration') plt.bar(index + bar_width, [tpr_lr_max_tpr, tnr_lr_max_tpr, tpr_ada_max_tpr, tnr_ada_max_tpr], - bar_width, color='b', label='After') + bar_width, color='b', label='After calibration') plt.xticks( index + bar_width / 2, - ('tpr logistic regression', - 'tnr logistic regression', - 'tpr adaboost', - 'tnr adaboost') -) + ('tpr logistic', 'tnr logistic', 'tpr adaboost', 'tnr adaboost')) plt.ylabel('scores') -plt.title('tpr and tnr before and after calibration') -plt.legend() +plt.title('true positive & true negative rate') +plt.subplots_adjust(hspace=0.6) plt.show() From ac0bf60db148d9407a0004357e64b6fadddf507a Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 29 Apr 2018 22:24:59 +0200 Subject: [PATCH 067/100] fix example --- doc/modules/calibration.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 0257ff46cb827..9f564b9e96bde 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -277,8 +277,6 @@ Here is a simple usage example:: >>> X_train, X_test, y_train, y_test = train_test_split( ... X, y, train_size=0.6, random_state=42) >>> clf = CutoffClassifier(GaussianNB(), cv=3).fit(X_train, y_train) - >>> clf.decision_threshold_ - 0.93244970838859154 .. topic:: Examples: * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py` From 00e110b234638101c9f0b0631a6ef1a3613b154b Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 11:36:39 +0200 Subject: [PATCH 068/100] improve docs and example --- doc/modules/calibration.rst | 42 ++++++++++--------- .../plot_decision_threshold_calibration.py | 14 ++++++- sklearn/calibration.py | 36 +++++++++------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 9f564b9e96bde..dc8b4195b8e75 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -238,33 +238,39 @@ To use the :class:`CutoffClassifier` you need to provide an estimator that has a ``decision_function`` or a ``predict_proba`` method. The ``scoring`` parameter controls whether the first will be preferred over the second if both are available. -The wrapped estimator can be pre-trained, in which case ``cv='prefit'``, or not. If +The wrapped estimator can be pre-trained, in which case ``cv = 'prefit'``, or not. If the classifier is not trained then a cross-validation loop specified by the parameter ``cv`` can be used to obtain a decision threshold by averaging all decision thresholds calculated on the hold-out parts of each cross validation iteration. Finally the model -is trained on all the provided data. When using ``cv='prefit'`` you need to make sure +is trained on all the provided data. When using ``cv = 'prefit'`` you need to make sure to use a hold-out part of your data for calibration. The methods for finding appropriate decision thresholds are based either on precision recall estimates or true positive and true negative rates. Specifically: +.. currentmodule:: sklearn.metrics + * ``f_beta`` - selects a decision threshold that maximizes the f_beta score. The value of - beta is specified by the parameter ``beta`` + selects a decision threshold that maximizes the :func:`fbeta_score`. The value of + beta is specified by the parameter ``beta``. The ``beta`` parameter determines + the weight of precision. When ``beta = 1`` both precision recall get the same + weight therefore the maximization target in this case is the :func:`f1_score`. + if ``beta < 1`` more weight is given to precision whereas if ``beta > 1`` more + weight is given to recall. * ``roc`` - selects the decision threshold for the point on the roc curve that is + selects the decision threshold for the point on the :func:`roc_curve` that is closest to the ideal corner (0, 1) * ``max_tpr`` - selects the decision threshold for the point that yields the highest true positive - rate while maintaining a minimum, specified by the parameter ``threshold``, for the - true negative rate + selects the decision threshold for the point that yields the highest true + positive rate while maintaining a minimum true negative rate, specified by + the parameter ``threshold`` * ``max_tnr`` selects the decision threshold for the point that yields the highest true - negative rate while maintaining a minimum, specified by the parameter ``threshold``, - for the true positive rate + negative rate while maintaining a minimum true positive rate, specified by + the parameter ``threshold`` Here is a simple usage example:: @@ -276,7 +282,8 @@ Here is a simple usage example:: >>> X, y = load_breast_cancer(return_X_y=True) >>> X_train, X_test, y_train, y_test = train_test_split( ... X, y, train_size=0.6, random_state=42) - >>> clf = CutoffClassifier(GaussianNB(), cv=3).fit(X_train, y_train) + >>> clf = CutoffClassifier(GaussianNB(), method='roc', cv=3).fit( + ... X_train, y_train) .. topic:: Examples: * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py` @@ -285,16 +292,11 @@ The following image shows the results of using the :class:`CutoffClassifier` for finding a decision threshold for a :class:`LogisticRegression` classifier and an :class:`AdaBoostClassifier` for two use cases. -In the first one we want to increase the overall accuracy of the classifiers on -the breast cancer dataset. As you can see after calibration the `f1 score` of -:class:`LogisticRegression` has increased slightly whereas the accuracy of -:class:`AdaBoostClassifier` has stayed the same. +In the first case we want to increase the overall accuracy of the classifiers on +the breast cancer dataset. In the second case we want to find a decision threshold +that yields maximum true positive rate while maintaining a minimum value of +``0.7`` for the true negative rate. -In the second case we want to find a decision threshold that yields maximum -true positive rate while maintaining a minimum value of ``0.7`` for the true negative -rate. As seen after calibration both classifiers achieve better true positive rate -while their respective true negative rates have decreased slightly or remained -stable. .. figure:: ../auto_examples/calibration/images/sphx_glr_plot_decision_threshold_calibration_001.png :target: ../auto_examples/calibration/plot_decision_threshold_calibration.html diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index 891207568a445..09369281fcd58 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -14,8 +14,18 @@ the true negative rate as well as the f beta score. In this example the decision threshold calibration is applied on two -classifiers trained on the breast cancer dataset. The goal is to maximize the -true positive rate while maintaining a minimum true negative rate. +classifiers trained on the breast cancer dataset. The goal in the first case is +to maximize the f1 score of the classifiers whereas in the second the goals is +to maximize the true positive rate while maintaining a minimum true negative +rate. + +As you can see after calibration the f1 score of the LogisticRegression +classifiers has increased slightly whereas the accuracy of the +AdaBoostClassifier classifier has stayed the same. + +For the second goal as seen after calibration both classifiers achieve better +true positive rate while their respective true negative rates have decreased +slightly or remained stable. """ # Author: Prokopios Gryllos diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 0c00b676a5602..3081c462751c4 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -54,19 +54,22 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): method : str The method to use for choosing the cutoff point - - 'roc', selects the point on the roc curve that is closest to the - ideal corner (0, 1) + 'roc' + selects the point on the roc curve that is closest to the ideal + corner (0, 1) - - 'f_beta', selects a decision threshold that maximizes the f_beta - score + 'f_beta' + selects a decision threshold that maximizes the f_beta score - - 'max_tpr', selects the point that yields the highest true positive - rate with true negative rate at least equal to the value of the - parameter threshold + 'max_tpr' + selects the point that yields the highest true positive rate with + true negative rate at least equal to the value of the parameter + threshold - - 'max_tnr', selects the point that yields the highest true negative - rate with true positive rate at least equal to the value of the - parameter threshold + 'max_tnr' + selects the point that yields the highest true negative rate with + true positive rate at least equal to the value of the parameter + threshold beta : float in [0, 1], optional (default=None) beta value to be used in case method == 'f_beta' @@ -74,14 +77,15 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): scoring : str or None, optional (default=None) The method to be used for acquiring the score - - 'decision_function'. base_estimator.decision_function will be used - for scoring + 'decision_function' + base_estimator.decision_function will be used for scoring - - 'predict_proba'. base_estimator.predict_proba will be used for - scoring + 'predict_proba' + base_estimator.predict_proba will be used for scoring - - None. base_estimator.decision_function will be used first and if not - available base_estimator.predict_proba + None + base_estimator.decision_function will be used first and if not + available base_estimator.predict_proba pos_label : object, optional (default=1) Object representing the positive label From 549b186f7d749e68f18a03f129f95a1889d206dd Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 11:44:02 +0200 Subject: [PATCH 069/100] rename scoring param to strategy --- doc/modules/calibration.rst | 2 +- .../plot_decision_threshold_calibration.py | 8 ++--- sklearn/calibration.py | 32 +++++++++---------- sklearn/tests/test_calibration.py | 14 ++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index dc8b4195b8e75..876f61eb5eac8 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -235,7 +235,7 @@ Usage ----- To use the :class:`CutoffClassifier` you need to provide an estimator that has -a ``decision_function`` or a ``predict_proba`` method. The ``scoring`` parameter +a ``decision_function`` or a ``predict_proba`` method. The ``strategy`` parameter controls whether the first will be preferred over the second if both are available. The wrapped estimator can be pre-trained, in which case ``cv = 'prefit'``, or not. If diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index 09369281fcd58..f5e58909412d9 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -76,13 +76,13 @@ # objective 1: we want to calibrate the decision threshold in order to achieve # better f1 score lr_f_beta = CutoffClassifier( - lr, method='f_beta', beta=1, cv='prefit', scoring='predict_proba' + lr, method='f_beta', beta=1, cv='prefit', strategy='predict_proba' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_lr_f_beta = lr_f_beta.predict(X_test) f_one_lr_f_beta = f1_score(y_test, y_pred_lr_f_beta) ada_f_beta = CutoffClassifier( - ada, method='f_beta', beta=1, cv='prefit', scoring='predict_proba' + ada, method='f_beta', beta=1, cv='prefit', strategy='predict_proba' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_ada_f_beta = ada_f_beta.predict(X_test) f_one_ada_f_beta = f1_score(y_test, y_pred_ada_f_beta) @@ -90,7 +90,7 @@ # objective 2: we want to maximize the true positive rate while the true # negative rate is at least 0.7 lr_max_tpr = CutoffClassifier( - lr, method='max_tpr', cv='prefit', threshold=0.7, scoring='predict_proba' + lr, method='max_tpr', cv='prefit', threshold=0.7, strategy='predict_proba' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_lr_max_tpr = lr_max_tpr.predict(X_test) tn_lr_max_tpr, fp_lr_max_tpr, fn_lr_max_tpr, tp_lr_max_tpr = \ @@ -99,7 +99,7 @@ tnr_lr_max_tpr = tn_lr_max_tpr / (tn_lr_max_tpr + fp_lr_max_tpr) ada_max_tpr = CutoffClassifier( - ada, method='max_tpr', cv='prefit', threshold=0.7, scoring='predict_proba' + ada, method='max_tpr', cv='prefit', threshold=0.7, strategy='predict_proba' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_ada_max_tpr = ada_max_tpr.predict(X_test) tn_ada_max_tpr, fp_ada_max_tpr, fn_ada_max_tpr, tp_ada_max_tpr = \ diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 3081c462751c4..1d41f321b0262 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -74,7 +74,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): beta : float in [0, 1], optional (default=None) beta value to be used in case method == 'f_beta' - scoring : str or None, optional (default=None) + strategy : str or None, optional (default=None) The method to be used for acquiring the score 'decision_function' @@ -113,12 +113,12 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Clinical chemistry, 1993 """ - def __init__(self, base_estimator, method='roc', beta=None, scoring=None, + def __init__(self, base_estimator, method='roc', beta=None, strategy=None, pos_label=1, cv=3, threshold=None): self.base_estimator = base_estimator self.method = method self.beta = beta - self.scoring = scoring + self.strategy = strategy self.pos_label = pos_label self.cv = cv self.threshold = threshold @@ -148,10 +148,10 @@ def fit(self, X, y): raise ValueError('method can either be "roc" or "max_tpr" or ' '"max_tnr. Got %s instead' % self.method) - if self.scoring not in [None, 'decision_function', 'predict_proba']: + if self.strategy not in [None, 'decision_function', 'predict_proba']: raise ValueError('scoring param can either be "decision_function" ' 'or "predict_proba" or None. Got %s instead' % - self.scoring) + self.strategy) if self.method == 'max_tpr' or self.method == 'max_tnr': if not self.threshold or not \ @@ -181,7 +181,7 @@ def fit(self, X, y): if self.cv == 'prefit': self.decision_threshold_ = _CutoffClassifier( - self.base_estimator, self.method, self.beta, self.scoring, + self.base_estimator, self.method, self.beta, self.strategy, self.pos_label, self.threshold ).fit(X, y).decision_threshold_ else: @@ -194,7 +194,7 @@ def fit(self, X, y): _CutoffClassifier(estimator, self.method, self.beta, - self.scoring, + self.strategy, self.pos_label, self.threshold).fit( X[test], y[test] @@ -221,7 +221,7 @@ def predict(self, X): X = check_array(X) check_is_fitted(self, ["label_encoder_", "decision_threshold_"]) - y_score = _get_binary_score(self.base_estimator, X, self.scoring, + y_score = _get_binary_score(self.base_estimator, X, self.strategy, self.pos_label) return self.label_encoder_.inverse_transform( (y_score > self.decision_threshold_).astype(int) @@ -249,7 +249,7 @@ class _CutoffClassifier(object): beta : float in [0, 1] beta value to be used in case method == 'f_beta' - scoring : str or None, optional (default=None) + strategy : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available @@ -268,12 +268,12 @@ class _CutoffClassifier(object): decision_threshold_ : float Acquired decision threshold for the positive class """ - def __init__(self, base_estimator, method, beta, scoring, pos_label, + def __init__(self, base_estimator, method, beta, strategy, pos_label, threshold): self.base_estimator = base_estimator self.method = method self.beta = beta - self.scoring = scoring + self.strategy = strategy self.pos_label = pos_label self.threshold = threshold @@ -294,7 +294,7 @@ def fit(self, X, y): self : object Instance of self """ - y_score = _get_binary_score(self.base_estimator, X, self.scoring, + y_score = _get_binary_score(self.base_estimator, X, self.strategy, self.pos_label) if self.method == 'f_beta': precision, recall, thresholds = precision_recall_curve( @@ -324,7 +324,7 @@ def fit(self, X, y): return self -def _get_binary_score(clf, X, scoring=None, pos_label=1): +def _get_binary_score(clf, X, strategy=None, pos_label=1): """Binary classification score for the positive label (0 or 1) Returns the score that a binary classifier outputs for the positive label @@ -342,20 +342,20 @@ def _get_binary_score(clf, X, scoring=None, pos_label=1): pos_label : int, optional (default=1) The positive label. Can either be 0 or 1 - scoring : str or None, optional (default=None) + strategy : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available predict_proba """ - if not scoring: + if not strategy: try: y_score = clf.decision_function(X) if pos_label == 0: y_score = - y_score except (NotImplementedError, AttributeError): y_score = clf.predict_proba(X)[:, pos_label] - elif scoring == 'decision_function': + elif strategy == 'decision_function': y_score = clf.decision_function(X) if pos_label == 0: y_score = - y_score diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 5945c79f88053..9568da4982db4 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -60,7 +60,7 @@ def test_cutoff_prefit(): assert_greater(tpr_roc + tnr_roc, tpr + tnr) clf_f1 = CutoffClassifier( - lr, method='f_beta', beta=1, cv='prefit', scoring='predict_proba').fit( + lr, method='f_beta', beta=1, cv='prefit', strategy='predict_proba').fit( X_test[:calibration_samples], y_test[:calibration_samples] ) @@ -165,27 +165,27 @@ def test_get_binary_score(): assert_array_equal( y_pred_score, - _get_binary_score(lr, X_test, scoring='decision_function', pos_label=1) + _get_binary_score(lr, X_test, strategy='decision_function', pos_label=1) ) assert_array_equal( - y_pred_score, - _get_binary_score(lr, X_test, scoring='decision_function', pos_label=0) + _get_binary_score(lr, X_test, strategy='decision_function', pos_label=0) ) assert_array_equal( y_pred_proba[:, 1], - _get_binary_score(lr, X_test, scoring='predict_proba', pos_label=1) + _get_binary_score(lr, X_test, strategy='predict_proba', pos_label=1) ) assert_array_equal( y_pred_proba[:, 0], - _get_binary_score(lr, X_test, scoring='predict_proba', pos_label=0) + _get_binary_score(lr, X_test, strategy='predict_proba', pos_label=0) ) assert_array_equal( y_pred_score, - _get_binary_score(lr, X_test, scoring=None, pos_label=1) + _get_binary_score(lr, X_test, strategy=None, pos_label=1) ) # classifier that does not have a decision_function @@ -193,7 +193,7 @@ def test_get_binary_score(): y_pred_proba_rf = rf.predict_proba(X_test) assert_array_equal( y_pred_proba_rf[:, 1], - _get_binary_score(rf, X_test, scoring=None, pos_label=1) + _get_binary_score(rf, X_test, strategy=None, pos_label=1) ) From bb6afee14863699afeeb2dced70f3f09a79f287f Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 12:02:40 +0200 Subject: [PATCH 070/100] change order of parameters --- .../plot_decision_threshold_calibration.py | 9 +-- sklearn/calibration.py | 55 +++++++++---------- sklearn/tests/test_calibration.py | 23 ++++---- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index f5e58909412d9..71c3c3d2a0696 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -76,13 +76,13 @@ # objective 1: we want to calibrate the decision threshold in order to achieve # better f1 score lr_f_beta = CutoffClassifier( - lr, method='f_beta', beta=1, cv='prefit', strategy='predict_proba' + lr, method='f_beta', strategy='predict_proba', beta=1, cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_lr_f_beta = lr_f_beta.predict(X_test) f_one_lr_f_beta = f1_score(y_test, y_pred_lr_f_beta) ada_f_beta = CutoffClassifier( - ada, method='f_beta', beta=1, cv='prefit', strategy='predict_proba' + ada, method='f_beta', strategy='predict_proba', beta=1, cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_ada_f_beta = ada_f_beta.predict(X_test) f_one_ada_f_beta = f1_score(y_test, y_pred_ada_f_beta) @@ -90,7 +90,7 @@ # objective 2: we want to maximize the true positive rate while the true # negative rate is at least 0.7 lr_max_tpr = CutoffClassifier( - lr, method='max_tpr', cv='prefit', threshold=0.7, strategy='predict_proba' + lr, method='max_tpr', strategy='predict_proba', threshold=0.7, cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_lr_max_tpr = lr_max_tpr.predict(X_test) tn_lr_max_tpr, fp_lr_max_tpr, fn_lr_max_tpr, tp_lr_max_tpr = \ @@ -99,7 +99,8 @@ tnr_lr_max_tpr = tn_lr_max_tpr / (tn_lr_max_tpr + fp_lr_max_tpr) ada_max_tpr = CutoffClassifier( - ada, method='max_tpr', cv='prefit', threshold=0.7, strategy='predict_proba' + ada, method='max_tpr', strategy='predict_proba', threshold=0.7, + cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) y_pred_ada_max_tpr = ada_max_tpr.predict(X_test) tn_ada_max_tpr, fp_ada_max_tpr, fn_ada_max_tpr, tp_ada_max_tpr = \ diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 1d41f321b0262..f54d5ff685248 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -71,9 +71,6 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): true positive rate at least equal to the value of the parameter threshold - beta : float in [0, 1], optional (default=None) - beta value to be used in case method == 'f_beta' - strategy : str or None, optional (default=None) The method to be used for acquiring the score @@ -87,6 +84,14 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): base_estimator.decision_function will be used first and if not available base_estimator.predict_proba + beta : float in [0, 1], optional (default=None) + beta value to be used in case method == 'f_beta' + + threshold : float in [0, 1] or None, (default=None) + In case method is 'max_tpr' or 'max_tnr' this parameter must be set to + specify the threshold for the true negative rate or true positive rate + respectively that needs to be achieved + pos_label : object, optional (default=1) Object representing the positive label @@ -95,11 +100,6 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): If cv='prefit' the base estimator is assumed to be fitted and all data will be used for the calibration of the probability threshold - threshold : float in [0, 1] or None, (default=None) - In case method is 'max_tpr' or 'max_tnr' this parameter must be set to - specify the threshold for the true negative rate or true positive rate - respectively that needs to be achieved - Attributes ---------- decision_threshold_ : float @@ -113,15 +113,15 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Clinical chemistry, 1993 """ - def __init__(self, base_estimator, method='roc', beta=None, strategy=None, - pos_label=1, cv=3, threshold=None): + def __init__(self, base_estimator, method='roc', strategy=None, beta=None, + threshold=None, pos_label=1, cv=3): self.base_estimator = base_estimator self.method = method - self.beta = beta self.strategy = strategy + self.beta = beta + self.threshold = threshold self.pos_label = pos_label self.cv = cv - self.threshold = threshold def fit(self, X, y): """Fit model @@ -181,8 +181,8 @@ def fit(self, X, y): if self.cv == 'prefit': self.decision_threshold_ = _CutoffClassifier( - self.base_estimator, self.method, self.beta, self.strategy, - self.pos_label, self.threshold + self.base_estimator, self.method, self.strategy, self.beta, + self.threshold, self.pos_label ).fit(X, y).decision_threshold_ else: cv = check_cv(self.cv, y, classifier=True) @@ -191,12 +191,9 @@ def fit(self, X, y): for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) decision_thresholds.append( - _CutoffClassifier(estimator, - self.method, - self.beta, - self.strategy, - self.pos_label, - self.threshold).fit( + _CutoffClassifier(estimator, self.method, self.strategy, + self.beta, self.threshold, + self.pos_label).fit( X[test], y[test] ).decision_threshold_ ) @@ -246,36 +243,36 @@ class _CutoffClassifier(object): method : 'roc' or 'f_beta' or 'max_tpr' or 'max_tnr' The method to use for choosing the cutoff point - beta : float in [0, 1] - beta value to be used in case method == 'f_beta' - strategy : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available predict_proba - pos_label : object - Label considered as positive during the roc_curve construction + beta : float in [0, 1] + beta value to be used in case method == 'f_beta' threshold : float in [0, 1] minimum required value for the true negative rate (specificity) in case method 'max_tpr' is used or for the true positive rate (sensitivity) in case method 'max_tnr' is used + pos_label : object + Label considered as positive during the roc_curve construction + Attributes ---------- decision_threshold_ : float Acquired decision threshold for the positive class """ - def __init__(self, base_estimator, method, beta, strategy, pos_label, - threshold): + def __init__(self, base_estimator, method, strategy, beta, threshold, + pos_label): self.base_estimator = base_estimator self.method = method - self.beta = beta self.strategy = strategy - self.pos_label = pos_label + self.beta = beta self.threshold = threshold + self.pos_label = pos_label def fit(self, X, y): """Select a decision threshold for the fitted model's positive class diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 9568da4982db4..b025dfd9a9e15 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -60,7 +60,8 @@ def test_cutoff_prefit(): assert_greater(tpr_roc + tnr_roc, tpr + tnr) clf_f1 = CutoffClassifier( - lr, method='f_beta', beta=1, cv='prefit', strategy='predict_proba').fit( + lr, method='f_beta', strategy='predict_proba', beta=1, + cv='prefit').fit( X_test[:calibration_samples], y_test[:calibration_samples] ) @@ -69,7 +70,7 @@ def test_cutoff_prefit(): f1_score(y_test[calibration_samples:], y_pred)) clf_max_tpr = CutoffClassifier( - lr, method='max_tpr', cv='prefit', threshold=0.7 + lr, method='max_tpr', threshold=0.7, cv='prefit' ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) y_pred_max_tpr = clf_max_tpr.predict(X_test[calibration_samples:]) @@ -86,7 +87,7 @@ def test_cutoff_prefit(): assert_greater_equal(tnr_max_tpr, 0.7) clf_max_tnr = CutoffClassifier( - lr, method='max_tnr', cv='prefit', threshold=0.7 + lr, method='max_tnr', threshold=0.7, cv='prefit' ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) y_pred_clf = clf_max_tnr.predict(X_test[calibration_samples:]) @@ -164,23 +165,23 @@ def test_get_binary_score(): y_pred_score = lr.decision_function(X_test) assert_array_equal( - y_pred_score, - _get_binary_score(lr, X_test, strategy='decision_function', pos_label=1) + y_pred_score, _get_binary_score( + lr, X_test, strategy='decision_function', pos_label=1) ) assert_array_equal( - - y_pred_score, - _get_binary_score(lr, X_test, strategy='decision_function', pos_label=0) + - y_pred_score, _get_binary_score( + lr, X_test, strategy='decision_function', pos_label=0) ) assert_array_equal( - y_pred_proba[:, 1], - _get_binary_score(lr, X_test, strategy='predict_proba', pos_label=1) + y_pred_proba[:, 1], _get_binary_score( + lr, X_test, strategy='predict_proba', pos_label=1) ) assert_array_equal( - y_pred_proba[:, 0], - _get_binary_score(lr, X_test, strategy='predict_proba', pos_label=0) + y_pred_proba[:, 0], _get_binary_score( + lr, X_test, strategy='predict_proba', pos_label=0) ) assert_array_equal( From 404ac2145d4c4b4749e708981b58cd8c70223e46 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 12:05:49 +0200 Subject: [PATCH 071/100] use positional args --- sklearn/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index f54d5ff685248..34c6ea1afd427 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -295,14 +295,14 @@ def fit(self, X, y): self.pos_label) if self.method == 'f_beta': precision, recall, thresholds = precision_recall_curve( - y, y_score, self.pos_label + y, y_score, pos_label=self.pos_label ) f_beta = (1 + self.beta**2) * (precision * recall) /\ (self.beta**2 * precision + recall) self.decision_threshold_ = thresholds[np.argmax(f_beta)] return self - fpr, tpr, thresholds = roc_curve(y, y_score, self.pos_label) + fpr, tpr, thresholds = roc_curve(y, y_score, pos_label=self.pos_label) if self.method == 'roc': # we find the threshold of the point (fpr, tpr) with the smallest From 595247e915e70ffc30df46ef2a544b2f1e535b4f Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 12:06:04 +0200 Subject: [PATCH 072/100] avoid backslash --- sklearn/calibration.py | 4 ++-- sklearn/tests/test_calibration.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 34c6ea1afd427..54d570b83ae23 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -297,8 +297,8 @@ def fit(self, X, y): precision, recall, thresholds = precision_recall_curve( y, y_score, pos_label=self.pos_label ) - f_beta = (1 + self.beta**2) * (precision * recall) /\ - (self.beta**2 * precision + recall) + f_beta = ((1 + self.beta**2) * (precision * recall) / + (self.beta**2 * precision + recall)) self.decision_threshold_ = thresholds[np.argmax(f_beta)] return self diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index b025dfd9a9e15..ac39e2a298f00 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -20,8 +20,8 @@ from sklearn.svm import LinearSVC from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer -from sklearn.metrics import brier_score_loss, log_loss, confusion_matrix,\ - f1_score +from sklearn.metrics import (brier_score_loss, log_loss, confusion_matrix, + f1_score) from sklearn.calibration import CalibratedClassifierCV, CutoffClassifier from sklearn.calibration import _get_binary_score from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration From ec19891f97cc982f67666373ed62c7b9ce8b4858 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 12:08:45 +0200 Subject: [PATCH 073/100] use np.mean instead of sum / n --- sklearn/calibration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 54d570b83ae23..2dd1ae578d07c 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -197,8 +197,7 @@ def fit(self, X, y): X[test], y[test] ).decision_threshold_ ) - self.decision_threshold_ = sum(decision_thresholds) /\ - len(decision_thresholds) + self.decision_threshold_ = np.mean(decision_thresholds) self.base_estimator.fit(X, y) return self From 9eb3f1638d48dd12a985e34f0bd7990350cbf36a Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 12:08:53 +0200 Subject: [PATCH 074/100] fix flake --- sklearn/tests/test_calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index ac39e2a298f00..050ddeb00ed87 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -21,7 +21,7 @@ from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer from sklearn.metrics import (brier_score_loss, log_loss, confusion_matrix, - f1_score) + f1_score) from sklearn.calibration import CalibratedClassifierCV, CutoffClassifier from sklearn.calibration import _get_binary_score from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration From 0aa70d4811e8adff9fa134aa9aa2681dfd830300 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 23:15:32 +0200 Subject: [PATCH 075/100] remove backslash --- sklearn/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 2dd1ae578d07c..02d7a29e39511 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -17,8 +17,8 @@ from scipy.optimize import fmin_bfgs from sklearn.preprocessing import LabelEncoder -from .base import BaseEstimator, ClassifierMixin, RegressorMixin,\ - MetaEstimatorMixin, clone +from .base import (BaseEstimator, ClassifierMixin, RegressorMixin, + MetaEstimatorMixin, clone) from .preprocessing import label_binarize, LabelBinarizer from .utils import check_X_y, check_array, indexable, column_or_1d from .utils.validation import check_is_fitted, check_consistent_length From 9d7f86139e55b705528689e9b97257b1e4afbed0 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 23:24:52 +0200 Subject: [PATCH 076/100] fix beta check --- sklearn/calibration.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 02d7a29e39511..25b685dfd681c 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -162,10 +162,9 @@ def fit(self, X, y): repr(self.threshold)) if self.method == 'f_beta': - if not self.beta or not isinstance(self.beta, (int, float)) \ - or not self.beta >= 0 or not self.beta <= 1: - raise ValueError('parameter beta must be a number in [0, 1]. ' - 'Got %s instead' % repr(self.beta)) + if not self.beta or not isinstance(self.beta, (int, float)): + raise ValueError('parameter beta must be a real number.' + 'Got %s instead' % type(self.beta)) X, y = check_X_y(X, y) From 2d5aca5b1ea08cc0287ec9ff2109a4122e7802a5 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 30 Apr 2018 23:59:37 +0200 Subject: [PATCH 077/100] extend fbeta asserts --- sklearn/tests/test_calibration.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 050ddeb00ed87..0bf48152731f3 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -21,7 +21,7 @@ from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer from sklearn.metrics import (brier_score_loss, log_loss, confusion_matrix, - f1_score) + f1_score, recall_score) from sklearn.calibration import CalibratedClassifierCV, CutoffClassifier from sklearn.calibration import _get_binary_score from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration @@ -69,6 +69,16 @@ def test_cutoff_prefit(): assert_greater(f1_score(y_test[calibration_samples:], y_pred_f1), f1_score(y_test[calibration_samples:], y_pred)) + clf_fbeta = CutoffClassifier( + lr, method='f_beta', strategy='predict_proba', beta=2, + cv='prefit').fit( + X_test[:calibration_samples], y_test[:calibration_samples] + ) + + y_pred_fbeta = clf_fbeta.predict(X_test[calibration_samples:]) + assert_greater(recall_score(y_test[calibration_samples:], y_pred_fbeta), + recall_score(y_test[calibration_samples:], y_pred)) + clf_max_tpr = CutoffClassifier( lr, method='max_tpr', threshold=0.7, cv='prefit' ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) From 9218c0b5a2c7ffcac4aeba40e2f13bc8a43c9298 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 17:58:13 +0200 Subject: [PATCH 078/100] fix negative label case --- sklearn/calibration.py | 14 +++++++++++--- sklearn/tests/test_calibration.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 25b685dfd681c..42ef6c4b434c5 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -343,16 +343,24 @@ def _get_binary_score(clf, X, strategy=None, pos_label=1): decision_function will be used first and if not available predict_proba """ + if len(clf.classes_) != 2: + raise ValueError('Expected binary classifier') + + if strategy not in (None, 'decision_function', 'predict_proba'): + raise ValueError('scoring param can either be "decision_function" ' + 'or "predict_proba" or None. ' + 'Got {} instead'.format(strategy)) + if not strategy: try: y_score = clf.decision_function(X) - if pos_label == 0: - y_score = - y_score + if pos_label == clf.classes_[0]: + y_score = -y_score except (NotImplementedError, AttributeError): y_score = clf.predict_proba(X)[:, pos_label] elif strategy == 'decision_function': y_score = clf.decision_function(X) - if pos_label == 0: + if pos_label == clf.classes_[0]: y_score = - y_score else: y_score = clf.predict_proba(X)[:, pos_label] diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 0bf48152731f3..5bad3b8c443b7 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -199,6 +199,8 @@ def test_get_binary_score(): _get_binary_score(lr, X_test, strategy=None, pos_label=1) ) + assert_raises(ValueError, _get_binary_score, lr, X_test, strategy='foo') + # classifier that does not have a decision_function rf = RandomForestClassifier().fit(X_train, y_train) y_pred_proba_rf = rf.predict_proba(X_test) @@ -207,6 +209,14 @@ def test_get_binary_score(): _get_binary_score(rf, X_test, strategy=None, pos_label=1) ) + X_non_binary, y_non_binary = make_classification( + n_samples=20, n_features=6, random_state=42, n_classes=4, + n_informative=4 + ) + + rf_non_bin = RandomForestClassifier().fit(X_non_binary, y_non_binary) + assert_raises(ValueError, _get_binary_score, rf_non_bin, X_non_binary) + @ignore_warnings def test_calibration(): From fab9022a7c7ef51a8cf893737d7178462c7928b5 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 18:01:52 +0200 Subject: [PATCH 079/100] use string formatting --- sklearn/calibration.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 42ef6c4b434c5..e4497e02ad2ed 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -146,32 +146,32 @@ def fit(self, X, y): if self.method not in ['roc', 'f_beta', 'max_tpr', 'max_tnr']: raise ValueError('method can either be "roc" or "max_tpr" or ' - '"max_tnr. Got %s instead' % self.method) + '"max_tnr. Got {} instead'.format(self.method)) if self.strategy not in [None, 'decision_function', 'predict_proba']: raise ValueError('scoring param can either be "decision_function" ' - 'or "predict_proba" or None. Got %s instead' % - self.strategy) + 'or "predict_proba" or None. ' + 'Got {} instead'.format(self.strategy)) if self.method == 'max_tpr' or self.method == 'max_tnr': if not self.threshold or not \ isinstance(self.threshold, (int, float)) \ or not self.threshold >= 0 or not self.threshold <= 1: raise ValueError('parameter threshold must be a number in' - '[0, 1]. Got %s instead' % - repr(self.threshold)) + '[0, 1]. ' + 'Got {} instead'.format(self.threshold)) if self.method == 'f_beta': if not self.beta or not isinstance(self.beta, (int, float)): raise ValueError('parameter beta must be a real number.' - 'Got %s instead' % type(self.beta)) + 'Got {} instead'.format(type(self.beta))) X, y = check_X_y(X, y) y_type = type_of_target(y) if y_type != 'binary': - raise ValueError('Expected target of binary type. Got %s ' % - y_type) + raise ValueError('Expected target of binary type. Got {}'.format( + y_type)) self.label_encoder_ = LabelEncoder().fit(y) From e2bf01946b8ca14df9bd19ce072306f51ed23dde Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 18:03:47 +0200 Subject: [PATCH 080/100] get rid of backslashes --- sklearn/calibration.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index e4497e02ad2ed..75f42f529503d 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -139,8 +139,8 @@ def fit(self, X, y): self : object Instance of self """ - if not hasattr(self.base_estimator, 'decision_function') and \ - not hasattr(self.base_estimator, 'predict_proba'): + if (not hasattr(self.base_estimator, 'decision_function') and + not hasattr(self.base_estimator, 'predict_proba')): raise TypeError('The base_estimator needs to implement either a ' 'decision_function or a predict_proba method') @@ -154,9 +154,9 @@ def fit(self, X, y): 'Got {} instead'.format(self.strategy)) if self.method == 'max_tpr' or self.method == 'max_tnr': - if not self.threshold or not \ - isinstance(self.threshold, (int, float)) \ - or not self.threshold >= 0 or not self.threshold <= 1: + if (not self.threshold or not + isinstance(self.threshold, (int, float)) + or not self.threshold >= 0 or not self.threshold <= 1): raise ValueError('parameter threshold must be a number in' '[0, 1]. ' 'Got {} instead'.format(self.threshold)) From febf9705fda9e06ddedcc8ec21fe94e02eea1844 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 19:15:40 +0200 Subject: [PATCH 081/100] update docstrings --- sklearn/calibration.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 75f42f529503d..1e97326833db7 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -47,9 +47,9 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Parameters ---------- base_estimator : obj - The classifier whose decision threshold will be adapted according to - the acquired cutoff point. The estimator must have a decision_function - or a predict_proba + The binary classifier whose decision threshold will be adapted + according to the acquired cutoff point. The estimator must have a + decision_function or a predict_proba method : str The method to use for choosing the cutoff point @@ -234,9 +234,9 @@ class _CutoffClassifier(object): Parameters ---------- base_estimator : obj - The classifier whose decision threshold will be adapted according to - the acquired cutoff point. The estimator must have a decision_function - or a predict_proba + The binary classifier whose decision threshold will be adapted + according to the acquired cutoff point. The estimator must have a + decision_function or a predict_proba method : 'roc' or 'f_beta' or 'max_tpr' or 'max_tnr' The method to use for choosing the cutoff point From d7c5943a9612a0ca1d634abfad489fcfef9d4594 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 19:16:08 +0200 Subject: [PATCH 082/100] minor update in error handlings --- sklearn/calibration.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 1e97326833db7..629175627e698 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -144,11 +144,11 @@ def fit(self, X, y): raise TypeError('The base_estimator needs to implement either a ' 'decision_function or a predict_proba method') - if self.method not in ['roc', 'f_beta', 'max_tpr', 'max_tnr']: + if self.method not in ('roc', 'f_beta', 'max_tpr', 'max_tnr'): raise ValueError('method can either be "roc" or "max_tpr" or ' '"max_tnr. Got {} instead'.format(self.method)) - if self.strategy not in [None, 'decision_function', 'predict_proba']: + if self.strategy not in (None, 'decision_function', 'predict_proba'): raise ValueError('scoring param can either be "decision_function" ' 'or "predict_proba" or None. ' 'Got {} instead'.format(self.strategy)) @@ -344,7 +344,9 @@ def _get_binary_score(clf, X, strategy=None, pos_label=1): predict_proba """ if len(clf.classes_) != 2: - raise ValueError('Expected binary classifier') + raise ValueError('Expected binary classifier. Found {} classes'.format( + len(clf.classes_) + )) if strategy not in (None, 'decision_function', 'predict_proba'): raise ValueError('scoring param can either be "decision_function" ' From 9601e2c1f446b202fe66b3680aca555d4328f165 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 19:25:08 +0200 Subject: [PATCH 083/100] add standard deviation as diagnostic --- sklearn/calibration.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 629175627e698..49185e031bc33 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -106,6 +106,13 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Decision threshold for the positive class. Determines the output of predict + std_ : float + Standard deviation of the obtained decision thresholds for when the + provided base estimator is not pre-trained and the decision_threshold_ + is computed as the mean of the decision threshold of each + cross-validation iteration. If the base estimator is pre-trained then + std_ = 0 + References ---------- .. [1] Receiver-operating characteristic (ROC) plots: a fundamental @@ -183,6 +190,7 @@ def fit(self, X, y): self.base_estimator, self.method, self.strategy, self.beta, self.threshold, self.pos_label ).fit(X, y).decision_threshold_ + self.std_ = .0 else: cv = check_cv(self.cv, y, classifier=True) decision_thresholds = [] @@ -197,6 +205,8 @@ def fit(self, X, y): ).decision_threshold_ ) self.decision_threshold_ = np.mean(decision_thresholds) + self.std_ = np.std(decision_thresholds) + self.base_estimator.fit(X, y) return self @@ -214,7 +224,8 @@ def predict(self, X): The predicted class """ X = check_array(X) - check_is_fitted(self, ["label_encoder_", "decision_threshold_"]) + check_is_fitted(self, + ["label_encoder_", "decision_threshold_", "std_"]) y_score = _get_binary_score(self.base_estimator, X, self.strategy, self.pos_label) From 3916f01ca5c4f9faf668cbbf27a2b0d3beb5fc22 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 19:47:33 +0200 Subject: [PATCH 084/100] dump commit to re-trigger build From d61977e30a4d94ccb93bd0215cbeb0e12b220112 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Tue, 1 May 2018 23:42:16 +0200 Subject: [PATCH 085/100] fix docs --- doc/modules/calibration.rst | 37 +++++++++++++++++++++++-------------- doc/modules/classes.rst | 23 +++++++++++++++++++++-- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 876f61eb5eac8..e888b09c738cf 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -1,4 +1,4 @@ -.. _calibration: +.. _probability_calibration: ======================= Probability calibration @@ -207,10 +207,15 @@ a similar decrease in log-loss. Wilks, D. S., 1990a +.. _decision_threshold_calibration: + +============================== Decision Threshold calibration ============================== -Often Machine Learning classifiers base their predictions on real-valued decision +.. currentmodule:: sklearn.calibration + +:class:`CutoffClassifier` Often Machine Learning classifiers base their predictions on real-valued decision functions or probability estimates that carry the inherited biases of their models. Additionally when using a machine learning model the evaluation criteria can differ from the optimisation objectives used by the model during training. @@ -223,10 +228,6 @@ provide better handling for sensitive classes. .. currentmodule:: sklearn.calibration -For example the :class:`LogisticRegression` classifier is predicting the class -for which the :func:`decision_function` returns the highest value. For a binary -classification task that sets decision threshold to ``0``. - :class:`CutoffClassifier` can be used as a wrapper around a model for binary classification to help obtain a more appropriate decision threshold and use it for predicting new samples. @@ -285,26 +286,34 @@ Here is a simple usage example:: >>> clf = CutoffClassifier(GaussianNB(), method='roc', cv=3).fit( ... X_train, y_train) + .. topic:: Examples: - * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py` + + * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py`: Decision + threshold calibration on the breast cancer dataset + The following image shows the results of using the :class:`CutoffClassifier` for finding a decision threshold for a :class:`LogisticRegression` classifier and an :class:`AdaBoostClassifier` for two use cases. -In the first case we want to increase the overall accuracy of the classifiers on -the breast cancer dataset. In the second case we want to find a decision threshold -that yields maximum true positive rate while maintaining a minimum value of -``0.7`` for the true negative rate. - - .. figure:: ../auto_examples/calibration/images/sphx_glr_plot_decision_threshold_calibration_001.png :target: ../auto_examples/calibration/plot_decision_threshold_calibration.html :align: center +In the first case we want to increase the overall accuracy of the classifiers on +the breast cancer dataset. In the second case we want to find a decision threshold +that yields maximum true positive rate while maintaining a minimum value +for the true negative rate. + +.. topic:: References: + + * Receiver-operating characteristic (ROC) plots: a fundamental + evaluation tool in clinical medicine, MH Zweig, G Campbell - + Clinical chemistry, 1993 Notes ----- Calibrating the decision threshold of a classifier does not guarantee increased performance. -The generalisation ability of the obtained decision threshold has to be evaluated. \ No newline at end of file +The generalisation ability of the obtained decision threshold has to be evaluated. diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 243c63ab0c7e2..b6ec26fafbf21 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -49,7 +49,7 @@ Functions get_config set_config -.. _calibration_ref: +.. _probability_calibration_ref: :mod:`sklearn.calibration`: Probability Calibration =================================================== @@ -58,7 +58,7 @@ Functions :no-members: :no-inherited-members: -**User guide:** See the :ref:`calibration` section for further details. +**User guide:** See the :ref:`probability_calibration` section for further details. .. currentmodule:: sklearn @@ -75,6 +75,25 @@ Functions calibration.calibration_curve +.. _decision_threshold_calibration_ref: + +:mod:`sklearn.calibration`: Decision Threshold Calibration +========================================================== + +.. automodule:: sklearn.calibration + :no-members: + :no-inherited-members: + +**User guide:** See the :ref:`decision_threshold_calibration` section for further details. + +.. currentmodule:: sklearn + +.. autosummary:: + :toctree: generated/ + :template: class.rst + + calibration.CutoffClassifier + .. _cluster_ref: :mod:`sklearn.cluster`: Clustering From 72c954ecc5511d2a4e020950a422827911ce7f16 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Wed, 2 May 2018 21:01:22 +0200 Subject: [PATCH 086/100] update docs & example & rename param --- doc/modules/calibration.rst | 85 ++++++++++--------- .../plot_decision_threshold_calibration.py | 23 +++-- sklearn/calibration.py | 74 ++++++++-------- sklearn/tests/test_calibration.py | 32 +++---- 4 files changed, 114 insertions(+), 100 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index e888b09c738cf..699784e7fb7a9 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -215,16 +215,17 @@ Decision Threshold calibration .. currentmodule:: sklearn.calibration -:class:`CutoffClassifier` Often Machine Learning classifiers base their predictions on real-valued decision -functions or probability estimates that carry the inherited biases of their models. -Additionally when using a machine learning model the evaluation criteria can differ -from the optimisation objectives used by the model during training. +Often Machine Learning classifiers base their +predictions on real-valued decision functions or probability estimates that +carry the inherited biases of their models. Additionally when using a machine +learning model the evaluation criteria can differ from the optimisation +objectives used by the model during training. When predicting between two classes it is commonly advised that an appropriate -decision threshold is estimated based on some cutoff criteria rather than arbitrarily -using the midpoint of the space of possible values. Estimating a decision threshold -for a specific use case can help to increase the overall accuracy of the model and -provide better handling for sensitive classes. +decision threshold is estimated based on some cutoff criteria rather than +arbitrarily using the midpoint of the space of possible values. Estimating a +decision threshold for a specific use case can help to increase the overall +accuracy of the model and provide better handling for sensitive classes. .. currentmodule:: sklearn.calibration @@ -236,32 +237,35 @@ Usage ----- To use the :class:`CutoffClassifier` you need to provide an estimator that has -a ``decision_function`` or a ``predict_proba`` method. The ``strategy`` parameter -controls whether the first will be preferred over the second if both are available. - -The wrapped estimator can be pre-trained, in which case ``cv = 'prefit'``, or not. If -the classifier is not trained then a cross-validation loop specified by the parameter -``cv`` can be used to obtain a decision threshold by averaging all decision thresholds -calculated on the hold-out parts of each cross validation iteration. Finally the model -is trained on all the provided data. When using ``cv = 'prefit'`` you need to make sure -to use a hold-out part of your data for calibration. - -The methods for finding appropriate decision thresholds are based either on precision -recall estimates or true positive and true negative rates. Specifically: +a ``decision_function`` or a ``predict_proba`` method. The ``method`` +parameter controls whether the first will be preferred over the second if both +are available. + +The wrapped estimator can be pre-trained, in which case ``cv = 'prefit'``, or +not. If the classifier is not trained then a cross-validation loop specified by +the parameter ``cv`` can be used to obtain a decision threshold by averaging +all decision thresholds calculated on the hold-out parts of each cross +validation iteration. Finally the model is trained on all the provided data. +When using ``cv = 'prefit'`` you need to make sure to use a hold-out part of +your data for calibration. + +The strategies, controlled by the parameter ``strategy``, for finding +appropriate decision thresholds are based either on precision recall estimates +or true positive and true negative rates. Specifically: .. currentmodule:: sklearn.metrics * ``f_beta`` - selects a decision threshold that maximizes the :func:`fbeta_score`. The value of - beta is specified by the parameter ``beta``. The ``beta`` parameter determines - the weight of precision. When ``beta = 1`` both precision recall get the same - weight therefore the maximization target in this case is the :func:`f1_score`. - if ``beta < 1`` more weight is given to precision whereas if ``beta > 1`` more - weight is given to recall. + selects a decision threshold that maximizes the :func:`fbeta_score`. The + value of beta is specified by the parameter ``beta``. The ``beta`` parameter + determines the weight of precision. When ``beta = 1`` both precision recall + get the same weight therefore the maximization target in this case is the + :func:`f1_score`. if ``beta < 1`` more weight is given to precision whereas + if ``beta > 1`` more weight is given to recall. * ``roc`` - selects the decision threshold for the point on the :func:`roc_curve` that is - closest to the ideal corner (0, 1) + selects the decision threshold for the point on the :func:`roc_curve` that + is closest to the ideal corner (0, 1) * ``max_tpr`` selects the decision threshold for the point that yields the highest true @@ -278,20 +282,24 @@ Here is a simple usage example:: >>> from sklearn.calibration import CutoffClassifier >>> from sklearn.datasets import load_breast_cancer >>> from sklearn.naive_bayes import GaussianNB + >>> from sklearn.metrics import precision_score >>> from sklearn.model_selection import train_test_split - >>> + >>> X, y = load_breast_cancer(return_X_y=True) >>> X_train, X_test, y_train, y_test = train_test_split( ... X, y, train_size=0.6, random_state=42) - >>> clf = CutoffClassifier(GaussianNB(), method='roc', cv=3).fit( - ... X_train, y_train) - + >>> clf = CutoffClassifier(GaussianNB(), strategy='f_beta', beta=0.6, + ... cv=3).fit(X_train, y_train) + >>> y_pred = clf.predict(X_test) + >>> precision_score(y_test, y_pred) # doctest: +ELLIPSIS + 0.95945945945945943 .. topic:: Examples: * :ref:`sphx_glr_auto_examples_calibration_plot_decision_threshold_calibration.py`: Decision threshold calibration on the breast cancer dataset +.. currentmodule:: sklearn.calibration The following image shows the results of using the :class:`CutoffClassifier` for finding a decision threshold for a :class:`LogisticRegression` classifier @@ -301,10 +309,10 @@ and an :class:`AdaBoostClassifier` for two use cases. :target: ../auto_examples/calibration/plot_decision_threshold_calibration.html :align: center -In the first case we want to increase the overall accuracy of the classifiers on -the breast cancer dataset. In the second case we want to find a decision threshold -that yields maximum true positive rate while maintaining a minimum value -for the true negative rate. +In the first case we want to increase the overall accuracy of the classifier on +the breast cancer dataset. In the second case we want to find a decision +threshold that yields maximum true positive rate while maintaining a minimum +value for the true negative rate. .. topic:: References: @@ -315,5 +323,6 @@ for the true negative rate. Notes ----- -Calibrating the decision threshold of a classifier does not guarantee increased performance. -The generalisation ability of the obtained decision threshold has to be evaluated. +Calibrating the decision threshold of a classifier does not guarantee increased +performance. The generalisation ability of the obtained decision threshold has +to be evaluated. diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index 71c3c3d2a0696..a60aa4ac90558 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -59,6 +59,7 @@ lr = LogisticRegression().fit( X_train[:-calibration_samples], y_train[:-calibration_samples]) + y_pred_lr = lr.predict(X_test) tn_lr, fp_lr, fn_lr, tp_lr = confusion_matrix(y_test, y_pred_lr).ravel() tpr_lr = tp_lr / (tp_lr + fn_lr) @@ -67,6 +68,7 @@ ada = AdaBoostClassifier().fit( X_train[:-calibration_samples], y_train[:-calibration_samples]) + y_pred_ada = ada.predict(X_test) tn_ada, fp_ada, fn_ada, tp_ada = confusion_matrix(y_test, y_pred_ada).ravel() tpr_ada = tp_ada / (tp_ada + fn_ada) @@ -76,22 +78,25 @@ # objective 1: we want to calibrate the decision threshold in order to achieve # better f1 score lr_f_beta = CutoffClassifier( - lr, method='f_beta', strategy='predict_proba', beta=1, cv='prefit' -).fit(X_train[calibration_samples:], y_train[calibration_samples:]) + lr, strategy='f_beta', method='predict_proba', beta=1, cv='prefit').fit( + X_train[calibration_samples:], y_train[calibration_samples:]) + y_pred_lr_f_beta = lr_f_beta.predict(X_test) f_one_lr_f_beta = f1_score(y_test, y_pred_lr_f_beta) ada_f_beta = CutoffClassifier( - ada, method='f_beta', strategy='predict_proba', beta=1, cv='prefit' + ada, strategy='f_beta', method='predict_proba', beta=1, cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) + y_pred_ada_f_beta = ada_f_beta.predict(X_test) f_one_ada_f_beta = f1_score(y_test, y_pred_ada_f_beta) # objective 2: we want to maximize the true positive rate while the true # negative rate is at least 0.7 lr_max_tpr = CutoffClassifier( - lr, method='max_tpr', strategy='predict_proba', threshold=0.7, cv='prefit' + lr, strategy='max_tpr', method='predict_proba', threshold=0.7, cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) + y_pred_lr_max_tpr = lr_max_tpr.predict(X_test) tn_lr_max_tpr, fp_lr_max_tpr, fn_lr_max_tpr, tp_lr_max_tpr = \ confusion_matrix(y_test, y_pred_lr_max_tpr).ravel() @@ -99,9 +104,9 @@ tnr_lr_max_tpr = tn_lr_max_tpr / (tn_lr_max_tpr + fp_lr_max_tpr) ada_max_tpr = CutoffClassifier( - ada, method='max_tpr', strategy='predict_proba', threshold=0.7, - cv='prefit' + ada, strategy='max_tpr', method='predict_proba', threshold=0.7, cv='prefit' ).fit(X_train[calibration_samples:], y_train[calibration_samples:]) + y_pred_ada_max_tpr = ada_max_tpr.predict(X_test) tn_ada_max_tpr, fp_ada_max_tpr, fn_ada_max_tpr, tp_ada_max_tpr = \ confusion_matrix(y_test, y_pred_ada_max_tpr).ravel() @@ -124,9 +129,9 @@ print('AdaBoost classifier: tpr = {}, tnr = {}, f1 = {}'.format( tpr_ada_max_tpr, tnr_ada_max_tpr, f_one_ada_f_beta)) -####### -# plots -####### +######### +# plots # +######### bar_width = 0.2 plt.subplot(2, 1, 1) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 49185e031bc33..a169158396a58 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -51,8 +51,8 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): according to the acquired cutoff point. The estimator must have a decision_function or a predict_proba - method : str - The method to use for choosing the cutoff point + strategy : str, optional (default='roc') + The strategy to use for choosing the cutoff point 'roc' selects the point on the roc curve that is closest to the ideal @@ -71,7 +71,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): true positive rate at least equal to the value of the parameter threshold - strategy : str or None, optional (default=None) + method : str or None, optional (default=None) The method to be used for acquiring the score 'decision_function' @@ -85,12 +85,12 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): available base_estimator.predict_proba beta : float in [0, 1], optional (default=None) - beta value to be used in case method == 'f_beta' + beta value to be used in case strategy == 'f_beta' threshold : float in [0, 1] or None, (default=None) - In case method is 'max_tpr' or 'max_tnr' this parameter must be set to - specify the threshold for the true negative rate or true positive rate - respectively that needs to be achieved + In case strategy is 'max_tpr' or 'max_tnr' this parameter must be set + to specify the threshold for the true negative rate or true positive + rate respectively that needs to be achieved pos_label : object, optional (default=1) Object representing the positive label @@ -120,11 +120,11 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Clinical chemistry, 1993 """ - def __init__(self, base_estimator, method='roc', strategy=None, beta=None, + def __init__(self, base_estimator, strategy='roc', method=None, beta=None, threshold=None, pos_label=1, cv=3): self.base_estimator = base_estimator - self.method = method self.strategy = strategy + self.method = method self.beta = beta self.threshold = threshold self.pos_label = pos_label @@ -151,16 +151,16 @@ def fit(self, X, y): raise TypeError('The base_estimator needs to implement either a ' 'decision_function or a predict_proba method') - if self.method not in ('roc', 'f_beta', 'max_tpr', 'max_tnr'): - raise ValueError('method can either be "roc" or "max_tpr" or ' - '"max_tnr. Got {} instead'.format(self.method)) + if self.strategy not in ('roc', 'f_beta', 'max_tpr', 'max_tnr'): + raise ValueError('strategy can either be "roc" or "max_tpr" or ' + '"max_tnr. Got {} instead'.format(self.strategy)) - if self.strategy not in (None, 'decision_function', 'predict_proba'): - raise ValueError('scoring param can either be "decision_function" ' + if self.method not in (None, 'decision_function', 'predict_proba'): + raise ValueError('method param can either be "decision_function" ' 'or "predict_proba" or None. ' - 'Got {} instead'.format(self.strategy)) + 'Got {} instead'.format(self.method)) - if self.method == 'max_tpr' or self.method == 'max_tnr': + if self.strategy == 'max_tpr' or self.strategy == 'max_tnr': if (not self.threshold or not isinstance(self.threshold, (int, float)) or not self.threshold >= 0 or not self.threshold <= 1): @@ -168,7 +168,7 @@ def fit(self, X, y): '[0, 1]. ' 'Got {} instead'.format(self.threshold)) - if self.method == 'f_beta': + if self.strategy == 'f_beta': if not self.beta or not isinstance(self.beta, (int, float)): raise ValueError('parameter beta must be a real number.' 'Got {} instead'.format(type(self.beta))) @@ -187,7 +187,7 @@ def fit(self, X, y): if self.cv == 'prefit': self.decision_threshold_ = _CutoffClassifier( - self.base_estimator, self.method, self.strategy, self.beta, + self.base_estimator, self.strategy, self.method, self.beta, self.threshold, self.pos_label ).fit(X, y).decision_threshold_ self.std_ = .0 @@ -198,7 +198,7 @@ def fit(self, X, y): for train, test in cv.split(X, y): estimator = clone(self.base_estimator).fit(X[train], y[train]) decision_thresholds.append( - _CutoffClassifier(estimator, self.method, self.strategy, + _CutoffClassifier(estimator, self.strategy, self.method, self.beta, self.threshold, self.pos_label).fit( X[test], y[test] @@ -227,7 +227,7 @@ def predict(self, X): check_is_fitted(self, ["label_encoder_", "decision_threshold_", "std_"]) - y_score = _get_binary_score(self.base_estimator, X, self.strategy, + y_score = _get_binary_score(self.base_estimator, X, self.method, self.pos_label) return self.label_encoder_.inverse_transform( (y_score > self.decision_threshold_).astype(int) @@ -249,22 +249,22 @@ class _CutoffClassifier(object): according to the acquired cutoff point. The estimator must have a decision_function or a predict_proba - method : 'roc' or 'f_beta' or 'max_tpr' or 'max_tnr' + strategy : 'roc' or 'f_beta' or 'max_tpr' or 'max_tnr' The method to use for choosing the cutoff point - strategy : str or None, optional (default=None) + method : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available predict_proba beta : float in [0, 1] - beta value to be used in case method == 'f_beta' + beta value to be used in case strategy == 'f_beta' threshold : float in [0, 1] minimum required value for the true negative rate (specificity) in case - method 'max_tpr' is used or for the true positive rate (sensitivity) in - case method 'max_tnr' is used + strategy 'max_tpr' is used or for the true positive rate (sensitivity) + in case method 'max_tnr' is used pos_label : object Label considered as positive during the roc_curve construction @@ -274,11 +274,11 @@ class _CutoffClassifier(object): decision_threshold_ : float Acquired decision threshold for the positive class """ - def __init__(self, base_estimator, method, strategy, beta, threshold, + def __init__(self, base_estimator, strategy, method, beta, threshold, pos_label): self.base_estimator = base_estimator - self.method = method self.strategy = strategy + self.method = method self.beta = beta self.threshold = threshold self.pos_label = pos_label @@ -300,9 +300,9 @@ def fit(self, X, y): self : object Instance of self """ - y_score = _get_binary_score(self.base_estimator, X, self.strategy, + y_score = _get_binary_score(self.base_estimator, X, self.method, self.pos_label) - if self.method == 'f_beta': + if self.strategy == 'f_beta': precision, recall, thresholds = precision_recall_curve( y, y_score, pos_label=self.pos_label ) @@ -313,13 +313,13 @@ def fit(self, X, y): fpr, tpr, thresholds = roc_curve(y, y_score, pos_label=self.pos_label) - if self.method == 'roc': + if self.strategy == 'roc': # we find the threshold of the point (fpr, tpr) with the smallest # euclidean distance from the "ideal" corner (0, 1) self.decision_threshold_ = thresholds[ np.argmin(fpr ** 2 + (tpr - 1) ** 2) ] - elif self.method == 'max_tpr': + elif self.strategy == 'max_tpr': indices = np.where(1 - fpr >= self.threshold)[0] max_tpr_index = np.argmax(tpr[indices]) self.decision_threshold_ = thresholds[indices[max_tpr_index]] @@ -330,7 +330,7 @@ def fit(self, X, y): return self -def _get_binary_score(clf, X, strategy=None, pos_label=1): +def _get_binary_score(clf, X, method=None, pos_label=1): """Binary classification score for the positive label (0 or 1) Returns the score that a binary classifier outputs for the positive label @@ -348,7 +348,7 @@ def _get_binary_score(clf, X, strategy=None, pos_label=1): pos_label : int, optional (default=1) The positive label. Can either be 0 or 1 - strategy : str or None, optional (default=None) + method : str or None, optional (default=None) The method to be used for acquiring the score. Can either be "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available @@ -359,19 +359,19 @@ def _get_binary_score(clf, X, strategy=None, pos_label=1): len(clf.classes_) )) - if strategy not in (None, 'decision_function', 'predict_proba'): + if method not in (None, 'decision_function', 'predict_proba'): raise ValueError('scoring param can either be "decision_function" ' 'or "predict_proba" or None. ' - 'Got {} instead'.format(strategy)) + 'Got {} instead'.format(method)) - if not strategy: + if not method: try: y_score = clf.decision_function(X) if pos_label == clf.classes_[0]: y_score = -y_score except (NotImplementedError, AttributeError): y_score = clf.predict_proba(X)[:, pos_label] - elif strategy == 'decision_function': + elif method == 'decision_function': y_score = clf.decision_function(X) if pos_label == clf.classes_[0]: y_score = - y_score diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 5bad3b8c443b7..a8200d3b4e7ca 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -38,7 +38,7 @@ def test_cutoff_prefit(): random_state=42) lr = LogisticRegression().fit(X_train, y_train) - clf_roc = CutoffClassifier(lr, method='roc', cv='prefit').fit( + clf_roc = CutoffClassifier(lr, strategy='roc', cv='prefit').fit( X_test[:calibration_samples], y_test[:calibration_samples] ) @@ -60,7 +60,7 @@ def test_cutoff_prefit(): assert_greater(tpr_roc + tnr_roc, tpr + tnr) clf_f1 = CutoffClassifier( - lr, method='f_beta', strategy='predict_proba', beta=1, + lr, strategy='f_beta', method='predict_proba', beta=1, cv='prefit').fit( X_test[:calibration_samples], y_test[:calibration_samples] ) @@ -70,7 +70,7 @@ def test_cutoff_prefit(): f1_score(y_test[calibration_samples:], y_pred)) clf_fbeta = CutoffClassifier( - lr, method='f_beta', strategy='predict_proba', beta=2, + lr, strategy='f_beta', method='predict_proba', beta=2, cv='prefit').fit( X_test[:calibration_samples], y_test[:calibration_samples] ) @@ -80,7 +80,7 @@ def test_cutoff_prefit(): recall_score(y_test[calibration_samples:], y_pred)) clf_max_tpr = CutoffClassifier( - lr, method='max_tpr', threshold=0.7, cv='prefit' + lr, strategy='max_tpr', threshold=0.7, cv='prefit' ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) y_pred_max_tpr = clf_max_tpr.predict(X_test[calibration_samples:]) @@ -97,7 +97,7 @@ def test_cutoff_prefit(): assert_greater_equal(tnr_max_tpr, 0.7) clf_max_tnr = CutoffClassifier( - lr, method='max_tnr', threshold=0.7, cv='prefit' + lr, strategy='max_tnr', threshold=0.7, cv='prefit' ).fit(X_test[:calibration_samples], y_test[:calibration_samples]) y_pred_clf = clf_max_tnr.predict(X_test[calibration_samples:]) @@ -123,14 +123,14 @@ def test_cutoff_prefit(): ) assert_raises(ValueError, clf_roc.fit, X_non_binary, y_non_binary) - clf_foo = CutoffClassifier(lr, method='f_beta', beta='foo') + clf_foo = CutoffClassifier(lr, strategy='f_beta', beta='foo') assert_raises(ValueError, clf_foo.fit, X_train, y_train) - clf_foo = CutoffClassifier(lr, method='foo') + clf_foo = CutoffClassifier(lr, strategy='foo') assert_raises(ValueError, clf_foo.fit, X_train, y_train) for method in ['max_tpr', 'max_tnr']: - clf_missing_info = CutoffClassifier(lr, method=method) + clf_missing_info = CutoffClassifier(lr, strategy=method) assert_raises(ValueError, clf_missing_info.fit, X_train, y_train) @@ -142,7 +142,7 @@ def test_cutoff_cv(): train_size=0.6, random_state=42) lr = LogisticRegression().fit(X_train, y_train) - clf_roc = CutoffClassifier(LogisticRegression(), method='roc', cv=3).fit( + clf_roc = CutoffClassifier(LogisticRegression(), strategy='roc', cv=3).fit( X_train, y_train ) @@ -176,37 +176,37 @@ def test_get_binary_score(): assert_array_equal( y_pred_score, _get_binary_score( - lr, X_test, strategy='decision_function', pos_label=1) + lr, X_test, method='decision_function', pos_label=1) ) assert_array_equal( - y_pred_score, _get_binary_score( - lr, X_test, strategy='decision_function', pos_label=0) + lr, X_test, method='decision_function', pos_label=0) ) assert_array_equal( y_pred_proba[:, 1], _get_binary_score( - lr, X_test, strategy='predict_proba', pos_label=1) + lr, X_test, method='predict_proba', pos_label=1) ) assert_array_equal( y_pred_proba[:, 0], _get_binary_score( - lr, X_test, strategy='predict_proba', pos_label=0) + lr, X_test, method='predict_proba', pos_label=0) ) assert_array_equal( y_pred_score, - _get_binary_score(lr, X_test, strategy=None, pos_label=1) + _get_binary_score(lr, X_test, method=None, pos_label=1) ) - assert_raises(ValueError, _get_binary_score, lr, X_test, strategy='foo') + assert_raises(ValueError, _get_binary_score, lr, X_test, method='foo') # classifier that does not have a decision_function rf = RandomForestClassifier().fit(X_train, y_train) y_pred_proba_rf = rf.predict_proba(X_test) assert_array_equal( y_pred_proba_rf[:, 1], - _get_binary_score(rf, X_test, strategy=None, pos_label=1) + _get_binary_score(rf, X_test, method=None, pos_label=1) ) X_non_binary, y_non_binary = make_classification( From 9260a5f8cafd7e5a544e65e07a892a5fb97b64b4 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Wed, 2 May 2018 21:15:33 +0200 Subject: [PATCH 087/100] fix typo --- examples/calibration/plot_decision_threshold_calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index a60aa4ac90558..5f0a92b8f9ad5 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -15,7 +15,7 @@ In this example the decision threshold calibration is applied on two classifiers trained on the breast cancer dataset. The goal in the first case is -to maximize the f1 score of the classifiers whereas in the second the goals is +to maximize the f1 score of the classifiers whereas in the second the goal is to maximize the true positive rate while maintaining a minimum true negative rate. From 76927112df69274e15223e7bac8e8416e250a8be Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Wed, 2 May 2018 22:25:32 +0200 Subject: [PATCH 088/100] fix doctest --- doc/modules/calibration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 699784e7fb7a9..92d85d39f508e 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -292,7 +292,7 @@ Here is a simple usage example:: ... cv=3).fit(X_train, y_train) >>> y_pred = clf.predict(X_test) >>> precision_score(y_test, y_pred) # doctest: +ELLIPSIS - 0.95945945945945943 + 0.959... .. topic:: Examples: From d3a3f3b42f9282cfc9c1bf900d0f6ef7d0888ca9 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 17:43:37 +0200 Subject: [PATCH 089/100] fix typo --- examples/calibration/plot_decision_threshold_calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/calibration/plot_decision_threshold_calibration.py b/examples/calibration/plot_decision_threshold_calibration.py index 5f0a92b8f9ad5..e14e680380e17 100644 --- a/examples/calibration/plot_decision_threshold_calibration.py +++ b/examples/calibration/plot_decision_threshold_calibration.py @@ -9,7 +9,7 @@ an arbitrary decision threshold as defined by the model can be not ideal. The CutoffClassifier can be used to calibrate the decision threshold of a model -in order to increase the classifiers trustworthiness. Optimization objectives +in order to increase the classifier's trustworthiness. Optimization objectives during the decision threshold calibration can be the true positive and / or the true negative rate as well as the f beta score. From d44ae62ab8d75fd7cdcab0d845fed2bdcf846469 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 17:43:49 +0200 Subject: [PATCH 090/100] fix docstring --- sklearn/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index a169158396a58..ad6c69533f1da 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -96,7 +96,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Object representing the positive label cv : int, cross-validation generator, iterable or 'prefit', optional - (default='prefit'). Determines the cross-validation splitting strategy. + (default=3). Determines the cross-validation splitting strategy. If cv='prefit' the base estimator is assumed to be fitted and all data will be used for the calibration of the probability threshold From 17b9eec9e1cc31058c6c8d8be2c8e2d584a25418 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 18:09:14 +0200 Subject: [PATCH 091/100] check decision threshold and std --- sklearn/tests/test_calibration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index a8200d3b4e7ca..e436db0d5ed43 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -146,6 +146,9 @@ def test_cutoff_cv(): X_train, y_train ) + assert clf_roc.decision_threshold_ != 0 + assert clf_roc.std_ != 0 + y_pred = lr.predict(X_test) y_pred_roc = clf_roc.predict(X_test) From a0f00ec1cb21820c9bb478561707b010df1aa99b Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 18:30:56 +0200 Subject: [PATCH 092/100] replace assert helpers --- sklearn/tests/test_calibration.py | 47 ++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index e436db0d5ed43..92ff6d70f5d2c 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -3,6 +3,8 @@ # License: BSD 3 clause from __future__ import division + +import pytest import numpy as np from scipy import sparse from sklearn.model_selection import LeaveOneOut, train_test_split @@ -57,7 +59,7 @@ def test_cutoff_prefit(): tnr_roc = tn_roc / (tn_roc + fp_roc) # check that the sum of tpr and tnr has improved - assert_greater(tpr_roc + tnr_roc, tpr + tnr) + assert tpr_roc + tnr_roc > tpr + tnr clf_f1 = CutoffClassifier( lr, strategy='f_beta', method='predict_proba', beta=1, @@ -66,8 +68,8 @@ def test_cutoff_prefit(): ) y_pred_f1 = clf_f1.predict(X_test[calibration_samples:]) - assert_greater(f1_score(y_test[calibration_samples:], y_pred_f1), - f1_score(y_test[calibration_samples:], y_pred)) + assert (f1_score(y_test[calibration_samples:], y_pred_f1) > + f1_score(y_test[calibration_samples:], y_pred)) clf_fbeta = CutoffClassifier( lr, strategy='f_beta', method='predict_proba', beta=2, @@ -76,8 +78,8 @@ def test_cutoff_prefit(): ) y_pred_fbeta = clf_fbeta.predict(X_test[calibration_samples:]) - assert_greater(recall_score(y_test[calibration_samples:], y_pred_fbeta), - recall_score(y_test[calibration_samples:], y_pred)) + assert (recall_score(y_test[calibration_samples:], y_pred_fbeta) > + recall_score(y_test[calibration_samples:], y_pred)) clf_max_tpr = CutoffClassifier( lr, strategy='max_tpr', threshold=0.7, cv='prefit' @@ -92,9 +94,9 @@ def test_cutoff_prefit(): tnr_max_tpr = tn_max_tpr / (tn_max_tpr + fp_max_tpr) # check that the tpr increases with tnr >= min_val_tnr - assert_greater(tpr_max_tpr, tpr) - assert_greater(tpr_max_tpr, tpr_roc) - assert_greater_equal(tnr_max_tpr, 0.7) + assert tpr_max_tpr > tpr + assert tpr_max_tpr > tpr_roc + assert tnr_max_tpr >= 0.7 clf_max_tnr = CutoffClassifier( lr, strategy='max_tnr', threshold=0.7, cv='prefit' @@ -109,29 +111,34 @@ def test_cutoff_prefit(): tpr_clf_max_tnr = tp_clf / (tp_clf + fn_clf) # check that the tnr increases with tpr >= min_val_tpr - assert_greater(tnr_clf_max_tnr, tnr) - assert_greater(tnr_clf_max_tnr, tnr_roc) - assert_greater_equal(tpr_clf_max_tnr, 0.7) + assert tnr_clf_max_tnr > tnr + assert tnr_clf_max_tnr > tnr_roc + assert tpr_clf_max_tnr >= 0.7 # check error cases clf_bad_base_estimator = CutoffClassifier([]) - assert_raises(TypeError, clf_bad_base_estimator.fit, X_train, y_train) + with pytest.raises(TypeError): + clf_bad_base_estimator.fit(X_train, y_train) X_non_binary, y_non_binary = make_classification( n_samples=20, n_features=6, random_state=42, n_classes=4, n_informative=4 ) - assert_raises(ValueError, clf_roc.fit, X_non_binary, y_non_binary) + with pytest.raises(ValueError): + clf_roc.fit(X_non_binary, y_non_binary) clf_foo = CutoffClassifier(lr, strategy='f_beta', beta='foo') - assert_raises(ValueError, clf_foo.fit, X_train, y_train) + with pytest.raises(ValueError): + clf_foo.fit(X_train, y_train) clf_foo = CutoffClassifier(lr, strategy='foo') - assert_raises(ValueError, clf_foo.fit, X_train, y_train) + with pytest.raises(ValueError): + clf_foo.fit(X_train, y_train) for method in ['max_tpr', 'max_tnr']: clf_missing_info = CutoffClassifier(lr, strategy=method) - assert_raises(ValueError, clf_missing_info.fit, X_train, y_train) + with pytest.raises(ValueError): + clf_missing_info.fit(X_train, y_train) def test_cutoff_cv(): @@ -164,7 +171,7 @@ def test_cutoff_cv(): tnr_roc = tn_roc / (tn_roc + fp_roc) # check that the sum of tpr + tnr has improved - assert_greater(tpr_roc + tnr_roc, tpr + tnr) + assert tpr_roc + tnr_roc > tpr + tnr def test_get_binary_score(): @@ -202,7 +209,8 @@ def test_get_binary_score(): _get_binary_score(lr, X_test, method=None, pos_label=1) ) - assert_raises(ValueError, _get_binary_score, lr, X_test, method='foo') + with pytest.raises(ValueError): + _get_binary_score(lr, X_test, method='foo') # classifier that does not have a decision_function rf = RandomForestClassifier().fit(X_train, y_train) @@ -218,7 +226,8 @@ def test_get_binary_score(): ) rf_non_bin = RandomForestClassifier().fit(X_non_binary, y_non_binary) - assert_raises(ValueError, _get_binary_score, rf_non_bin, X_non_binary) + with pytest.raises(ValueError): + _get_binary_score(rf_non_bin, X_non_binary) @ignore_warnings From f8a07a2161e3137f24d21784571ef67f4b8bb0ed Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 18:41:59 +0200 Subject: [PATCH 093/100] update docstring --- sklearn/calibration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index ad6c69533f1da..5d0d69ab4f45b 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -353,6 +353,12 @@ def _get_binary_score(clf, X, method=None, pos_label=1): "decision_function" or "predict_proba" or None. If None then decision_function will be used first and if not available predict_proba + + Returns + ------- + y_score : array-like, shape (n_samples,) + The return value of the provided classifier's decision_function or + predict_proba depending on the method used. """ if len(clf.classes_) != 2: raise ValueError('Expected binary classifier. Found {} classes'.format( From ab6e3fd873381ba3f45adb3ba344165dc6a17efb Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 18:46:15 +0200 Subject: [PATCH 094/100] add classes_ --- sklearn/calibration.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 5d0d69ab4f45b..d4e0ad8cce105 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -113,6 +113,9 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): cross-validation iteration. If the base estimator is pre-trained then std_ = 0 + classes_ : array, shape (n_classes) + The class labels. + References ---------- .. [1] Receiver-operating characteristic (ROC) plots: a fundamental @@ -181,6 +184,7 @@ def fit(self, X, y): y_type)) self.label_encoder_ = LabelEncoder().fit(y) + self.classes_ = self.label_encoder_.classes_ y = self.label_encoder_.transform(y) self.pos_label = self.label_encoder_.transform([self.pos_label])[0] @@ -224,8 +228,9 @@ def predict(self, X): The predicted class """ X = check_array(X) - check_is_fitted(self, - ["label_encoder_", "decision_threshold_", "std_"]) + check_is_fitted( + self, ["label_encoder_", "decision_threshold_", "std_", "classes_"] + ) y_score = _get_binary_score(self.base_estimator, X, self.method, self.pos_label) From 67af28a61d3cc6bdec52f08cbd1118e1e326897f Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 19:09:44 +0200 Subject: [PATCH 095/100] make std None in prefit case --- sklearn/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/calibration.py b/sklearn/calibration.py index d4e0ad8cce105..16899fc45be5c 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -111,7 +111,7 @@ class CutoffClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): provided base estimator is not pre-trained and the decision_threshold_ is computed as the mean of the decision threshold of each cross-validation iteration. If the base estimator is pre-trained then - std_ = 0 + std_ = None classes_ : array, shape (n_classes) The class labels. @@ -194,7 +194,7 @@ def fit(self, X, y): self.base_estimator, self.strategy, self.method, self.beta, self.threshold, self.pos_label ).fit(X, y).decision_threshold_ - self.std_ = .0 + self.std_ = None else: cv = check_cv(self.cv, y, classifier=True) decision_thresholds = [] From 4e1f70a6ef5e19c3496bec84657906d3f41a52a1 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Thu, 10 May 2018 19:16:57 +0200 Subject: [PATCH 096/100] test for std not None --- sklearn/tests/test_calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 92ff6d70f5d2c..ed9887849c1a2 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -154,7 +154,7 @@ def test_cutoff_cv(): ) assert clf_roc.decision_threshold_ != 0 - assert clf_roc.std_ != 0 + assert clf_roc.std_ is not None and clf_roc.std_ != 0 y_pred = lr.predict(X_test) y_pred_roc = clf_roc.predict(X_test) From d1228ddb2b46abb42a9f08f5cb79342eeb15feb8 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Sun, 7 Oct 2018 20:44:50 +0200 Subject: [PATCH 097/100] use test_size in train_test_split --- sklearn/tests/test_calibration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index da4bd364d4d18..b7decb6636ee7 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -35,7 +35,7 @@ def test_cutoff_prefit(): n_classes=2) X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.6, + test_size=0.4, random_state=42) lr = LogisticRegression().fit(X_train, y_train) @@ -145,7 +145,7 @@ def test_cutoff_cv(): n_classes=2) X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.6, + test_size=0.4, random_state=42) lr = LogisticRegression().fit(X_train, y_train) clf_roc = CutoffClassifier(LogisticRegression(), strategy='roc', cv=3).fit( @@ -177,7 +177,7 @@ def test_get_binary_score(): X, y = make_classification(n_samples=200, n_features=6, random_state=42, n_classes=2) - X_train, X_test, y_train, _ = train_test_split(X, y, train_size=0.6, + X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.4, random_state=42) lr = LogisticRegression().fit(X_train, y_train) y_pred_proba = lr.predict_proba(X_test) From 8315204e3c7892c1b75ab39c2e07e81de1c98a5d Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Oct 2018 09:37:38 +0200 Subject: [PATCH 098/100] change solver to liblinear --- sklearn/tests/test_calibration.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index b7decb6636ee7..4269c86a46c1e 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -37,7 +37,7 @@ def test_cutoff_prefit(): X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=42) - lr = LogisticRegression().fit(X_train, y_train) + lr = LogisticRegression(solver='liblinear').fit(X_train, y_train) clf_roc = CutoffClassifier(lr, strategy='roc', cv='prefit').fit( X_test[:calibration_samples], y_test[:calibration_samples] @@ -147,8 +147,10 @@ def test_cutoff_cv(): X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=42) - lr = LogisticRegression().fit(X_train, y_train) - clf_roc = CutoffClassifier(LogisticRegression(), strategy='roc', cv=3).fit( + lr = LogisticRegression(solver='liblinear').fit(X_train, y_train) + clf_roc = CutoffClassifier(LogisticRegression(solver='liblinear'), + strategy='roc', + cv=3).fit( X_train, y_train ) @@ -179,7 +181,7 @@ def test_get_binary_score(): X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.4, random_state=42) - lr = LogisticRegression().fit(X_train, y_train) + lr = LogisticRegression(solver='liblinear').fit(X_train, y_train) y_pred_proba = lr.predict_proba(X_test) y_pred_score = lr.decision_function(X_test) From a995da62fdcf12142df71ef5c58f6026c836e299 Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Oct 2018 09:37:59 +0200 Subject: [PATCH 099/100] remove unused import --- sklearn/tests/test_calibration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 4269c86a46c1e..401f8723ec806 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -12,8 +12,7 @@ assert_greater, assert_almost_equal, assert_greater_equal, assert_array_equal, - assert_raises, - ignore_warnings) + assert_raises) from sklearn.datasets import make_classification, make_blobs from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import MultinomialNB From e36a892836c862a1e8bd4dbce736bda232ce44cd Mon Sep 17 00:00:00 2001 From: Prokopios Gryllos Date: Mon, 8 Oct 2018 10:10:33 +0200 Subject: [PATCH 100/100] fix n_estimators for RF --- sklearn/tests/test_calibration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py index 401f8723ec806..dfea4fbc76c7f 100644 --- a/sklearn/tests/test_calibration.py +++ b/sklearn/tests/test_calibration.py @@ -213,7 +213,7 @@ def test_get_binary_score(): _get_binary_score(lr, X_test, method='foo') # classifier that does not have a decision_function - rf = RandomForestClassifier().fit(X_train, y_train) + rf = RandomForestClassifier(n_estimators=10).fit(X_train, y_train) y_pred_proba_rf = rf.predict_proba(X_test) assert_array_equal( y_pred_proba_rf[:, 1], @@ -225,7 +225,8 @@ def test_get_binary_score(): n_informative=4 ) - rf_non_bin = RandomForestClassifier().fit(X_non_binary, y_non_binary) + rf_non_bin = RandomForestClassifier(n_estimators=10).fit(X_non_binary, + y_non_binary) with pytest.raises(ValueError): _get_binary_score(rf_non_bin, X_non_binary)