From 29ca13348c3ee92df19801d2ac18f7165ae77344 Mon Sep 17 00:00:00 2001 From: Matthew Mulholland Date: Mon, 18 Jul 2022 13:48:54 -0400 Subject: [PATCH 1/2] Integrate feature preprocessor as step in SKLL learner pipeline and also save pipeline separately --- rsmtool/modeler.py | 102 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 4 deletions(-) diff --git a/rsmtool/modeler.py b/rsmtool/modeler.py index f36641a1d..142189b03 100644 --- a/rsmtool/modeler.py +++ b/rsmtool/modeler.py @@ -8,17 +8,22 @@ :organization: ETS """ +import copy import logging import pickle from math import log10, sqrt from os.path import join +import warnings +import joblib import numpy as np import pandas as pd import statsmodels.api as sm from numpy.random import RandomState from scipy.optimize import nnls +from sklearn.base import BaseEstimator, TransformerMixin from sklearn.linear_model import LassoCV +from sklearn.pipeline import Pipeline from skll.data import FeatureSet from skll.learner import Learner @@ -30,6 +35,75 @@ from .writer import DataWriter +class RSMToolFeaturePreprocessor(BaseEstimator, TransformerMixin): + """ + `scikit-learn`-compatible feature processor that uses RSMTool. + + This processor can be fit with some training data and then used to + transform new features as part of a pipeline. The input and output + of the `transform` function are iterables of JSON feature + dictionaries. + """ + + def __init__(self, standardize_features=True): + """ + Initialize object with list of feature names, etc. + + Parameters + ---------- + standardize_features : bool, optional + Whether or not to standardize feature values + """ + + self.standardize_features = standardize_features + self.processor = FeaturePreprocessor() + + def fit(self, X=None, y=None, feature_info=None): + """ + Fit the feature preprocessor with some training data. + + Parameters + ---------- + X : iterable of feature dictionaries + Not used, only here for compatibility + y : iterable of target variable values + Not used, only here for compatibility + feature_info : pd.DataFrame + Feature info frame + """ + + self.df_feature_info = feature_info.copy() + self.df_feature_info.set_index("feature", inplace=True) + return self + + def transform(self, X, y=None): + """ + Parameters + ---------- + X : iterable of feature dictionaries + Feature dictionaries + y : None + This parameter exists only to conform with the API. It is + not used. + + Returns + ------- + List[Dict[str, Any]] + iterable of transformed feature dictionaries + """ + + features = pd.DataFrame([{"spkitemid": i, **x} for i, x in enumerate(X)]) + with warnings.catch_warnings(): + warnings.simplefilter(action="ignore", category=FutureWarning) + features, _ = self.processor.preprocess_new_data( + features, + self.df_feature_info, + standardize_features=self.standardize_features, + ) + del features["spkitemid"] + return features.to_dict(orient="records") + + class Modeler: """ Class to train model and generate predictions with built-in or SKLL models. @@ -256,6 +330,13 @@ def create_fake_skll_learner(self, df_coefficients): # now create its parameters from the coefficients from the built-in model learner.model.coef_ = learner.feat_vectorizer.transform(coefdict).toarray()[0] learner.model.intercept_ = intercept + learner.pipeline = Pipeline( + [ + ('vectorizer', copy.deepcopy(learner.feat_vectorizer)), + ('estimator', copy.deepcopy(learner.model)), + ] + ) + return learner def train_linear_regression(self, df_train, feature_columns): @@ -463,7 +544,7 @@ def train_lasso_fixed_lambda_then_lr(self, df_train, feature_columns): # note that 'alpha' in sklearn is different from this lambda # so we need to normalize looking at the sklearn objective equation p_alpha = p_lambda / len(df_train) - l_lasso = Learner('Lasso', model_kwargs={'alpha': p_alpha, 'positive': True}) + l_lasso = Learner('Lasso', model_kwargs={'alpha': p_alpha, 'positive': True}, pipeline=True) l_lasso.train(fs_train, grid_search=False) # get the feature names that have the non-zero coefficients @@ -716,7 +797,7 @@ def train_lasso_fixed_lambda_then_non_negative_lr(self, df_train, feature_column # note that 'alpha' in sklearn is different from this lambda # so we need to normalize looking at the sklearn objective equation p_alpha = p_lambda / len(df_train) - l_lasso = Learner('Lasso', model_kwargs={'alpha': p_alpha, 'positive': True}) + l_lasso = Learner('Lasso', model_kwargs={'alpha': p_alpha, 'positive': True}, pipeline=True) l_lasso.train(fs_train, grid_search=False) # get the feature names that have the non-zero coefficients @@ -805,7 +886,7 @@ def train_lasso_fixed_lambda(self, df_train, feature_columns): # note that 'alpha' in sklearn is different from this lambda # so we need to normalize looking at the sklearn objective equation alpha = p_lambda / len(df_train) - learner = Learner('Lasso', model_kwargs={'alpha': alpha, 'positive': True}) + learner = Learner('Lasso', model_kwargs={'alpha': alpha, 'positive': True}, pipeline=True) learner.train(fs_train, grid_search=False) # convert this model's parameters to a data frame @@ -1112,7 +1193,8 @@ def train_skll_model(self, model_kwargs = custom_fixed_parameters if custom_fixed_parameters is not None else {} learner = Learner(model_name, model_kwargs=model_kwargs, - probability=predict_expected_scores) + probability=predict_expected_scores, + pipeline=True) # get the features, IDs, and labels from the given data frame feature_columns = [c for c in df_train.columns if c not in ['spkitemid', 'sc1']] @@ -1200,6 +1282,18 @@ def train(self, else: model = self.train_builtin_model(*args, **kwargs) + # Generate an `sklearn`-style feature preprocessor to be used + # later in the model's pipeline + processor = RSMToolFeaturePreprocessor(standardize_features=True) + processor.fit(feature_info=data_container.feature_info) + pipeline = Pipeline( + [("rsmtool_feature_preprocessor", processor)] + + list(model.pipeline.named_steps.items()) + ) + pipeline_file_path = join(filedir, '{}.pipeline.model'.format(experiment_id)) + with open(pipeline_file_path, "wb") as pipeline_file: + joblib.dump(pipeline, pipeline_file) + return model def predict(self, df, min_score, max_score, predict_expected=False): From 9c2b5469bd05639ef4fbdbe0890382b7e7f35109 Mon Sep 17 00:00:00 2001 From: Matthew Mulholland Date: Mon, 18 Jul 2022 15:11:09 -0400 Subject: [PATCH 2/2] Remove fit method in RSMToolFeaturePreprocessor and move what it did into the __init__ method --- rsmtool/modeler.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/rsmtool/modeler.py b/rsmtool/modeler.py index 142189b03..d3b4299aa 100644 --- a/rsmtool/modeler.py +++ b/rsmtool/modeler.py @@ -45,36 +45,22 @@ class RSMToolFeaturePreprocessor(BaseEstimator, TransformerMixin): dictionaries. """ - def __init__(self, standardize_features=True): + def __init__(self, feature_info, standardize_features=True): """ Initialize object with list of feature names, etc. Parameters ---------- + feature_info : pd.DataFrame + Frame containing RSMTool feature preprocessing information standardize_features : bool, optional Whether or not to standardize feature values """ self.standardize_features = standardize_features self.processor = FeaturePreprocessor() - - def fit(self, X=None, y=None, feature_info=None): - """ - Fit the feature preprocessor with some training data. - - Parameters - ---------- - X : iterable of feature dictionaries - Not used, only here for compatibility - y : iterable of target variable values - Not used, only here for compatibility - feature_info : pd.DataFrame - Feature info frame - """ - self.df_feature_info = feature_info.copy() self.df_feature_info.set_index("feature", inplace=True) - return self def transform(self, X, y=None): """ @@ -1284,8 +1270,9 @@ def train(self, # Generate an `sklearn`-style feature preprocessor to be used # later in the model's pipeline - processor = RSMToolFeaturePreprocessor(standardize_features=True) - processor.fit(feature_info=data_container.feature_info) + processor = RSMToolFeaturePreprocessor( + feature_info=data_container.feature_info, standardize_features=True + ) pipeline = Pipeline( [("rsmtool_feature_preprocessor", processor)] + list(model.pipeline.named_steps.items())