diff --git a/sklearn/inspection/_partial_dependence.py b/sklearn/inspection/_partial_dependence.py index 98786550425bd..fa32ea6e69553 100644 --- a/sklearn/inspection/_partial_dependence.py +++ b/sklearn/inspection/_partial_dependence.py @@ -11,12 +11,14 @@ from sklearn.base import is_classifier, is_regressor from sklearn.ensemble import RandomForestRegressor +from sklearn.ensemble._forest import ForestClassifier, ForestRegressor from sklearn.ensemble._gb import BaseGradientBoosting from sklearn.ensemble._hist_gradient_boosting.gradient_boosting import ( BaseHistGradientBoosting, ) from sklearn.inspection._pd_utils import _check_feature_names, _get_feature_index from sklearn.tree import DecisionTreeRegressor +from sklearn.tree._classes import BaseDecisionTree from sklearn.utils import Bunch, _safe_indexing, check_array from sklearn.utils._indexing import ( _determine_key_type, @@ -218,6 +220,53 @@ def _partial_dependence_recursion(est, grid, features): return averaged_predictions +def _partial_dependence_tree_accurate(est, X, feature, grid): + """Calculate partial dependence for a single feature using tree_accurate. + + Performs a single O(m * D²) background pass over ``X`` per tree to collect + per-leaf statistics, then combines them with a vectorised pass over the grid. + Only ``kind='average'`` is supported. + + Parameters + ---------- + est : BaseDecisionTree, ForestRegressor or ForestClassifier + A fitted single tree (``DecisionTree*``, ``ExtraTree*``) or forest of + trees (``RandomForest*``, ``ExtraTrees*``). + X : ndarray of shape (n_samples, n_features), dtype=np.float32 + Background dataset used for marginalisation. + feature : int + Global column index of the target feature. + grid : ndarray of shape (n_grid,), dtype=np.float32 + Grid of values for the target feature. + + Returns + ------- + averaged_predictions : ndarray of shape (n_outputs, n_grid) + """ + m = X.shape[0] + + if is_classifier(est): + n_classes = est.n_classes_ + n_effective_outputs = 1 if n_classes == 2 else n_classes + else: + n_effective_outputs = est.n_outputs_ + + out = np.zeros((n_effective_outputs, len(grid)), dtype=np.float64) + + if isinstance(est, BaseDecisionTree): + est.tree_.compute_partial_dependence_tree_accurate(X, grid, feature, out) + out /= m + elif isinstance(est, (ForestRegressor, ForestClassifier)): + n_trees = len(est.estimators_) + for tree_est in est.estimators_: + tree_est.tree_.compute_partial_dependence_tree_accurate( + X, grid, feature, out + ) + out /= m * n_trees + + return out + + def _partial_dependence_brute( est, grid, features, X, response_method, sample_weight=None ): @@ -361,7 +410,7 @@ def _partial_dependence_brute( "response_method": [StrOptions({"auto", "predict_proba", "decision_function"})], "percentiles": [tuple], "grid_resolution": [Interval(Integral, 1, None, closed="left")], - "method": [StrOptions({"auto", "recursion", "brute"})], + "method": [StrOptions({"auto", "recursion", "brute", "tree_accurate"})], "kind": [StrOptions({"average", "individual", "both"})], "custom_values": [dict, None], }, @@ -487,7 +536,7 @@ def partial_dependence( .. versionadded:: 1.7 - method : {'auto', 'recursion', 'brute'}, default='auto' + method : {'auto', 'recursion', 'brute', 'tree_accurate'}, default='auto' The method used to calculate the averaged predictions: - `'recursion'` is only supported for some tree-based estimators @@ -510,6 +559,20 @@ def partial_dependence( - `'brute'` is supported for any estimator, but is more computationally intensive. + - `'tree_accurate'` is supported for the single-tree estimators + :class:`~sklearn.tree.DecisionTreeRegressor`, + :class:`~sklearn.tree.DecisionTreeClassifier`, + :class:`~sklearn.tree.ExtraTreeRegressor`, + :class:`~sklearn.tree.ExtraTreeClassifier` and the forests + :class:`~sklearn.ensemble.RandomForestRegressor`, + :class:`~sklearn.ensemble.RandomForestClassifier`, + :class:`~sklearn.ensemble.ExtraTreesRegressor`, + :class:`~sklearn.ensemble.ExtraTreesClassifier`, + when `kind='average'` and `response_method='decision_function'`. + Joint PDP (tuples in `features`) is not supported. + This method is equivalent to the `'brute'` method, + but significantly faster for tree-based estimators. + - `'auto'`: the `'recursion'` is used for estimators that support it, and `'brute'` is used otherwise. If `sample_weight` is not `None`, then `'brute'` is used regardless of the estimator. @@ -522,10 +585,10 @@ def partial_dependence( samples in the dataset or one value per sample or both. See Returns below. - Note that the fast `method='recursion'` option is only available for - `kind='average'` and `sample_weights=None`. Computing individual - dependencies and doing weighted averages requires using the slower - `method='brute'`. + Note that the fast `method='recursion'` and `method='tree_accurate'` + options are only available for `kind='average'`. + Computing individual dependencies and doing weighted averages requires + using the slower `method='brute'`. .. versionadded:: 0.24 @@ -601,6 +664,8 @@ def partial_dependence( raise ValueError( "The 'recursion' method only applies when 'kind' is set to 'average'" ) + if method == "tree_accurate": + raise ValueError("The 'tree_accurate' method only supports kind='average'.") method = "brute" if method == "recursion" and sample_weight is not None: @@ -608,6 +673,38 @@ def partial_dependence( "The 'recursion' method can only be applied when sample_weight is None." ) + if method == "tree_accurate": + _features_iter = [features] if isinstance(features, (str, int)) else features + if any(isinstance(f, (list, tuple)) for f in _features_iter): + raise ValueError( + "The 'tree_accurate' method does not support joint PDP " + "(tuples in features). Use method='brute' instead." + ) + if sample_weight is not None: + raise ValueError( + "The 'tree_accurate' method can only be applied when " + "sample_weight is None." + ) + if not isinstance( + estimator, (BaseDecisionTree, ForestRegressor, ForestClassifier) + ): + raise ValueError( + "The 'tree_accurate' method only supports DecisionTreeRegressor, " + "DecisionTreeClassifier, ExtraTreeRegressor, ExtraTreeClassifier, " + "RandomForestRegressor, RandomForestClassifier, " + "ExtraTreesRegressor and ExtraTreesClassifier. " + "Use method='brute' for other estimators." + ) + if is_classifier(estimator): + if response_method == "auto": + response_method = "decision_function" + if response_method != "decision_function": + raise ValueError( + "With the 'tree_accurate' method, the response_method for " + "classifiers must be 'decision_function'. " + "Got {}.".format(response_method) + ) + if method == "auto": if sample_weight is not None: method = "brute" @@ -750,6 +847,13 @@ def partial_dependence( predictions = predictions.reshape( -1, X.shape[0], *[val.shape[0] for val in values] ) + elif method == "tree_accurate": + X_bg = np.asarray(X, dtype=np.float32, order="C") + feature = int(features_indices[0]) + grid_1d = np.asarray(values[0], dtype=np.float32) + averaged_predictions = _partial_dependence_tree_accurate( + estimator, X_bg, feature, grid_1d + ) else: averaged_predictions = _partial_dependence_recursion( estimator, grid, features_indices diff --git a/sklearn/inspection/_plot/partial_dependence.py b/sklearn/inspection/_plot/partial_dependence.py index 958f988ff98ac..1418aa28a0908 100644 --- a/sklearn/inspection/_plot/partial_dependence.py +++ b/sklearn/inspection/_plot/partial_dependence.py @@ -432,6 +432,20 @@ def from_estimator( the average of the ICEs by design, it is not compatible with ICE and thus `kind` must be `'average'`. + - `'tree_accurate'` is supported for the single-tree estimators + :class:`~sklearn.tree.DecisionTreeRegressor`, + :class:`~sklearn.tree.DecisionTreeClassifier`, + :class:`~sklearn.tree.ExtraTreeRegressor`, + :class:`~sklearn.tree.ExtraTreeClassifier` and the forests + :class:`~sklearn.ensemble.RandomForestRegressor`, + :class:`~sklearn.ensemble.RandomForestClassifier`, + :class:`~sklearn.ensemble.ExtraTreesRegressor`, + :class:`~sklearn.ensemble.ExtraTreesClassifier`, + when `kind='average'` and `response_method='decision_function'`. + Joint PDP (tuples in `features`) is not supported. + This method is equivalent to the `'brute'` method, + but faster for tree-based estimators. + - `'brute'` is supported for any estimator, but is more computationally intensive. diff --git a/sklearn/inspection/tests/test_partial_dependence.py b/sklearn/inspection/tests/test_partial_dependence.py index 8de472c6e7114..1fb0c918fc0fb 100644 --- a/sklearn/inspection/tests/test_partial_dependence.py +++ b/sklearn/inspection/tests/test_partial_dependence.py @@ -38,7 +38,13 @@ StandardScaler, scale, ) -from sklearn.tree import DecisionTreeRegressor +from sklearn.tree import ( + DecisionTreeClassifier, + DecisionTreeRegressor, + ExtraTreeClassifier, + ExtraTreeRegressor, +) +from sklearn.tree._utils import SPLIT_CATEGORICAL_BITSET, SPLIT_CATEGORICAL_HASH from sklearn.tree.tests.test_tree import assert_is_subtree from sklearn.utils._testing import assert_allclose, assert_array_equal from sklearn.utils.fixes import _IS_32BIT @@ -1253,3 +1259,209 @@ def test_partial_dependence_empty_categorical_features(): partial_dependence( estimator=clf, X=iris.data, features=[0], categorical_features=[] ) + + +# ============================================================================= +# tree_accurate method tests +# ============================================================================= + + +def _assert_tree_accurate_matches_brute(est, X, feature, **pd_kwargs): + """Assert tree_accurate reproduces brute for one feature of ``X``. + + ``pd_kwargs`` is forwarded to both calls, so the two runs differ only in + ``method``. Returns the tree_accurate result so callers can assert more. + """ + result_ta = partial_dependence( + est, X, features=[feature], method="tree_accurate", **pd_kwargs + ) + result_br = partial_dependence( + est, X, features=[feature], method="brute", **pd_kwargs + ) + np.testing.assert_allclose( + result_ta["average"], + result_br["average"], + rtol=1e-7, + atol=1e-9, + err_msg=f"tree_accurate does not match brute for feature {feature}", + ) + return result_ta + + +@pytest.mark.parametrize("seed", range(3)) +@pytest.mark.parametrize( + "Estimator", + [ + DecisionTreeRegressor, + RandomForestRegressor, + ], +) +def test_tree_accurate_matches_brute(Estimator, seed): + """tree_accurate must give the same averaged predictions as brute.""" + rng = np.random.RandomState(seed) + n_samples, n_features = 200, 5 + X = rng.randn(n_samples, n_features).astype(np.float64) + y = rng.randn(n_samples) + + kwargs = dict(max_depth=4, random_state=seed) + if Estimator is RandomForestRegressor: + kwargs["n_estimators"] = 3 + kwargs["max_features"] = "sqrt" + est = Estimator(**kwargs).fit(X, y) + + for feature in range(n_features): + _assert_tree_accurate_matches_brute(est, X, feature, grid_resolution=20) + + +def test_tree_accurate_repeated_feature_in_path(): + """tree_accurate must give correct values when a feature splits multiple + times on the same root-to-leaf path.""" + # Build a tree that is guaranteed to reuse feature 0 at multiple depths + # by restricting max_features so it can only choose feature 0. + rng = np.random.RandomState(42) + n_samples = 500 + # Single informative feature so the tree will reuse it + X = rng.randn(n_samples, 3).astype(np.float64) + y = X[:, 0] ** 2 # target depends on feature 0 only + + est = DecisionTreeRegressor(max_depth=6, random_state=0).fit(X, y) + + _assert_tree_accurate_matches_brute(est, X, 0, grid_resolution=30) + + +def test_tree_accurate_kind_not_average_and_sample_weight_raise(): + """kind != 'average' and sample_weight != None must raise ValueError.""" + rng = np.random.RandomState(0) + X = rng.randn(50, 3).astype(np.float64) + y = rng.randn(50) + est = DecisionTreeRegressor(max_depth=3, random_state=0).fit(X, y) + + for kind in ("individual", "both"): + with pytest.raises(ValueError, match="'tree_accurate' method only supports"): + partial_dependence(est, X, features=[0], method="tree_accurate", kind=kind) + + with pytest.raises(ValueError, match="'tree_accurate' method can only be applied"): + partial_dependence( + est, X, features=[0], method="tree_accurate", sample_weight=np.ones(50) + ) + + +def test_tree_accurate_unsupported_estimator_raises(): + """Non-tree estimators must raise ValueError.""" + X, y = make_regression(n_samples=50, random_state=0) + est = LinearRegression().fit(X, y) + + with pytest.raises(ValueError, match="'tree_accurate' method only supports"): + partial_dependence(est, X, features=[0], method="tree_accurate") + + +def test_tree_accurate_binary_classifier_matches_brute(): + """Binary DecisionTreeClassifier: tree_accurate PDP must match brute.""" + X, y = make_classification(n_samples=100, n_features=4, random_state=0) + est = DecisionTreeClassifier(max_depth=3, random_state=0).fit(X, y) + _assert_tree_accurate_matches_brute(est, X, 0, grid_resolution=10) + + +def test_tree_accurate_multiclass_classifier_matches_brute(): + """Multiclass DecisionTreeClassifier: tree_accurate PDP must match brute.""" + X, y = make_classification( + n_samples=100, + n_features=4, + n_classes=3, + n_informative=3, + n_redundant=0, + random_state=0, + ) + est = DecisionTreeClassifier(max_depth=3, random_state=0).fit(X, y) + _assert_tree_accurate_matches_brute(est, X, 0, grid_resolution=10) + + +def test_tree_accurate_multi_output_matches_brute(): + """Multi-output DecisionTreeRegressor: tree_accurate must match brute per output.""" + rng = np.random.RandomState(42) + X = rng.randn(120, 4).astype(np.float64) + # Two independent regression targets. + Y = np.column_stack([X[:, 0] + X[:, 1], X[:, 2] - X[:, 3]]) + + est = DecisionTreeRegressor(max_depth=4, random_state=0).fit(X, Y) + assert est.n_outputs_ == 2 + + result_accurate = _assert_tree_accurate_matches_brute(est, X, 0, grid_resolution=10) + + # Both methods should return shape (n_outputs, n_grid) = (2, 10). + assert result_accurate["average"].shape == (2, 10) + + +def _make_categorical_data(seed=0, n_samples=2000): + """Data whose first column is an unordered category with a non-monotone effect. + + The effect is deliberately not monotone in the category code, so a split test + that compares the code against a numeric threshold cannot reproduce it. + """ + rng = np.random.RandomState(seed) + category = rng.randint(0, 6, n_samples) + effect = np.array([0.0, 5.0, 1.0, 6.0, 2.0, 7.0]) + x1 = rng.normal(size=n_samples) + X = np.column_stack([category.astype(float), x1]) + y = effect[category] + 2.0 * x1 + 0.01 * rng.normal(size=n_samples) + return X, y + + +def _assert_split_kinds_present(est, expected_kind): + """Guard that the fitted tree really contains the split kind under test.""" + tree = est.tree_ if hasattr(est, "tree_") else est.estimators_[0].tree_ + internal = tree.children_left != -1 + assert expected_kind in set(np.asarray(tree.split_kind)[internal]) + + +@pytest.mark.parametrize( + "estimator_cls, make_target, split_kind, n_outputs", + [ + pytest.param( + DecisionTreeRegressor, + lambda y: y, + SPLIT_CATEGORICAL_BITSET, + 1, + id="bitset-category-as-target", + ), + pytest.param( + ExtraTreeRegressor, + lambda y: y, + SPLIT_CATEGORICAL_HASH, + 1, + id="hash-regressor", + ), + pytest.param( + DecisionTreeClassifier, + lambda y: (y > np.median(y)).astype(int), + SPLIT_CATEGORICAL_BITSET, + 1, + id="bitset-binary-classifier", + ), + pytest.param( + ExtraTreeClassifier, + lambda y: np.digitize(y, np.quantile(y, [0.33, 0.66])), + SPLIT_CATEGORICAL_HASH, + 3, + id="hash-multiclass-classifier", + ), + ], +) +def test_tree_accurate_categorical_matches_brute( + estimator_cls, make_target, split_kind, n_outputs +): + """Categorical splits must be routed with the same split test as prediction. + + Covers both routing kinds, regression and classification, and the category + as the PDP target as well as a feature that is marginalised over. + ``splitter='best'`` rejects categorical features for multiclass, so the + multiclass case uses ExtraTreeClassifier and its hash routing. + """ + X, y = _make_categorical_data() + est = estimator_cls(categorical_features=[0], max_depth=8, random_state=0) + est.fit(X, make_target(y)) + _assert_split_kinds_present(est, split_kind) + + result = _assert_tree_accurate_matches_brute(est, X, 0, categorical_features=[0]) + result = _assert_tree_accurate_matches_brute(est, X, 1, categorical_features=[0]) + assert result["average"].shape[0] == n_outputs diff --git a/sklearn/tree/_tree.pyx b/sklearn/tree/_tree.pyx index d587400bba80a..278550aaa745c 100644 --- a/sklearn/tree/_tree.pyx +++ b/sklearn/tree/_tree.pyx @@ -1532,6 +1532,148 @@ cdef class Tree: raise ValueError("Total weight should be 1.0 but was %.9f" % total_weight) + def compute_partial_dependence_tree_accurate( + self, + float32_t[:, ::1] X_bg, + float32_t[::1] grid, + intp_t required_feature, + float64_t[:, ::1] out, + ): + """Partial dependence via background-data tree traversal (tree_accurate). + + Two traversals are performed, both routing through ``goes_left`` -- + the same split test as prediction -- so numeric, categorical and + missing-value routing stay consistent automatically: + + * Background pass -- the heavy, ``O(n_background * depth)`` part. Each + sample is routed through the tree, branching both ways at splits on + ``required_feature`` and following the sample everywhere else; the + number of arrivals per leaf is accumulated into ``count``. + ``count[leaf]`` is thus the number of background samples routed to + that leaf when ``required_feature`` is treated as a wildcard. + + * Grid pass -- ``O(n_grid * node_count)``. Each grid value is routed + through the tree, following it at ``required_feature`` splits and + branching both ways elsewhere; ``value * count`` is accumulated at + every leaf reached. The result for grid value ``g`` equals + ``n_background * PDP(g)``. + + ``required_feature`` of each sample is never read -- marginalising it is + exactly the both-ways branching above. Working memory is + ``O(node_count)``. + + Accumulates raw sums into ``out``; the caller divides by the number of + background samples (and, for forests, by n_estimators). + + Parameters + ---------- + X_bg : float32 C-contiguous array of shape (n_background, n_features) + Background dataset used for marginalisation. + grid : float32 C-contiguous array of shape (n_grid,) + Grid values for the required feature. + required_feature : intp + Global index of the feature whose PDP is being computed. + out : float64 array of shape (n_outputs, n_grid) + Output array; sums are accumulated in-place. + """ + cdef: + intp_t n_grid = grid.shape[0] + intp_t n_background = X_bg.shape[0] + intp_t _TREE_LEAF_SENTINEL = TREE_LEAF + intp_t K = out.shape[0] + intp_t stack_size, sample_idx, node_idx, j, k, cnt + float32_t g + bint go_left + Node* node + float64_t[:, ::1] node_values + + if n_background == 0 or n_grid == 0 or self.node_count == 0: + return + + node_values = self._tree_accurate_node_values() + + # ---- Pass A: wildcard routing counts (heavy, m-dependent) ----------- + cdef intp_t[::1] count = np.zeros(self.node_count, dtype=np.intp) + cdef intp_t stack_capacity = 2 * (self.max_depth + 2) + 4 + cdef intp_t[::1] stack_node = np.empty(stack_capacity, dtype=np.intp) + + for sample_idx in range(n_background): + stack_size = 1 + stack_node[0] = 0 + while stack_size > 0: + stack_size -= 1 + node_idx = stack_node[stack_size] + node = &self.nodes[node_idx] + if node.left_child == _TREE_LEAF_SENTINEL: + count[node_idx] += 1 + elif node.feature == required_feature: + stack_node[stack_size] = node.left_child + stack_size += 1 + stack_node[stack_size] = node.right_child + stack_size += 1 + else: + go_left = goes_left( + node.threshold, + node.left_cat_bitset, + node.missing_go_to_left, + node.split_kind, + X_bg[sample_idx, node.feature], + ) + if go_left: + stack_node[stack_size] = node.left_child + else: + stack_node[stack_size] = node.right_child + stack_size += 1 + + # ---- Pass B: per grid value, accumulate value * count --------------- + for j in range(n_grid): + g = grid[j] + stack_size = 1 + stack_node[0] = 0 + while stack_size > 0: + stack_size -= 1 + node_idx = stack_node[stack_size] + node = &self.nodes[node_idx] + if node.left_child == _TREE_LEAF_SENTINEL: + cnt = count[node_idx] + if cnt != 0: + for k in range(K): + out[k, j] += node_values[node_idx, k] * cnt + elif node.feature == required_feature: + go_left = goes_left( + node.threshold, + node.left_cat_bitset, + node.missing_go_to_left, + node.split_kind, + g, + ) + if go_left: + stack_node[stack_size] = node.left_child + else: + stack_node[stack_size] = node.right_child + stack_size += 1 + else: + stack_node[stack_size] = node.left_child + stack_size += 1 + stack_node[stack_size] = node.right_child + stack_size += 1 + + def _tree_accurate_node_values(self): + """Per-node output values matching predict / decision_function. + + Returns a C-contiguous ``(node_count, K)`` float64 array: regression + values, all class probabilities for multiclass, or the positive-class + probability only for binary classification. + """ + value_3d = np.asarray(self._get_value_ndarray()) + if self.max_n_classes > 1: + class_counts = value_3d[:, 0, :] + probs = class_counts / class_counts.sum(axis=1, keepdims=True) + if self.max_n_classes == 2: + return np.ascontiguousarray(probs[:, 1:2]) + return np.ascontiguousarray(probs) + return np.ascontiguousarray(value_3d[:, :, 0]) + def _check_n_classes(n_classes, expected_dtype): if n_classes.ndim != 1: