scikit-learn · thomasjpfan · Jan 11, 2022 · Jan 2, 2021 · Jan 2, 2021 · Apr 2, 2021
diff --git a/doc/whats_new/v1.1.rst b/doc/whats_new/v1.1.rst
@@ -210,6 +210,12 @@ Changelog
 :mod:`sklearn.ensemble`
 .......................
 
+- |Enhancement| :class:`ensemble.HistGradientBoostingClassifier` is faster,
+  for binary and in particular for multiclass problems thanks to the new private loss
+  function module.
+  :pr:`20811`, :pr:`20567` and :pr:`21814` by
+  :user:`Christian Lorentzen <lorentzenchr>`.
+
 - |API| Changed the default of :func:`max_features` to 1.0 for
   :class:`ensemble.RandomForestRegressor` and to `"sqrt"` for
   :class:`ensemble.RandomForestClassifier`. Note that these give the same fit

diff --git a/sklearn/_loss/loss.py b/sklearn/_loss/loss.py
@@ -427,7 +427,7 @@ def fit_intercept_only(self, y_true, sample_weight=None):
 
         Returns
         -------
-        raw_prediction : float or (n_classes,)
+        raw_prediction : numpy scalar or array of shape (n_classes,)
             Raw predictions of an intercept-only model.
         """
         # As default, take weighted average of the target over the samples
@@ -461,6 +461,57 @@ def constant_to_optimal_zero(self, y_true, sample_weight=None):
         """
         return np.zeros_like(y_true)
 
+    def init_gradient_and_hessian(self, n_samples, dtype=np.float64, order="F"):
+        """Initialize arrays for gradients and hessians.
+
+        Unless hessians are constant, arrays are initialized with undefined values.
+
+        Parameters
+        ----------
+        n_samples : int
+            The number of samples, usually passed to `fit()`.
+        dtype : {np.float64, np.float32}, default=np.float64
+            The dtype of the arrays gradient and hessian.
+        order : {'C', 'F'}, default='F'
+            Order of the arrays gradient and hessian. The default 'F' makes the arrays
+            contiguous along samples.
+
+        Returns
+        -------
+        gradient : C-contiguous array of shape (n_samples,) or array of shape \
+            (n_samples, n_classes)
+            Empty array (allocated but not initialized) to be used as argument
+            gradient_out.
+        hessian : C-contiguous array of shape (n_samples,), array of shape
+            (n_samples, n_classes) or shape (1,)
+            Empty (allocated but not initialized) array to be used as argument
+            hessian_out.
+            If constant_hessian is True (e.g. `HalfSquaredError`), the array is
+            initialized to ``1``.
+        """
+        if dtype not in (np.float32, np.float64):
+            raise ValueError(
+                "Valid options for 'dtype' are np.float32 and np.float64. "
+                f"Got dtype={dtype} instead."
+            )
+
+        if self.is_multiclass:
+            shape = (n_samples, self.n_classes)
+        else:
+            shape = (n_samples,)
+        gradient = np.empty(shape=shape, dtype=dtype, order=order)
+
+        if self.constant_hessian:
+            # If the hessians are constant, we consider them equal to 1.
+            # - This is correct for HalfSquaredError
+            # - For AbsoluteError, hessians are actually 0, but they are
+            #   always ignored anyway.
+            hessian = np.ones(shape=(1,), dtype=dtype)
+        else:
+            hessian = np.empty(shape=shape, dtype=dtype, order=order)
+
+        return gradient, hessian
+
 
 # Note: Naturally, we would inherit in the following order
 #         class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss)

diff --git a/sklearn/_loss/tests/test_loss.py b/sklearn/_loss/tests/test_loss.py
@@ -991,6 +991,63 @@ def test_predict_proba(loss):
             )
 
 
+@pytest.mark.parametrize("loss", ALL_LOSSES)
+@pytest.mark.parametrize("sample_weight", [None, "range"])
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
+@pytest.mark.parametrize("order", ("C", "F"))
+def test_init_gradient_and_hessians(loss, sample_weight, dtype, order):
+    """Test that init_gradient_and_hessian works as expected.
+
+    passing sample_weight to a loss correctly influences the constant_hessian
+    attribute, and consequently the shape of the hessian array.
+    """
+    n_samples = 5
+    if sample_weight == "range":
+        sample_weight = np.ones(n_samples)
+    loss = loss(sample_weight=sample_weight)
+    gradient, hessian = loss.init_gradient_and_hessian(
+        n_samples=n_samples,
+        dtype=dtype,
+        order=order,
+    )
+    if loss.constant_hessian:
+        assert gradient.shape == (n_samples,)
+        assert hessian.shape == (1,)
+    elif loss.is_multiclass:
+        assert gradient.shape == (n_samples, loss.n_classes)
+        assert hessian.shape == (n_samples, loss.n_classes)
+    else:
+        assert hessian.shape == (n_samples,)
+        assert hessian.shape == (n_samples,)
+
+    assert gradient.dtype == dtype
+    assert hessian.dtype == dtype
+
+    if order == "C":
+        assert gradient.flags.c_contiguous
+        assert hessian.flags.c_contiguous
+    else:
+        assert gradient.flags.f_contiguous
+        assert hessian.flags.f_contiguous
+
+
+@pytest.mark.parametrize("loss", ALL_LOSSES)
+@pytest.mark.parametrize(
+    "params, err_msg",
+    [
+        (
+            {"dtype": np.int64},
+            f"Valid options for 'dtype' are .* Got dtype={np.int64} instead.",
+        ),
+    ],
+)
+def test_init_gradient_and_hessian_raises(loss, params, err_msg):
+    """Test that init_gradient_and_hessian raises errors for invalid input."""
+    loss = loss()
+    with pytest.raises((ValueError, TypeError), match=err_msg):
+        gradient, hessian = loss.init_gradient_and_hessian(n_samples=5, **params)
+
+
 @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
 def test_loss_pickle(loss):
     """Test that losses can be pickled."""

diff --git a/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx b/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx