From 41f3f393135476b7e6f3ef1ec052036e66b78180 Mon Sep 17 00:00:00 2001 From: Noel Dawe Date: Sun, 3 Feb 2013 19:58:30 +0100 Subject: [PATCH 1/4] minor edits in docs --- doc/modules/ensemble.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ensemble.rst b/doc/modules/ensemble.rst index 79b220aea4884..4fba191e4fe8a 100644 --- a/doc/modules/ensemble.rst +++ b/doc/modules/ensemble.rst @@ -253,7 +253,7 @@ AdaBoost The module :mod:`sklearn.ensemble` implements the popular boosting algorithm known as AdaBoost. This algorithm was first introduced by Freud and Schapire -[FS1995]_ back in 1995. +[FS1995]_ in 1995. The core principle of AdaBoost is to fit a sequence of weak learners (i.e., models that are only slightly better than random guessing, such as small @@ -324,7 +324,7 @@ of decision trees). .. [ZZRH2009] J. Zhu, H. Zou, S. Rosset, T. Hastie. "Multi-class AdaBoost", 2009. - .. [D1997] H. Drucker. "Improving Regressor using Boosting Techniques", 1997. + .. [D1997] H. Drucker. "Improving Regressors using Boosting Techniques", 1997. .. [HTF2009] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical Learning Ed. 2", Springer, 2009. From be2394a219bf54b45d0d73cfa05a3cb3d6c7eac7 Mon Sep 17 00:00:00 2001 From: Noel Dawe Date: Sun, 3 Feb 2013 21:36:36 +0100 Subject: [PATCH 2/4] DOC: notes about examples and minor edits --- doc/modules/ensemble.rst | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/doc/modules/ensemble.rst b/doc/modules/ensemble.rst index 4fba191e4fe8a..9e493d9d0df93 100644 --- a/doc/modules/ensemble.rst +++ b/doc/modules/ensemble.rst @@ -251,9 +251,9 @@ the transformation performs an implicit, non-parametric density estimation. AdaBoost ======== -The module :mod:`sklearn.ensemble` implements the popular boosting algorithm -known as AdaBoost. This algorithm was first introduced by Freud and Schapire -[FS1995]_ in 1995. +The module :mod:`sklearn.ensemble.weight_boosting` implements the popular +boosting algorithm known as AdaBoost introduced in 1995 by Freud and +Schapire [FS1995]_. The core principle of AdaBoost is to fit a sequence of weak learners (i.e., models that are only slightly better than random guessing, such as small @@ -266,7 +266,7 @@ to each of the training samples. Initially, those weights are all set to original data. For each successive iteration, the sample weights are individually modified and the learning algorithm is reapplied to the reweighted data. At a given step, those training examples that were incorrectly predicted -by the boosting model induced at the previous step have their weights increased, +by the boosted model induced at the previous step have their weights increased, whereas the weights are decreased for those that were predicted correctly. As iterations proceed, examples that are difficult to predict receive ever-increasing influence. Each subsequent weak learner is thereby forced to @@ -306,15 +306,25 @@ The number of weak learners is controlled by the parameter ``n_estimators``. The the final combination. By default, weak learners are decision stumps. Different weak learners can be specified through the ``base_estimator`` parameter. The main parameters to tune to obtain good results are ``n_estimators`` and -the complexity of the base estimators (e.g., its depth ``max_depth`` in case -of decision trees). +the complexity of the base estimators (e.g., its depth ``max_depth`` or +minimum required number of samples at a leaf ``min_samples_leaf`` in case of +decision trees). .. topic:: Examples: - * :ref:`example_ensemble_plot_adaboost_hastie_10_2.py` - * :ref:`example_ensemble_plot_adaboost_multiclass.py` - * :ref:`example_ensemble_plot_adaboost_regression.py` - * :ref:`example_ensemble_plot_adaboost_twoclass.py` + * :ref:`example_ensemble_plot_adaboost_hastie_10_2.py` compares the + classification error of a decision stump, decision tree, and a boosted + decision stump using AdaBoost-SAMME and AdaBoost-SAMME.R. + + * :ref:`example_ensemble_plot_adaboost_multiclass.py` shows the performance + of AdaBoost-SAMME and AdaBoost-SAMME.R on a multi-class problem. + + * :ref:`example_ensemble_plot_adaboost_twoclass.py` shows the decision boundary + and decision function values for a non-linearly separable two-class problem + using AdaBoost-SAMME. + + * :ref:`example_ensemble_plot_adaboost_regression.py` demonstrates regression + with the AdaBoost.R2 algorithm. .. topic:: References From f4a283a16e1f2eddd7ebbffca6921ec18c7dc085 Mon Sep 17 00:00:00 2001 From: Gael Varoquaux Date: Sun, 3 Feb 2013 17:16:36 +0100 Subject: [PATCH 3/4] COSMIT: prettify code and figure in example --- examples/ensemble/plot_adaboost_twoclass.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/ensemble/plot_adaboost_twoclass.py b/examples/ensemble/plot_adaboost_twoclass.py index af0acc6eca2d8..18afe629f978e 100644 --- a/examples/ensemble/plot_adaboost_twoclass.py +++ b/examples/ensemble/plot_adaboost_twoclass.py @@ -37,10 +37,9 @@ y = np.concatenate((y1, - y2 + 1)) # Create and fit an AdaBoosted decision tree -bdt = AdaBoostClassifier( - DecisionTreeClassifier(max_depth=1), - algorithm="SAMME", - n_estimators=200) +bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), + algorithm="SAMME", + n_estimators=200) bdt.fit(X, y) @@ -68,7 +67,8 @@ pl.scatter(X[idx, 0], X[idx, 1], c=c, cmap=pl.cm.Paired, label="Class %s" % n) -pl.axis("tight") +pl.xlim(x_min, x_max) +pl.ylim(y_min, y_max) pl.legend(loc='upper right') pl.xlabel("Decision Boundary") From 1f95b18c2719a796f43992974821844ee965c3fd Mon Sep 17 00:00:00 2001 From: Gael Varoquaux Date: Mon, 4 Feb 2013 00:11:17 +0100 Subject: [PATCH 4/4] COSMIT --- examples/ensemble/plot_adaboost_regression.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/ensemble/plot_adaboost_regression.py b/examples/ensemble/plot_adaboost_regression.py index 506ba2b5d2190..92bdc0253d16b 100644 --- a/examples/ensemble/plot_adaboost_regression.py +++ b/examples/ensemble/plot_adaboost_regression.py @@ -27,10 +27,8 @@ clf_1 = DecisionTreeRegressor(max_depth=4) -clf_2 = AdaBoostRegressor( - DecisionTreeRegressor(max_depth=4), - n_estimators=300, - random_state=rng) +clf_2 = AdaBoostRegressor(DecisionTreeRegressor(max_depth=4), + n_estimators=300, random_state=rng) clf_1.fit(X, y) clf_2.fit(X, y)